hl-valdemar/honey
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Wiki Activity

parse const decls

Browse Source
This commit is contained in:
Valdemar Høgsvig Lorenzen
2026-07-25 00:26:47 +02:00
parent c7f527e3c3
commit 90c7195d4b
28 changed files with 1452 additions and 1134 deletions
Download Patch File Download Diff File Expand all files Collapse all files
source/lexer/lexer.hon
+127 -57
View File
@@ -1,29 +1,52 @@
import "@std"
import "@std/mem"
import "@std/arraylist"
import "@std/strmap"
import "@std/enums/enummap"
import "@std/arraylist"
import "@std/debug"
import "@source/strpool"
ErrorCode :: enum {
invalid_character,
float_must_end_with_digit,
}
ErrorDetails :: struct {
name []u8
message []u8
}
error_msg_map std.EnumMap(ErrorCode, ErrorDetails) :: enummap.init({
invalid_character = ErrorDetails {
name = "L0",
message = "invalid character",
},
float_must_end_with_digit = ErrorDetails {
name = "L1",
message = "float must end with a digit",
},
})
keywords std.StringMap(TokenKind) :: strmap.init([
{ "proc", .proc },
{ "return", .return },
{ "if", .if },
{ "for", .for },
{ "else", .else },
{ "while", .while },
{ "proc", .proc },
{ "return", .return },
{ "if", .if },
{ "for", .for },
{ "else", .else },
{ "while", .while },
])
TokenIndex :: alias usize
TokenId :: distinct u32
ScanDiagnostic :: struct {
token TokenIndex
message []u8
Diagnostic :: struct {
code ErrorCode
token TokenId
}
State :: struct {
tokens std.ArrayList(Token)
diagnostics std.ArrayList(ScanDiagnostic)
diagnostics std.ArrayList(Diagnostic)
}
init proc(allocator mem.Allocator) State {
@@ -39,8 +62,8 @@ deinit proc(state @mut State) void {
}
scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternError) {
tokens :: &state.tokens
diagnostics :: &state.diagnostics
tokens :: &state.tokens
diagnostics :: &state.diagnostics
cursor usize = 0
while cursor < input.len {
@@ -48,9 +71,9 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE
# whitespace
if char == '\n' {
try arraylist.append(tokens, Token{ kind = .newline, start = cursor })
cursor += 1
continue
try add_token(tokens, Token{ kind = .newline, start = cursor })
cursor += 1
continue
} else if is_whitespace(char) {
cursor += 1
continue
@@ -58,10 +81,8 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE
# comments
if char == '#' {
# fixme(brolang): while currently requires curly braces but this should be legal
#while (cursor < input.len and input[cursor] != '\n') cursor += 1
while cursor < input.len and input[cursor] != '\n' : cursor += 1 {}
while (cursor < input.len and input[cursor] != '\n') cursor += 1
cursor += 1 # also skip newline
continue
}
@@ -75,7 +96,7 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE
is_alpha(input[cursor]) or
is_digit(input[cursor]) or
input[cursor] == '_'
)) : cursor += 1 {}
)) cursor += 1
kind :: strmap.get(&keywords, input[start..cursor]) orelse .ident
@@ -83,9 +104,9 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE
str_id :: if (kind == .ident)
try strpool.intern(&strpool.strings, input[start..cursor])
else
strpool.NoId
strpool.NO_ID
try arraylist.append(tokens, Token{
try add_token(tokens, Token{
kind = kind,
start = start,
str_id = str_id
@@ -99,7 +120,7 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE
has_decimal bool = false
# scan integer part
while cursor < input.len and is_digit(input[cursor]) : cursor += 1 {}
while (cursor < input.len and is_digit(input[cursor])) cursor += 1
# check for decimal point
if cursor < input.len and input[cursor] == '.' {
@@ -109,22 +130,17 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE
# assert that decimals follow the decimal point
if has_decimal and (cursor >= input.len or !is_digit(input[cursor])) {
token :: tokens.items.len
try arraylist.append(tokens, Token{ kind = .invalid, start = start })
try arraylist.append(diagnostics, ScanDiagnostic{
token = token,
message = "float must end with a digit",
})
continue
token :: token_id(tokens.items.len)
try arraylist.append(diagnostics, Diagnostic{ token = token, code = .float_must_end_with_digit })
try add_token(tokens, Token{ kind = .invalid, start = start })
continue
}
# scan decimal part
while cursor < input.len and is_digit(input[cursor]) : cursor += 1 {}
while (cursor < input.len and is_digit(input[cursor])) cursor += 1
if (has_decimal)
try arraylist.append(tokens, Token{ kind = .float, start = start })
else
try arraylist.append(tokens, Token{ kind = .int, start = start })
kind :: if (has_decimal) .float else .int
try add_token(tokens, Token{ kind = kind, start = start })
continue
}
@@ -135,15 +151,15 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE
cursor += 1
while cursor < input.len and input[cursor] != '"' and input[cursor] != '\n' : cursor += 1 {
# ignore escaped characters
if (input[cursor] == '\\' and cursor + 1 < input.len) cursor += 1
# ignore escaped characters
if (input[cursor] == '\\' and cursor + 1 < input.len) cursor += 1
}
if cursor < input.len and input[cursor] == '"' {
cursor += 1
try arraylist.append(tokens, Token{ kind = .string, start = start })
try add_token(tokens, Token{ kind = .string, start = start })
} else {
try arraylist.append(tokens, Token{ kind = .invalid, start = start })
try add_token(tokens, Token{ kind = .invalid, start = start })
}
continue
@@ -151,51 +167,95 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE
# mutable assignment
if char == '=' {
try arraylist.append(tokens, Token{ kind = .equal, start = cursor })
try add_token(tokens, Token{ kind = .equal, start = cursor })
cursor += 1
continue
}
# immutable assignment
if cursor + 1 < input.len and char == ':' and input[cursor + 1] == ':' {
try arraylist.append(tokens, Token{ kind = .double_colon, start = cursor })
cursor += 2
# immutable assignment or single colon
if char == ':' {
if cursor + 1 < input.len and input[cursor + 1] == ':' {
try add_token(tokens, Token{ kind = .double_colon, start = cursor })
cursor += 2
continue
}
try add_token(tokens, Token{ kind = .colon, start = cursor })
cursor += 1
continue
}
# parentheses
if char == '(' {
try arraylist.append(tokens, Token{ kind = .open_paren, start = cursor })
try add_token(tokens, Token{ kind = .open_paren, start = cursor })
cursor += 1
continue
} else if char == ')' {
try arraylist.append(tokens, Token{ kind = .close_paren, start = cursor })
try add_token(tokens, Token{ kind = .close_paren, start = cursor })
cursor += 1
continue
}
# curly braces
if char == '{' {
try arraylist.append(tokens, Token{ kind = .open_curly, start = cursor })
try add_token(tokens, Token{ kind = .open_curly, start = cursor })
cursor += 1
continue
} else if char == '}' {
try arraylist.append(tokens, Token{ kind = .close_curly, start = cursor })
try add_token(tokens, Token{ kind = .close_curly, start = cursor })
cursor += 1
continue
}
# invalid character
token :: tokens.items.len
try arraylist.append(tokens, Token{ kind = .invalid, start = cursor })
try arraylist.append(diagnostics, ScanDiagnostic{
token = token,
message = "invalid character",
})
token :: token_id(tokens.items.len)
try arraylist.append(diagnostics, Diagnostic{ token = token, code = .invalid_character })
try add_token(tokens, Token{ kind = .invalid, start = cursor })
cursor += 1
}
try arraylist.append(tokens, Token{ kind = .eof, start = cursor })
try add_token(tokens, Token{ kind = .eof, start = cursor })
}
#! returns the cursor position after scanning a number.
scan_number proc(start usize, input []u8) usize {
debug.assert(is_digit(input[start]))
has_decimal bool = false
cursor usize = start
# scan integer part
while cursor < input.len and is_digit(input[cursor]) : cursor += 1 {}
# check for decimal point
if cursor < input.len and input[cursor] == '.' {
has_decimal = true
cursor += 1
}
# assert that decimals follow the decimal point
debug.assert(!(has_decimal and (cursor >= input.len or !is_digit(input[cursor]))))
# scan decimal part
while cursor < input.len and is_digit(input[cursor]) : cursor += 1 {}
return cursor
}
#! returns the cursor position after scanning a string.
scan_string proc(start usize, input []u8) usize {
debug.assert(input[start] == '"')
cursor usize = start + 1
while cursor < input.len and input[cursor] != '"' and input[cursor] != '\n' : cursor += 1 {
# ignore escaped characters
if (input[cursor] == '\\' and cursor + 1 < input.len) cursor += 1
}
debug.assert(cursor < input.len and input[cursor] == '"')
cursor += 1
return cursor
}
hide is_whitespace proc(char u8) bool {
@@ -215,3 +275,13 @@ hide is_digit proc(char u8) bool {
else: false
}
}
hide add_token proc(tokens @mut std.ArrayList(Token), token Token) void ! mem.AllocError {
_ = token_id(tokens.items.len)
try arraylist.append(tokens, token)
}
hide token_id proc(idx uint) TokenId {
debug.assert(u64(idx) < u64(maxval!(TokenId)))
return TokenId(idx)
}
source/lexer/lexer.test.hon
+13 -14
View File
@@ -3,20 +3,19 @@ import "@std/mem"
import "@std/testing"
handles_keywords_identifiers_and_error_progress test {
strpool.strings = strpool.init(mem.c_allocator)
defer strpool.deinit(&strpool.strings)
strpool.strings = strpool.init(mem.c_allocator)
defer strpool.deinit(&strpool.strings)
state State = init(mem.c_allocator)
defer deinit(&state)
try scan(&state, "if name # comment\n@1.")
state State = init(mem.c_allocator)
defer deinit(&state)
try scan(&state, "if name # comment\n@1.")
try testing.expect_equal(6, state.tokens.items.len)
try testing.expect_equal(TokenKind.if, state.tokens.items[0].kind)
try testing.expect_equal(TokenKind.ident, state.tokens.items[1].kind)
try testing.expect_equal(TokenKind.newline, state.tokens.items[2].kind)
try testing.expect_equal(TokenKind.invalid, state.tokens.items[3].kind)
try testing.expect_equal(TokenKind.invalid, state.tokens.items[4].kind)
try testing.expect_equal(TokenKind.eof, state.tokens.items[5].kind)
try testing.expect_equal(2, state.diagnostics.items.len)
try testing.expect_equal("name", strpool.get_str(&strpool.strings, state.tokens.items[1].str_id)?)
try testing.expect_equal(5, state.tokens.items.len)
try testing.expect_equal(TokenKind.if, state.tokens.items[0].kind)
try testing.expect_equal(TokenKind.ident, state.tokens.items[1].kind)
try testing.expect_equal(TokenKind.invalid, state.tokens.items[2].kind)
try testing.expect_equal(TokenKind.invalid, state.tokens.items[3].kind)
try testing.expect_equal(TokenKind.eof, state.tokens.items[4].kind)
try testing.expect_equal(2, state.diagnostics.items.len)
try testing.expect_equal("name", strpool.get_str(&strpool.strings, state.tokens.items[1].str_id)?)
}
source/lexer/token.hon
+32 -28
View File
@@ -1,34 +1,38 @@
import "@source/strpool"
TokenKind :: enum {
if
else
for
while
proc
return
ident
int
float
string
equal
double_colon
open_paren
close_paren
open_curly
close_curly
newline
invalid
eof
}
Token :: struct {
kind TokenKind
start int
str_id strpool.StringId = strpool.NoId
str_id strpool.StringId = strpool.NO_ID
}
TokenKind :: enum {
# keywords
if, else
for, while
proc, return
# literals
ident, string
int, float
# comparison
equal_equal, not_equal
less, less_equal
greater, greater_equal
# assignment
equal, double_colon
# delimiters
open_paren, close_paren
open_bracket, close_bracket
open_curly, close_curly
colon
newline
# special
eof
invalid
}