hl-valdemar/honey
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Wiki Activity

refactor: encapsulate compact syntax tree storage

Browse Source
This commit is contained in:
Valdemar Høgsvig Lorenzen
2026-09-19 17:39:28 +02:00
parent e9c82a39aa
commit 93c238470f
9 changed files with 962 additions and 1136 deletions
Download Patch File Download Diff File Expand all files Collapse all files
source/lexer/lexer.hon
+84 -103
View File
@@ -1,9 +1,8 @@
import "@std"
import "@std/mem"
import "@std/strmap"
import "@std/enums/enummap"
import "@std/arraylist"
import "@source/diagnostics"
import "@source/strpool"
ScanError :: struct {
@@ -11,29 +10,15 @@ ScanError :: struct {
end usize
}
@hide:file
ErrorCode :: enum {
invalid_character,
float_must_end_with_digit,
unterminated_string,
}
SourceError :: enum { source_too_large }
ErrorDetails :: struct {
name []u8
message []u8
}
error_msg_map std.EnumMap(ErrorCode, ErrorDetails) :: enummap.init({
invalid_character = ErrorDetails {
name = "L0",
message = "invalid character",
},
float_must_end_with_digit = ErrorDetails {
name = "L1",
message = "float must end with a digit",
},
})
keywords std.StringMap(TokenKind) :: strmap.init([
keywords strmap.StringMap(TokenKind) :: strmap.init([
{ "proc", .proc },
{ "return", .return },
{ "if", .if },
@@ -42,38 +27,31 @@ keywords std.StringMap(TokenKind) :: strmap.init([
{ "while", .while },
])
Diagnostic :: struct {
token TokenId
code ErrorCode
}
State :: struct {
tokens std.ArrayList(Token)
diagnostics std.ArrayList(Diagnostic)
tokens arraylist.ArrayList(Token)
}
init proc(allocator mem.Allocator) State {
return State {
tokens = arraylist.init(allocator),
diagnostics = arraylist.init(allocator),
}
return State{ tokens = arraylist.init(allocator) }
}
deinit proc(state @mut State) void {
arraylist.deinit(&state.tokens)
arraylist.deinit(&state.diagnostics)
}
scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.InternError) {
diagnostics :: &state.diagnostics
scan proc(
state @mut State,
program []u8,
diagnostic_store @mut diagnostics.Store,
) void ! (mem.AllocError | strpool.InternError | diagnostics.CapacityError | SourceError) {
if (program.len > usize(maxval!(u32))) return .source_too_large
cursor := 0
while cursor < program.len {
char :: program[cursor]
# whitespace
if char == '\n' {
try add_token(state, Token{ start = cursor, kind = .newline })
try add_token(state, Token{ span = make_span(cursor, cursor + 1), kind = .newline })
cursor += 1
continue
} else if is_whitespace(char) {
@@ -84,7 +62,7 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter
# comments
if char == '#' {
while (cursor < program.len and program[cursor] != '\n') cursor += 1
cursor += 1 # also skip newline
if (cursor < program.len) cursor += 1
continue
}
@@ -93,9 +71,9 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter
start :: cursor
result :: scan_ident(start, program)
try add_token(state, Token{
start = start,
span = make_span(start, result.end),
kind = result.kind,
str_id = result.str_id,
data = TokenData{ str_id = result.str_id },
})
cursor = result.end
continue
@@ -105,15 +83,21 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter
if is_digit(char) {
start :: cursor
result :: scan_number(start, program) catch |err| {
token :: token_id(state.tokens.items.len)
try add_token(state, Token{ start = start, kind = .invalid })
try arraylist.append(diagnostics, Diagnostic{ token = token, code = err.code })
id :: try diagnostics.add(
diagnostic_store,
diagnostic_code(err.code),
make_span(start, err.end),
)
try add_token(state, Token{
span = make_span(start, err.end),
kind = .invalid,
data = TokenData{ diagnostic = id },
})
cursor = err.end
continue
}
kind :: if (result.has_decimal) .float else .int
try add_token(state, Token{ start = start, kind = kind })
try add_token(state, Token{ span = make_span(start, result.end), kind = kind })
cursor = result.end
continue
}
@@ -122,14 +106,20 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter
if char == '"' {
start :: cursor
result :: scan_string(start, program) catch |err| {
token :: token_id(state.tokens.items.len)
try add_token(state, Token{ start = start, kind = .invalid })
try arraylist.append(diagnostics, Diagnostic{ token = token, code = err.code })
id :: try diagnostics.add(
diagnostic_store,
diagnostic_code(err.code),
make_span(start, err.end),
)
try add_token(state, Token{
span = make_span(start, err.end),
kind = .invalid,
data = TokenData{ diagnostic = id },
})
cursor = err.end
continue
}
try add_token(state, Token{ start = start, kind = .string })
try add_token(state, Token{ span = make_span(start, result.end), kind = .string })
cursor = result.end
continue
}
@@ -137,18 +127,16 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter
# immutable decl, mutable decl, or single colon
if char == ':' {
if cursor + 1 < program.len and program[cursor + 1] == ':' {
try add_token(state, Token{ start = cursor, kind = .double_colon })
try add_token(state, Token{ span = make_span(cursor, cursor + 2), kind = .double_colon })
cursor += 2
continue
}
if cursor + 1 < program.len and program[cursor + 1] == '=' {
try add_token(state, Token{ start = cursor, kind = .colon_equal })
try add_token(state, Token{ span = make_span(cursor, cursor + 2), kind = .colon_equal })
cursor += 2
continue
}
try add_token(state, Token{ start = cursor, kind = .colon })
try add_token(state, Token{ span = make_span(cursor, cursor + 1), kind = .colon })
cursor += 1
continue
}
@@ -156,45 +144,41 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter
# compound arithmetic assignment
if char == '+' {
if cursor + 1 < program.len and program[cursor + 1] == '=' {
try add_token(state, Token{ start = cursor, kind = .plus_equal })
try add_token(state, Token{ span = make_span(cursor, cursor + 2), kind = .plus_equal })
cursor += 2
continue
}
try add_token(state, Token{ start = cursor, kind = .plus })
try add_token(state, Token{ span = make_span(cursor, cursor + 1), kind = .plus })
cursor += 1
continue
}
if char == '-' {
if cursor + 1 < program.len and program[cursor + 1] == '=' {
try add_token(state, Token{ start = cursor, kind = .minus_equal })
try add_token(state, Token{ span = make_span(cursor, cursor + 2), kind = .minus_equal })
cursor += 2
continue
}
try add_token(state, Token{ start = cursor, kind = .minus })
try add_token(state, Token{ span = make_span(cursor, cursor + 1), kind = .minus })
cursor += 1
continue
}
if char == '*' {
if cursor + 1 < program.len and program[cursor + 1] == '=' {
try add_token(state, Token{ start = cursor, kind = .star_equal })
try add_token(state, Token{ span = make_span(cursor, cursor + 2), kind = .star_equal })
cursor += 2
continue
}
try add_token(state, Token{ start = cursor, kind = .star })
try add_token(state, Token{ span = make_span(cursor, cursor + 1), kind = .star })
cursor += 1
continue
}
if char == '/' {
if cursor + 1 < program.len and program[cursor + 1] == '=' {
try add_token(state, Token{ start = cursor, kind = .slash_equal })
try add_token(state, Token{ span = make_span(cursor, cursor + 2), kind = .slash_equal })
cursor += 2
continue
}
try add_token(state, Token{ start = cursor, kind = .slash })
try add_token(state, Token{ span = make_span(cursor, cursor + 1), kind = .slash })
cursor += 1
continue
}
@@ -205,25 +189,33 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter
',': .comma
'(': .open_paren
')': .close_paren
'[': .open_bracket
']': .close_bracket
'{': .open_curly
'}': .close_curly
else: null
}
if (kind) |k| {
try add_token(state, Token{ start = cursor, kind = k })
if (kind) |value| {
try add_token(state, Token{ span = make_span(cursor, cursor + 1), kind = value })
cursor += 1
continue
}
# invalid character
token :: token_id(state.tokens.items.len)
try add_token(state, Token{ start = cursor, kind = .invalid })
try arraylist.append(diagnostics, Diagnostic{ token = token, code = .invalid_character })
id :: try diagnostics.add(
diagnostic_store,
.invalid_character,
make_span(cursor, cursor + 1),
)
try add_token(state, Token{
span = make_span(cursor, cursor + 1),
kind = .invalid,
data = TokenData{ diagnostic = id },
})
cursor += 1
}
try add_token(state, Token{ start = cursor, kind = .eof })
try add_token(state, Token{ span = make_span(cursor, cursor), kind = .eof })
}
ScanIdentResult :: struct {
@@ -234,7 +226,6 @@ ScanIdentResult :: struct {
scan_ident proc(start usize, program []u8) ScanIdentResult {
cursor := start + 1
# scan whole identifier
while (cursor < program.len and (
is_alpha(program[cursor]) or
@@ -242,19 +233,13 @@ scan_ident proc(start usize, program []u8) ScanIdentResult {
program[cursor] == '_'
)) cursor += 1
kind :: strmap.get(&keywords, program[start..cursor]) orelse .ident
# don't intern keywords (already O(1) lookup via token kind)
kind :: strmap.get(&keywords, program[start..cursor]) orelse .ident
str_id :: if (kind == .ident)
strpool.intern(&strpool.STRINGS, program[start..cursor]) catch strpool.NO_STR
else
strpool.NO_STR
return ScanIdentResult{
end = cursor,
kind = kind,
str_id = str_id,
}
return ScanIdentResult{ end = cursor, kind = kind, str_id = str_id }
}
ScanNumResult :: struct {
@@ -265,53 +250,36 @@ ScanNumResult :: struct {
scan_number proc(start usize, program []u8) ScanNumResult ! ScanError {
cursor := start
has_decimal := false
# scan integer part
while (cursor < program.len and is_digit(program[cursor])) cursor += 1
# check for decimal
if cursor < program.len and program[cursor] == '.' {
has_decimal = true
cursor += 1
}
# assert non-terminating decimal
if has_decimal and (cursor >= program.len or !is_digit(program[cursor])) {
return ScanError{
code = .float_must_end_with_digit,
end = cursor,
}
return ScanError{ code = .float_must_end_with_digit, end = cursor }
}
# scan fractional part
while (cursor < program.len and is_digit(program[cursor])) cursor += 1
return ScanNumResult{
end = cursor,
has_decimal = has_decimal,
}
return ScanNumResult{ end = cursor, has_decimal = has_decimal }
}
ScanStrResult :: struct { end usize }
scan_string proc(start usize, program []u8) ScanStrResult ! ScanError {
cursor := start + 1 # skip first `"`
# scan entire string
while cursor < program.len and program[cursor] != '"' and program[cursor] != '\n' : cursor += 1 {
# ignore escaped characters
if (program[cursor] == '\\' and cursor + 1 < program.len) cursor += 1
}
# assert string terminal
if (cursor >= program.len or program[cursor] != '"') return ScanError{
code = .unterminated_string,
end = cursor,
}
return ScanStrResult{
end = cursor + 1, # skip last `"`
if (cursor >= program.len or program[cursor] != '"') {
return ScanError{ code = .unterminated_string, end = cursor }
}
return ScanStrResult{ end = cursor + 1 } # skip last `"`
}
@hide
@@ -335,6 +303,19 @@ is_digit proc(char u8) bool {
}
}
@hide:file
diagnostic_code proc(code ErrorCode) diagnostics.Code {
return match code {
.float_must_end_with_digit: .float_must_end_with_digit
.unterminated_string: .unterminated_string
}
}
@hide:file
make_span proc(start, end usize) diagnostics.Span {
return diagnostics.Span{ start = u32(start), end = u32(end) }
}
@hide
add_token proc(state @mut State, token Token) void ! mem.AllocError {
_ = token_id(state.tokens.items.len)
source/lexer/lexer.test.hon
+12 -7
View File
@@ -1,14 +1,17 @@
import "@source/strpool"
import "@std/mem"
import "@std/testing"
import "@source/diagnostics"
import "@source/strpool"
handles_keywords_identifiers_and_error_progress test {
reports_invalid_input_and_keeps_scanning test {
strpool.STRINGS = strpool.init(mem.c_allocator)
defer strpool.deinit(&strpool.STRINGS)
state State := init(mem.c_allocator)
diagnostic_store := diagnostics.init(mem.c_allocator)
defer diagnostics.deinit(&diagnostic_store)
state := init(mem.c_allocator)
defer deinit(&state)
try scan(&state, "if name # comment\n@1. 2 3.5 \"hi\"")
try scan(&state, "if name # comment\n@1. 2 3.5 \"hi\"", &diagnostic_store)
try testing.expect_equal(8, state.tokens.items.len)
try testing.expect_equal(TokenKind.if, state.tokens.items[0].kind)
@@ -19,6 +22,8 @@ handles_keywords_identifiers_and_error_progress test {
try testing.expect_equal(TokenKind.float, state.tokens.items[5].kind)
try testing.expect_equal(TokenKind.string, state.tokens.items[6].kind)
try testing.expect_equal(TokenKind.eof, state.tokens.items[7].kind)
try testing.expect_equal(2, state.diagnostics.items.len)
try testing.expect_equal("name", strpool.get_str(&strpool.STRINGS, state.tokens.items[1].str_id)?)
try testing.expect_equal(2, diagnostics.len(&diagnostic_store))
try testing.expect_equal("name", strpool.get_str(&strpool.STRINGS, state.tokens.items[1].data.str_id)?)
try testing.expect_equal(u32(3), state.tokens.items[1].span.start)
try testing.expect_equal(u32(7), state.tokens.items[1].span.end)
}
source/lexer/token.hon
+12 -20
View File
@@ -1,14 +1,19 @@
import "@std/debug"
import "@source/diagnostics"
import "@source/strpool"
TokenId :: distinct u32
NO_TOKEN :: maxval!(TokenId)
TokenData :: union {
str_id strpool.StringId
diagnostic diagnostics.DiagnosticId
}
Token :: struct {
start uint
span diagnostics.Span
kind TokenKind
str_id strpool.StringId = strpool.NO_STR
data TokenData = TokenData{ str_id = strpool.NO_STR }
}
TokenKind :: enum {
@@ -55,27 +60,14 @@ TokenKind :: enum {
render_token proc(tok Token, program []u8) void {
match tok.kind {
.ident: {
if (strpool.get_str(&strpool.STRINGS, tok.str_id)) |str| {
if (strpool.get_str(&strpool.STRINGS, tok.data.str_id)) |str| {
debug.print("{}({})\n", {tok.kind, str})
} else {
res :: scan_ident(tok.start, program)
debug.print("{}({})\n", {tok.kind, program[tok.start..res.end]})
debug.print("{}({})\n", {tok.kind, program[tok.span.start..tok.span.end]})
}
}
.string: {
res :: scan_string(tok.start, program) catch |_| {
debug.print("{}({})\n", {tok.kind, "null"})
return
}
debug.print("{}({})\n", {tok.kind, program[tok.start..res.end]})
}
.int, .float: {
res :: scan_number(tok.start, program) catch |_| {
debug.print("{}({})\n", {tok.kind, "null"})
return
}
debug.print("{}({})\n", {tok.kind, program[tok.start..res.end]})
}
.string, .int, .float:
debug.print("{}({})\n", {tok.kind, program[tok.span.start..tok.span.end]})
else: debug.print("{}\n", {tok.kind})
}
}