From 425ce38011aded02e065c064990cc13cf6af39ab Mon Sep 17 00:00:00 2001 From: hl-valdemar Date: Wed, 5 Aug 2026 22:11:06 +0200 Subject: [PATCH] lexer refactor --- source/lexer/lexer.hon | 101 +++++++++++++++++++++++++---------------- source/lexer/token.hon | 2 +- 2 files changed, 64 insertions(+), 39 deletions(-) diff --git a/source/lexer/lexer.hon b/source/lexer/lexer.hon index e55bdeb..6ee7ca3 100644 --- a/source/lexer/lexer.hon +++ b/source/lexer/lexer.hon @@ -7,7 +7,12 @@ import "@std/debug" import "@source/strpool" -ScanError :: enum { +ScanError :: struct { + code ErrorCode + end usize +} + +ErrorCode :: enum { invalid_character, float_must_end_with_digit, unterminated_string, @@ -18,7 +23,7 @@ ErrorDetails :: struct { message []u8 } -error_msg_map std.EnumMap(ScanError, ErrorDetails) :: enummap.init({ +error_msg_map std.EnumMap(ErrorCode, ErrorDetails) :: enummap.init({ invalid_character = ErrorDetails { name = "L0", message = "invalid character", @@ -42,7 +47,7 @@ TokenId :: distinct u32 Diagnostic :: struct { token TokenId - code ScanError + code ErrorCode } State :: struct { @@ -66,7 +71,7 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter tokens :: &state.tokens diagnostics :: &state.diagnostics - cursor usize := 0 + cursor := 0 while cursor < program.len { char :: program[cursor] @@ -90,28 +95,13 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter # identifiers and keywords if is_alpha(char) or char == '_' { start :: cursor - cursor += 1 - - # scan whole identifier - while (cursor < program.len and ( - is_alpha(program[cursor]) or - is_digit(program[cursor]) or - program[cursor] == '_' - )) cursor += 1 - - kind :: strmap.get(&keywords, program[start..cursor]) orelse .ident - - # don't intern keywords (already O(1) lookup via token kind) - str_id :: if (kind == .ident) - try strpool.intern(&strpool.strings, program[start..cursor]) - else - strpool.NO_ID - + result :: scan_ident(start, program) try add_token(tokens, Token{ - kind = kind, + kind = result.kind, start = start, - str_id = str_id + str_id = result.str_id, }) + cursor = result.end continue } @@ -121,9 +111,8 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter result :: scan_number(start, program) catch |err| { token :: token_id(tokens.items.len) try add_token(tokens, Token{ kind = .invalid, start = start }) - try arraylist.append(diagnostics, Diagnostic{ token = token, code = err }) - cursor += 1 - while (cursor < program.len and (is_digit(program[cursor]) or program[cursor] == '.')) cursor += 1 + try arraylist.append(diagnostics, Diagnostic{ token = token, code = err.code }) + cursor = err.end continue } @@ -139,9 +128,8 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter result :: scan_string(start, program) catch |err| { token :: token_id(tokens.items.len) try add_token(tokens, Token{ kind = .invalid, start = start }) - try arraylist.append(diagnostics, Diagnostic{ token = token, code = err }) - cursor += 1 - while cursor < program.len and program[cursor] != '\n' : cursor += 1 {} + try arraylist.append(diagnostics, Diagnostic{ token = token, code = err.code }) + cursor = err.end continue } @@ -150,13 +138,6 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter continue } - # mutable assignment - if char == '=' { - try add_token(tokens, Token{ kind = .equal, start = cursor }) - cursor += 1 - continue - } - # immutable assignment or single colon if char == ':' { if cursor + 1 < program.len and program[cursor + 1] == ':' { @@ -170,6 +151,13 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter continue } + # mutable reassignment + if char == '=' { + try add_token(tokens, Token{ kind = .equal, start = cursor }) + cursor += 1 + continue + } + # parentheses if char == '(' { try add_token(tokens, Token{ kind = .open_paren, start = cursor }) @@ -202,6 +190,37 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter try add_token(tokens, Token{ kind = .eof, start = cursor }) } +ScanIdentResult :: struct { + end usize + kind TokenKind + str_id strpool.StringId +} + +scan_ident proc(start usize, program []u8) ScanIdentResult { + cursor := start + 1 + + # scan whole identifier + while (cursor < program.len and ( + is_alpha(program[cursor]) or + is_digit(program[cursor]) or + program[cursor] == '_' + )) cursor += 1 + + kind :: strmap.get(&keywords, program[start..cursor]) orelse .ident + + # don't intern keywords (already O(1) lookup via token kind) + str_id :: if (kind == .ident) + strpool.intern(&strpool.strings, program[start..cursor]) catch strpool.NO_ID + else + strpool.NO_ID + + return ScanIdentResult{ + end = cursor, + kind = kind, + str_id = str_id, + } +} + ScanNumResult :: struct { end usize has_decimal bool @@ -222,7 +241,10 @@ scan_number proc(start usize, program []u8) ScanNumResult ! ScanError { # assert non-terminating decimal if has_decimal and (cursor >= program.len or !is_digit(program[cursor])) { - return .float_must_end_with_digit + return ScanError{ + code = .float_must_end_with_digit, + end = cursor + 1, + } } # scan fractional part @@ -246,7 +268,10 @@ scan_string proc(start usize, program []u8) ScanStrResult ! ScanError { } # assert string terminal - if (cursor >= program.len or program[cursor] != '"') return .unterminated_string + if (cursor >= program.len or program[cursor] != '"') return ScanError{ + code = .unterminated_string, + end = cursor, + } return ScanStrResult{ end = cursor + 1, # skip last `"` diff --git a/source/lexer/token.hon b/source/lexer/token.hon index cc0339e..957d3b6 100644 --- a/source/lexer/token.hon +++ b/source/lexer/token.hon @@ -2,7 +2,7 @@ import "@source/strpool" Token :: struct { kind TokenKind - start int + start uint str_id strpool.StringId = strpool.NO_ID }