lexer refactor

source/lexer/lexer.hon

@@ -7,7 +7,12 @@ import "@std/debug"
 
 import "@source/strpool"
 
-ScanError :: enum {
+ScanError :: struct {
+    code ErrorCode
+    end usize
+}
+
+ErrorCode :: enum {
     invalid_character,
     float_must_end_with_digit,
     unterminated_string,
@@ -18,7 +23,7 @@ ErrorDetails :: struct {
     message []u8
 }
 
-error_msg_map std.EnumMap(ScanError, ErrorDetails) :: enummap.init({
+error_msg_map std.EnumMap(ErrorCode, ErrorDetails) :: enummap.init({
     invalid_character = ErrorDetails {
         name = "L0",
         message = "invalid character",
@@ -42,7 +47,7 @@ TokenId :: distinct u32
 
 Diagnostic :: struct {
     token TokenId
-    code ScanError
+    code ErrorCode
 }
 
 State :: struct {
@@ -66,7 +71,7 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter
     tokens :: &state.tokens
     diagnostics :: &state.diagnostics
 
-    cursor usize := 0
+    cursor := 0
     while cursor < program.len {
         char :: program[cursor]
 
@@ -90,28 +95,13 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter
         # identifiers and keywords
         if is_alpha(char) or char == '_' {
             start :: cursor
-            cursor += 1
+            result :: scan_ident(start, program)
-
-            # scan whole identifier
-            while (cursor < program.len and (
-                is_alpha(program[cursor]) or
-                is_digit(program[cursor]) or
-                program[cursor] == '_'
-            )) cursor += 1
-
-            kind :: strmap.get(&keywords, program[start..cursor]) orelse .ident
-
-            # don't intern keywords (already O(1) lookup via token kind)
-            str_id :: if (kind == .ident)
-                try strpool.intern(&strpool.strings, program[start..cursor])
-            else
-                strpool.NO_ID
-
             try add_token(tokens, Token{
-                kind = kind,
+                kind = result.kind,
                 start = start,
-                str_id = str_id
+                str_id = result.str_id,
             })
+            cursor = result.end
             continue
         }
 
@@ -121,9 +111,8 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter
             result :: scan_number(start, program) catch |err| {
                 token :: token_id(tokens.items.len)
                 try add_token(tokens, Token{ kind = .invalid, start = start })
-                try arraylist.append(diagnostics, Diagnostic{ token = token, code = err })
+                try arraylist.append(diagnostics, Diagnostic{ token = token, code = err.code })
-                cursor += 1
+                cursor = err.end
-                while (cursor < program.len and (is_digit(program[cursor]) or program[cursor] == '.')) cursor += 1
                 continue
             }
 
@@ -139,9 +128,8 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter
             result :: scan_string(start, program) catch |err| {
                 token :: token_id(tokens.items.len)
                 try add_token(tokens, Token{ kind = .invalid, start = start })
-                try arraylist.append(diagnostics, Diagnostic{ token = token, code = err })
+                try arraylist.append(diagnostics, Diagnostic{ token = token, code = err.code })
-                cursor += 1
+                cursor = err.end
-                while cursor < program.len and program[cursor] != '\n' : cursor += 1 {}
                 continue
             }
 
@@ -150,13 +138,6 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter
             continue
         }
 
-        # mutable assignment
-        if char == '=' {
-            try add_token(tokens, Token{ kind = .equal, start = cursor })
-            cursor += 1
-            continue
-        }
-
         # immutable assignment or single colon
         if char == ':' {
             if cursor + 1 < program.len and program[cursor + 1] == ':' {
@@ -170,6 +151,13 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter
             continue
         }
 
+        # mutable reassignment
+        if char == '=' {
+            try add_token(tokens, Token{ kind = .equal, start = cursor })
+            cursor += 1
+            continue
+        }
+
         # parentheses
         if char == '(' {
             try add_token(tokens, Token{ kind = .open_paren, start = cursor })
@@ -202,6 +190,37 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter
     try add_token(tokens, Token{ kind = .eof, start = cursor })
 }
 
+ScanIdentResult :: struct {
+    end usize
+    kind TokenKind
+    str_id strpool.StringId
+}
+
+scan_ident proc(start usize, program []u8) ScanIdentResult {
+    cursor := start + 1
+
+    # scan whole identifier
+    while (cursor < program.len and (
+        is_alpha(program[cursor]) or
+        is_digit(program[cursor]) or
+        program[cursor] == '_'
+    )) cursor += 1
+
+    kind :: strmap.get(&keywords, program[start..cursor]) orelse .ident
+
+    # don't intern keywords (already O(1) lookup via token kind)
+    str_id :: if (kind == .ident)
+        strpool.intern(&strpool.strings, program[start..cursor]) catch strpool.NO_ID
+    else
+        strpool.NO_ID
+
+    return ScanIdentResult{
+        end = cursor,
+        kind = kind,
+        str_id = str_id,
+    }
+}
+
 ScanNumResult :: struct {
     end usize
     has_decimal bool
@@ -222,7 +241,10 @@ scan_number proc(start usize, program []u8) ScanNumResult ! ScanError {
 
     # assert non-terminating decimal
     if has_decimal and (cursor >= program.len or !is_digit(program[cursor])) {
-        return .float_must_end_with_digit
+        return ScanError{
+            code = .float_must_end_with_digit,
+            end = cursor + 1,
+        }
     }
 
     # scan fractional part
@@ -246,7 +268,10 @@ scan_string proc(start usize, program []u8) ScanStrResult ! ScanError {
     }
 
     # assert string terminal
-    if (cursor >= program.len or program[cursor] != '"') return .unterminated_string
+    if (cursor >= program.len or program[cursor] != '"') return ScanError{
+        code = .unterminated_string,
+        end = cursor,
+    }
 
     return ScanStrResult{
         end = cursor + 1, # skip last `"`

source/lexer/token.hon

@@ -2,7 +2,7 @@ import "@source/strpool"
 
 Token :: struct {
     kind TokenKind
-    start int
+    start uint
     str_id strpool.StringId = strpool.NO_ID
 }