lexer touchups

source/ast/ast.hon

@@ -48,3 +48,22 @@ NodeKind :: enum {
 
     invalid
 }
+
+# todo: render ast nodes as source code
+
+render proc(node @Node, tokens []Token) void { }
+
+render_literal proc(node @Node, tokens []Token, program []u8) []u8 ! lexer.ScanError {
+    tok :: tokens[node.main_token]
+    match node.kind {
+        .literal_int, .literal_float: {
+            result :: try lexer.scan_number(tok.start, program)
+            return program[tok.start..result.end]
+        }
+        .literal_string: {
+            result :: try lexer.scan_string(tok.start, program)
+            return program[tok.start..result.end]
+        }
+        else: unreachable
+    }
+}

source/lexer/lexer.hon

@@ -7,9 +7,10 @@ import "@std/debug"
 
 import "@source/strpool"
 
-ErrorCode :: enum {
+ScanError :: enum {
     invalid_character,
     float_must_end_with_digit,
+    unterminated_string,
 }
 
 ErrorDetails :: struct {
@@ -17,7 +18,7 @@ ErrorDetails :: struct {
     message []u8
 }
 
-error_msg_map std.EnumMap(ErrorCode, ErrorDetails) :: enummap.init({
+error_msg_map std.EnumMap(ScanError, ErrorDetails) :: enummap.init({
     invalid_character = ErrorDetails {
         name = "L0",
         message = "invalid character",
@@ -40,8 +41,8 @@ keywords std.StringMap(TokenKind) :: strmap.init([
 TokenId :: distinct u32
 
 Diagnostic :: struct {
-    code ErrorCode
     token TokenId
+    code ScanError
 }
 
 State :: struct {
@@ -61,13 +62,13 @@ deinit proc(state @mut State) void {
     arraylist.deinit(&state.diagnostics)
 }
 
-scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternError) {
+scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.InternError) {
     tokens :: &state.tokens
     diagnostics :: &state.diagnostics
 
     cursor usize = 0
-    while cursor < input.len {
+    while cursor < program.len {
-        char :: input[cursor]
+        char :: program[cursor]
 
         # whitespace
         if char == '\n' {
@@ -81,7 +82,7 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE
 
         # comments
         if char == '#' {
-            while (cursor < input.len and input[cursor] != '\n') cursor += 1
+            while (cursor < program.len and program[cursor] != '\n') cursor += 1
             cursor += 1 # also skip newline
             continue
         }
@@ -92,17 +93,17 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE
             cursor += 1
 
             # scan whole identifier
-            while (cursor < input.len and (
+            while (cursor < program.len and (
-                is_alpha(input[cursor]) or
+                is_alpha(program[cursor]) or
-                is_digit(input[cursor]) or
+                is_digit(program[cursor]) or
-                input[cursor] == '_'
+                program[cursor] == '_'
             )) cursor += 1
 
-            kind :: strmap.get(&keywords, input[start..cursor]) orelse .ident
+            kind :: strmap.get(&keywords, program[start..cursor]) orelse .ident
 
             # don't intern keywords (already O(1) lookup via token kind)
             str_id :: if (kind == .ident)
-                try strpool.intern(&strpool.strings, input[start..cursor])
+                try strpool.intern(&strpool.strings, program[start..cursor])
             else
                 strpool.NO_ID
 
@@ -117,51 +118,35 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE
         # numeric literals
         if is_digit(char) {
             start :: cursor
-            has_decimal bool = false
+            result :: scan_number(start, program) catch |err| {
-
-            # scan integer part
-            while (cursor < input.len and is_digit(input[cursor])) cursor += 1
-
-            # check for decimal point
-            if cursor < input.len and input[cursor] == '.' {
-                has_decimal = true
-                cursor += 1
-            }
-
-            # assert that decimals follow the decimal point
-            if has_decimal and (cursor >= input.len or !is_digit(input[cursor])) {
                 token :: token_id(tokens.items.len)
-                try arraylist.append(diagnostics, Diagnostic{ token = token, code = .float_must_end_with_digit })
                 try add_token(tokens, Token{ kind = .invalid, start = start })
+                try arraylist.append(diagnostics, Diagnostic{ token = token, code = err })
+                cursor += 1
+                while (cursor < program.len and (is_digit(program[cursor]) or program[cursor] == '.')) cursor += 1
                 continue
             }
 
-            # scan decimal part
+            kind :: if (result.has_decimal) .float else .int
-            while (cursor < input.len and is_digit(input[cursor])) cursor += 1
-
-            kind :: if (has_decimal) .float else .int
             try add_token(tokens, Token{ kind = kind, start = start })
-
+            cursor = result.end
             continue
         }
 
         # string literals
         if char == '"' {
             start :: cursor
-            cursor += 1
+            result :: scan_string(start, program) catch |err| {
-
+                token :: token_id(tokens.items.len)
-            while cursor < input.len and input[cursor] != '"' and input[cursor] != '\n' : cursor += 1 {
-                # ignore escaped characters
-                if (input[cursor] == '\\' and cursor + 1 < input.len) cursor += 1
-            }
-
-            if cursor < input.len and input[cursor] == '"' {
-                cursor += 1
-                try add_token(tokens, Token{ kind = .string, start = start })
-            } else {
                 try add_token(tokens, Token{ kind = .invalid, start = start })
+                try arraylist.append(diagnostics, Diagnostic{ token = token, code = err })
+                cursor += 1
+                while cursor < program.len and program[cursor] != '\n' : cursor += 1 {}
+                continue
             }
 
+            try add_token(tokens, Token{ kind = .string, start = start })
+            cursor = result.end
             continue
         }
 
@@ -174,7 +159,7 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE
 
         # immutable assignment or single colon
         if char == ':' {
-            if cursor + 1 < input.len and input[cursor + 1] == ':' {
+            if cursor + 1 < program.len and program[cursor + 1] == ':' {
                 try add_token(tokens, Token{ kind = .double_colon, start = cursor })
                 cursor += 2
                 continue
@@ -217,45 +202,55 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE
     try add_token(tokens, Token{ kind = .eof, start = cursor })
 }
 
-#! returns the cursor position after scanning a number.
+ScanNumResult :: struct {
-scan_number proc(start usize, input []u8) usize {
+    end usize
-    debug.assert(is_digit(input[start]))
+    has_decimal bool
+}
 
-    has_decimal bool = false
+scan_number proc(start usize, program []u8) ScanNumResult ! ScanError {
     cursor usize = start
+    has_decimal bool = false
 
     # scan integer part
-    while cursor < input.len and is_digit(input[cursor]) : cursor += 1 {}
+    while (cursor < program.len and is_digit(program[cursor])) cursor += 1
 
-    # check for decimal point
+    # check for decimal
-    if cursor < input.len and input[cursor] == '.' {
+    if cursor < program.len and program[cursor] == '.' {
         has_decimal = true
         cursor += 1
     }
 
-    # assert that decimals follow the decimal point
+    # assert non-terminating decimal
-    debug.assert(!(has_decimal and (cursor >= input.len or !is_digit(input[cursor]))))
+    if has_decimal and (cursor >= program.len or !is_digit(program[cursor])) {
-
+        return .float_must_end_with_digit
-    # scan decimal part
-    while cursor < input.len and is_digit(input[cursor]) : cursor += 1 {}
-
-    return cursor
-}
-
-#! returns the cursor position after scanning a string.
-scan_string proc(start usize, input []u8) usize {
-    debug.assert(input[start] == '"')
-
-    cursor usize = start + 1
-    while cursor < input.len and input[cursor] != '"' and input[cursor] != '\n' : cursor += 1 {
-        # ignore escaped characters
-        if (input[cursor] == '\\' and cursor + 1 < input.len) cursor += 1
     }
 
-    debug.assert(cursor < input.len and input[cursor] == '"')
+    # scan fractional part
-    cursor += 1
+    while (cursor < program.len and is_digit(program[cursor])) cursor += 1
 
-    return cursor
+    return ScanNumResult{
+        end = cursor,
+        has_decimal = has_decimal,
+    }
+}
+
+ScanStrResult :: struct { end usize }
+
+scan_string proc(start usize, program []u8) ScanStrResult ! ScanError {
+    cursor usize = start + 1 # skip first `"`
+
+    # scan entire string
+    while cursor < program.len and program[cursor] != '"' and program[cursor] != '\n' : cursor += 1 {
+        # ignore escaped characters
+        if (program[cursor] == '\\' and cursor + 1 < program.len) cursor += 1
+    }
+
+    # assert string terminal
+    if (cursor >= program.len or program[cursor] != '"') return .unterminated_string
+
+    return ScanStrResult{
+        end = cursor + 1, # skip last `"`
+    }
 }
 
 hide is_whitespace proc(char u8) bool {

source/lexer/lexer.test.hon

@@ -8,14 +8,17 @@ handles_keywords_identifiers_and_error_progress test {
 
     state State = init(mem.c_allocator)
     defer deinit(&state)
-    try scan(&state, "if name # comment\n@1.")
+    try scan(&state, "if name # comment\n@1. 2 3.5 \"hi\"")
 
-    try testing.expect_equal(5, state.tokens.items.len)
+    try testing.expect_equal(8, state.tokens.items.len)
     try testing.expect_equal(TokenKind.if, state.tokens.items[0].kind)
     try testing.expect_equal(TokenKind.ident, state.tokens.items[1].kind)
     try testing.expect_equal(TokenKind.invalid, state.tokens.items[2].kind)
     try testing.expect_equal(TokenKind.invalid, state.tokens.items[3].kind)
-    try testing.expect_equal(TokenKind.eof, state.tokens.items[4].kind)
+    try testing.expect_equal(TokenKind.int, state.tokens.items[4].kind)
+    try testing.expect_equal(TokenKind.float, state.tokens.items[5].kind)
+    try testing.expect_equal(TokenKind.string, state.tokens.items[6].kind)
+    try testing.expect_equal(TokenKind.eof, state.tokens.items[7].kind)
     try testing.expect_equal(2, state.diagnostics.items.len)
     try testing.expect_equal("name", strpool.get_str(&strpool.strings, state.tokens.items[1].str_id)?)
 }

source/main.hon

@@ -16,7 +16,8 @@ test import "@source/lexer"
 
 program ::
     `# literals
-    `x int :: 123
+    `x int :: 123.9
+    `y :: "hello"
 
 main proc() void {
     strpool.strings = strpool.init(mem.c_allocator)
@@ -56,12 +57,17 @@ main proc() void {
     for parse_state.nodes.items |node, id| {
         token :: scan_state.tokens.items[node.main_token]
 
-        end :: if (token.kind == .int or token.kind == .float)
+        end :: if (token.kind == .int or token.kind == .float) lbl: {
-            lexer.scan_number(token.start, program)
+            res :: lexer.scan_number(token.start, program) catch {
-        else if (token.kind == .string)
+                yield :lbl token.start
-            lexer.scan_string(token.start, program)
+            }
-        else
+            yield :lbl res.end
-            token.start
+        } else if (token.kind == .string) lbl: {
+            res :: lexer.scan_string(token.start, program) catch {
+                yield :lbl token.start
+            }
+            yield :lbl res.end
+        } else token.start
 
         debug.print("{} (id = {}): {}\n", {
             node.kind,
@@ -71,5 +77,14 @@ main proc() void {
             #node.data1,
         })
     }
+    debug.print("]]\n\n", {})
+
+    debug.print("AST Render::[[\n", {})
+    literal :: ast.render_literal(
+        &parse_state.nodes.items[1],
+        scan_state.tokens.items,
+        program,
+    ) catch "<invalid>"
+    debug.print("generated literal: {}\n", { literal })
     debug.print("]]\n", {})
 }