cleanup (isle 6 again?)

source/ast/ast.hon

@@ -13,13 +13,13 @@ import "@source/lexer"
 TokenId :: alias lexer.TokenId
 
 @hide:file
-NO_ID_TOKEN :: alias lexer.NO_ID_TOKEN
+NO_TOKEN :: alias lexer.NO_TOKEN
 
 NodeId :: distinct u32
 ExtraId :: distinct u32
 
-NO_ID_NODE :: maxval!(NodeId)
+NO_NODE :: maxval!(NodeId)
-NO_ID_EXTRA :: maxval!(ExtraId)
+NO_EXTRA :: maxval!(ExtraId)
 
 # unsafe access on its own; NodeKind serves as a the tag
 # that determines the variant.
@@ -31,10 +31,10 @@ NodeData :: union {
 
 Node :: struct {
     kind NodeKind
-    main_token TokenId = NO_ID_TOKEN
+    main_token TokenId = NO_TOKEN
 
-    data0 NodeData = NodeData{ node_id = NO_ID_NODE }
+    data0 NodeData = NodeData{ node_id = NO_NODE }
-    data1 NodeData = NodeData{ node_id = NO_ID_NODE }
+    data1 NodeData = NodeData{ node_id = NO_NODE }
 }
 
 NodeKind :: enum {
@@ -49,7 +49,8 @@ NodeKind :: enum {
     expr_identifier
 
     # UNARY:
-    # todo: document
+    # * main_token: operator
+    # * data0: NodeId - expression
     expr_unary
 
     # BINARY:
@@ -77,16 +78,16 @@ NodeKind :: enum {
 }
 
 node_id proc(idx uint) NodeId {
-    debug.assert(u64(idx) < u64(NO_ID_NODE))
+    debug.assert(u64(idx) < u64(NO_NODE))
     return NodeId(idx)
 }
 
 maybe_node_id proc(idx uint) ?NodeId {
-    if (u64(idx) >= u64(NO_ID_NODE)) return null
+    if (u64(idx) >= u64(NO_NODE)) return null
     return NodeId(idx)
 }
 
 extra_id proc(idx uint) ExtraId {
-    debug.assert(u64(idx) < u64(NO_ID_EXTRA))
+    debug.assert(u64(idx) < u64(NO_EXTRA))
     return ExtraId(idx)
 }

source/lexer/lexer.hon

@@ -245,9 +245,9 @@ scan_ident proc(start usize, program []u8) ScanIdentResult {
 
     # don't intern keywords (already O(1) lookup via token kind)
     str_id :: if (kind == .ident)
-        strpool.intern(&strpool.strings, program[start..cursor]) catch strpool.NO_ID_STR
+        strpool.intern(&strpool.STRINGS, program[start..cursor]) catch strpool.NO_STR
     else
-        strpool.NO_ID_STR
+        strpool.NO_STR
 
     return ScanIdentResult{
         end = cursor,

source/lexer/lexer.test.hon

@@ -3,8 +3,8 @@ import "@std/mem"
 import "@std/testing"
 
 handles_keywords_identifiers_and_error_progress test {
-    strpool.strings = strpool.init(mem.c_allocator)
+    strpool.STRINGS = strpool.init(mem.c_allocator)
-    defer strpool.deinit(&strpool.strings)
+    defer strpool.deinit(&strpool.STRINGS)
 
     state State := init(mem.c_allocator)
     defer deinit(&state)
@@ -20,5 +20,5 @@ handles_keywords_identifiers_and_error_progress test {
     try testing.expect_equal(TokenKind.string, state.tokens.items[6].kind)
     try testing.expect_equal(TokenKind.eof, state.tokens.items[7].kind)
     try testing.expect_equal(2, state.diagnostics.items.len)
-    try testing.expect_equal("name", strpool.get_str(&strpool.strings, state.tokens.items[1].str_id)?)
+    try testing.expect_equal("name", strpool.get_str(&strpool.STRINGS, state.tokens.items[1].str_id)?)
 }

source/lexer/token.hon

@@ -3,12 +3,12 @@ import "@source/strpool"
 
 TokenId :: distinct u32
 
-NO_ID_TOKEN :: maxval!(TokenId)
+NO_TOKEN :: maxval!(TokenId)
 
 Token :: struct {
     start uint
     kind TokenKind
-    str_id strpool.StringId = strpool.NO_ID_STR
+    str_id strpool.StringId = strpool.NO_STR
 }
 
 TokenKind :: enum {
@@ -51,8 +51,32 @@ TokenKind :: enum {
     invalid
 }
 
+render_token proc(tok Token, program []u8) void {
+    match tok.kind {
+        .ident: {
+            str :: strpool.get_str(&strpool.STRINGS, tok.str_id) orelse "null"
+            debug.print("{}({})\n", {tok.kind, str})
+        }
+        .string: {
+            res :: scan_string(tok.start, program) catch |_| {
+                debug.print("{}({})\n", {tok.kind, "null"})
+                return
+            }
+            debug.print("{}({})\n", {tok.kind, program[tok.start..res.end]})
+        }
+        .int, .float: {
+            res :: scan_number(tok.start, program) catch |_| {
+                debug.print("{}({})\n", {tok.kind, "null"})
+                return
+            }
+            debug.print("{}({})\n", {tok.kind, program[tok.start..res.end]})
+        }
+        else: debug.print("{}\n", {tok.kind})
+    }
+}
+
 @hide
 token_id proc(idx uint) TokenId {
-    debug.assert(u64(idx) < u64(NO_ID_TOKEN))
+    debug.assert(u64(idx) < u64(NO_TOKEN))
     return TokenId(idx)
 }

source/main.hon

@@ -26,8 +26,8 @@ program ::
     `a := 1 + 2 * 3
 
 main proc() void! {
-    strpool.strings = strpool.init(mem.c_allocator)
+    strpool.STRINGS = strpool.init(mem.c_allocator)
-    defer strpool.deinit(&strpool.strings)
+    defer strpool.deinit(&strpool.STRINGS)
 
     debug.print("PROGRAM::[[\n{}\n]]\n\n", {program})
 
@@ -39,9 +39,7 @@ main proc() void! {
     }
 
     debug.print("TOKENS::[[\n", {})
-    for scan_state.tokens.items |token| {
+    for (scan_state.tokens.items) |tok| lexer.render_token(tok, program)
-        debug.print("{}\n", {token.kind})
-    }
     debug.print("]]\n\n", {})
 
     parse_state := parser.init(mem.c_allocator)
@@ -52,9 +50,7 @@ main proc() void! {
     }
 
     debug.print("AST::[[\n", {})
-    for parse_state.nodes.items |node, id| {
+    for (parse_state.nodes.items) |node| debug.print("{}\n", { node.kind })
-        debug.print("{} (id = {})\n", { node.kind, id })
-    }
     debug.print("]]\n\n", {})
 
     renderer := ast_renderer.init(mem.c_allocator)
@@ -62,6 +58,6 @@ main proc() void! {
     try ast_renderer.render(&renderer, root, &parse_state, scan_state.tokens.items)
 
     debug.print("AST Render::[[\n", {})
-    for (renderer.tokens.items) |tok| debug.print("{}\n", {tok.kind})
+    for (renderer.tokens.items) |tok| lexer.render_token(tok, program)
     debug.print("]]\n", {})
 }

source/parser/parser.hon

@@ -53,10 +53,10 @@ ExtraId :: alias ast.ExtraId
 NodeData :: alias ast.NodeData
 
 @hide:file
-NO_ID_NODE :: alias ast.NO_ID_NODE
+NO_NODE :: alias ast.NO_NODE
 
 @hide:file
-NO_ID_EXTRA :: alias ast.NO_ID_EXTRA
+NO_EXTRA :: alias ast.NO_EXTRA
 
 BindingPowerScalar :: distinct u32
 
@@ -103,7 +103,7 @@ parse proc(state @mut State, tokens []Token) NodeId ! ParseError {
 parse_pkg proc(state @mut State) NodeId ! ParseError {
     if (state.next_token < state.tokens.len
         and state.tokens[state.next_token].kind == .eof
-    ) return NO_ID_NODE
+    ) return NO_NODE
 
     stmt_ids std.ArrayList(NodeId) := arraylist.init(state.nodes.allocator)
     defer arraylist.deinit(&stmt_ids)
@@ -141,7 +141,7 @@ parse_decl proc(state @mut State) NodeId ! ParseError {
     type_id :: if (next_is(state, .ident))
         try parse_primary(state)
     else
-        NO_ID_NODE
+        NO_NODE
 
     # expect `::` or `:=` (immutable or mutable assignment)
     mutability_tok_id :: try consume_either(state, &[.double_colon, .colon_equal])

source/parser/parser.test.hon

@@ -7,8 +7,8 @@ import "@source/lexer"
 ast_renderer :: import "@source/ast/renderer"
 
 handles_package_declarations test {
-    strpool.strings = strpool.init(mem.c_allocator)
+    strpool.STRINGS = strpool.init(mem.c_allocator)
-    defer strpool.deinit(&strpool.strings)
+    defer strpool.deinit(&strpool.STRINGS)
 
     source ::
         `first int :: 42
@@ -37,8 +37,8 @@ handles_package_declarations test {
 }
 
 handles_binary_arithmetic_expression test {
-    strpool.strings = strpool.init(mem.c_allocator)
+    strpool.STRINGS = strpool.init(mem.c_allocator)
-    defer strpool.deinit(&strpool.strings)
+    defer strpool.deinit(&strpool.STRINGS)
 
     source ::
         `first int :: 42 + 3
@@ -68,8 +68,8 @@ handles_binary_arithmetic_expression test {
 }
 
 handles_statement_declaration test {
-    strpool.strings = strpool.init(mem.c_allocator)
+    strpool.STRINGS = strpool.init(mem.c_allocator)
-    defer strpool.deinit(&strpool.strings)
+    defer strpool.deinit(&strpool.STRINGS)
 
     source ::
         `value int :: 42

source/strpool/strpool.hon

@@ -2,14 +2,14 @@ import "@std/mem"
 import "@std/arraylist"
 import "@std/hashmap"
 
-# cross-cutting concern, hence global singleton (owned by main.hon)
-strings StringPool := undefined
-
 InternError :: enum { out_of_space }
 
 StringId :: alias u32
 
-NO_ID_STR :: maxval!(StringId)
+NO_STR :: maxval!(StringId)
+
+# cross-cutting concern, hence global singleton (owned by main.hon)
+STRINGS StringPool := undefined
 
 StringPool :: struct {
     ids hashmap.StringHashMap(StringId) # string → id
@@ -36,7 +36,7 @@ deinit proc(pool @mut StringPool) void {
 intern proc(pool @mut StringPool, str []u8) StringId ! (mem.AllocError | InternError) {
     if hashmap.get(&pool.ids, str) |id| return id
 
-    if (pool.strings.items.len >= usize(NO_ID_STR)) return .out_of_space
+    if (pool.strings.items.len >= usize(NO_STR)) return .out_of_space
     id StringId :: StringId(pool.strings.items.len)
 
     owned_str []mut u8 :: try mem.alloc(u8, pool.allocator, str.len)