diff --git a/source/ast/ast.hon b/source/ast/ast.hon index 693b16a..a554c43 100644 --- a/source/ast/ast.hon +++ b/source/ast/ast.hon @@ -13,13 +13,13 @@ import "@source/lexer" TokenId :: alias lexer.TokenId @hide:file -NO_ID_TOKEN :: alias lexer.NO_ID_TOKEN +NO_TOKEN :: alias lexer.NO_TOKEN NodeId :: distinct u32 ExtraId :: distinct u32 -NO_ID_NODE :: maxval!(NodeId) -NO_ID_EXTRA :: maxval!(ExtraId) +NO_NODE :: maxval!(NodeId) +NO_EXTRA :: maxval!(ExtraId) # unsafe access on its own; NodeKind serves as a the tag # that determines the variant. @@ -31,10 +31,10 @@ NodeData :: union { Node :: struct { kind NodeKind - main_token TokenId = NO_ID_TOKEN + main_token TokenId = NO_TOKEN - data0 NodeData = NodeData{ node_id = NO_ID_NODE } - data1 NodeData = NodeData{ node_id = NO_ID_NODE } + data0 NodeData = NodeData{ node_id = NO_NODE } + data1 NodeData = NodeData{ node_id = NO_NODE } } NodeKind :: enum { @@ -49,7 +49,8 @@ NodeKind :: enum { expr_identifier # UNARY: - # todo: document + # * main_token: operator + # * data0: NodeId - expression expr_unary # BINARY: @@ -77,16 +78,16 @@ NodeKind :: enum { } node_id proc(idx uint) NodeId { - debug.assert(u64(idx) < u64(NO_ID_NODE)) + debug.assert(u64(idx) < u64(NO_NODE)) return NodeId(idx) } maybe_node_id proc(idx uint) ?NodeId { - if (u64(idx) >= u64(NO_ID_NODE)) return null + if (u64(idx) >= u64(NO_NODE)) return null return NodeId(idx) } extra_id proc(idx uint) ExtraId { - debug.assert(u64(idx) < u64(NO_ID_EXTRA)) + debug.assert(u64(idx) < u64(NO_EXTRA)) return ExtraId(idx) } diff --git a/source/lexer/lexer.hon b/source/lexer/lexer.hon index a9338c4..7661297 100644 --- a/source/lexer/lexer.hon +++ b/source/lexer/lexer.hon @@ -245,9 +245,9 @@ scan_ident proc(start usize, program []u8) ScanIdentResult { # don't intern keywords (already O(1) lookup via token kind) str_id :: if (kind == .ident) - strpool.intern(&strpool.strings, program[start..cursor]) catch strpool.NO_ID_STR + strpool.intern(&strpool.STRINGS, program[start..cursor]) catch strpool.NO_STR else - strpool.NO_ID_STR + strpool.NO_STR return ScanIdentResult{ end = cursor, diff --git a/source/lexer/lexer.test.hon b/source/lexer/lexer.test.hon index 9a00027..e6813ac 100644 --- a/source/lexer/lexer.test.hon +++ b/source/lexer/lexer.test.hon @@ -3,8 +3,8 @@ import "@std/mem" import "@std/testing" handles_keywords_identifiers_and_error_progress test { - strpool.strings = strpool.init(mem.c_allocator) - defer strpool.deinit(&strpool.strings) + strpool.STRINGS = strpool.init(mem.c_allocator) + defer strpool.deinit(&strpool.STRINGS) state State := init(mem.c_allocator) defer deinit(&state) @@ -20,5 +20,5 @@ handles_keywords_identifiers_and_error_progress test { try testing.expect_equal(TokenKind.string, state.tokens.items[6].kind) try testing.expect_equal(TokenKind.eof, state.tokens.items[7].kind) try testing.expect_equal(2, state.diagnostics.items.len) - try testing.expect_equal("name", strpool.get_str(&strpool.strings, state.tokens.items[1].str_id)?) + try testing.expect_equal("name", strpool.get_str(&strpool.STRINGS, state.tokens.items[1].str_id)?) } diff --git a/source/lexer/token.hon b/source/lexer/token.hon index a08b1f6..14092ce 100644 --- a/source/lexer/token.hon +++ b/source/lexer/token.hon @@ -3,12 +3,12 @@ import "@source/strpool" TokenId :: distinct u32 -NO_ID_TOKEN :: maxval!(TokenId) +NO_TOKEN :: maxval!(TokenId) Token :: struct { start uint kind TokenKind - str_id strpool.StringId = strpool.NO_ID_STR + str_id strpool.StringId = strpool.NO_STR } TokenKind :: enum { @@ -51,8 +51,32 @@ TokenKind :: enum { invalid } +render_token proc(tok Token, program []u8) void { + match tok.kind { + .ident: { + str :: strpool.get_str(&strpool.STRINGS, tok.str_id) orelse "null" + debug.print("{}({})\n", {tok.kind, str}) + } + .string: { + res :: scan_string(tok.start, program) catch |_| { + debug.print("{}({})\n", {tok.kind, "null"}) + return + } + debug.print("{}({})\n", {tok.kind, program[tok.start..res.end]}) + } + .int, .float: { + res :: scan_number(tok.start, program) catch |_| { + debug.print("{}({})\n", {tok.kind, "null"}) + return + } + debug.print("{}({})\n", {tok.kind, program[tok.start..res.end]}) + } + else: debug.print("{}\n", {tok.kind}) + } +} + @hide token_id proc(idx uint) TokenId { - debug.assert(u64(idx) < u64(NO_ID_TOKEN)) + debug.assert(u64(idx) < u64(NO_TOKEN)) return TokenId(idx) } diff --git a/source/main.hon b/source/main.hon index 3c012fc..10dc76f 100644 --- a/source/main.hon +++ b/source/main.hon @@ -26,8 +26,8 @@ program :: `a := 1 + 2 * 3 main proc() void! { - strpool.strings = strpool.init(mem.c_allocator) - defer strpool.deinit(&strpool.strings) + strpool.STRINGS = strpool.init(mem.c_allocator) + defer strpool.deinit(&strpool.STRINGS) debug.print("PROGRAM::[[\n{}\n]]\n\n", {program}) @@ -39,9 +39,7 @@ main proc() void! { } debug.print("TOKENS::[[\n", {}) - for scan_state.tokens.items |token| { - debug.print("{}\n", {token.kind}) - } + for (scan_state.tokens.items) |tok| lexer.render_token(tok, program) debug.print("]]\n\n", {}) parse_state := parser.init(mem.c_allocator) @@ -52,9 +50,7 @@ main proc() void! { } debug.print("AST::[[\n", {}) - for parse_state.nodes.items |node, id| { - debug.print("{} (id = {})\n", { node.kind, id }) - } + for (parse_state.nodes.items) |node| debug.print("{}\n", { node.kind }) debug.print("]]\n\n", {}) renderer := ast_renderer.init(mem.c_allocator) @@ -62,6 +58,6 @@ main proc() void! { try ast_renderer.render(&renderer, root, &parse_state, scan_state.tokens.items) debug.print("AST Render::[[\n", {}) - for (renderer.tokens.items) |tok| debug.print("{}\n", {tok.kind}) + for (renderer.tokens.items) |tok| lexer.render_token(tok, program) debug.print("]]\n", {}) } diff --git a/source/parser/parser.hon b/source/parser/parser.hon index a5988cd..c821365 100644 --- a/source/parser/parser.hon +++ b/source/parser/parser.hon @@ -53,10 +53,10 @@ ExtraId :: alias ast.ExtraId NodeData :: alias ast.NodeData @hide:file -NO_ID_NODE :: alias ast.NO_ID_NODE +NO_NODE :: alias ast.NO_NODE @hide:file -NO_ID_EXTRA :: alias ast.NO_ID_EXTRA +NO_EXTRA :: alias ast.NO_EXTRA BindingPowerScalar :: distinct u32 @@ -103,7 +103,7 @@ parse proc(state @mut State, tokens []Token) NodeId ! ParseError { parse_pkg proc(state @mut State) NodeId ! ParseError { if (state.next_token < state.tokens.len and state.tokens[state.next_token].kind == .eof - ) return NO_ID_NODE + ) return NO_NODE stmt_ids std.ArrayList(NodeId) := arraylist.init(state.nodes.allocator) defer arraylist.deinit(&stmt_ids) @@ -141,7 +141,7 @@ parse_decl proc(state @mut State) NodeId ! ParseError { type_id :: if (next_is(state, .ident)) try parse_primary(state) else - NO_ID_NODE + NO_NODE # expect `::` or `:=` (immutable or mutable assignment) mutability_tok_id :: try consume_either(state, &[.double_colon, .colon_equal]) diff --git a/source/parser/parser.test.hon b/source/parser/parser.test.hon index c817e28..742a45f 100644 --- a/source/parser/parser.test.hon +++ b/source/parser/parser.test.hon @@ -7,8 +7,8 @@ import "@source/lexer" ast_renderer :: import "@source/ast/renderer" handles_package_declarations test { - strpool.strings = strpool.init(mem.c_allocator) - defer strpool.deinit(&strpool.strings) + strpool.STRINGS = strpool.init(mem.c_allocator) + defer strpool.deinit(&strpool.STRINGS) source :: `first int :: 42 @@ -37,8 +37,8 @@ handles_package_declarations test { } handles_binary_arithmetic_expression test { - strpool.strings = strpool.init(mem.c_allocator) - defer strpool.deinit(&strpool.strings) + strpool.STRINGS = strpool.init(mem.c_allocator) + defer strpool.deinit(&strpool.STRINGS) source :: `first int :: 42 + 3 @@ -68,8 +68,8 @@ handles_binary_arithmetic_expression test { } handles_statement_declaration test { - strpool.strings = strpool.init(mem.c_allocator) - defer strpool.deinit(&strpool.strings) + strpool.STRINGS = strpool.init(mem.c_allocator) + defer strpool.deinit(&strpool.STRINGS) source :: `value int :: 42 diff --git a/source/strpool/strpool.hon b/source/strpool/strpool.hon index d26447d..f2b60e3 100644 --- a/source/strpool/strpool.hon +++ b/source/strpool/strpool.hon @@ -2,14 +2,14 @@ import "@std/mem" import "@std/arraylist" import "@std/hashmap" -# cross-cutting concern, hence global singleton (owned by main.hon) -strings StringPool := undefined - InternError :: enum { out_of_space } StringId :: alias u32 -NO_ID_STR :: maxval!(StringId) +NO_STR :: maxval!(StringId) + +# cross-cutting concern, hence global singleton (owned by main.hon) +STRINGS StringPool := undefined StringPool :: struct { ids hashmap.StringHashMap(StringId) # string → id @@ -36,7 +36,7 @@ deinit proc(pool @mut StringPool) void { intern proc(pool @mut StringPool, str []u8) StringId ! (mem.AllocError | InternError) { if hashmap.get(&pool.ids, str) |id| return id - if (pool.strings.items.len >= usize(NO_ID_STR)) return .out_of_space + if (pool.strings.items.len >= usize(NO_STR)) return .out_of_space id StringId :: StringId(pool.strings.items.len) owned_str []mut u8 :: try mem.alloc(u8, pool.allocator, str.len)