basic ast renderer

source/ast/ast.hon

@@ -1,7 +1,15 @@
+#! GRAMMAR:
+#! * type ..... -> identifier
+#! * assignment -> :: | :=
+#! * literal .. -> int | float | string
+#! * expr ..... -> identifier | literal
+#! * decl ..... -> identifier [type] assignment expr
+#! * stmt ..... -> decl
+
+import "@std/debug"
 import "@source/lexer"
 
-hide TokenId :: alias lexer.TokenId
+hide(file) TokenId :: alias lexer.TokenId
-hide Token :: alias lexer.Token
 
 NodeId :: distinct u32
 ExtraId :: distinct u32
@@ -44,27 +52,18 @@ NodeKind :: enum {
     # * data0: ExtraId
     #   + extra[data0]: NodeId - type identifier node
     #   + extra[data0 + 1]: NodeId - initializer expression
-    # * data1: TokenId of either `::` or `=` indicating mutability
+    # * data1: TokenId of either `::` or `:=` indicating mutability
     stmt_decl
 
     invalid
 }
 
-# todo: render ast nodes as source code
+node_id proc(idx uint) NodeId {
-
+    debug.assert(u64(idx) < u64(NO_ID_NODE))
-render proc(node @Node, tokens []Token) void { }
+    return NodeId(idx)
-
+}
-render_literal proc(node @Node, tokens []Token, program []u8) []u8 ! lexer.ScanError {
+
-    tok :: tokens[node.main_token]
+extra_id proc(idx uint) ExtraId {
-    match node.kind {
+    debug.assert(u64(idx) < u64(NO_ID_EXTRA))
-        .literal_int, .literal_float: {
+    return ExtraId(idx)
-            result :: try lexer.scan_number(tok.start, program)
-            return program[tok.start..result.end]
-        }
-        .literal_string: {
-            result :: try lexer.scan_string(tok.start, program)
-            return program[tok.start..result.end]
-        }
-        else: unreachable
-    }
 }

source/ast/renderer/renderer.hon

@@ -0,0 +1,119 @@
+import "@std"
+import "@std/mem"
+import "@std/arraylist"
+
+import "@source/lexer"
+import "@source/parser"
+import "@source/ast"
+
+hide(file) TokenId :: alias lexer.TokenId
+hide(file) Token :: alias lexer.Token
+hide(file) Node :: alias ast.Node
+hide(file) NodeId :: alias ast.NodeId
+hide(file) ParseState :: alias parser.State
+
+Renderer :: struct { tokens std.ArrayList(Token) }
+
+init proc(allocator mem.Allocator) Renderer {
+    return Renderer{ tokens = arraylist.init(allocator) }
+}
+
+deinit proc(renderer @mut Renderer) void {
+    arraylist.deinit(&renderer.tokens)
+}
+
+render proc(
+    renderer @mut Renderer,
+    node_id NodeId,
+    parse_state @ParseState,
+    tokens []Token,
+) void ! mem.AllocError {
+    node :: parse_state.nodes.items[usize(node_id)]
+    match node.kind {
+        .stmt_decl: try render_stmt_decl(renderer, node_id, parse_state, tokens)
+        else: unreachable
+    }
+}
+
+render_stmt_decl proc(
+    renderer @mut Renderer,
+    node_id NodeId,
+    parse_state @ParseState,
+    tokens []Token,
+) void ! mem.AllocError {
+    node :: parse_state.nodes.items[usize(node_id)]
+    tok_ident :: tokens[usize(node.main_token)]
+    try arraylist.append(&renderer.tokens, tok_ident)
+
+    node_id_type :: ast.node_id(parse_state.extra.items[usize(node.data0.extra_id)])
+    try render_type(renderer, node_id_type, parse_state, tokens)
+
+    tok_assign :: tokens[usize(node.data1.token_id)]
+    try arraylist.append(&renderer.tokens, tok_assign)
+
+    node_id_expr :: ast.node_id(parse_state.extra.items[usize(node.data0.extra_id + 1)])
+    last_expr_token :: try render_expr(renderer, node_id_expr, parse_state, tokens)
+
+    terminator_idx :: usize(last_expr_token) + 1
+    tok_terminator :: if (terminator_idx < tokens.len) tokens[terminator_idx] else return
+    try arraylist.append(&renderer.tokens, tok_terminator)
+
+    eof_idx :: terminator_idx + 1
+    if (tok_terminator.kind == .newline and
+        eof_idx < tokens.len and
+        tokens[eof_idx].kind == .eof
+    ) try arraylist.append(&renderer.tokens, tokens[eof_idx])
+}
+
+render_expr proc(renderer @mut Renderer,
+    node_id NodeId,
+    parse_state @ParseState,
+    tokens []Token,
+) TokenId ! mem.AllocError {
+    # todo: support unary and binary expressions
+    node :: parse_state.nodes.items[usize(node_id)]
+    match node.kind {
+        .expr_identifier: {
+            tok :: tokens[usize(node.main_token)]
+            try arraylist.append(&renderer.tokens, tok)
+            return node.main_token
+        }
+        .literal_int, .literal_float, .literal_string:
+            return try render_lit(renderer, node_id, parse_state, tokens)
+        else: unreachable
+    }
+}
+
+render_type proc(
+    renderer @mut Renderer,
+    node_id NodeId,
+    parse_state @ParseState,
+    tokens []Token,
+) void ! mem.AllocError {
+    # todo: support more expressive type expressions...
+    node :: parse_state.nodes.items[usize(node_id)]
+    match node.kind {
+        .expr_identifier: {
+            tok :: tokens[usize(node.main_token)]
+            try arraylist.append(&renderer.tokens, tok)
+        }
+        else: unreachable
+    }
+}
+
+render_lit proc(
+    renderer @mut Renderer,
+    node_id NodeId,
+    parse_state @ParseState,
+    tokens []Token,
+) TokenId ! mem.AllocError {
+    node :: parse_state.nodes.items[usize(node_id)]
+    match node.kind {
+        .literal_int, .literal_float, .literal_string: {
+            tok :: tokens[node.main_token]
+            try arraylist.append(&renderer.tokens, tok)
+            return node.main_token
+        }
+        else: unreachable
+    }
+}

source/main.hon

@@ -10,10 +10,12 @@ test import "@std/strmap"
 import "@source/strpool"
 import "@source/lexer"
 import "@source/parser"
-import "@source/ast"
+
+ast_renderer :: import "@source/ast/renderer"
 
 test import "@source/strpool"
 test import "@source/lexer"
+test import "@source/parser"
 
 
 program ::
@@ -21,7 +23,7 @@ program ::
     `x int :: 123.9
     `y :: "hello"
 
-main proc() void {
+main proc() void! {
     strpool.strings = strpool.init(mem.c_allocator)
     defer strpool.deinit(&strpool.strings)
 
@@ -29,7 +31,6 @@ main proc() void {
 
     scan_state := lexer.init(mem.c_allocator)
     defer lexer.deinit(&scan_state)
-
     lexer.scan(&scan_state, program) catch |err| {
         debug.print("failed to scan: {}\n", {err})
         return
@@ -50,7 +51,7 @@ main proc() void {
 
     parse_state := parser.init(mem.c_allocator)
     defer parser.deinit(&parse_state)
-    parser.parse(&parse_state, scan_state.tokens.items) catch |err| {
+    root :: parser.parse(&parse_state, scan_state.tokens.items) catch |err| {
         debug.print("failed to parse: {}\n", {err})
         return
     }
@@ -71,23 +72,17 @@ main proc() void {
             yield :lbl res.end
         } else token.start
 
-        debug.print("{} (id = {}): {}\n", {
+        debug.print("{} (id = {}): {}\n", { node.kind, id, program[token.start..end] })
-            node.kind,
-            id,
-            program[token.start..end],
-            #node.data0,
-            #node.data1,
-        })
     }
     debug.print("]]\n\n", {})
 
-    debug.print("AST Render::[[\n", {})
+    renderer := ast_renderer.init(mem.c_allocator)
-    literal :: ast.render_literal(
+    defer ast_renderer.deinit(&renderer)
-        &parse_state.nodes.items[1],
+    try ast_renderer.render(&renderer, root, &parse_state, scan_state.tokens.items)
-        scan_state.tokens.items,
-        program,
-    ) catch "<invalid>"
 
-    debug.print("generated literal: {}\n", { literal })
+    debug.print("AST Render::[[\n", {})
+    for renderer.tokens.items |tok| {
+        debug.print("{}\n", {tok.kind})
+    }
     debug.print("]]\n", {})
 }

source/parser/parser.hon

@@ -1,7 +1,6 @@
 import "@std"
 import "@std/mem"
 import "@std/arraylist"
-import "@std/debug"
 import "@std/enums/enummap"
 
 import "@source/lexer"
@@ -32,16 +31,16 @@ error_msg_map std.EnumMap(ErrorCode, ErrorDetails) :: enummap.init({
     },
 })
 
-hide Token :: alias lexer.Token
+hide(file) Token :: alias lexer.Token
-hide TokenId :: alias lexer.TokenId
+hide(file) TokenId :: alias lexer.TokenId
-hide TokenKind :: alias lexer.TokenKind
+hide(file) TokenKind :: alias lexer.TokenKind
-hide Node :: alias ast.Node
+hide(file) Node :: alias ast.Node
-hide NodeId :: alias ast.NodeId
+hide(file) NodeId :: alias ast.NodeId
-hide ExtraId :: alias ast.ExtraId
+hide(file) ExtraId :: alias ast.ExtraId
-hide NodeData :: alias ast.NodeData
+hide(file) NodeData :: alias ast.NodeData
 
-hide NO_ID_NODE :: alias ast.NO_ID_NODE
+hide(file) NO_ID_NODE :: alias ast.NO_ID_NODE
-hide NO_ID_EXTRA :: alias ast.NO_ID_EXTRA
+hide(file) NO_ID_EXTRA :: alias ast.NO_ID_EXTRA
 
 State :: struct {
     nodes std.ArrayList(Node)
@@ -65,9 +64,9 @@ deinit proc(state @mut State) void {
     arraylist.deinit(&state.extra)
 }
 
-parse proc(state @mut State, tokens []Token) void ! ParseError {
+parse proc(state @mut State, tokens []Token) NodeId ! ParseError {
     state.tokens = tokens
-    _ = try parse_decl(state)
+    return try parse_decl(state)
 }
 
 parse_decl proc(state @mut State) NodeId ! ParseError {
@@ -87,7 +86,7 @@ parse_decl proc(state @mut State) NodeId ! ParseError {
     mutability_tok_id :: try consume(state, .double_colon)
 
     # expect expression
-    expr_id :: try parse_primary(state)
+    expr_id :: try parse_expr(state)
 
     # expect statement terminator (newline)
     _ = try consume_either(state, &[.newline, .eof])
@@ -105,6 +104,10 @@ parse_decl proc(state @mut State) NodeId ! ParseError {
     return decl
 }
 
+parse_expr proc(state @mut State) NodeId ! mem.AllocError {
+    return try parse_primary(state)
+}
+
 parse_primary proc(state @mut State) NodeId ! mem.AllocError {
     start_token :: state.next_token
     state.next_token += 1
@@ -154,23 +157,13 @@ hide consume_either proc(state @mut State, token_kinds []TokenKind) TokenId ! Er
 }
 
 hide add_node proc(nodes @mut std.ArrayList(Node), node Node) NodeId ! mem.AllocError {
-    id :: node_id(nodes.items.len)
+    id :: ast.node_id(nodes.items.len)
     try arraylist.append(nodes, node)
     return id
 }
 
-hide node_id proc(idx uint) NodeId {
+hide add_extra proc(extra @mut std.ArrayList(u32), data u32) ExtraId ! mem.AllocError {
-    debug.assert(u64(idx) < u64(NO_ID_NODE))
+    id :: ast.extra_id(extra.items.len)
-    return NodeId(idx)
+    try arraylist.append(extra, data)
-}
-
-hide add_extra proc(extra @mut std.ArrayList(u32), idx u32) ExtraId ! mem.AllocError {
-    id :: extra_id(extra.items.len)
-    try arraylist.append(extra, idx)
     return id
 }
-
-hide extra_id proc(idx uint) ExtraId {
-    debug.assert(u64(idx) < u64(NO_ID_EXTRA))
-    return ExtraId(idx)
-}

source/parser/parser.test.hon

@@ -0,0 +1,35 @@
+import "@std/mem"
+import "@std/testing"
+
+import "@source/strpool"
+import "@source/lexer"
+
+ast_renderer :: import "@source/ast/renderer"
+
+handles_statement_declaration test {
+    strpool.strings = strpool.init(mem.c_allocator)
+    defer strpool.deinit(&strpool.strings)
+
+    source ::
+        `value int :: 42
+
+    scan_state := lexer.init(mem.c_allocator)
+    defer lexer.deinit(&scan_state)
+    try lexer.scan(&scan_state, source)
+
+    parse_state := init(mem.c_allocator)
+    defer deinit(&parse_state)
+    root :: try parse(&parse_state, scan_state.tokens.items)
+
+    renderer := ast_renderer.init(mem.c_allocator)
+    defer ast_renderer.deinit(&renderer)
+    try ast_renderer.render(&renderer, root, &parse_state, scan_state.tokens.items)
+
+    try testing.expect_equal(scan_state.tokens.items.len, renderer.tokens.items.len)
+    for scan_state.tokens.items |expected, i| {
+        actual :: renderer.tokens.items[i]
+        try testing.expect_equal(expected.kind, actual.kind)
+        try testing.expect_equal(expected.start, actual.start)
+        try testing.expect_equal(expected.str_id, actual.str_id)
+    }
+}

source/strpool/strpool.test.hon

@@ -1,7 +1,7 @@
 import "@std/mem"
 import "@std/testing"
 
-handles_intern test {
+handles_interning test {
     pool StringPool := init(mem.c_allocator)
     defer deinit(&pool)