diff --git a/source/ast/ast.hon b/source/ast/ast.hon index a554815..b69afe1 100644 --- a/source/ast/ast.hon +++ b/source/ast/ast.hon @@ -1,7 +1,15 @@ +#! GRAMMAR: +#! * type ..... -> identifier +#! * assignment -> :: | := +#! * literal .. -> int | float | string +#! * expr ..... -> identifier | literal +#! * decl ..... -> identifier [type] assignment expr +#! * stmt ..... -> decl + +import "@std/debug" import "@source/lexer" -hide TokenId :: alias lexer.TokenId -hide Token :: alias lexer.Token +hide(file) TokenId :: alias lexer.TokenId NodeId :: distinct u32 ExtraId :: distinct u32 @@ -44,27 +52,18 @@ NodeKind :: enum { # * data0: ExtraId # + extra[data0]: NodeId - type identifier node # + extra[data0 + 1]: NodeId - initializer expression - # * data1: TokenId of either `::` or `=` indicating mutability + # * data1: TokenId of either `::` or `:=` indicating mutability stmt_decl invalid } -# todo: render ast nodes as source code - -render proc(node @Node, tokens []Token) void { } - -render_literal proc(node @Node, tokens []Token, program []u8) []u8 ! lexer.ScanError { - tok :: tokens[node.main_token] - match node.kind { - .literal_int, .literal_float: { - result :: try lexer.scan_number(tok.start, program) - return program[tok.start..result.end] - } - .literal_string: { - result :: try lexer.scan_string(tok.start, program) - return program[tok.start..result.end] - } - else: unreachable - } +node_id proc(idx uint) NodeId { + debug.assert(u64(idx) < u64(NO_ID_NODE)) + return NodeId(idx) +} + +extra_id proc(idx uint) ExtraId { + debug.assert(u64(idx) < u64(NO_ID_EXTRA)) + return ExtraId(idx) } diff --git a/source/ast/renderer/renderer.hon b/source/ast/renderer/renderer.hon new file mode 100644 index 0000000..bc7e057 --- /dev/null +++ b/source/ast/renderer/renderer.hon @@ -0,0 +1,119 @@ +import "@std" +import "@std/mem" +import "@std/arraylist" + +import "@source/lexer" +import "@source/parser" +import "@source/ast" + +hide(file) TokenId :: alias lexer.TokenId +hide(file) Token :: alias lexer.Token +hide(file) Node :: alias ast.Node +hide(file) NodeId :: alias ast.NodeId +hide(file) ParseState :: alias parser.State + +Renderer :: struct { tokens std.ArrayList(Token) } + +init proc(allocator mem.Allocator) Renderer { + return Renderer{ tokens = arraylist.init(allocator) } +} + +deinit proc(renderer @mut Renderer) void { + arraylist.deinit(&renderer.tokens) +} + +render proc( + renderer @mut Renderer, + node_id NodeId, + parse_state @ParseState, + tokens []Token, +) void ! mem.AllocError { + node :: parse_state.nodes.items[usize(node_id)] + match node.kind { + .stmt_decl: try render_stmt_decl(renderer, node_id, parse_state, tokens) + else: unreachable + } +} + +render_stmt_decl proc( + renderer @mut Renderer, + node_id NodeId, + parse_state @ParseState, + tokens []Token, +) void ! mem.AllocError { + node :: parse_state.nodes.items[usize(node_id)] + tok_ident :: tokens[usize(node.main_token)] + try arraylist.append(&renderer.tokens, tok_ident) + + node_id_type :: ast.node_id(parse_state.extra.items[usize(node.data0.extra_id)]) + try render_type(renderer, node_id_type, parse_state, tokens) + + tok_assign :: tokens[usize(node.data1.token_id)] + try arraylist.append(&renderer.tokens, tok_assign) + + node_id_expr :: ast.node_id(parse_state.extra.items[usize(node.data0.extra_id + 1)]) + last_expr_token :: try render_expr(renderer, node_id_expr, parse_state, tokens) + + terminator_idx :: usize(last_expr_token) + 1 + tok_terminator :: if (terminator_idx < tokens.len) tokens[terminator_idx] else return + try arraylist.append(&renderer.tokens, tok_terminator) + + eof_idx :: terminator_idx + 1 + if (tok_terminator.kind == .newline and + eof_idx < tokens.len and + tokens[eof_idx].kind == .eof + ) try arraylist.append(&renderer.tokens, tokens[eof_idx]) +} + +render_expr proc(renderer @mut Renderer, + node_id NodeId, + parse_state @ParseState, + tokens []Token, +) TokenId ! mem.AllocError { + # todo: support unary and binary expressions + node :: parse_state.nodes.items[usize(node_id)] + match node.kind { + .expr_identifier: { + tok :: tokens[usize(node.main_token)] + try arraylist.append(&renderer.tokens, tok) + return node.main_token + } + .literal_int, .literal_float, .literal_string: + return try render_lit(renderer, node_id, parse_state, tokens) + else: unreachable + } +} + +render_type proc( + renderer @mut Renderer, + node_id NodeId, + parse_state @ParseState, + tokens []Token, +) void ! mem.AllocError { + # todo: support more expressive type expressions... + node :: parse_state.nodes.items[usize(node_id)] + match node.kind { + .expr_identifier: { + tok :: tokens[usize(node.main_token)] + try arraylist.append(&renderer.tokens, tok) + } + else: unreachable + } +} + +render_lit proc( + renderer @mut Renderer, + node_id NodeId, + parse_state @ParseState, + tokens []Token, +) TokenId ! mem.AllocError { + node :: parse_state.nodes.items[usize(node_id)] + match node.kind { + .literal_int, .literal_float, .literal_string: { + tok :: tokens[node.main_token] + try arraylist.append(&renderer.tokens, tok) + return node.main_token + } + else: unreachable + } +} diff --git a/source/main.hon b/source/main.hon index a820505..f8b51e8 100644 --- a/source/main.hon +++ b/source/main.hon @@ -10,10 +10,12 @@ test import "@std/strmap" import "@source/strpool" import "@source/lexer" import "@source/parser" -import "@source/ast" + +ast_renderer :: import "@source/ast/renderer" test import "@source/strpool" test import "@source/lexer" +test import "@source/parser" program :: @@ -21,7 +23,7 @@ program :: `x int :: 123.9 `y :: "hello" -main proc() void { +main proc() void! { strpool.strings = strpool.init(mem.c_allocator) defer strpool.deinit(&strpool.strings) @@ -29,7 +31,6 @@ main proc() void { scan_state := lexer.init(mem.c_allocator) defer lexer.deinit(&scan_state) - lexer.scan(&scan_state, program) catch |err| { debug.print("failed to scan: {}\n", {err}) return @@ -50,7 +51,7 @@ main proc() void { parse_state := parser.init(mem.c_allocator) defer parser.deinit(&parse_state) - parser.parse(&parse_state, scan_state.tokens.items) catch |err| { + root :: parser.parse(&parse_state, scan_state.tokens.items) catch |err| { debug.print("failed to parse: {}\n", {err}) return } @@ -71,23 +72,17 @@ main proc() void { yield :lbl res.end } else token.start - debug.print("{} (id = {}): {}\n", { - node.kind, - id, - program[token.start..end], - #node.data0, - #node.data1, - }) + debug.print("{} (id = {}): {}\n", { node.kind, id, program[token.start..end] }) } debug.print("]]\n\n", {}) + renderer := ast_renderer.init(mem.c_allocator) + defer ast_renderer.deinit(&renderer) + try ast_renderer.render(&renderer, root, &parse_state, scan_state.tokens.items) + debug.print("AST Render::[[\n", {}) - literal :: ast.render_literal( - &parse_state.nodes.items[1], - scan_state.tokens.items, - program, - ) catch "" - - debug.print("generated literal: {}\n", { literal }) + for renderer.tokens.items |tok| { + debug.print("{}\n", {tok.kind}) + } debug.print("]]\n", {}) } diff --git a/source/parser/parser.hon b/source/parser/parser.hon index 7ad7ae8..fe46397 100644 --- a/source/parser/parser.hon +++ b/source/parser/parser.hon @@ -1,7 +1,6 @@ import "@std" import "@std/mem" import "@std/arraylist" -import "@std/debug" import "@std/enums/enummap" import "@source/lexer" @@ -32,16 +31,16 @@ error_msg_map std.EnumMap(ErrorCode, ErrorDetails) :: enummap.init({ }, }) -hide Token :: alias lexer.Token -hide TokenId :: alias lexer.TokenId -hide TokenKind :: alias lexer.TokenKind -hide Node :: alias ast.Node -hide NodeId :: alias ast.NodeId -hide ExtraId :: alias ast.ExtraId -hide NodeData :: alias ast.NodeData +hide(file) Token :: alias lexer.Token +hide(file) TokenId :: alias lexer.TokenId +hide(file) TokenKind :: alias lexer.TokenKind +hide(file) Node :: alias ast.Node +hide(file) NodeId :: alias ast.NodeId +hide(file) ExtraId :: alias ast.ExtraId +hide(file) NodeData :: alias ast.NodeData -hide NO_ID_NODE :: alias ast.NO_ID_NODE -hide NO_ID_EXTRA :: alias ast.NO_ID_EXTRA +hide(file) NO_ID_NODE :: alias ast.NO_ID_NODE +hide(file) NO_ID_EXTRA :: alias ast.NO_ID_EXTRA State :: struct { nodes std.ArrayList(Node) @@ -65,9 +64,9 @@ deinit proc(state @mut State) void { arraylist.deinit(&state.extra) } -parse proc(state @mut State, tokens []Token) void ! ParseError { +parse proc(state @mut State, tokens []Token) NodeId ! ParseError { state.tokens = tokens - _ = try parse_decl(state) + return try parse_decl(state) } parse_decl proc(state @mut State) NodeId ! ParseError { @@ -87,7 +86,7 @@ parse_decl proc(state @mut State) NodeId ! ParseError { mutability_tok_id :: try consume(state, .double_colon) # expect expression - expr_id :: try parse_primary(state) + expr_id :: try parse_expr(state) # expect statement terminator (newline) _ = try consume_either(state, &[.newline, .eof]) @@ -105,6 +104,10 @@ parse_decl proc(state @mut State) NodeId ! ParseError { return decl } +parse_expr proc(state @mut State) NodeId ! mem.AllocError { + return try parse_primary(state) +} + parse_primary proc(state @mut State) NodeId ! mem.AllocError { start_token :: state.next_token state.next_token += 1 @@ -154,23 +157,13 @@ hide consume_either proc(state @mut State, token_kinds []TokenKind) TokenId ! Er } hide add_node proc(nodes @mut std.ArrayList(Node), node Node) NodeId ! mem.AllocError { - id :: node_id(nodes.items.len) + id :: ast.node_id(nodes.items.len) try arraylist.append(nodes, node) return id } -hide node_id proc(idx uint) NodeId { - debug.assert(u64(idx) < u64(NO_ID_NODE)) - return NodeId(idx) -} - -hide add_extra proc(extra @mut std.ArrayList(u32), idx u32) ExtraId ! mem.AllocError { - id :: extra_id(extra.items.len) - try arraylist.append(extra, idx) +hide add_extra proc(extra @mut std.ArrayList(u32), data u32) ExtraId ! mem.AllocError { + id :: ast.extra_id(extra.items.len) + try arraylist.append(extra, data) return id } - -hide extra_id proc(idx uint) ExtraId { - debug.assert(u64(idx) < u64(NO_ID_EXTRA)) - return ExtraId(idx) -} diff --git a/source/parser/parser.test.hon b/source/parser/parser.test.hon new file mode 100644 index 0000000..e2d97fe --- /dev/null +++ b/source/parser/parser.test.hon @@ -0,0 +1,35 @@ +import "@std/mem" +import "@std/testing" + +import "@source/strpool" +import "@source/lexer" + +ast_renderer :: import "@source/ast/renderer" + +handles_statement_declaration test { + strpool.strings = strpool.init(mem.c_allocator) + defer strpool.deinit(&strpool.strings) + + source :: + `value int :: 42 + + scan_state := lexer.init(mem.c_allocator) + defer lexer.deinit(&scan_state) + try lexer.scan(&scan_state, source) + + parse_state := init(mem.c_allocator) + defer deinit(&parse_state) + root :: try parse(&parse_state, scan_state.tokens.items) + + renderer := ast_renderer.init(mem.c_allocator) + defer ast_renderer.deinit(&renderer) + try ast_renderer.render(&renderer, root, &parse_state, scan_state.tokens.items) + + try testing.expect_equal(scan_state.tokens.items.len, renderer.tokens.items.len) + for scan_state.tokens.items |expected, i| { + actual :: renderer.tokens.items[i] + try testing.expect_equal(expected.kind, actual.kind) + try testing.expect_equal(expected.start, actual.start) + try testing.expect_equal(expected.str_id, actual.str_id) + } +} diff --git a/source/strpool/strpool.test.hon b/source/strpool/strpool.test.hon index f3fb6c4..7fe9855 100644 --- a/source/strpool/strpool.test.hon +++ b/source/strpool/strpool.test.hon @@ -1,7 +1,7 @@ import "@std/mem" import "@std/testing" -handles_intern test { +handles_interning test { pool StringPool := init(mem.c_allocator) defer deinit(&pool)