From cfd668b729c193d39f0621cf147ca49c9ed25224 Mon Sep 17 00:00:00 2001 From: hl-valdemar Date: Fri, 14 Aug 2026 20:04:22 +0200 Subject: [PATCH] parse func decls --- source/ast/ast.hon | 23 +++- source/ast/renderer/renderer.hon | 159 +++++++++++++++++++++++---- source/lexer/lexer.hon | 1 + source/lexer/token.hon | 9 +- source/main.hon | 33 ++++-- source/parser/parser.hon | 179 ++++++++++++++++++++++--------- source/parser/parser.test.hon | 78 ++++++++++++++ 7 files changed, 397 insertions(+), 85 deletions(-) diff --git a/source/ast/ast.hon b/source/ast/ast.hon index a554c43..6aebb56 100644 --- a/source/ast/ast.hon +++ b/source/ast/ast.hon @@ -59,14 +59,33 @@ NodeKind :: enum { # * data1: NodeId - right-hand side expr_binary - # DECL: + # STATEMENT DECLS: # * main_token: symbol name (identifier) # * data0: ExtraId - # + extra[data0]: NodeId - type identifier node + # + extra[data0]: NodeId - type expr or NO_NODE # + extra[data0 + 1]: NodeId - initializer expression # * data1: TokenId of either `::` or `:=` indicating mutability stmt_decl + # FUNC DECLS: + # * main_token: function name (identifier) + # * data0: NodeId - body block (or NO_NODE for extern) + # * data1: ExtraId + # + extra[data1]: NodeId - return type + # + extra[data1 + 1]: u32 - param count + # + extra[data1 + 2..]: NodeId - param decls + func_decl + + # PARAM DECLS: + # * main_token: param name (identifier) + # * data0: NodeId - param type + param_decl + + # BLOCK: + # * data0: ExtraId - statement range start + # * data1: ExtraId - statement range end + block + # PACKAGE: # * data0: ExtraId # * data1: ExtraId diff --git a/source/ast/renderer/renderer.hon b/source/ast/renderer/renderer.hon index feb4399..9af71a5 100644 --- a/source/ast/renderer/renderer.hon +++ b/source/ast/renderer/renderer.hon @@ -6,20 +6,11 @@ import "@source/lexer" import "@source/parser" import "@source/ast" -@hide:file -TokenId :: alias lexer.TokenId - -@hide:file -Token :: alias lexer.Token - -@hide:file -Node :: alias ast.Node - -@hide:file -NodeId :: alias ast.NodeId - -@hide:file -ParseState :: alias parser.State +@hide:file TokenId :: alias lexer.TokenId +@hide:file Token :: alias lexer.Token +@hide:file Node :: alias ast.Node +@hide:file NodeId :: alias ast.NodeId +@hide:file ParseState :: alias parser.State Renderer :: struct { tokens std.ArrayList(Token) } @@ -42,7 +33,9 @@ render_token_stream proc( .literal_int, .literal_float, .literal_string: _ = try render_lit(renderer, node_id, parse_state, tokens) .stmt_decl: - try render_decl(renderer, node_id, parse_state, tokens) + try render_stmt_decl(renderer, node_id, parse_state, tokens) + .func_decl: + try render_func_decl(renderer, node_id, parse_state, tokens) .package: try render_pkg(renderer, node_id, parse_state, tokens) else: unreachable @@ -64,14 +57,16 @@ render_pkg proc( stmt_node :: parse_state.nodes.items[usize(stmt_node_id)] match stmt_node.kind { .stmt_decl: - try render_decl(renderer, stmt_node_id, parse_state, tokens) + try render_stmt_decl(renderer, stmt_node_id, parse_state, tokens) + .func_decl: + try render_func_decl(renderer, stmt_node_id, parse_state, tokens) else: unreachable } } } @hide:file -render_decl proc( +render_stmt_decl proc( renderer @mut Renderer, node_id NodeId, parse_state @ParseState, @@ -93,6 +88,7 @@ render_decl proc( terminator_idx :: usize(last_expr_token) + 1 tok_terminator :: if (terminator_idx < tokens.len) tokens[terminator_idx] else return + if (tok_terminator.kind != .newline and tok_terminator.kind != .eof) return try arraylist.append(&renderer.tokens, tok_terminator) eof_idx :: terminator_idx + 1 @@ -103,7 +99,134 @@ render_decl proc( } @hide:file -render_expr proc(renderer @mut Renderer, +render_func_decl proc( + renderer @mut Renderer, + node_id NodeId, + parse_state @ParseState, + tokens []Token, +) void ! mem.AllocError { + node :: parse_state.nodes.items[usize(node_id)] + ident_idx :: usize(node.main_token) + try arraylist.append(&renderer.tokens, tokens[ident_idx]) + try arraylist.append(&renderer.tokens, tokens[ident_idx + 1]) + try arraylist.append(&renderer.tokens, tokens[ident_idx + 2]) + + payload_start :: usize(node.data1.extra_id) + param_count :: usize(parse_state.extra.items[payload_start + 1]) + first_param :: payload_start + 2 + for 0..param_count |i| { + param_id :: ast.node_id(parse_state.extra.items[first_param + i]) + param_node :: parse_state.nodes.items[usize(param_id)] + type_id :: param_node.data0.node_id + type_node :: parse_state.nodes.items[usize(type_id)] + try arraylist.append(&renderer.tokens, tokens[usize(param_node.main_token)]) + + has_next :: i + 1 < param_count + if has_next { + next_param_id :: ast.node_id(parse_state.extra.items[first_param + i + 1]) + next_param :: parse_state.nodes.items[usize(next_param_id)] + if (next_param.data0.node_id == type_id) { + try arraylist.append( + &renderer.tokens, + tokens[usize(param_node.main_token) + 1], + ) + continue + } + } + + try render_type(renderer, type_id, parse_state, tokens) + if has_next { + try arraylist.append( + &renderer.tokens, + tokens[usize(type_node.main_token) + 1], + ) + } + } + + return_type_id :: ast.node_id(parse_state.extra.items[payload_start]) + return_type_node :: parse_state.nodes.items[usize(return_type_id)] + try arraylist.append( + &renderer.tokens, + tokens[usize(return_type_node.main_token) - 1], + ) + try render_type(renderer, return_type_id, parse_state, tokens) + + open_curly_id :: TokenId(usize(return_type_node.main_token) + 1) + close_curly_id :: try render_block( + renderer, + node.data0.node_id, + open_curly_id, + parse_state, + tokens, + ) + + terminator_idx :: usize(close_curly_id) + 1 + if (terminator_idx >= tokens.len) return + tok_terminator :: tokens[terminator_idx] + if (tok_terminator.kind != .newline and tok_terminator.kind != .eof) return + try arraylist.append(&renderer.tokens, tok_terminator) + + eof_idx :: terminator_idx + 1 + if (tok_terminator.kind == .newline and + eof_idx < tokens.len and + tokens[eof_idx].kind == .eof + ) try arraylist.append(&renderer.tokens, tokens[eof_idx]) +} + +@hide:file +render_block proc( + renderer @mut Renderer, + node_id NodeId, + open_curly_id TokenId, + parse_state @ParseState, + tokens []Token, +) TokenId ! mem.AllocError { + node :: parse_state.nodes.items[usize(node_id)] + open_curly_idx :: usize(open_curly_id) + try arraylist.append(&renderer.tokens, tokens[open_curly_idx]) + + leading_token := open_curly_idx + 1 + while (leading_token < tokens.len and tokens[leading_token].kind == .newline) { + try arraylist.append(&renderer.tokens, tokens[leading_token]) + leading_token += 1 + } + + extra_start :: usize(node.data0.extra_id) + extra_end :: usize(node.data1.extra_id) + for extra_start..extra_end |i| { + stmt_id :: ast.node_id(parse_state.extra.items[i]) + stmt :: parse_state.nodes.items[usize(stmt_id)] + match stmt.kind { + .stmt_decl: + try render_stmt_decl(renderer, stmt_id, parse_state, tokens) + else: unreachable + } + } + + # ponytail: linear delimiter scan; store brace tokens if nested blocks make rendering hot. + depth usize := 1 + close_curly_idx := open_curly_idx + 1 + while close_curly_idx < tokens.len { + match tokens[close_curly_idx].kind { + .open_curly: depth += 1 + .close_curly: { + depth -= 1 + if (depth == 0) { + try arraylist.append(&renderer.tokens, tokens[close_curly_idx]) + return TokenId(close_curly_idx) + } + } + else: {} + } + close_curly_idx += 1 + } + unreachable +} + + +@hide:file +render_expr proc( + renderer @mut Renderer, node_id NodeId, parse_state @ParseState, tokens []Token, diff --git a/source/lexer/lexer.hon b/source/lexer/lexer.hon index 7661297..5d7ceb7 100644 --- a/source/lexer/lexer.hon +++ b/source/lexer/lexer.hon @@ -202,6 +202,7 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter # single character tokens kind ?TokenKind :: match char { '=': .equal + ',': .comma '(': .open_paren ')': .close_paren '{': .open_curly diff --git a/source/lexer/token.hon b/source/lexer/token.hon index 2b29f30..64644f4 100644 --- a/source/lexer/token.hon +++ b/source/lexer/token.hon @@ -43,6 +43,7 @@ TokenKind :: enum { open_bracket, close_bracket open_curly, close_curly + comma colon newline @@ -54,8 +55,12 @@ TokenKind :: enum { render_token proc(tok Token, program []u8) void { match tok.kind { .ident: { - str :: strpool.get_str(&strpool.STRINGS, tok.str_id) orelse "null" - debug.print("{}({})\n", {tok.kind, str}) + if (strpool.get_str(&strpool.STRINGS, tok.str_id)) |str| { + debug.print("{}({})\n", {tok.kind, str}) + } else { + res :: scan_ident(tok.start, program) + debug.print("{}({})\n", {tok.kind, program[tok.start..res.end]}) + } } .string: { res :: scan_string(tok.start, program) catch |_| { diff --git a/source/main.hon b/source/main.hon index c842b1a..cc77504 100644 --- a/source/main.hon +++ b/source/main.hon @@ -25,6 +25,11 @@ program :: ` `a := 1 + 2 * 3 `b :: -3 / 2 + 1 + ` + `empty proc() void {} + `grouped proc(a, b T, c U) void { + ` value int :: 1 + `} main proc() void! { strpool.STRINGS = strpool.init(mem.c_allocator) @@ -51,39 +56,47 @@ main proc() void! { } debug.print("AST::[[\n", {}) - for parse_state.nodes.items |node| { + for parse_state.nodes.items |node, i| { match node.kind { .expr_identifier: { ident :: scan_state.tokens.items[usize(node.main_token)] res :: lexer.scan_ident(ident.start, program) - debug.print("{}({})\n", { node.kind, program[ident.start..res.end] }) + debug.print("{}: {}({})\n", { i, node.kind, program[ident.start..res.end] }) } .literal_int, .literal_float: { lit :: scan_state.tokens.items[usize(node.main_token)] res :: lexer.scan_number(lit.start, program) catch |_| { - debug.print("{}\n", { node.kind }) + debug.print("{}: {}\n", { i, node.kind }) continue } - debug.print("{}({})\n", { node.kind, program[lit.start..res.end] }) + debug.print("{}: {}({})\n", { i, node.kind, program[lit.start..res.end] }) } .literal_string: { str :: scan_state.tokens.items[usize(node.main_token)] res :: lexer.scan_string(str.start, program) catch |_| { - debug.print("{}\n", { node.kind }) + debug.print("{}: {}\n", { i, node.kind }) continue } - debug.print("{}({})\n", { node.kind, program[str.start..res.end] }) + debug.print("{}: {}({})\n", { i, node.kind, program[str.start..res.end] }) } - .expr_unary, .expr_binary: { + .expr_unary: { operator :: scan_state.tokens.items[usize(node.main_token)] - debug.print("{}({})\n", { node.kind, operator.kind }) + operand :: node.data0.node_id + debug.print("{}: {}({}, operand={})\n", { i, node.kind, operator.kind, operand }) + } + .expr_binary: { + operator :: scan_state.tokens.items[usize(node.main_token)] + lhs :: node.data0.node_id + rhs :: node.data1.node_id + debug.print("{}: {}({}, lhs={}, rhs={})\n", { i, node.kind, operator.kind, lhs, rhs }) } .stmt_decl: { ident :: scan_state.tokens.items[usize(node.main_token)] + init :: parse_state.extra.items[usize(node.data0.extra_id) + 1] res :: lexer.scan_ident(ident.start, program) - debug.print("{}({})\n", { node.kind, program[ident.start..res.end] }) + debug.print("{}: {}({}, init={})\n", { i, node.kind, program[ident.start..res.end], init }) } - else: debug.print("{}\n", { node.kind }) + else: debug.print("{}: {}\n", { i, node.kind }) } } debug.print("]]\n\n", {}) diff --git a/source/parser/parser.hon b/source/parser/parser.hon index 405b42d..e88cb23 100644 --- a/source/parser/parser.hon +++ b/source/parser/parser.hon @@ -32,35 +32,16 @@ error_msg_map std.EnumMap(ErrorCode, ErrorDetails) :: enummap.init({ }, }) -@hide:file -Token :: alias lexer.Token - -@hide:file -TokenId :: alias lexer.TokenId - -@hide:file -TokenKind :: alias lexer.TokenKind - -@hide:file -Node :: alias ast.Node - -@hide:file -NodeId :: alias ast.NodeId - -@hide:file -ExtraId :: alias ast.ExtraId - -@hide:file -NodeData :: alias ast.NodeData - -@hide:file -NO_NODE :: alias ast.NO_NODE - -@hide:file -NO_EXTRA :: alias ast.NO_EXTRA - -@hide:file -token_id :: alias lexer.token_id +@hide:file Token :: alias lexer.Token +@hide:file TokenId :: alias lexer.TokenId +@hide:file TokenKind :: alias lexer.TokenKind +@hide:file Node :: alias ast.Node +@hide:file NodeId :: alias ast.NodeId +@hide:file ExtraId :: alias ast.ExtraId +@hide:file NodeData :: alias ast.NodeData +@hide:file NO_NODE :: alias ast.NO_NODE +@hide:file NO_EXTRA :: alias ast.NO_EXTRA +@hide:file token_id :: alias lexer.token_id BindingPowerScalar :: distinct u32 @@ -129,7 +110,7 @@ parse_pkg proc(state @mut State) NodeId ! ParseError { while (state.next_token < state.tokens.len and state.tokens[state.next_token].kind != .eof ) { - stmt_id :: try parse_stmt(state) + stmt_id :: try parse_decl(state) try arraylist.append(&stmt_ids, stmt_id) } @@ -137,22 +118,112 @@ parse_pkg proc(state @mut State) NodeId ! ParseError { for (stmt_ids.items) |stmt_id| _ = try add_extra(state, u32(stmt_id)) pkg_end :: ast.extra_id(state.extra.items.len) - package :: try add_node(state, Node{ + return try add_node(state, Node{ kind = .package, data0 = NodeData{ extra_id = pkg_start }, data1 = NodeData{ extra_id = pkg_end }, }) - - return package -} - -parse_stmt proc(state @mut State) NodeId ! ParseError { - return try parse_decl(state) } parse_decl proc(state @mut State) NodeId ! ParseError { + if next_n_is(state, 1, .proc) + return try parse_func_decl(state) + else + return try parse_stmt_decl(state) +} + +parse_func_decl proc(state @mut State) NodeId ! ParseError { + ident_tok_id :: try consume(state, .ident) + _ = try consume(state, .proc) + _ = try consume(state, .open_paren) + + param_decls std.ArrayList(NodeId) := arraylist.init(state.nodes.allocator) + defer arraylist.deinit(¶m_decls) + + ident_ids std.ArrayList(TokenId) := arraylist.init(state.nodes.allocator) + defer arraylist.deinit(&ident_ids) + + while !next_is(state, .close_paren) { + arraylist.clear(&ident_ids) + + # collect param group + while true { + param_tok_id :: try consume(state, .ident) + try arraylist.append(&ident_ids, param_tok_id) + + if next_is(state, .ident) break + _ = try consume(state, .comma) + } + + # collect type + try expect(state, .ident) + type_id :: try parse_primary(state) + for (ident_ids.items) |id| { + param_decl :: try add_node(state, Node{ + kind = .param_decl, + main_token = id, + data0 = NodeData{ node_id = type_id }, + }) + try arraylist.append(¶m_decls, param_decl) + } + + if next_is(state, .comma) _ = try consume(state, .comma) + } + + _ = try consume(state, .close_paren) + + try expect(state, .ident) + return_type_id :: try parse_primary(state) + block_id :: try parse_block(state) + + while next_is(state, .newline) _ = try consume(state, .newline) + + extra_id :: try add_extra(state, u32(return_type_id)) + _ = try add_extra(state, u32(param_decls.items.len)) + for (param_decls.items) |param| _ = try add_extra(state, u32(param)) + + return try add_node(state, Node{ + kind = .func_decl, + main_token = ident_tok_id, + data0 = NodeData{ node_id = block_id }, + data1 = NodeData{ extra_id = extra_id }, + }) +} + +parse_block proc(state @mut State) NodeId ! ParseError { + stmt_ids std.ArrayList(NodeId) := arraylist.init(state.nodes.allocator) + defer arraylist.deinit(&stmt_ids) + + _ = try consume(state, .open_curly) + while next_is(state, .newline) _ = try consume(state, .newline) + + while !next_is(state, .close_curly) { + stmt_id :: try parse_stmt(state) + try arraylist.append(&stmt_ids, stmt_id) + while next_is(state, .newline) _ = try consume(state, .newline) + } + + _ = try consume(state, .close_curly) + + block_start :: ast.extra_id(state.extra.items.len) + for (stmt_ids.items) |id| _ = try add_extra(state, u32(id)) + block_end :: ast.extra_id(state.extra.items.len) + + return try add_node(state, Node{ + kind = .block, + data0 = NodeData{ extra_id = block_start }, + data1 = NodeData{ extra_id = block_end }, + }) +} + +parse_stmt proc(state @mut State) NodeId ! ParseError { + if next_is(state, .ident) return try parse_stmt_decl(state) + return .unexpected_token +} + +parse_stmt_decl proc(state @mut State) NodeId ! ParseError { # expect identifier - ident_token :: try consume(state, .ident) + ident_tok_id :: try consume(state, .ident) # check for type # todo: make a parse_type proc for this @@ -170,17 +241,15 @@ parse_decl proc(state @mut State) NodeId ! ParseError { # expect statement terminator (newline) while (next_is(state, .newline)) _ = try consume(state, .newline) - extra_start :: try add_extra(state, u32(type_id)) + extra_start_id :: try add_extra(state, u32(type_id)) _ = try add_extra(state, u32(expr_id)) - decl :: try add_node(state, Node{ + return try add_node(state, Node{ kind = .stmt_decl, - main_token = ident_token, - data0 = NodeData{ extra_id = extra_start }, + main_token = ident_tok_id, + data0 = NodeData{ extra_id = extra_start_id }, data1 = NodeData{ token_id = mutability_tok_id }, }) - - return decl } parse_expr proc(state @mut State) NodeId ! mem.AllocError { @@ -248,28 +317,28 @@ parse_expr_bp proc(state @mut State, min_bp BindingPowerScalar) NodeId ! mem.All parse_primary proc(state @mut State) NodeId ! mem.AllocError { start_idx :: state.next_token - start_token :: TokenId(start_idx) + start_tok_id :: TokenId(start_idx) state.next_token += 1 return match state.tokens[start_idx].kind { .int: try add_node(state, Node{ kind = .literal_int, - main_token = start_token, + main_token = start_tok_id, }) .float: try add_node(state, Node{ kind = .literal_float, - main_token = start_token, + main_token = start_tok_id, }) .string: try add_node(state, Node{ kind = .literal_string, - main_token = start_token, + main_token = start_tok_id, }) .ident: try add_node(state, Node{ kind = .expr_identifier, - main_token = start_token, + main_token = start_tok_id, }) else: try add_node(state, Node{ kind = .invalid, - main_token = start_token, + main_token = start_tok_id, }) } } @@ -281,9 +350,13 @@ next_is proc(state @mut State, token_kind TokenKind) bool { @hide next_is_either proc(state @mut State, token_kinds []TokenKind) bool { - is_either := false - for (token_kinds) |kind| is_either = is_either or next_is(state, kind) - return is_either + for (token_kinds) |kind| if next_is(state, kind) return true + return false +} + +@hide +next_n_is proc(state @mut State, n usize, token_kind TokenKind) bool { + return state.tokens[state.next_token + n].kind == token_kind } @hide diff --git a/source/parser/parser.test.hon b/source/parser/parser.test.hon index 497f47e..60fe943 100644 --- a/source/parser/parser.test.hon +++ b/source/parser/parser.test.hon @@ -3,6 +3,7 @@ import "@std/testing" import "@source/strpool" import "@source/lexer" +import "@source/ast" ast_renderer :: import "@source/ast/renderer" @@ -122,3 +123,80 @@ handles_statement_declaration test { try testing.expect_equal(expected.str_id, actual.str_id) } } + +handles_function_declarations test { + strpool.STRINGS = strpool.init(mem.c_allocator) + defer strpool.deinit(&strpool.STRINGS) + + source :: + `empty proc() void {} + `grouped proc(a, b T, c U) void { + ` value int :: 1 + `} + + scan_state := lexer.init(mem.c_allocator) + defer lexer.deinit(&scan_state) + try lexer.scan(&scan_state, source) + + parse_state := init(mem.c_allocator) + defer deinit(&parse_state) + root_id :: try parse(&parse_state, scan_state.tokens.items) + root :: parse_state.nodes.items[usize(root_id)] + + package_start :: usize(root.data0.extra_id) + package_end :: usize(root.data1.extra_id) + try testing.expect_equal(usize(2), package_end - package_start) + + empty_id :: ast.node_id(parse_state.extra.items[package_start]) + empty :: parse_state.nodes.items[usize(empty_id)] + empty_payload :: usize(empty.data1.extra_id) + empty_block :: parse_state.nodes.items[usize(empty.data0.node_id)] + try testing.expect_equal(ast.NodeKind.func_decl, empty.kind) + try testing.expect_equal(u32(0), parse_state.extra.items[empty_payload + 1]) + try testing.expect_equal( + usize(empty_block.data0.extra_id), + usize(empty_block.data1.extra_id), + ) + + grouped_id :: ast.node_id(parse_state.extra.items[package_start + 1]) + grouped :: parse_state.nodes.items[usize(grouped_id)] + grouped_payload :: usize(grouped.data1.extra_id) + try testing.expect_equal(ast.NodeKind.func_decl, grouped.kind) + try testing.expect_equal(u32(3), parse_state.extra.items[grouped_payload + 1]) + + param_a_id :: ast.node_id(parse_state.extra.items[grouped_payload + 2]) + param_b_id :: ast.node_id(parse_state.extra.items[grouped_payload + 3]) + param_c_id :: ast.node_id(parse_state.extra.items[grouped_payload + 4]) + param_a :: parse_state.nodes.items[usize(param_a_id)] + param_b :: parse_state.nodes.items[usize(param_b_id)] + param_c :: parse_state.nodes.items[usize(param_c_id)] + try testing.expect_equal(ast.NodeKind.param_decl, param_a.kind) + try testing.expect_equal(ast.NodeKind.param_decl, param_b.kind) + try testing.expect_equal(ast.NodeKind.param_decl, param_c.kind) + try testing.expect(param_a.data0.node_id == param_b.data0.node_id) + try testing.expect(param_a.data0.node_id != param_c.data0.node_id) + + grouped_block :: parse_state.nodes.items[usize(grouped.data0.node_id)] + try testing.expect_equal(ast.NodeKind.block, grouped_block.kind) + try testing.expect_equal( + usize(1), + usize(grouped_block.data1.extra_id) - usize(grouped_block.data0.extra_id), + ) + + renderer := ast_renderer.init(mem.c_allocator) + defer ast_renderer.deinit(&renderer) + try ast_renderer.render_token_stream( + &renderer, + root_id, + &parse_state, + scan_state.tokens.items, + ) + + try testing.expect_equal(scan_state.tokens.items.len, renderer.tokens.items.len) + for scan_state.tokens.items |expected, i| { + actual :: renderer.tokens.items[i] + try testing.expect_equal(expected.kind, actual.kind) + try testing.expect_equal(expected.start, actual.start) + try testing.expect_equal(expected.str_id, actual.str_id) + } +}