hl-valdemar/honey
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Wiki Activity

parse func decls

Browse Source
This commit is contained in:
Valdemar Høgsvig Lorenzen
2026-08-14 20:04:22 +02:00
parent 518037e7b7
commit cfd668b729
7 changed files with 397 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files
source/ast/ast.hon
+21 -2
View File
@@ -59,14 +59,33 @@ NodeKind :: enum {
# * data1: NodeId - right-hand side
expr_binary
# DECL:
# STATEMENT DECLS:
# * main_token: symbol name (identifier)
# * data0: ExtraId
# + extra[data0]: NodeId - type identifier node
# + extra[data0]: NodeId - type expr or NO_NODE
# + extra[data0 + 1]: NodeId - initializer expression
# * data1: TokenId of either `::` or `:=` indicating mutability
stmt_decl
# FUNC DECLS:
# * main_token: function name (identifier)
# * data0: NodeId - body block (or NO_NODE for extern)
# * data1: ExtraId
# + extra[data1]: NodeId - return type
# + extra[data1 + 1]: u32 - param count
# + extra[data1 + 2..]: NodeId - param decls
func_decl
# PARAM DECLS:
# * main_token: param name (identifier)
# * data0: NodeId - param type
param_decl
# BLOCK:
# * data0: ExtraId - statement range start
# * data1: ExtraId - statement range end
block
# PACKAGE:
# * data0: ExtraId
# * data1: ExtraId
source/ast/renderer/renderer.hon
+141 -18
View File
@@ -6,20 +6,11 @@ import "@source/lexer"
import "@source/parser"
import "@source/ast"
@hide:file
TokenId :: alias lexer.TokenId
@hide:file
Token :: alias lexer.Token
@hide:file
Node :: alias ast.Node
@hide:file
NodeId :: alias ast.NodeId
@hide:file
ParseState :: alias parser.State
@hide:file TokenId :: alias lexer.TokenId
@hide:file Token :: alias lexer.Token
@hide:file Node :: alias ast.Node
@hide:file NodeId :: alias ast.NodeId
@hide:file ParseState :: alias parser.State
Renderer :: struct { tokens std.ArrayList(Token) }
@@ -42,7 +33,9 @@ render_token_stream proc(
.literal_int, .literal_float, .literal_string:
_ = try render_lit(renderer, node_id, parse_state, tokens)
.stmt_decl:
try render_decl(renderer, node_id, parse_state, tokens)
try render_stmt_decl(renderer, node_id, parse_state, tokens)
.func_decl:
try render_func_decl(renderer, node_id, parse_state, tokens)
.package:
try render_pkg(renderer, node_id, parse_state, tokens)
else: unreachable
@@ -64,14 +57,16 @@ render_pkg proc(
stmt_node :: parse_state.nodes.items[usize(stmt_node_id)]
match stmt_node.kind {
.stmt_decl:
try render_decl(renderer, stmt_node_id, parse_state, tokens)
try render_stmt_decl(renderer, stmt_node_id, parse_state, tokens)
.func_decl:
try render_func_decl(renderer, stmt_node_id, parse_state, tokens)
else: unreachable
}
}
}
@hide:file
render_decl proc(
render_stmt_decl proc(
renderer @mut Renderer,
node_id NodeId,
parse_state @ParseState,
@@ -93,6 +88,7 @@ render_decl proc(
terminator_idx :: usize(last_expr_token) + 1
tok_terminator :: if (terminator_idx < tokens.len) tokens[terminator_idx] else return
if (tok_terminator.kind != .newline and tok_terminator.kind != .eof) return
try arraylist.append(&renderer.tokens, tok_terminator)
eof_idx :: terminator_idx + 1
@@ -103,7 +99,134 @@ render_decl proc(
}
@hide:file
render_expr proc(renderer @mut Renderer,
render_func_decl proc(
renderer @mut Renderer,
node_id NodeId,
parse_state @ParseState,
tokens []Token,
) void ! mem.AllocError {
node :: parse_state.nodes.items[usize(node_id)]
ident_idx :: usize(node.main_token)
try arraylist.append(&renderer.tokens, tokens[ident_idx])
try arraylist.append(&renderer.tokens, tokens[ident_idx + 1])
try arraylist.append(&renderer.tokens, tokens[ident_idx + 2])
payload_start :: usize(node.data1.extra_id)
param_count :: usize(parse_state.extra.items[payload_start + 1])
first_param :: payload_start + 2
for 0..param_count |i| {
param_id :: ast.node_id(parse_state.extra.items[first_param + i])
param_node :: parse_state.nodes.items[usize(param_id)]
type_id :: param_node.data0.node_id
type_node :: parse_state.nodes.items[usize(type_id)]
try arraylist.append(&renderer.tokens, tokens[usize(param_node.main_token)])
has_next :: i + 1 < param_count
if has_next {
next_param_id :: ast.node_id(parse_state.extra.items[first_param + i + 1])
next_param :: parse_state.nodes.items[usize(next_param_id)]
if (next_param.data0.node_id == type_id) {
try arraylist.append(
&renderer.tokens,
tokens[usize(param_node.main_token) + 1],
)
continue
}
}
try render_type(renderer, type_id, parse_state, tokens)
if has_next {
try arraylist.append(
&renderer.tokens,
tokens[usize(type_node.main_token) + 1],
)
}
}
return_type_id :: ast.node_id(parse_state.extra.items[payload_start])
return_type_node :: parse_state.nodes.items[usize(return_type_id)]
try arraylist.append(
&renderer.tokens,
tokens[usize(return_type_node.main_token) - 1],
)
try render_type(renderer, return_type_id, parse_state, tokens)
open_curly_id :: TokenId(usize(return_type_node.main_token) + 1)
close_curly_id :: try render_block(
renderer,
node.data0.node_id,
open_curly_id,
parse_state,
tokens,
)
terminator_idx :: usize(close_curly_id) + 1
if (terminator_idx >= tokens.len) return
tok_terminator :: tokens[terminator_idx]
if (tok_terminator.kind != .newline and tok_terminator.kind != .eof) return
try arraylist.append(&renderer.tokens, tok_terminator)
eof_idx :: terminator_idx + 1
if (tok_terminator.kind == .newline and
eof_idx < tokens.len and
tokens[eof_idx].kind == .eof
) try arraylist.append(&renderer.tokens, tokens[eof_idx])
}
@hide:file
render_block proc(
renderer @mut Renderer,
node_id NodeId,
open_curly_id TokenId,
parse_state @ParseState,
tokens []Token,
) TokenId ! mem.AllocError {
node :: parse_state.nodes.items[usize(node_id)]
open_curly_idx :: usize(open_curly_id)
try arraylist.append(&renderer.tokens, tokens[open_curly_idx])
leading_token := open_curly_idx + 1
while (leading_token < tokens.len and tokens[leading_token].kind == .newline) {
try arraylist.append(&renderer.tokens, tokens[leading_token])
leading_token += 1
}
extra_start :: usize(node.data0.extra_id)
extra_end :: usize(node.data1.extra_id)
for extra_start..extra_end |i| {
stmt_id :: ast.node_id(parse_state.extra.items[i])
stmt :: parse_state.nodes.items[usize(stmt_id)]
match stmt.kind {
.stmt_decl:
try render_stmt_decl(renderer, stmt_id, parse_state, tokens)
else: unreachable
}
}
# ponytail: linear delimiter scan; store brace tokens if nested blocks make rendering hot.
depth usize := 1
close_curly_idx := open_curly_idx + 1
while close_curly_idx < tokens.len {
match tokens[close_curly_idx].kind {
.open_curly: depth += 1
.close_curly: {
depth -= 1
if (depth == 0) {
try arraylist.append(&renderer.tokens, tokens[close_curly_idx])
return TokenId(close_curly_idx)
}
}
else: {}
}
close_curly_idx += 1
}
unreachable
}
@hide:file
render_expr proc(
renderer @mut Renderer,
node_id NodeId,
parse_state @ParseState,
tokens []Token,
source/lexer/lexer.hon
+1
View File
@@ -202,6 +202,7 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter
# single character tokens
kind ?TokenKind :: match char {
'=': .equal
',': .comma
'(': .open_paren
')': .close_paren
'{': .open_curly
source/lexer/token.hon
+6 -1
View File
@@ -43,6 +43,7 @@ TokenKind :: enum {
open_bracket, close_bracket
open_curly, close_curly
comma
colon
newline
@@ -54,8 +55,12 @@ TokenKind :: enum {
render_token proc(tok Token, program []u8) void {
match tok.kind {
.ident: {
str :: strpool.get_str(&strpool.STRINGS, tok.str_id) orelse "null"
if (strpool.get_str(&strpool.STRINGS, tok.str_id)) |str| {
debug.print("{}({})\n", {tok.kind, str})
} else {
res :: scan_ident(tok.start, program)
debug.print("{}({})\n", {tok.kind, program[tok.start..res.end]})
}
}
.string: {
res :: scan_string(tok.start, program) catch |_| {
source/main.hon
+23 -10
View File
@@ -25,6 +25,11 @@ program ::
`
`a := 1 + 2 * 3
`b :: -3 / 2 + 1
`
`empty proc() void {}
`grouped proc(a, b T, c U) void {
` value int :: 1
`}
main proc() void! {
strpool.STRINGS = strpool.init(mem.c_allocator)
@@ -51,39 +56,47 @@ main proc() void! {
}
debug.print("AST::[[\n", {})
for parse_state.nodes.items |node| {
for parse_state.nodes.items |node, i| {
match node.kind {
.expr_identifier: {
ident :: scan_state.tokens.items[usize(node.main_token)]
res :: lexer.scan_ident(ident.start, program)
debug.print("{}({})\n", { node.kind, program[ident.start..res.end] })
debug.print("{}: {}({})\n", { i, node.kind, program[ident.start..res.end] })
}
.literal_int, .literal_float: {
lit :: scan_state.tokens.items[usize(node.main_token)]
res :: lexer.scan_number(lit.start, program) catch |_| {
debug.print("{}\n", { node.kind })
debug.print("{}: {}\n", { i, node.kind })
continue
}
debug.print("{}({})\n", { node.kind, program[lit.start..res.end] })
debug.print("{}: {}({})\n", { i, node.kind, program[lit.start..res.end] })
}
.literal_string: {
str :: scan_state.tokens.items[usize(node.main_token)]
res :: lexer.scan_string(str.start, program) catch |_| {
debug.print("{}\n", { node.kind })
debug.print("{}: {}\n", { i, node.kind })
continue
}
debug.print("{}({})\n", { node.kind, program[str.start..res.end] })
debug.print("{}: {}({})\n", { i, node.kind, program[str.start..res.end] })
}
.expr_unary, .expr_binary: {
.expr_unary: {
operator :: scan_state.tokens.items[usize(node.main_token)]
debug.print("{}({})\n", { node.kind, operator.kind })
operand :: node.data0.node_id
debug.print("{}: {}({}, operand={})\n", { i, node.kind, operator.kind, operand })
}
.expr_binary: {
operator :: scan_state.tokens.items[usize(node.main_token)]
lhs :: node.data0.node_id
rhs :: node.data1.node_id
debug.print("{}: {}({}, lhs={}, rhs={})\n", { i, node.kind, operator.kind, lhs, rhs })
}
.stmt_decl: {
ident :: scan_state.tokens.items[usize(node.main_token)]
init :: parse_state.extra.items[usize(node.data0.extra_id) + 1]
res :: lexer.scan_ident(ident.start, program)
debug.print("{}({})\n", { node.kind, program[ident.start..res.end] })
debug.print("{}: {}({}, init={})\n", { i, node.kind, program[ident.start..res.end], init })
}
else: debug.print("{}\n", { node.kind })
else: debug.print("{}: {}\n", { i, node.kind })
}
}
debug.print("]]\n\n", {})
source/parser/parser.hon
+126 -53
View File
@@ -32,35 +32,16 @@ error_msg_map std.EnumMap(ErrorCode, ErrorDetails) :: enummap.init({
},
})
@hide:file
Token :: alias lexer.Token
@hide:file
TokenId :: alias lexer.TokenId
@hide:file
TokenKind :: alias lexer.TokenKind
@hide:file
Node :: alias ast.Node
@hide:file
NodeId :: alias ast.NodeId
@hide:file
ExtraId :: alias ast.ExtraId
@hide:file
NodeData :: alias ast.NodeData
@hide:file
NO_NODE :: alias ast.NO_NODE
@hide:file
NO_EXTRA :: alias ast.NO_EXTRA
@hide:file
token_id :: alias lexer.token_id
@hide:file Token :: alias lexer.Token
@hide:file TokenId :: alias lexer.TokenId
@hide:file TokenKind :: alias lexer.TokenKind
@hide:file Node :: alias ast.Node
@hide:file NodeId :: alias ast.NodeId
@hide:file ExtraId :: alias ast.ExtraId
@hide:file NodeData :: alias ast.NodeData
@hide:file NO_NODE :: alias ast.NO_NODE
@hide:file NO_EXTRA :: alias ast.NO_EXTRA
@hide:file token_id :: alias lexer.token_id
BindingPowerScalar :: distinct u32
@@ -129,7 +110,7 @@ parse_pkg proc(state @mut State) NodeId ! ParseError {
while (state.next_token < state.tokens.len
and state.tokens[state.next_token].kind != .eof
) {
stmt_id :: try parse_stmt(state)
stmt_id :: try parse_decl(state)
try arraylist.append(&stmt_ids, stmt_id)
}
@@ -137,22 +118,112 @@ parse_pkg proc(state @mut State) NodeId ! ParseError {
for (stmt_ids.items) |stmt_id| _ = try add_extra(state, u32(stmt_id))
pkg_end :: ast.extra_id(state.extra.items.len)
package :: try add_node(state, Node{
return try add_node(state, Node{
kind = .package,
data0 = NodeData{ extra_id = pkg_start },
data1 = NodeData{ extra_id = pkg_end },
})
return package
}
parse_stmt proc(state @mut State) NodeId ! ParseError {
return try parse_decl(state)
}
parse_decl proc(state @mut State) NodeId ! ParseError {
if next_n_is(state, 1, .proc)
return try parse_func_decl(state)
else
return try parse_stmt_decl(state)
}
parse_func_decl proc(state @mut State) NodeId ! ParseError {
ident_tok_id :: try consume(state, .ident)
_ = try consume(state, .proc)
_ = try consume(state, .open_paren)
param_decls std.ArrayList(NodeId) := arraylist.init(state.nodes.allocator)
defer arraylist.deinit(&param_decls)
ident_ids std.ArrayList(TokenId) := arraylist.init(state.nodes.allocator)
defer arraylist.deinit(&ident_ids)
while !next_is(state, .close_paren) {
arraylist.clear(&ident_ids)
# collect param group
while true {
param_tok_id :: try consume(state, .ident)
try arraylist.append(&ident_ids, param_tok_id)
if next_is(state, .ident) break
_ = try consume(state, .comma)
}
# collect type
try expect(state, .ident)
type_id :: try parse_primary(state)
for (ident_ids.items) |id| {
param_decl :: try add_node(state, Node{
kind = .param_decl,
main_token = id,
data0 = NodeData{ node_id = type_id },
})
try arraylist.append(&param_decls, param_decl)
}
if next_is(state, .comma) _ = try consume(state, .comma)
}
_ = try consume(state, .close_paren)
try expect(state, .ident)
return_type_id :: try parse_primary(state)
block_id :: try parse_block(state)
while next_is(state, .newline) _ = try consume(state, .newline)
extra_id :: try add_extra(state, u32(return_type_id))
_ = try add_extra(state, u32(param_decls.items.len))
for (param_decls.items) |param| _ = try add_extra(state, u32(param))
return try add_node(state, Node{
kind = .func_decl,
main_token = ident_tok_id,
data0 = NodeData{ node_id = block_id },
data1 = NodeData{ extra_id = extra_id },
})
}
parse_block proc(state @mut State) NodeId ! ParseError {
stmt_ids std.ArrayList(NodeId) := arraylist.init(state.nodes.allocator)
defer arraylist.deinit(&stmt_ids)
_ = try consume(state, .open_curly)
while next_is(state, .newline) _ = try consume(state, .newline)
while !next_is(state, .close_curly) {
stmt_id :: try parse_stmt(state)
try arraylist.append(&stmt_ids, stmt_id)
while next_is(state, .newline) _ = try consume(state, .newline)
}
_ = try consume(state, .close_curly)
block_start :: ast.extra_id(state.extra.items.len)
for (stmt_ids.items) |id| _ = try add_extra(state, u32(id))
block_end :: ast.extra_id(state.extra.items.len)
return try add_node(state, Node{
kind = .block,
data0 = NodeData{ extra_id = block_start },
data1 = NodeData{ extra_id = block_end },
})
}
parse_stmt proc(state @mut State) NodeId ! ParseError {
if next_is(state, .ident) return try parse_stmt_decl(state)
return .unexpected_token
}
parse_stmt_decl proc(state @mut State) NodeId ! ParseError {
# expect identifier
ident_token :: try consume(state, .ident)
ident_tok_id :: try consume(state, .ident)
# check for type
# todo: make a parse_type proc for this
@@ -170,17 +241,15 @@ parse_decl proc(state @mut State) NodeId ! ParseError {
# expect statement terminator (newline)
while (next_is(state, .newline)) _ = try consume(state, .newline)
extra_start :: try add_extra(state, u32(type_id))
extra_start_id :: try add_extra(state, u32(type_id))
_ = try add_extra(state, u32(expr_id))
decl :: try add_node(state, Node{
return try add_node(state, Node{
kind = .stmt_decl,
main_token = ident_token,
data0 = NodeData{ extra_id = extra_start },
main_token = ident_tok_id,
data0 = NodeData{ extra_id = extra_start_id },
data1 = NodeData{ token_id = mutability_tok_id },
})
return decl
}
parse_expr proc(state @mut State) NodeId ! mem.AllocError {
@@ -248,28 +317,28 @@ parse_expr_bp proc(state @mut State, min_bp BindingPowerScalar) NodeId ! mem.All
parse_primary proc(state @mut State) NodeId ! mem.AllocError {
start_idx :: state.next_token
start_token :: TokenId(start_idx)
start_tok_id :: TokenId(start_idx)
state.next_token += 1
return match state.tokens[start_idx].kind {
.int: try add_node(state, Node{
kind = .literal_int,
main_token = start_token,
main_token = start_tok_id,
})
.float: try add_node(state, Node{
kind = .literal_float,
main_token = start_token,
main_token = start_tok_id,
})
.string: try add_node(state, Node{
kind = .literal_string,
main_token = start_token,
main_token = start_tok_id,
})
.ident: try add_node(state, Node{
kind = .expr_identifier,
main_token = start_token,
main_token = start_tok_id,
})
else: try add_node(state, Node{
kind = .invalid,
main_token = start_token,
main_token = start_tok_id,
})
}
}
@@ -281,9 +350,13 @@ next_is proc(state @mut State, token_kind TokenKind) bool {
@hide
next_is_either proc(state @mut State, token_kinds []TokenKind) bool {
is_either := false
for (token_kinds) |kind| is_either = is_either or next_is(state, kind)
return is_either
for (token_kinds) |kind| if next_is(state, kind) return true
return false
}
@hide
next_n_is proc(state @mut State, n usize, token_kind TokenKind) bool {
return state.tokens[state.next_token + n].kind == token_kind
}
@hide
source/parser/parser.test.hon
+78
View File
@@ -3,6 +3,7 @@ import "@std/testing"
import "@source/strpool"
import "@source/lexer"
import "@source/ast"
ast_renderer :: import "@source/ast/renderer"
@@ -122,3 +123,80 @@ handles_statement_declaration test {
try testing.expect_equal(expected.str_id, actual.str_id)
}
}
handles_function_declarations test {
strpool.STRINGS = strpool.init(mem.c_allocator)
defer strpool.deinit(&strpool.STRINGS)
source ::
`empty proc() void {}
`grouped proc(a, b T, c U) void {
` value int :: 1
`}
scan_state := lexer.init(mem.c_allocator)
defer lexer.deinit(&scan_state)
try lexer.scan(&scan_state, source)
parse_state := init(mem.c_allocator)
defer deinit(&parse_state)
root_id :: try parse(&parse_state, scan_state.tokens.items)
root :: parse_state.nodes.items[usize(root_id)]
package_start :: usize(root.data0.extra_id)
package_end :: usize(root.data1.extra_id)
try testing.expect_equal(usize(2), package_end - package_start)
empty_id :: ast.node_id(parse_state.extra.items[package_start])
empty :: parse_state.nodes.items[usize(empty_id)]
empty_payload :: usize(empty.data1.extra_id)
empty_block :: parse_state.nodes.items[usize(empty.data0.node_id)]
try testing.expect_equal(ast.NodeKind.func_decl, empty.kind)
try testing.expect_equal(u32(0), parse_state.extra.items[empty_payload + 1])
try testing.expect_equal(
usize(empty_block.data0.extra_id),
usize(empty_block.data1.extra_id),
)
grouped_id :: ast.node_id(parse_state.extra.items[package_start + 1])
grouped :: parse_state.nodes.items[usize(grouped_id)]
grouped_payload :: usize(grouped.data1.extra_id)
try testing.expect_equal(ast.NodeKind.func_decl, grouped.kind)
try testing.expect_equal(u32(3), parse_state.extra.items[grouped_payload + 1])
param_a_id :: ast.node_id(parse_state.extra.items[grouped_payload + 2])
param_b_id :: ast.node_id(parse_state.extra.items[grouped_payload + 3])
param_c_id :: ast.node_id(parse_state.extra.items[grouped_payload + 4])
param_a :: parse_state.nodes.items[usize(param_a_id)]
param_b :: parse_state.nodes.items[usize(param_b_id)]
param_c :: parse_state.nodes.items[usize(param_c_id)]
try testing.expect_equal(ast.NodeKind.param_decl, param_a.kind)
try testing.expect_equal(ast.NodeKind.param_decl, param_b.kind)
try testing.expect_equal(ast.NodeKind.param_decl, param_c.kind)
try testing.expect(param_a.data0.node_id == param_b.data0.node_id)
try testing.expect(param_a.data0.node_id != param_c.data0.node_id)
grouped_block :: parse_state.nodes.items[usize(grouped.data0.node_id)]
try testing.expect_equal(ast.NodeKind.block, grouped_block.kind)
try testing.expect_equal(
usize(1),
usize(grouped_block.data1.extra_id) - usize(grouped_block.data0.extra_id),
)
renderer := ast_renderer.init(mem.c_allocator)
defer ast_renderer.deinit(&renderer)
try ast_renderer.render_token_stream(
&renderer,
root_id,
&parse_state,
scan_state.tokens.items,
)
try testing.expect_equal(scan_state.tokens.items.len, renderer.tokens.items.len)
for scan_state.tokens.items |expected, i| {
actual :: renderer.tokens.items[i]
try testing.expect_equal(expected.kind, actual.kind)
try testing.expect_equal(expected.start, actual.start)
try testing.expect_equal(expected.str_id, actual.str_id)
}
}