hl-valdemar/honey
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Wiki Activity

basic ast renderer

Browse Source
This commit is contained in:
Valdemar Høgsvig Lorenzen
2026-08-05 22:11:06 +02:00
parent 425ce38011
commit 6293c93c3c
6 changed files with 207 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
source/ast/ast.hon
+19 -20
View File
@@ -1,7 +1,15 @@
#! GRAMMAR:
#! * type ..... -> identifier
#! * assignment -> :: | :=
#! * literal .. -> int | float | string
#! * expr ..... -> identifier | literal
#! * decl ..... -> identifier [type] assignment expr
#! * stmt ..... -> decl
import "@std/debug"
import "@source/lexer"
hide TokenId :: alias lexer.TokenId
hide Token :: alias lexer.Token
hide(file) TokenId :: alias lexer.TokenId
NodeId :: distinct u32
ExtraId :: distinct u32
@@ -44,27 +52,18 @@ NodeKind :: enum {
# * data0: ExtraId
# + extra[data0]: NodeId - type identifier node
# + extra[data0 + 1]: NodeId - initializer expression
# * data1: TokenId of either `::` or `=` indicating mutability
# * data1: TokenId of either `::` or `:=` indicating mutability
stmt_decl
invalid
}
# todo: render ast nodes as source code
render proc(node @Node, tokens []Token) void { }
render_literal proc(node @Node, tokens []Token, program []u8) []u8 ! lexer.ScanError {
tok :: tokens[node.main_token]
match node.kind {
.literal_int, .literal_float: {
result :: try lexer.scan_number(tok.start, program)
return program[tok.start..result.end]
}
.literal_string: {
result :: try lexer.scan_string(tok.start, program)
return program[tok.start..result.end]
}
else: unreachable
}
node_id proc(idx uint) NodeId {
debug.assert(u64(idx) < u64(NO_ID_NODE))
return NodeId(idx)
}
extra_id proc(idx uint) ExtraId {
debug.assert(u64(idx) < u64(NO_ID_EXTRA))
return ExtraId(idx)
}
source/ast/renderer/renderer.hon
+119
View File
@@ -0,0 +1,119 @@
import "@std"
import "@std/mem"
import "@std/arraylist"
import "@source/lexer"
import "@source/parser"
import "@source/ast"
hide(file) TokenId :: alias lexer.TokenId
hide(file) Token :: alias lexer.Token
hide(file) Node :: alias ast.Node
hide(file) NodeId :: alias ast.NodeId
hide(file) ParseState :: alias parser.State
Renderer :: struct { tokens std.ArrayList(Token) }
init proc(allocator mem.Allocator) Renderer {
return Renderer{ tokens = arraylist.init(allocator) }
}
deinit proc(renderer @mut Renderer) void {
arraylist.deinit(&renderer.tokens)
}
render proc(
renderer @mut Renderer,
node_id NodeId,
parse_state @ParseState,
tokens []Token,
) void ! mem.AllocError {
node :: parse_state.nodes.items[usize(node_id)]
match node.kind {
.stmt_decl: try render_stmt_decl(renderer, node_id, parse_state, tokens)
else: unreachable
}
}
render_stmt_decl proc(
renderer @mut Renderer,
node_id NodeId,
parse_state @ParseState,
tokens []Token,
) void ! mem.AllocError {
node :: parse_state.nodes.items[usize(node_id)]
tok_ident :: tokens[usize(node.main_token)]
try arraylist.append(&renderer.tokens, tok_ident)
node_id_type :: ast.node_id(parse_state.extra.items[usize(node.data0.extra_id)])
try render_type(renderer, node_id_type, parse_state, tokens)
tok_assign :: tokens[usize(node.data1.token_id)]
try arraylist.append(&renderer.tokens, tok_assign)
node_id_expr :: ast.node_id(parse_state.extra.items[usize(node.data0.extra_id + 1)])
last_expr_token :: try render_expr(renderer, node_id_expr, parse_state, tokens)
terminator_idx :: usize(last_expr_token) + 1
tok_terminator :: if (terminator_idx < tokens.len) tokens[terminator_idx] else return
try arraylist.append(&renderer.tokens, tok_terminator)
eof_idx :: terminator_idx + 1
if (tok_terminator.kind == .newline and
eof_idx < tokens.len and
tokens[eof_idx].kind == .eof
) try arraylist.append(&renderer.tokens, tokens[eof_idx])
}
render_expr proc(renderer @mut Renderer,
node_id NodeId,
parse_state @ParseState,
tokens []Token,
) TokenId ! mem.AllocError {
# todo: support unary and binary expressions
node :: parse_state.nodes.items[usize(node_id)]
match node.kind {
.expr_identifier: {
tok :: tokens[usize(node.main_token)]
try arraylist.append(&renderer.tokens, tok)
return node.main_token
}
.literal_int, .literal_float, .literal_string:
return try render_lit(renderer, node_id, parse_state, tokens)
else: unreachable
}
}
render_type proc(
renderer @mut Renderer,
node_id NodeId,
parse_state @ParseState,
tokens []Token,
) void ! mem.AllocError {
# todo: support more expressive type expressions...
node :: parse_state.nodes.items[usize(node_id)]
match node.kind {
.expr_identifier: {
tok :: tokens[usize(node.main_token)]
try arraylist.append(&renderer.tokens, tok)
}
else: unreachable
}
}
render_lit proc(
renderer @mut Renderer,
node_id NodeId,
parse_state @ParseState,
tokens []Token,
) TokenId ! mem.AllocError {
node :: parse_state.nodes.items[usize(node_id)]
match node.kind {
.literal_int, .literal_float, .literal_string: {
tok :: tokens[node.main_token]
try arraylist.append(&renderer.tokens, tok)
return node.main_token
}
else: unreachable
}
}
source/main.hon
+13 -18
View File
@@ -10,10 +10,12 @@ test import "@std/strmap"
import "@source/strpool"
import "@source/lexer"
import "@source/parser"
import "@source/ast"
ast_renderer :: import "@source/ast/renderer"
test import "@source/strpool"
test import "@source/lexer"
test import "@source/parser"
program ::
@@ -21,7 +23,7 @@ program ::
`x int :: 123.9
`y :: "hello"
main proc() void {
main proc() void! {
strpool.strings = strpool.init(mem.c_allocator)
defer strpool.deinit(&strpool.strings)
@@ -29,7 +31,6 @@ main proc() void {
scan_state := lexer.init(mem.c_allocator)
defer lexer.deinit(&scan_state)
lexer.scan(&scan_state, program) catch |err| {
debug.print("failed to scan: {}\n", {err})
return
@@ -50,7 +51,7 @@ main proc() void {
parse_state := parser.init(mem.c_allocator)
defer parser.deinit(&parse_state)
parser.parse(&parse_state, scan_state.tokens.items) catch |err| {
root :: parser.parse(&parse_state, scan_state.tokens.items) catch |err| {
debug.print("failed to parse: {}\n", {err})
return
}
@@ -71,23 +72,17 @@ main proc() void {
yield :lbl res.end
} else token.start
debug.print("{} (id = {}): {}\n", {
node.kind,
id,
program[token.start..end],
#node.data0,
#node.data1,
})
debug.print("{} (id = {}): {}\n", { node.kind, id, program[token.start..end] })
}
debug.print("]]\n\n", {})
renderer := ast_renderer.init(mem.c_allocator)
defer ast_renderer.deinit(&renderer)
try ast_renderer.render(&renderer, root, &parse_state, scan_state.tokens.items)
debug.print("AST Render::[[\n", {})
literal :: ast.render_literal(
&parse_state.nodes.items[1],
scan_state.tokens.items,
program,
) catch "<invalid>"
debug.print("generated literal: {}\n", { literal })
for renderer.tokens.items |tok| {
debug.print("{}\n", {tok.kind})
}
debug.print("]]\n", {})
}
source/parser/parser.hon
+20 -27
View File
@@ -1,7 +1,6 @@
import "@std"
import "@std/mem"
import "@std/arraylist"
import "@std/debug"
import "@std/enums/enummap"
import "@source/lexer"
@@ -32,16 +31,16 @@ error_msg_map std.EnumMap(ErrorCode, ErrorDetails) :: enummap.init({
},
})
hide Token :: alias lexer.Token
hide TokenId :: alias lexer.TokenId
hide TokenKind :: alias lexer.TokenKind
hide Node :: alias ast.Node
hide NodeId :: alias ast.NodeId
hide ExtraId :: alias ast.ExtraId
hide NodeData :: alias ast.NodeData
hide(file) Token :: alias lexer.Token
hide(file) TokenId :: alias lexer.TokenId
hide(file) TokenKind :: alias lexer.TokenKind
hide(file) Node :: alias ast.Node
hide(file) NodeId :: alias ast.NodeId
hide(file) ExtraId :: alias ast.ExtraId
hide(file) NodeData :: alias ast.NodeData
hide NO_ID_NODE :: alias ast.NO_ID_NODE
hide NO_ID_EXTRA :: alias ast.NO_ID_EXTRA
hide(file) NO_ID_NODE :: alias ast.NO_ID_NODE
hide(file) NO_ID_EXTRA :: alias ast.NO_ID_EXTRA
State :: struct {
nodes std.ArrayList(Node)
@@ -65,9 +64,9 @@ deinit proc(state @mut State) void {
arraylist.deinit(&state.extra)
}
parse proc(state @mut State, tokens []Token) void ! ParseError {
parse proc(state @mut State, tokens []Token) NodeId ! ParseError {
state.tokens = tokens
_ = try parse_decl(state)
return try parse_decl(state)
}
parse_decl proc(state @mut State) NodeId ! ParseError {
@@ -87,7 +86,7 @@ parse_decl proc(state @mut State) NodeId ! ParseError {
mutability_tok_id :: try consume(state, .double_colon)
# expect expression
expr_id :: try parse_primary(state)
expr_id :: try parse_expr(state)
# expect statement terminator (newline)
_ = try consume_either(state, &[.newline, .eof])
@@ -105,6 +104,10 @@ parse_decl proc(state @mut State) NodeId ! ParseError {
return decl
}
parse_expr proc(state @mut State) NodeId ! mem.AllocError {
return try parse_primary(state)
}
parse_primary proc(state @mut State) NodeId ! mem.AllocError {
start_token :: state.next_token
state.next_token += 1
@@ -154,23 +157,13 @@ hide consume_either proc(state @mut State, token_kinds []TokenKind) TokenId ! Er
}
hide add_node proc(nodes @mut std.ArrayList(Node), node Node) NodeId ! mem.AllocError {
id :: node_id(nodes.items.len)
id :: ast.node_id(nodes.items.len)
try arraylist.append(nodes, node)
return id
}
hide node_id proc(idx uint) NodeId {
debug.assert(u64(idx) < u64(NO_ID_NODE))
return NodeId(idx)
}
hide add_extra proc(extra @mut std.ArrayList(u32), idx u32) ExtraId ! mem.AllocError {
id :: extra_id(extra.items.len)
try arraylist.append(extra, idx)
hide add_extra proc(extra @mut std.ArrayList(u32), data u32) ExtraId ! mem.AllocError {
id :: ast.extra_id(extra.items.len)
try arraylist.append(extra, data)
return id
}
hide extra_id proc(idx uint) ExtraId {
debug.assert(u64(idx) < u64(NO_ID_EXTRA))
return ExtraId(idx)
}
source/parser/parser.test.hon
+35
View File
@@ -0,0 +1,35 @@
import "@std/mem"
import "@std/testing"
import "@source/strpool"
import "@source/lexer"
ast_renderer :: import "@source/ast/renderer"
handles_statement_declaration test {
strpool.strings = strpool.init(mem.c_allocator)
defer strpool.deinit(&strpool.strings)
source ::
`value int :: 42
scan_state := lexer.init(mem.c_allocator)
defer lexer.deinit(&scan_state)
try lexer.scan(&scan_state, source)
parse_state := init(mem.c_allocator)
defer deinit(&parse_state)
root :: try parse(&parse_state, scan_state.tokens.items)
renderer := ast_renderer.init(mem.c_allocator)
defer ast_renderer.deinit(&renderer)
try ast_renderer.render(&renderer, root, &parse_state, scan_state.tokens.items)
try testing.expect_equal(scan_state.tokens.items.len, renderer.tokens.items.len)
for scan_state.tokens.items |expected, i| {
actual :: renderer.tokens.items[i]
try testing.expect_equal(expected.kind, actual.kind)
try testing.expect_equal(expected.start, actual.start)
try testing.expect_equal(expected.str_id, actual.str_id)
}
}
source/strpool/strpool.test.hon
+1 -1
View File
@@ -1,7 +1,7 @@
import "@std/mem"
import "@std/testing"
handles_intern test {
handles_interning test {
pool StringPool := init(mem.c_allocator)
defer deinit(&pool)