From 8dd06afde25dc9d145690a471a8fc21c497c39aa Mon Sep 17 00:00:00 2001 From: hl-valdemar Date: Mon, 10 Aug 2026 23:35:10 +0200 Subject: [PATCH] parse arithmetic binary expressions --- source/ast/ast.hon | 24 +++++- source/ast/renderer/renderer.hon | 60 ++++++++++++-- source/lexer/lexer.hon | 129 ++++++++++++++++++----------- source/lexer/token.hon | 26 +++++- source/main.hon | 31 ++----- source/parser/parser.hon | 137 ++++++++++++++++++++++++------- source/parser/parser.test.hon | 61 ++++++++++++++ source/strpool/strpool.hon | 4 +- std/io/io.hon | 2 +- std/testing/testing.hon | 4 +- 10 files changed, 358 insertions(+), 120 deletions(-) diff --git a/source/ast/ast.hon b/source/ast/ast.hon index 2b0af89..693b16a 100644 --- a/source/ast/ast.hon +++ b/source/ast/ast.hon @@ -12,6 +12,9 @@ import "@source/lexer" @hide:file TokenId :: alias lexer.TokenId +@hide:file +NO_ID_TOKEN :: alias lexer.NO_ID_TOKEN + NodeId :: distinct u32 ExtraId :: distinct u32 @@ -28,7 +31,7 @@ NodeData :: union { Node :: struct { kind NodeKind - main_token TokenId + main_token TokenId = NO_ID_TOKEN data0 NodeData = NodeData{ node_id = NO_ID_NODE } data1 NodeData = NodeData{ node_id = NO_ID_NODE } @@ -45,7 +48,14 @@ NodeKind :: enum { # * main_token: token literal expr_identifier + # UNARY: + # todo: document expr_unary + + # BINARY: + # * main_token: operator + # * data0: NodeId - left-hand side + # * data1: NodeId - right-hand side expr_binary # DECL: @@ -56,6 +66,13 @@ NodeKind :: enum { # * data1: TokenId of either `::` or `:=` indicating mutability stmt_decl + # PACKAGE: + # * data0: ExtraId + # * data1: ExtraId + # + # extra[data0..data1]: NodeId - all statement node ids in package + package + invalid } @@ -64,6 +81,11 @@ node_id proc(idx uint) NodeId { return NodeId(idx) } +maybe_node_id proc(idx uint) ?NodeId { + if (u64(idx) >= u64(NO_ID_NODE)) return null + return NodeId(idx) +} + extra_id proc(idx uint) ExtraId { debug.assert(u64(idx) < u64(NO_ID_EXTRA)) return ExtraId(idx) diff --git a/source/ast/renderer/renderer.hon b/source/ast/renderer/renderer.hon index d2fface..773b7a1 100644 --- a/source/ast/renderer/renderer.hon +++ b/source/ast/renderer/renderer.hon @@ -43,10 +43,33 @@ render proc( _ = try render_lit(renderer, node_id, parse_state, tokens) .stmt_decl: try render_decl(renderer, node_id, parse_state, tokens) + .package: + try render_pkg(renderer, node_id, parse_state, tokens) else: unreachable } } +@hide:file +render_pkg proc( + renderer @mut Renderer, + node_id NodeId, + parse_state @ParseState, + tokens []Token, +) void ! mem.AllocError { + node :: parse_state.nodes.items[usize(node_id)] + extra_start :: usize(node.data0.extra_id) + extra_end :: usize(node.data1.extra_id) + for extra_start..extra_end |i| { + stmt_node_id :: ast.node_id(parse_state.extra.items[i]) + stmt_node :: parse_state.nodes.items[usize(stmt_node_id)] + match stmt_node.kind { + .stmt_decl: + try render_decl(renderer, stmt_node_id, parse_state, tokens) + else: unreachable + } + } +} + @hide:file render_decl proc( renderer @mut Renderer, @@ -58,8 +81,9 @@ render_decl proc( tok_ident :: tokens[usize(node.main_token)] try arraylist.append(&renderer.tokens, tok_ident) - node_id_type :: ast.node_id(parse_state.extra.items[usize(node.data0.extra_id)]) - try render_type(renderer, node_id_type, parse_state, tokens) + raw_node_id_type :: parse_state.extra.items[usize(node.data0.extra_id)] + node_id_type :: ast.maybe_node_id(raw_node_id_type) + if (node_id_type) |id| try render_type(renderer, id, parse_state, tokens) tok_assign :: tokens[usize(node.data1.token_id)] try arraylist.append(&renderer.tokens, tok_assign) @@ -84,20 +108,42 @@ render_expr proc(renderer @mut Renderer, parse_state @ParseState, tokens []Token, ) TokenId ! mem.AllocError { - # todo: support unary and binary expressions + # todo: support unary expressions node :: parse_state.nodes.items[usize(node_id)] - match node.kind { + return match node.kind { .expr_identifier: { tok :: tokens[usize(node.main_token)] try arraylist.append(&renderer.tokens, tok) - return node.main_token + yield node.main_token } + .expr_binary: + try render_expr_binary(renderer, node_id, parse_state, tokens) .literal_int, .literal_float, .literal_string: - return try render_lit(renderer, node_id, parse_state, tokens) + try render_lit(renderer, node_id, parse_state, tokens) else: unreachable } } +@hide:file +render_expr_binary proc( + renderer @mut Renderer, + node_id NodeId, + parse_state @ParseState, + tokens []Token, +) TokenId ! mem.AllocError { + node :: parse_state.nodes.items[usize(node_id)] + operator :: node.main_token + lhs :: node.data0.node_id + rhs :: node.data1.node_id + + # render lhs operator rhs + _ = try render_expr(renderer, lhs, parse_state, tokens) + try arraylist.append(&renderer.tokens, tokens[usize(operator)]) + last_token :: try render_expr(renderer, rhs, parse_state, tokens) + + return last_token +} + @hide:file render_type proc( renderer @mut Renderer, @@ -124,7 +170,7 @@ render_lit proc( tokens []Token, ) TokenId ! mem.AllocError { node :: parse_state.nodes.items[usize(node_id)] - tok :: tokens[node.main_token] + tok :: tokens[usize(node.main_token)] try arraylist.append(&renderer.tokens, tok) return node.main_token } diff --git a/source/lexer/lexer.hon b/source/lexer/lexer.hon index 13471f6..a9338c4 100644 --- a/source/lexer/lexer.hon +++ b/source/lexer/lexer.hon @@ -3,7 +3,6 @@ import "@std/mem" import "@std/strmap" import "@std/enums/enummap" import "@std/arraylist" -import "@std/debug" import "@source/strpool" @@ -43,8 +42,6 @@ keywords std.StringMap(TokenKind) :: strmap.init([ { "while", .while }, ]) -TokenId :: distinct u32 - Diagnostic :: struct { token TokenId code ErrorCode @@ -68,7 +65,6 @@ deinit proc(state @mut State) void { } scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.InternError) { - tokens :: &state.tokens diagnostics :: &state.diagnostics cursor := 0 @@ -77,7 +73,7 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter # whitespace if char == '\n' { - try add_token(tokens, Token{ kind = .newline, start = cursor }) + try add_token(state, Token{ start = cursor, kind = .newline }) cursor += 1 continue } else if is_whitespace(char) { @@ -96,9 +92,9 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter if is_alpha(char) or char == '_' { start :: cursor result :: scan_ident(start, program) - try add_token(tokens, Token{ - kind = result.kind, + try add_token(state, Token{ start = start, + kind = result.kind, str_id = result.str_id, }) cursor = result.end @@ -109,15 +105,15 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter if is_digit(char) { start :: cursor result :: scan_number(start, program) catch |err| { - token :: token_id(tokens.items.len) - try add_token(tokens, Token{ kind = .invalid, start = start }) + token :: token_id(state.tokens.items.len) + try add_token(state, Token{ start = start, kind = .invalid }) try arraylist.append(diagnostics, Diagnostic{ token = token, code = err.code }) cursor = err.end continue } kind :: if (result.has_decimal) .float else .int - try add_token(tokens, Token{ kind = kind, start = start }) + try add_token(state, Token{ start = start, kind = kind }) cursor = result.end continue } @@ -126,68 +122,107 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter if char == '"' { start :: cursor result :: scan_string(start, program) catch |err| { - token :: token_id(tokens.items.len) - try add_token(tokens, Token{ kind = .invalid, start = start }) + token :: token_id(state.tokens.items.len) + try add_token(state, Token{ start = start, kind = .invalid }) try arraylist.append(diagnostics, Diagnostic{ token = token, code = err.code }) cursor = err.end continue } - try add_token(tokens, Token{ kind = .string, start = start }) + try add_token(state, Token{ start = start, kind = .string }) cursor = result.end continue } - # immutable assignment or single colon + # immutable decl, mutable decl, or single colon if char == ':' { if cursor + 1 < program.len and program[cursor + 1] == ':' { - try add_token(tokens, Token{ kind = .double_colon, start = cursor }) + try add_token(state, Token{ start = cursor, kind = .double_colon }) cursor += 2 continue } - try add_token(tokens, Token{ kind = .colon, start = cursor }) + if cursor + 1 < program.len and program[cursor + 1] == '=' { + try add_token(state, Token{ start = cursor, kind = .colon_equal }) + cursor += 2 + continue + } + + try add_token(state, Token{ start = cursor, kind = .colon }) cursor += 1 continue } - # mutable reassignment - if char == '=' { - try add_token(tokens, Token{ kind = .equal, start = cursor }) + # compound arithmetic assignment + if char == '+' { + if cursor + 1 < program.len and program[cursor + 1] == '=' { + try add_token(state, Token{ start = cursor, kind = .plus_equal }) + cursor += 2 + continue + } + + try add_token(state, Token{ start = cursor, kind = .plus }) + cursor += 1 + continue + } + if char == '-' { + if cursor + 1 < program.len and program[cursor + 1] == '=' { + try add_token(state, Token{ start = cursor, kind = .minus_equal }) + cursor += 2 + continue + } + + try add_token(state, Token{ start = cursor, kind = .minus }) + cursor += 1 + continue + } + if char == '*' { + if cursor + 1 < program.len and program[cursor + 1] == '=' { + try add_token(state, Token{ start = cursor, kind = .star_equal }) + cursor += 2 + continue + } + + try add_token(state, Token{ start = cursor, kind = .star }) + cursor += 1 + continue + } + if char == '/' { + if cursor + 1 < program.len and program[cursor + 1] == '=' { + try add_token(state, Token{ start = cursor, kind = .slash_equal }) + cursor += 2 + continue + } + + try add_token(state, Token{ start = cursor, kind = .slash }) cursor += 1 continue } - # parentheses - if char == '(' { - try add_token(tokens, Token{ kind = .open_paren, start = cursor }) - cursor += 1 - continue - } else if char == ')' { - try add_token(tokens, Token{ kind = .close_paren, start = cursor }) - cursor += 1 - continue + # single character tokens + kind ?TokenKind :: match char { + '=': .equal + '(': .open_paren + ')': .close_paren + '{': .open_curly + '}': .close_curly + else: null } - # curly braces - if char == '{' { - try add_token(tokens, Token{ kind = .open_curly, start = cursor }) - cursor += 1 - continue - } else if char == '}' { - try add_token(tokens, Token{ kind = .close_curly, start = cursor }) + if (kind) |k| { + try add_token(state, Token{ start = cursor, kind = k }) cursor += 1 continue } # invalid character - token :: token_id(tokens.items.len) + token :: token_id(state.tokens.items.len) + try add_token(state, Token{ start = cursor, kind = .invalid }) try arraylist.append(diagnostics, Diagnostic{ token = token, code = .invalid_character }) - try add_token(tokens, Token{ kind = .invalid, start = cursor }) cursor += 1 } - try add_token(tokens, Token{ kind = .eof, start = cursor }) + try add_token(state, Token{ start = cursor, kind = .eof }) } ScanIdentResult :: struct { @@ -210,9 +245,9 @@ scan_ident proc(start usize, program []u8) ScanIdentResult { # don't intern keywords (already O(1) lookup via token kind) str_id :: if (kind == .ident) - strpool.intern(&strpool.strings, program[start..cursor]) catch strpool.NO_ID + strpool.intern(&strpool.strings, program[start..cursor]) catch strpool.NO_ID_STR else - strpool.NO_ID + strpool.NO_ID_STR return ScanIdentResult{ end = cursor, @@ -243,7 +278,7 @@ scan_number proc(start usize, program []u8) ScanNumResult ! ScanError { if has_decimal and (cursor >= program.len or !is_digit(program[cursor])) { return ScanError{ code = .float_must_end_with_digit, - end = cursor + 1, + end = cursor, } } @@ -300,13 +335,7 @@ is_digit proc(char u8) bool { } @hide -add_token proc(tokens @mut std.ArrayList(Token), token Token) void ! mem.AllocError { - _ = token_id(tokens.items.len) - try arraylist.append(tokens, token) -} - -@hide -token_id proc(idx uint) TokenId { - debug.assert(u64(idx) < u64(maxval!(TokenId))) - return TokenId(idx) +add_token proc(state @mut State, token Token) void ! mem.AllocError { + _ = token_id(state.tokens.items.len) + try arraylist.append(&state.tokens, token) } diff --git a/source/lexer/token.hon b/source/lexer/token.hon index 957d3b6..a08b1f6 100644 --- a/source/lexer/token.hon +++ b/source/lexer/token.hon @@ -1,9 +1,14 @@ +import "@std/debug" import "@source/strpool" +TokenId :: distinct u32 + +NO_ID_TOKEN :: maxval!(TokenId) + Token :: struct { - kind TokenKind start uint - str_id strpool.StringId = strpool.NO_ID + kind TokenKind + str_id strpool.StringId = strpool.NO_ID_STR } TokenKind :: enum { @@ -22,7 +27,16 @@ TokenKind :: enum { greater, greater_equal # assignment - equal, double_colon + double_colon, colon_equal + equal + + # arithmetic + plus, minus + star, slash + + # assignment & arithmetic + plus_equal, minus_equal + star_equal, slash_equal # delimiters open_paren, close_paren @@ -36,3 +50,9 @@ TokenKind :: enum { eof invalid } + +@hide +token_id proc(idx uint) TokenId { + debug.assert(u64(idx) < u64(NO_ID_TOKEN)) + return TokenId(idx) +} diff --git a/source/main.hon b/source/main.hon index 96c91ef..3c012fc 100644 --- a/source/main.hon +++ b/source/main.hon @@ -1,6 +1,5 @@ import "@std/debug" import "@std/mem" -import "@std/enums/enummap" test import "@std/enums/enummap" test import "@std/arraylist" @@ -20,8 +19,11 @@ test import "@source/parser" program :: `# literals - `x int :: 123.9 - `y :: "hello" + `x float :: 123.9 + `y int := 43 + `z :: "hello" + ` + `a := 1 + 2 * 3 main proc() void! { strpool.strings = strpool.init(mem.c_allocator) @@ -42,13 +44,6 @@ main proc() void! { } debug.print("]]\n\n", {}) - debug.print("TOKENS::DIAGNOSTICS::[[\n", {}) - for scan_state.diagnostics.items |diagnostic| { - details :: enummap.get(&lexer.error_msg_map, diagnostic.code) - debug.print("{}: {}\n", {details?.name, details?.message}) - } - debug.print("]]\n\n", {}) - parse_state := parser.init(mem.c_allocator) defer parser.deinit(&parse_state) root :: parser.parse(&parse_state, scan_state.tokens.items) catch |err| { @@ -58,21 +53,7 @@ main proc() void! { debug.print("AST::[[\n", {}) for parse_state.nodes.items |node, id| { - token :: scan_state.tokens.items[node.main_token] - - end :: if (token.kind == .int or token.kind == .float) lbl: { - res :: lexer.scan_number(token.start, program) catch { - yield :lbl token.start - } - yield :lbl res.end - } else if (token.kind == .string) lbl: { - res :: lexer.scan_string(token.start, program) catch { - yield :lbl token.start - } - yield :lbl res.end - } else token.start - - debug.print("{} (id = {}): {}\n", { node.kind, id, program[token.start..end] }) + debug.print("{} (id = {})\n", { node.kind, id }) } debug.print("]]\n\n", {}) diff --git a/source/parser/parser.hon b/source/parser/parser.hon index a9486cc..a5988cd 100644 --- a/source/parser/parser.hon +++ b/source/parser/parser.hon @@ -58,6 +58,21 @@ NO_ID_NODE :: alias ast.NO_ID_NODE @hide:file NO_ID_EXTRA :: alias ast.NO_ID_EXTRA +BindingPowerScalar :: distinct u32 + +BindingPower :: struct { + left BindingPowerScalar + right BindingPowerScalar +} + +infix_binding_power proc(token_kind TokenKind) ?BindingPower { + return match token_kind { + .plus, .minus: BindingPower{ left = BindingPowerScalar(10), right = BindingPowerScalar(11) } + .star, .slash: BindingPower{ left = BindingPowerScalar(20), right = BindingPowerScalar(21) } + else: null + } +} + State :: struct { nodes std.ArrayList(Node) @@ -65,7 +80,7 @@ State :: struct { extra std.ArrayList(u32) tokens []Token = &[] - next_token TokenId = TokenId(0) + next_token usize = 0 } init proc(allocator mem.Allocator) State { @@ -82,35 +97,65 @@ deinit proc(state @mut State) void { parse proc(state @mut State, tokens []Token) NodeId ! ParseError { state.tokens = tokens + return try parse_pkg(state) +} + +parse_pkg proc(state @mut State) NodeId ! ParseError { + if (state.next_token < state.tokens.len + and state.tokens[state.next_token].kind == .eof + ) return NO_ID_NODE + + stmt_ids std.ArrayList(NodeId) := arraylist.init(state.nodes.allocator) + defer arraylist.deinit(&stmt_ids) + + while (state.next_token < state.tokens.len + and state.tokens[state.next_token].kind != .eof + ) { + stmt_id :: try parse_stmt(state) + try arraylist.append(&stmt_ids, stmt_id) + } + + pkg_start :: ast.extra_id(state.extra.items.len) + for (stmt_ids.items) |stmt_id| _ = try add_extra(state, u32(stmt_id)) + pkg_end :: ast.extra_id(state.extra.items.len) + + package :: try add_node(state, Node{ + kind = .package, + data0 = NodeData{ extra_id = pkg_start }, + data1 = NodeData{ extra_id = pkg_end }, + }) + + return package +} + +parse_stmt proc(state @mut State) NodeId ! ParseError { return try parse_decl(state) } parse_decl proc(state @mut State) NodeId ! ParseError { # expect identifier - try expect(state, .ident) - ident_token :: state.next_token - state.next_token += 1 + ident_token :: try consume(state, .ident) # check for type # todo: make a parse_type proc for this - type_id :: if (state.tokens[state.next_token].kind == .ident) + type_id :: if (next_is(state, .ident)) try parse_primary(state) else NO_ID_NODE - # expect `::` (immutable assignment) - mutability_tok_id :: try consume(state, .double_colon) + # expect `::` or `:=` (immutable or mutable assignment) + mutability_tok_id :: try consume_either(state, &[.double_colon, .colon_equal]) # expect expression expr_id :: try parse_expr(state) # expect statement terminator (newline) - _ = try consume_either(state, &[.newline, .eof]) + while (next_is(state, .newline)) _ = try consume(state, .newline) - extra_start :: try add_extra(&state.extra, u32(type_id)) - _ = try add_extra(&state.extra, u32(expr_id)) + extra_start :: try add_extra(state, u32(type_id)) + _ = try add_extra(state, u32(expr_id)) - decl :: try add_node(&state.nodes, Node{ + decl :: try add_node(state, Node{ kind = .stmt_decl, main_token = ident_token, data0 = NodeData{ extra_id = extra_start }, @@ -121,36 +166,68 @@ parse_decl proc(state @mut State) NodeId ! ParseError { } parse_expr proc(state @mut State) NodeId ! mem.AllocError { - return try parse_primary(state) + return try parse_expr_bp(state, BindingPowerScalar(0)) +} + +parse_expr_bp proc(state @mut State, min_bp BindingPowerScalar) NodeId ! mem.AllocError { + lhs := try parse_primary(state) + + while infix_binding_power(state.tokens[state.next_token].kind) |bp| { + if (bp.left < min_bp) break + + operator :: TokenId(state.next_token) + state.next_token += 1 + + rhs :: try parse_expr_bp(state, bp.right) + lhs = try add_node(state, Node{ + kind = .expr_binary, + main_token = operator, + data0 = NodeData{ node_id = lhs }, + data1 = NodeData{ node_id = rhs }, + }) + } + + return lhs } parse_primary proc(state @mut State) NodeId ! mem.AllocError { - start_token :: state.next_token + start_idx :: state.next_token + start_token :: TokenId(start_idx) state.next_token += 1 - return match state.tokens[start_token].kind { - .int: try add_node(&state.nodes, Node{ + return match state.tokens[start_idx].kind { + .int: try add_node(state, Node{ kind = .literal_int, main_token = start_token, }) - .float: try add_node(&state.nodes, Node{ + .float: try add_node(state, Node{ kind = .literal_float, main_token = start_token, }) - .string: try add_node(&state.nodes, Node{ + .string: try add_node(state, Node{ kind = .literal_string, main_token = start_token, }) - .ident: try add_node(&state.nodes, Node{ + .ident: try add_node(state, Node{ kind = .expr_identifier, main_token = start_token, }) - else: try add_node(&state.nodes, Node{ + else: try add_node(state, Node{ kind = .invalid, main_token = start_token, }) } } +next_is proc(state @mut State, token_kind TokenKind) bool { + return state.tokens[state.next_token].kind == token_kind +} + +next_is_either proc(state @mut State, token_kinds []TokenKind) bool { + is_either := false + for (token_kinds) |kind| is_either = is_either or next_is(state, kind) + return is_either +} + expect proc(state @mut State, token_kind TokenKind) void ! ErrorCode { if (state.tokens[state.next_token].kind != token_kind) return .unexpected_token } @@ -162,24 +239,26 @@ expect_either proc(state @mut State, token_kinds []TokenKind) void ! ErrorCode { consume proc(state @mut State, token_kind TokenKind) TokenId ! ErrorCode { try expect(state, token_kind) + id :: TokenId(state.next_token) state.next_token += 1 - return state.next_token - 1 + return id } consume_either proc(state @mut State, token_kinds []TokenKind) TokenId ! ErrorCode { try expect_either(state, token_kinds) + id :: TokenId(state.next_token) state.next_token += 1 - return state.next_token - 1 -} - -add_node proc(nodes @mut std.ArrayList(Node), node Node) NodeId ! mem.AllocError { - id :: ast.node_id(nodes.items.len) - try arraylist.append(nodes, node) return id } -add_extra proc(extra @mut std.ArrayList(u32), data u32) ExtraId ! mem.AllocError { - id :: ast.extra_id(extra.items.len) - try arraylist.append(extra, data) +add_node proc(state @mut State, node Node) NodeId ! mem.AllocError { + id :: ast.node_id(state.nodes.items.len) + try arraylist.append(&state.nodes, node) + return id +} + +add_extra proc(state @mut State, data u32) ExtraId ! mem.AllocError { + id :: ast.extra_id(state.extra.items.len) + try arraylist.append(&state.extra, data) return id } diff --git a/source/parser/parser.test.hon b/source/parser/parser.test.hon index e2d97fe..c817e28 100644 --- a/source/parser/parser.test.hon +++ b/source/parser/parser.test.hon @@ -6,6 +6,67 @@ import "@source/lexer" ast_renderer :: import "@source/ast/renderer" +handles_package_declarations test { + strpool.strings = strpool.init(mem.c_allocator) + defer strpool.deinit(&strpool.strings) + + source :: + `first int :: 42 + `second float :: 2.15 + 3 + `message :: "hello" + + scan_state := lexer.init(mem.c_allocator) + defer lexer.deinit(&scan_state) + try lexer.scan(&scan_state, source) + + parse_state := init(mem.c_allocator) + defer deinit(&parse_state) + root :: try parse(&parse_state, scan_state.tokens.items) + + renderer := ast_renderer.init(mem.c_allocator) + defer ast_renderer.deinit(&renderer) + try ast_renderer.render(&renderer, root, &parse_state, scan_state.tokens.items) + + try testing.expect_equal(scan_state.tokens.items.len, renderer.tokens.items.len) + for scan_state.tokens.items |expected, i| { + actual :: renderer.tokens.items[i] + try testing.expect_equal(expected.kind, actual.kind) + try testing.expect_equal(expected.start, actual.start) + try testing.expect_equal(expected.str_id, actual.str_id) + } +} + +handles_binary_arithmetic_expression test { + strpool.strings = strpool.init(mem.c_allocator) + defer strpool.deinit(&strpool.strings) + + source :: + `first int :: 42 + 3 + `second :: 134 + 5 + `third float :: 1.35 + 2 + `fourth :: first + second + + scan_state := lexer.init(mem.c_allocator) + defer lexer.deinit(&scan_state) + try lexer.scan(&scan_state, source) + + parse_state := init(mem.c_allocator) + defer deinit(&parse_state) + root :: try parse(&parse_state, scan_state.tokens.items) + + renderer := ast_renderer.init(mem.c_allocator) + defer ast_renderer.deinit(&renderer) + try ast_renderer.render(&renderer, root, &parse_state, scan_state.tokens.items) + + try testing.expect_equal(scan_state.tokens.items.len, renderer.tokens.items.len) + for scan_state.tokens.items |expected, i| { + actual :: renderer.tokens.items[i] + try testing.expect_equal(expected.kind, actual.kind) + try testing.expect_equal(expected.start, actual.start) + try testing.expect_equal(expected.str_id, actual.str_id) + } +} + handles_statement_declaration test { strpool.strings = strpool.init(mem.c_allocator) defer strpool.deinit(&strpool.strings) diff --git a/source/strpool/strpool.hon b/source/strpool/strpool.hon index 9520a87..d26447d 100644 --- a/source/strpool/strpool.hon +++ b/source/strpool/strpool.hon @@ -9,7 +9,7 @@ InternError :: enum { out_of_space } StringId :: alias u32 -NO_ID :: maxval!(StringId) +NO_ID_STR :: maxval!(StringId) StringPool :: struct { ids hashmap.StringHashMap(StringId) # string → id @@ -36,7 +36,7 @@ deinit proc(pool @mut StringPool) void { intern proc(pool @mut StringPool, str []u8) StringId ! (mem.AllocError | InternError) { if hashmap.get(&pool.ids, str) |id| return id - if (pool.strings.items.len >= usize(NO_ID)) return .out_of_space + if (pool.strings.items.len >= usize(NO_ID_STR)) return .out_of_space id StringId :: StringId(pool.strings.items.len) owned_str []mut u8 :: try mem.alloc(u8, pool.allocator, str.len) diff --git a/std/io/io.hon b/std/io/io.hon index 8c7f4a0..48f552f 100644 --- a/std/io/io.hon +++ b/std/io/io.hon @@ -217,7 +217,7 @@ parse_format proc($N usize, $format []u8, $Args type) [N]mut FormatToken { token_count := 0 argument_count := 0 literal_start := 0 - cursor usize := 0 + cursor := 0 while cursor < format.len { byte :: format[cursor] if byte == '{' { diff --git a/std/testing/testing.hon b/std/testing/testing.hon index 2fa24c3..7bd72c1 100644 --- a/std/testing/testing.hon +++ b/std/testing/testing.hon @@ -73,10 +73,10 @@ expect_type proc($Expected, $Actual type, _ Actual, location SourceLocation) voi run proc(name []u8, callback *proc() void ! Error) bool { callback() catch |_| { - debug.print("{s}...[failed]\n", {name,}) + debug.print("{s} ... [\x1b[91mfailed\x1b[0m]\n", {name}) return false } - debug.print("{s}...[ok]\n", {name,}) + debug.print("{s} ... [\x1b[92mok\x1b[0m]\n", {name}) return true }