parse arithmetic binary expressions

source/ast/ast.hon

@@ -12,6 +12,9 @@ import "@source/lexer"
 @hide:file
 TokenId :: alias lexer.TokenId
 
+@hide:file
+NO_ID_TOKEN :: alias lexer.NO_ID_TOKEN
+
 NodeId :: distinct u32
 ExtraId :: distinct u32
 
@@ -28,7 +31,7 @@ NodeData :: union {
 
 Node :: struct {
     kind NodeKind
-    main_token TokenId
+    main_token TokenId = NO_ID_TOKEN
 
     data0 NodeData = NodeData{ node_id = NO_ID_NODE }
     data1 NodeData = NodeData{ node_id = NO_ID_NODE }
@@ -45,7 +48,14 @@ NodeKind :: enum {
     # * main_token: token literal
     expr_identifier
 
+    # UNARY:
+    # todo: document
     expr_unary
+
+    # BINARY:
+    # * main_token: operator
+    # * data0: NodeId - left-hand side
+    # * data1: NodeId - right-hand side
     expr_binary
 
     # DECL:
@@ -56,6 +66,13 @@ NodeKind :: enum {
     # * data1: TokenId of either `::` or `:=` indicating mutability
     stmt_decl
 
+    # PACKAGE:
+    # * data0: ExtraId
+    # * data1: ExtraId
+    # 
+    # extra[data0..data1]: NodeId - all statement node ids in package
+    package
+
     invalid
 }
 
@@ -64,6 +81,11 @@ node_id proc(idx uint) NodeId {
     return NodeId(idx)
 }
 
+maybe_node_id proc(idx uint) ?NodeId {
+    if (u64(idx) >= u64(NO_ID_NODE)) return null
+    return NodeId(idx)
+}
+
 extra_id proc(idx uint) ExtraId {
     debug.assert(u64(idx) < u64(NO_ID_EXTRA))
     return ExtraId(idx)

source/ast/renderer/renderer.hon

@@ -43,10 +43,33 @@ render proc(
             _ = try render_lit(renderer, node_id, parse_state, tokens)
         .stmt_decl:
             try render_decl(renderer, node_id, parse_state, tokens)
+        .package:
+            try render_pkg(renderer, node_id, parse_state, tokens)
         else: unreachable
     }
 }
 
+@hide:file
+render_pkg proc(
+    renderer @mut Renderer,
+    node_id NodeId,
+    parse_state @ParseState,
+    tokens []Token,
+) void ! mem.AllocError {
+    node :: parse_state.nodes.items[usize(node_id)]
+    extra_start :: usize(node.data0.extra_id)
+    extra_end :: usize(node.data1.extra_id)
+    for extra_start..extra_end |i| {
+        stmt_node_id :: ast.node_id(parse_state.extra.items[i])
+        stmt_node :: parse_state.nodes.items[usize(stmt_node_id)]
+        match stmt_node.kind {
+            .stmt_decl:
+                try render_decl(renderer, stmt_node_id, parse_state, tokens)
+            else: unreachable
+        }
+    }
+}
+
 @hide:file
 render_decl proc(
     renderer @mut Renderer,
@@ -58,8 +81,9 @@ render_decl proc(
     tok_ident :: tokens[usize(node.main_token)]
     try arraylist.append(&renderer.tokens, tok_ident)
 
-    node_id_type :: ast.node_id(parse_state.extra.items[usize(node.data0.extra_id)])
+    raw_node_id_type :: parse_state.extra.items[usize(node.data0.extra_id)]
-    try render_type(renderer, node_id_type, parse_state, tokens)
+    node_id_type :: ast.maybe_node_id(raw_node_id_type)
+    if (node_id_type) |id| try render_type(renderer, id, parse_state, tokens)
 
     tok_assign :: tokens[usize(node.data1.token_id)]
     try arraylist.append(&renderer.tokens, tok_assign)
@@ -84,20 +108,42 @@ render_expr proc(renderer @mut Renderer,
     parse_state @ParseState,
     tokens []Token,
 ) TokenId ! mem.AllocError {
-    # todo: support unary and binary expressions
+    # todo: support unary expressions
     node :: parse_state.nodes.items[usize(node_id)]
-    match node.kind {
+    return match node.kind {
         .expr_identifier: {
             tok :: tokens[usize(node.main_token)]
             try arraylist.append(&renderer.tokens, tok)
-            return node.main_token
+            yield node.main_token
         }
+        .expr_binary:
+            try render_expr_binary(renderer, node_id, parse_state, tokens)
         .literal_int, .literal_float, .literal_string:
-            return try render_lit(renderer, node_id, parse_state, tokens)
+            try render_lit(renderer, node_id, parse_state, tokens)
         else: unreachable
     }
 }
 
+@hide:file
+render_expr_binary proc(
+    renderer @mut Renderer,
+    node_id NodeId,
+    parse_state @ParseState,
+    tokens []Token,
+) TokenId ! mem.AllocError {
+    node :: parse_state.nodes.items[usize(node_id)]
+    operator :: node.main_token
+    lhs :: node.data0.node_id
+    rhs :: node.data1.node_id
+
+    # render lhs operator rhs
+    _ = try render_expr(renderer, lhs, parse_state, tokens)
+    try arraylist.append(&renderer.tokens, tokens[usize(operator)])
+    last_token :: try render_expr(renderer, rhs, parse_state, tokens)
+
+    return last_token
+}
+
 @hide:file
 render_type proc(
     renderer @mut Renderer,
@@ -124,7 +170,7 @@ render_lit proc(
     tokens []Token,
 ) TokenId ! mem.AllocError {
     node :: parse_state.nodes.items[usize(node_id)]
-    tok :: tokens[node.main_token]
+    tok :: tokens[usize(node.main_token)]
     try arraylist.append(&renderer.tokens, tok)
     return node.main_token
 }

source/lexer/lexer.hon

@@ -3,7 +3,6 @@ import "@std/mem"
 import "@std/strmap"
 import "@std/enums/enummap"
 import "@std/arraylist"
-import "@std/debug"
 
 import "@source/strpool"
 
@@ -43,8 +42,6 @@ keywords std.StringMap(TokenKind) :: strmap.init([
     { "while", .while },
 ])
 
-TokenId :: distinct u32
-
 Diagnostic :: struct {
     token TokenId
     code ErrorCode
@@ -68,7 +65,6 @@ deinit proc(state @mut State) void {
 }
 
 scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.InternError) {
-    tokens :: &state.tokens
     diagnostics :: &state.diagnostics
 
     cursor := 0
@@ -77,7 +73,7 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter
 
         # whitespace
         if char == '\n' {
-            try add_token(tokens, Token{ kind = .newline, start = cursor })
+            try add_token(state, Token{ start = cursor, kind = .newline })
             cursor += 1
             continue
         } else if is_whitespace(char) {
@@ -96,9 +92,9 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter
         if is_alpha(char) or char == '_' {
             start :: cursor
             result :: scan_ident(start, program)
-            try add_token(tokens, Token{
+            try add_token(state, Token{
-                kind = result.kind,
                 start = start,
+                kind = result.kind,
                 str_id = result.str_id,
             })
             cursor = result.end
@@ -109,15 +105,15 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter
         if is_digit(char) {
             start :: cursor
             result :: scan_number(start, program) catch |err| {
-                token :: token_id(tokens.items.len)
+                token :: token_id(state.tokens.items.len)
-                try add_token(tokens, Token{ kind = .invalid, start = start })
+                try add_token(state, Token{ start = start, kind = .invalid })
                 try arraylist.append(diagnostics, Diagnostic{ token = token, code = err.code })
                 cursor = err.end
                 continue
             }
 
             kind :: if (result.has_decimal) .float else .int
-            try add_token(tokens, Token{ kind = kind, start = start })
+            try add_token(state, Token{ start = start, kind = kind })
             cursor = result.end
             continue
         }
@@ -126,68 +122,107 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter
         if char == '"' {
             start :: cursor
             result :: scan_string(start, program) catch |err| {
-                token :: token_id(tokens.items.len)
+                token :: token_id(state.tokens.items.len)
-                try add_token(tokens, Token{ kind = .invalid, start = start })
+                try add_token(state, Token{ start = start, kind = .invalid })
                 try arraylist.append(diagnostics, Diagnostic{ token = token, code = err.code })
                 cursor = err.end
                 continue
             }
 
-            try add_token(tokens, Token{ kind = .string, start = start })
+            try add_token(state, Token{ start = start, kind = .string })
             cursor = result.end
             continue
         }
 
-        # immutable assignment or single colon
+        # immutable decl, mutable decl, or single colon
         if char == ':' {
             if cursor + 1 < program.len and program[cursor + 1] == ':' {
-                try add_token(tokens, Token{ kind = .double_colon, start = cursor })
+                try add_token(state, Token{ start = cursor, kind = .double_colon })
                 cursor += 2
                 continue
             }
 
-            try add_token(tokens, Token{ kind = .colon, start = cursor })
+            if cursor + 1 < program.len and program[cursor + 1] == '=' {
+                try add_token(state, Token{ start = cursor, kind = .colon_equal })
+                cursor += 2
+                continue
+            }
+
+            try add_token(state, Token{ start = cursor, kind = .colon })
             cursor += 1
             continue
         }
 
-        # mutable reassignment
+        # compound arithmetic assignment
-        if char == '=' {
+        if char == '+' {
-            try add_token(tokens, Token{ kind = .equal, start = cursor })
+            if cursor + 1 < program.len and program[cursor + 1] == '=' {
+                try add_token(state, Token{ start = cursor, kind = .plus_equal })
+                cursor += 2
+                continue
+            }
+
+            try add_token(state, Token{ start = cursor, kind = .plus })
+            cursor += 1
+            continue
+        }
+        if char == '-' {
+            if cursor + 1 < program.len and program[cursor + 1] == '=' {
+                try add_token(state, Token{ start = cursor, kind = .minus_equal })
+                cursor += 2
+                continue
+            }
+
+            try add_token(state, Token{ start = cursor, kind = .minus })
+            cursor += 1
+            continue
+        }
+        if char == '*' {
+            if cursor + 1 < program.len and program[cursor + 1] == '=' {
+                try add_token(state, Token{ start = cursor, kind = .star_equal })
+                cursor += 2
+                continue
+            }
+
+            try add_token(state, Token{ start = cursor, kind = .star })
+            cursor += 1
+            continue
+        }
+        if char == '/' {
+            if cursor + 1 < program.len and program[cursor + 1] == '=' {
+                try add_token(state, Token{ start = cursor, kind = .slash_equal })
+                cursor += 2
+                continue
+            }
+
+            try add_token(state, Token{ start = cursor, kind = .slash })
             cursor += 1
             continue
         }
 
-        # parentheses
+        # single character tokens
-        if char == '(' {
+        kind ?TokenKind :: match char {
-            try add_token(tokens, Token{ kind = .open_paren, start = cursor })
+            '=': .equal
-            cursor += 1
+            '(': .open_paren
-            continue
+            ')': .close_paren
-        } else if char == ')' {
+            '{': .open_curly
-            try add_token(tokens, Token{ kind = .close_paren, start = cursor })
+            '}': .close_curly
-            cursor += 1
+            else: null
-            continue
         }
 
-        # curly braces
+        if (kind) |k| {
-        if char == '{' {
+            try add_token(state, Token{ start = cursor, kind = k })
-            try add_token(tokens, Token{ kind = .open_curly, start = cursor })
-            cursor += 1
-            continue
-        } else if char == '}' {
-            try add_token(tokens, Token{ kind = .close_curly, start = cursor })
             cursor += 1
             continue
         }
 
         # invalid character
-        token :: token_id(tokens.items.len)
+        token :: token_id(state.tokens.items.len)
+        try add_token(state, Token{ start = cursor, kind = .invalid })
         try arraylist.append(diagnostics, Diagnostic{ token = token, code = .invalid_character })
-        try add_token(tokens, Token{ kind = .invalid, start = cursor })
         cursor += 1
     }
 
-    try add_token(tokens, Token{ kind = .eof, start = cursor })
+    try add_token(state, Token{ start = cursor, kind = .eof })
 }
 
 ScanIdentResult :: struct {
@@ -210,9 +245,9 @@ scan_ident proc(start usize, program []u8) ScanIdentResult {
 
     # don't intern keywords (already O(1) lookup via token kind)
     str_id :: if (kind == .ident)
-        strpool.intern(&strpool.strings, program[start..cursor]) catch strpool.NO_ID
+        strpool.intern(&strpool.strings, program[start..cursor]) catch strpool.NO_ID_STR
     else
-        strpool.NO_ID
+        strpool.NO_ID_STR
 
     return ScanIdentResult{
         end = cursor,
@@ -243,7 +278,7 @@ scan_number proc(start usize, program []u8) ScanNumResult ! ScanError {
     if has_decimal and (cursor >= program.len or !is_digit(program[cursor])) {
         return ScanError{
             code = .float_must_end_with_digit,
-            end = cursor + 1,
+            end = cursor,
         }
     }
 
@@ -300,13 +335,7 @@ is_digit proc(char u8) bool {
 }
 
 @hide
-add_token proc(tokens @mut std.ArrayList(Token), token Token) void ! mem.AllocError {
+add_token proc(state @mut State, token Token) void ! mem.AllocError {
-    _ = token_id(tokens.items.len)
+    _ = token_id(state.tokens.items.len)
-    try arraylist.append(tokens, token)
+    try arraylist.append(&state.tokens, token)
-}
-
-@hide
-token_id proc(idx uint) TokenId {
-    debug.assert(u64(idx) < u64(maxval!(TokenId)))
-    return TokenId(idx)
 }

source/lexer/token.hon

@@ -1,9 +1,14 @@
+import "@std/debug"
 import "@source/strpool"
 
+TokenId :: distinct u32
+
+NO_ID_TOKEN :: maxval!(TokenId)
+
 Token :: struct {
-    kind TokenKind
     start uint
-    str_id strpool.StringId = strpool.NO_ID
+    kind TokenKind
+    str_id strpool.StringId = strpool.NO_ID_STR
 }
 
 TokenKind :: enum {
@@ -22,7 +27,16 @@ TokenKind :: enum {
     greater, greater_equal
 
     # assignment
-    equal, double_colon
+    double_colon, colon_equal
+    equal
+
+    # arithmetic
+    plus, minus
+    star, slash
+
+    # assignment & arithmetic
+    plus_equal, minus_equal
+    star_equal, slash_equal
 
     # delimiters
     open_paren, close_paren
@@ -36,3 +50,9 @@ TokenKind :: enum {
     eof
     invalid
 }
+
+@hide
+token_id proc(idx uint) TokenId {
+    debug.assert(u64(idx) < u64(NO_ID_TOKEN))
+    return TokenId(idx)
+}

source/main.hon

@@ -1,6 +1,5 @@
 import "@std/debug"
 import "@std/mem"
-import "@std/enums/enummap"
 
 test import "@std/enums/enummap"
 test import "@std/arraylist"
@@ -20,8 +19,11 @@ test import "@source/parser"
 
 program ::
     `# literals
-    `x int :: 123.9
+    `x float :: 123.9
-    `y :: "hello"
+    `y int := 43
+    `z :: "hello"
+    `
+    `a := 1 + 2 * 3
 
 main proc() void! {
     strpool.strings = strpool.init(mem.c_allocator)
@@ -42,13 +44,6 @@ main proc() void! {
     }
     debug.print("]]\n\n", {})
 
-    debug.print("TOKENS::DIAGNOSTICS::[[\n", {})
-    for scan_state.diagnostics.items |diagnostic| {
-        details :: enummap.get(&lexer.error_msg_map, diagnostic.code)
-        debug.print("{}: {}\n", {details?.name, details?.message})
-    }
-    debug.print("]]\n\n", {})
-
     parse_state := parser.init(mem.c_allocator)
     defer parser.deinit(&parse_state)
     root :: parser.parse(&parse_state, scan_state.tokens.items) catch |err| {
@@ -58,21 +53,7 @@ main proc() void! {
 
     debug.print("AST::[[\n", {})
     for parse_state.nodes.items |node, id| {
-        token :: scan_state.tokens.items[node.main_token]
+        debug.print("{} (id = {})\n", { node.kind, id })
-
-        end :: if (token.kind == .int or token.kind == .float) lbl: {
-            res :: lexer.scan_number(token.start, program) catch {
-                yield :lbl token.start
-            }
-            yield :lbl res.end
-        } else if (token.kind == .string) lbl: {
-            res :: lexer.scan_string(token.start, program) catch {
-                yield :lbl token.start
-            }
-            yield :lbl res.end
-        } else token.start
-
-        debug.print("{} (id = {}): {}\n", { node.kind, id, program[token.start..end] })
     }
     debug.print("]]\n\n", {})
 

source/parser/parser.hon

@@ -58,6 +58,21 @@ NO_ID_NODE :: alias ast.NO_ID_NODE
 @hide:file
 NO_ID_EXTRA :: alias ast.NO_ID_EXTRA
 
+BindingPowerScalar :: distinct u32
+
+BindingPower :: struct {
+    left BindingPowerScalar
+    right BindingPowerScalar
+}
+
+infix_binding_power proc(token_kind TokenKind) ?BindingPower {
+    return match token_kind {
+        .plus, .minus: BindingPower{ left = BindingPowerScalar(10), right = BindingPowerScalar(11) }
+        .star, .slash: BindingPower{ left = BindingPowerScalar(20), right = BindingPowerScalar(21) }
+        else: null
+    }
+}
+
 State :: struct {
     nodes std.ArrayList(Node)
 
@@ -65,7 +80,7 @@ State :: struct {
     extra std.ArrayList(u32)
 
     tokens []Token = &[]
-    next_token TokenId = TokenId(0)
+    next_token usize = 0
 }
 
 init proc(allocator mem.Allocator) State {
@@ -82,35 +97,65 @@ deinit proc(state @mut State) void {
 
 parse proc(state @mut State, tokens []Token) NodeId ! ParseError {
     state.tokens = tokens
+    return try parse_pkg(state)
+}
+
+parse_pkg proc(state @mut State) NodeId ! ParseError {
+    if (state.next_token < state.tokens.len
+        and state.tokens[state.next_token].kind == .eof
+    ) return NO_ID_NODE
+
+    stmt_ids std.ArrayList(NodeId) := arraylist.init(state.nodes.allocator)
+    defer arraylist.deinit(&stmt_ids)
+
+    while (state.next_token < state.tokens.len
+        and state.tokens[state.next_token].kind != .eof
+    ) {
+        stmt_id :: try parse_stmt(state)
+        try arraylist.append(&stmt_ids, stmt_id)
+    }
+
+    pkg_start :: ast.extra_id(state.extra.items.len)
+    for (stmt_ids.items) |stmt_id| _ = try add_extra(state, u32(stmt_id))
+    pkg_end :: ast.extra_id(state.extra.items.len)
+
+    package :: try add_node(state, Node{
+        kind = .package,
+        data0 = NodeData{ extra_id = pkg_start },
+        data1 = NodeData{ extra_id = pkg_end },
+    })
+
+    return package
+}
+
+parse_stmt proc(state @mut State) NodeId ! ParseError {
     return try parse_decl(state)
 }
 
 parse_decl proc(state @mut State) NodeId ! ParseError {
     # expect identifier
-    try expect(state, .ident)
+    ident_token :: try consume(state, .ident)
-    ident_token :: state.next_token
-    state.next_token += 1
 
     # check for type
     # todo: make a parse_type proc for this
-    type_id :: if (state.tokens[state.next_token].kind == .ident)
+    type_id :: if (next_is(state, .ident))
         try parse_primary(state)
     else
         NO_ID_NODE
 
-    # expect `::` (immutable assignment)
+    # expect `::` or `:=` (immutable or mutable assignment)
-    mutability_tok_id :: try consume(state, .double_colon)
+    mutability_tok_id :: try consume_either(state, &[.double_colon, .colon_equal])
 
     # expect expression
     expr_id :: try parse_expr(state)
 
     # expect statement terminator (newline)
-    _ = try consume_either(state, &[.newline, .eof])
+    while (next_is(state, .newline)) _ = try consume(state, .newline)
 
-    extra_start :: try add_extra(&state.extra, u32(type_id))
+    extra_start :: try add_extra(state, u32(type_id))
-    _ = try add_extra(&state.extra, u32(expr_id))
+    _ = try add_extra(state, u32(expr_id))
 
-    decl :: try add_node(&state.nodes, Node{
+    decl :: try add_node(state, Node{
         kind = .stmt_decl,
         main_token = ident_token,
         data0 = NodeData{ extra_id = extra_start },
@@ -121,36 +166,68 @@ parse_decl proc(state @mut State) NodeId ! ParseError {
 }
 
 parse_expr proc(state @mut State) NodeId ! mem.AllocError {
-    return try parse_primary(state)
+    return try parse_expr_bp(state, BindingPowerScalar(0))
+}
+
+parse_expr_bp proc(state @mut State, min_bp BindingPowerScalar) NodeId ! mem.AllocError {
+    lhs := try parse_primary(state)
+
+    while infix_binding_power(state.tokens[state.next_token].kind) |bp| {
+        if (bp.left < min_bp) break
+
+        operator :: TokenId(state.next_token)
+        state.next_token += 1
+
+        rhs :: try parse_expr_bp(state, bp.right)
+        lhs = try add_node(state, Node{
+            kind = .expr_binary,
+            main_token = operator,
+            data0 = NodeData{ node_id = lhs },
+            data1 = NodeData{ node_id = rhs },
+        })
+    }
+
+    return lhs
 }
 
 parse_primary proc(state @mut State) NodeId ! mem.AllocError {
-    start_token :: state.next_token
+    start_idx :: state.next_token
+    start_token :: TokenId(start_idx)
     state.next_token += 1
-    return match state.tokens[start_token].kind {
+    return match state.tokens[start_idx].kind {
-        .int: try add_node(&state.nodes, Node{
+        .int: try add_node(state, Node{
             kind = .literal_int,
             main_token = start_token,
         })
-        .float: try add_node(&state.nodes, Node{
+        .float: try add_node(state, Node{
             kind = .literal_float,
             main_token = start_token,
         })
-        .string: try add_node(&state.nodes, Node{
+        .string: try add_node(state, Node{
             kind = .literal_string,
             main_token = start_token,
         })
-        .ident: try add_node(&state.nodes, Node{
+        .ident: try add_node(state, Node{
             kind = .expr_identifier,
             main_token = start_token,
         })
-        else: try add_node(&state.nodes, Node{
+        else: try add_node(state, Node{
             kind = .invalid,
             main_token = start_token,
         })
     }
 }
 
+next_is proc(state @mut State, token_kind TokenKind) bool {
+    return state.tokens[state.next_token].kind == token_kind
+}
+
+next_is_either proc(state @mut State, token_kinds []TokenKind) bool {
+    is_either := false
+    for (token_kinds) |kind| is_either = is_either or next_is(state, kind)
+    return is_either
+}
+
 expect proc(state @mut State, token_kind TokenKind) void ! ErrorCode {
     if (state.tokens[state.next_token].kind != token_kind) return .unexpected_token
 }
@@ -162,24 +239,26 @@ expect_either proc(state @mut State, token_kinds []TokenKind) void ! ErrorCode {
 
 consume proc(state @mut State, token_kind TokenKind) TokenId ! ErrorCode {
     try expect(state, token_kind)
+    id :: TokenId(state.next_token)
     state.next_token += 1
-    return state.next_token - 1
+    return id
 }
 
 consume_either proc(state @mut State, token_kinds []TokenKind) TokenId ! ErrorCode {
     try expect_either(state, token_kinds)
+    id :: TokenId(state.next_token)
     state.next_token += 1
-    return state.next_token - 1
-}
-
-add_node proc(nodes @mut std.ArrayList(Node), node Node) NodeId ! mem.AllocError {
-    id :: ast.node_id(nodes.items.len)
-    try arraylist.append(nodes, node)
     return id
 }
 
-add_extra proc(extra @mut std.ArrayList(u32), data u32) ExtraId ! mem.AllocError {
+add_node proc(state @mut State, node Node) NodeId ! mem.AllocError {
-    id :: ast.extra_id(extra.items.len)
+    id :: ast.node_id(state.nodes.items.len)
-    try arraylist.append(extra, data)
+    try arraylist.append(&state.nodes, node)
+    return id
+}
+
+add_extra proc(state @mut State, data u32) ExtraId ! mem.AllocError {
+    id :: ast.extra_id(state.extra.items.len)
+    try arraylist.append(&state.extra, data)
     return id
 }

source/parser/parser.test.hon

@@ -6,6 +6,67 @@ import "@source/lexer"
 
 ast_renderer :: import "@source/ast/renderer"
 
+handles_package_declarations test {
+    strpool.strings = strpool.init(mem.c_allocator)
+    defer strpool.deinit(&strpool.strings)
+
+    source ::
+        `first int :: 42
+        `second float :: 2.15 + 3
+        `message :: "hello"
+
+    scan_state := lexer.init(mem.c_allocator)
+    defer lexer.deinit(&scan_state)
+    try lexer.scan(&scan_state, source)
+
+    parse_state := init(mem.c_allocator)
+    defer deinit(&parse_state)
+    root :: try parse(&parse_state, scan_state.tokens.items)
+
+    renderer := ast_renderer.init(mem.c_allocator)
+    defer ast_renderer.deinit(&renderer)
+    try ast_renderer.render(&renderer, root, &parse_state, scan_state.tokens.items)
+
+    try testing.expect_equal(scan_state.tokens.items.len, renderer.tokens.items.len)
+    for scan_state.tokens.items |expected, i| {
+        actual :: renderer.tokens.items[i]
+        try testing.expect_equal(expected.kind, actual.kind)
+        try testing.expect_equal(expected.start, actual.start)
+        try testing.expect_equal(expected.str_id, actual.str_id)
+    }
+}
+
+handles_binary_arithmetic_expression test {
+    strpool.strings = strpool.init(mem.c_allocator)
+    defer strpool.deinit(&strpool.strings)
+
+    source ::
+        `first int :: 42 + 3
+        `second :: 134 + 5
+        `third float :: 1.35 + 2
+        `fourth :: first + second
+
+    scan_state := lexer.init(mem.c_allocator)
+    defer lexer.deinit(&scan_state)
+    try lexer.scan(&scan_state, source)
+
+    parse_state := init(mem.c_allocator)
+    defer deinit(&parse_state)
+    root :: try parse(&parse_state, scan_state.tokens.items)
+
+    renderer := ast_renderer.init(mem.c_allocator)
+    defer ast_renderer.deinit(&renderer)
+    try ast_renderer.render(&renderer, root, &parse_state, scan_state.tokens.items)
+
+    try testing.expect_equal(scan_state.tokens.items.len, renderer.tokens.items.len)
+    for scan_state.tokens.items |expected, i| {
+        actual :: renderer.tokens.items[i]
+        try testing.expect_equal(expected.kind, actual.kind)
+        try testing.expect_equal(expected.start, actual.start)
+        try testing.expect_equal(expected.str_id, actual.str_id)
+    }
+}
+
 handles_statement_declaration test {
     strpool.strings = strpool.init(mem.c_allocator)
     defer strpool.deinit(&strpool.strings)

source/strpool/strpool.hon

@@ -9,7 +9,7 @@ InternError :: enum { out_of_space }
 
 StringId :: alias u32
 
-NO_ID :: maxval!(StringId)
+NO_ID_STR :: maxval!(StringId)
 
 StringPool :: struct {
     ids hashmap.StringHashMap(StringId) # string → id
@@ -36,7 +36,7 @@ deinit proc(pool @mut StringPool) void {
 intern proc(pool @mut StringPool, str []u8) StringId ! (mem.AllocError | InternError) {
     if hashmap.get(&pool.ids, str) |id| return id
 
-    if (pool.strings.items.len >= usize(NO_ID)) return .out_of_space
+    if (pool.strings.items.len >= usize(NO_ID_STR)) return .out_of_space
     id StringId :: StringId(pool.strings.items.len)
 
     owned_str []mut u8 :: try mem.alloc(u8, pool.allocator, str.len)

std/io/io.hon

@@ -217,7 +217,7 @@ parse_format proc($N usize, $format []u8, $Args type) [N]mut FormatToken {
     token_count := 0
     argument_count := 0
     literal_start := 0
-    cursor usize := 0
+    cursor := 0
     while cursor < format.len {
         byte :: format[cursor]
         if byte == '{' {

std/testing/testing.hon

@@ -73,10 +73,10 @@ expect_type proc($Expected, $Actual type, _ Actual, location SourceLocation) voi
 
 run proc(name []u8, callback *proc() void ! Error) bool {
     callback() catch |_| {
-        debug.print("{s}...[failed]\n", {name,})
+        debug.print("{s} ... [\x1b[91mfailed\x1b[0m]\n", {name})
         return false
     }
-    debug.print("{s}...[ok]\n", {name,})
+    debug.print("{s} ... [\x1b[92mok\x1b[0m]\n", {name})
     return true
 }