parse func decls

source/ast/ast.hon

@@ -1,12 +1,25 @@
+#! GRAMMAR:
+#! * type ..... -> identifier
+#! * assignment -> :: | :=
+#! * literal .. -> int | float | string
+#! * expr ..... -> identifier | literal
+#! * decl ..... -> identifier [type] assignment expr
+#! * stmt ..... -> decl
+
+import "@std/debug"
 import "@source/lexer"
 
-hide TokenId :: alias lexer.TokenId
+@hide:file
+TokenId :: alias lexer.TokenId
+
+@hide:file
+NO_TOKEN :: alias lexer.NO_TOKEN
 
 NodeId :: distinct u32
 ExtraId :: distinct u32
 
-NO_ID_NODE :: maxval!(NodeId)
-NO_ID_EXTRA :: maxval!(ExtraId)
+NO_NODE :: maxval!(NodeId)
+NO_EXTRA :: maxval!(ExtraId)
 
 # unsafe access on its own; NodeKind serves as a the tag
 # that determines the variant.
@@ -18,10 +31,10 @@ NodeData :: union {
 
 Node :: struct {
     kind NodeKind
-    main_token TokenId
+    main_token TokenId = NO_TOKEN
 
-    data0 NodeData = NodeData{ node_id = NO_ID_NODE }
-    data1 NodeData = NodeData{ node_id = NO_ID_NODE }
+    data0 NodeData = NodeData{ node_id = NO_NODE }
+    data1 NodeData = NodeData{ node_id = NO_NODE }
 }
 
 NodeKind :: enum {
@@ -35,16 +48,65 @@ NodeKind :: enum {
     # * main_token: token literal
     expr_identifier
 
+    # UNARY:
+    # * main_token: operator
+    # * data0: NodeId - expression
     expr_unary
+
+    # BINARY:
+    # * main_token: operator
+    # * data0: NodeId - left-hand side
+    # * data1: NodeId - right-hand side
     expr_binary
 
-    # DECL:
+    # STATEMENT DECLS:
     # * main_token: symbol name (identifier)
     # * data0: ExtraId
-    #   + extra[data0]: NodeId - type identifier node
+    #   + extra[data0]: NodeId - type expr or NO_NODE
     #   + extra[data0 + 1]: NodeId - initializer expression
-    # * data1: TokenId of either `::` or `=` indicating mutability
+    # * data1: TokenId of either `::` or `:=` indicating mutability
     stmt_decl
 
+    # FUNC DECLS:
+    # * main_token: function name (identifier)
+    # * data0: NodeId - body block (or NO_NODE for extern)
+    # * data1: ExtraId
+    #   + extra[data1]: NodeId - return type
+    #   + extra[data1 + 1]: u32 - param count
+    #   + extra[data1 + 2..]: NodeId - param decls
+    func_decl
+
+    # PARAM DECLS:
+    # * main_token: param name (identifier)
+    # * data0: NodeId - param type
+    param_decl
+
+    # BLOCK:
+    # * data0: ExtraId - statement range start
+    # * data1: ExtraId - statement range end
+    block
+
+    # PACKAGE:
+    # * data0: ExtraId
+    # * data1: ExtraId
+    # 
+    # extra[data0..data1]: NodeId - all statement node ids in package
+    package
+
     invalid
 }
+
+node_id proc(idx uint) NodeId {
+    debug.assert(u64(idx) < u64(NO_NODE))
+    return NodeId(idx)
+}
+
+maybe_node_id proc(idx uint) ?NodeId {
+    if (u64(idx) >= u64(NO_NODE)) return null
+    return NodeId(idx)
+}
+
+extra_id proc(idx uint) ExtraId {
+    debug.assert(u64(idx) < u64(NO_EXTRA))
+    return ExtraId(idx)
+}

source/ast/renderer/renderer.hon

@@ -0,0 +1,319 @@
+import "@std"
+import "@std/mem"
+import "@std/arraylist"
+
+import "@source/lexer"
+import "@source/parser"
+import "@source/ast"
+
+@hide:file TokenId :: alias lexer.TokenId
+@hide:file Token :: alias lexer.Token
+@hide:file Node :: alias ast.Node
+@hide:file NodeId :: alias ast.NodeId
+@hide:file ParseState :: alias parser.State
+
+Renderer :: struct { tokens std.ArrayList(Token) }
+
+init proc(allocator mem.Allocator) Renderer {
+    return Renderer{ tokens = arraylist.init(allocator) }
+}
+
+deinit proc(renderer @mut Renderer) void {
+    arraylist.deinit(&renderer.tokens)
+}
+
+render_token_stream proc(
+    renderer @mut Renderer,
+    node_id NodeId,
+    parse_state @ParseState,
+    tokens []Token
+) void ! mem.AllocError {
+    node :: parse_state.nodes.items[usize(node_id)]
+    match node.kind {
+        .literal_int, .literal_float, .literal_string:
+            _ = try render_lit(renderer, node_id, parse_state, tokens)
+        .stmt_decl:
+            try render_stmt_decl(renderer, node_id, parse_state, tokens)
+        .func_decl:
+            try render_func_decl(renderer, node_id, parse_state, tokens)
+        .package:
+            try render_pkg(renderer, node_id, parse_state, tokens)
+        else: unreachable
+    }
+}
+
+@hide:file
+render_pkg proc(
+    renderer @mut Renderer,
+    node_id NodeId,
+    parse_state @ParseState,
+    tokens []Token,
+) void ! mem.AllocError {
+    node :: parse_state.nodes.items[usize(node_id)]
+    extra_start :: usize(node.data0.extra_id)
+    extra_end :: usize(node.data1.extra_id)
+    for extra_start..extra_end |i| {
+        stmt_node_id :: ast.node_id(parse_state.extra.items[i])
+        stmt_node :: parse_state.nodes.items[usize(stmt_node_id)]
+        match stmt_node.kind {
+            .stmt_decl:
+                try render_stmt_decl(renderer, stmt_node_id, parse_state, tokens)
+            .func_decl:
+                try render_func_decl(renderer, stmt_node_id, parse_state, tokens)
+            else: unreachable
+        }
+    }
+}
+
+@hide:file
+render_stmt_decl proc(
+    renderer @mut Renderer,
+    node_id NodeId,
+    parse_state @ParseState,
+    tokens []Token,
+) void ! mem.AllocError {
+    node :: parse_state.nodes.items[usize(node_id)]
+    tok_ident :: tokens[usize(node.main_token)]
+    try arraylist.append(&renderer.tokens, tok_ident)
+
+    raw_node_id_type :: parse_state.extra.items[usize(node.data0.extra_id)]
+    node_id_type :: ast.maybe_node_id(raw_node_id_type)
+    if (node_id_type) |id| try render_type(renderer, id, parse_state, tokens)
+
+    tok_assign :: tokens[usize(node.data1.token_id)]
+    try arraylist.append(&renderer.tokens, tok_assign)
+
+    node_id_expr :: ast.node_id(parse_state.extra.items[usize(node.data0.extra_id + 1)])
+    last_expr_token :: try render_expr(renderer, node_id_expr, parse_state, tokens)
+
+    terminator_idx :: usize(last_expr_token) + 1
+    tok_terminator :: if (terminator_idx < tokens.len) tokens[terminator_idx] else return
+    if (tok_terminator.kind != .newline and tok_terminator.kind != .eof) return
+    try arraylist.append(&renderer.tokens, tok_terminator)
+
+    eof_idx :: terminator_idx + 1
+    if (tok_terminator.kind == .newline and
+        eof_idx < tokens.len and
+        tokens[eof_idx].kind == .eof
+    ) try arraylist.append(&renderer.tokens, tokens[eof_idx])
+}
+
+@hide:file
+render_func_decl proc(
+    renderer @mut Renderer,
+    node_id NodeId,
+    parse_state @ParseState,
+    tokens []Token,
+) void ! mem.AllocError {
+    node :: parse_state.nodes.items[usize(node_id)]
+    ident_idx :: usize(node.main_token)
+    try arraylist.append(&renderer.tokens, tokens[ident_idx])
+    try arraylist.append(&renderer.tokens, tokens[ident_idx + 1])
+    try arraylist.append(&renderer.tokens, tokens[ident_idx + 2])
+
+    payload_start :: usize(node.data1.extra_id)
+    param_count :: usize(parse_state.extra.items[payload_start + 1])
+    first_param :: payload_start + 2
+    for 0..param_count |i| {
+        param_id :: ast.node_id(parse_state.extra.items[first_param + i])
+        param_node :: parse_state.nodes.items[usize(param_id)]
+        type_id :: param_node.data0.node_id
+        type_node :: parse_state.nodes.items[usize(type_id)]
+        try arraylist.append(&renderer.tokens, tokens[usize(param_node.main_token)])
+
+        has_next :: i + 1 < param_count
+        if has_next {
+            next_param_id :: ast.node_id(parse_state.extra.items[first_param + i + 1])
+            next_param :: parse_state.nodes.items[usize(next_param_id)]
+            if (next_param.data0.node_id == type_id) {
+                try arraylist.append(
+                    &renderer.tokens,
+                    tokens[usize(param_node.main_token) + 1],
+                )
+                continue
+            }
+        }
+
+        try render_type(renderer, type_id, parse_state, tokens)
+        if has_next {
+            try arraylist.append(
+                &renderer.tokens,
+                tokens[usize(type_node.main_token) + 1],
+            )
+        }
+    }
+
+    return_type_id :: ast.node_id(parse_state.extra.items[payload_start])
+    return_type_node :: parse_state.nodes.items[usize(return_type_id)]
+    try arraylist.append(
+        &renderer.tokens,
+        tokens[usize(return_type_node.main_token) - 1],
+    )
+    try render_type(renderer, return_type_id, parse_state, tokens)
+
+    open_curly_id :: TokenId(usize(return_type_node.main_token) + 1)
+    close_curly_id :: try render_block(
+        renderer,
+        node.data0.node_id,
+        open_curly_id,
+        parse_state,
+        tokens,
+    )
+
+    terminator_idx :: usize(close_curly_id) + 1
+    if (terminator_idx >= tokens.len) return
+    tok_terminator :: tokens[terminator_idx]
+    if (tok_terminator.kind != .newline and tok_terminator.kind != .eof) return
+    try arraylist.append(&renderer.tokens, tok_terminator)
+
+    eof_idx :: terminator_idx + 1
+    if (tok_terminator.kind == .newline and
+        eof_idx < tokens.len and
+        tokens[eof_idx].kind == .eof
+    ) try arraylist.append(&renderer.tokens, tokens[eof_idx])
+}
+
+@hide:file
+render_block proc(
+    renderer @mut Renderer,
+    node_id NodeId,
+    open_curly_id TokenId,
+    parse_state @ParseState,
+    tokens []Token,
+) TokenId ! mem.AllocError {
+    node :: parse_state.nodes.items[usize(node_id)]
+    open_curly_idx :: usize(open_curly_id)
+    try arraylist.append(&renderer.tokens, tokens[open_curly_idx])
+
+    leading_token := open_curly_idx + 1
+    while (leading_token < tokens.len and tokens[leading_token].kind == .newline) {
+        try arraylist.append(&renderer.tokens, tokens[leading_token])
+        leading_token += 1
+    }
+
+    extra_start :: usize(node.data0.extra_id)
+    extra_end :: usize(node.data1.extra_id)
+    for extra_start..extra_end |i| {
+        stmt_id :: ast.node_id(parse_state.extra.items[i])
+        stmt :: parse_state.nodes.items[usize(stmt_id)]
+        match stmt.kind {
+            .stmt_decl:
+                try render_stmt_decl(renderer, stmt_id, parse_state, tokens)
+            else: unreachable
+        }
+    }
+
+    # ponytail: linear delimiter scan; store brace tokens if nested blocks make rendering hot.
+    depth usize := 1
+    close_curly_idx := open_curly_idx + 1
+    while close_curly_idx < tokens.len {
+        match tokens[close_curly_idx].kind {
+            .open_curly: depth += 1
+            .close_curly: {
+                depth -= 1
+                if (depth == 0) {
+                    try arraylist.append(&renderer.tokens, tokens[close_curly_idx])
+                    return TokenId(close_curly_idx)
+                }
+            }
+            else: {}
+        }
+        close_curly_idx += 1
+    }
+    unreachable
+}
+
+
+@hide:file
+render_expr proc(
+    renderer @mut Renderer,
+    node_id NodeId,
+    parse_state @ParseState,
+    tokens []Token,
+) TokenId ! mem.AllocError {
+    # todo: support unary expressions
+    node :: parse_state.nodes.items[usize(node_id)]
+    return match node.kind {
+        .expr_identifier: {
+            tok :: tokens[usize(node.main_token)]
+            try arraylist.append(&renderer.tokens, tok)
+            yield node.main_token
+        }
+        .expr_unary:
+            try render_expr_unary(renderer, node_id, parse_state, tokens)
+        .expr_binary:
+            try render_expr_binary(renderer, node_id, parse_state, tokens)
+        .literal_int, .literal_float, .literal_string:
+            try render_lit(renderer, node_id, parse_state, tokens)
+        else: unreachable
+    }
+}
+
+@hide:file
+render_expr_unary proc(
+    renderer @mut Renderer,
+    node_id NodeId,
+    parse_state @ParseState,
+    tokens []Token,
+) TokenId ! mem.AllocError {
+    node :: parse_state.nodes.items[usize(node_id)]
+    operator :: node.main_token
+    operand :: node.data0.node_id
+
+    # render operator operand
+    try arraylist.append(&renderer.tokens, tokens[usize(operator)])
+    last_token :: try render_expr(renderer, operand, parse_state, tokens)
+
+    return last_token
+}
+
+@hide:file
+render_expr_binary proc(
+    renderer @mut Renderer,
+    node_id NodeId,
+    parse_state @ParseState,
+    tokens []Token,
+) TokenId ! mem.AllocError {
+    node :: parse_state.nodes.items[usize(node_id)]
+    operator :: node.main_token
+    lhs :: node.data0.node_id
+    rhs :: node.data1.node_id
+
+    # render lhs operator rhs
+    _ = try render_expr(renderer, lhs, parse_state, tokens)
+    try arraylist.append(&renderer.tokens, tokens[usize(operator)])
+    last_token :: try render_expr(renderer, rhs, parse_state, tokens)
+
+    return last_token
+}
+
+@hide:file
+render_type proc(
+    renderer @mut Renderer,
+    node_id NodeId,
+    parse_state @ParseState,
+    tokens []Token,
+) void ! mem.AllocError {
+    # todo: support more expressive type expressions...
+    node :: parse_state.nodes.items[usize(node_id)]
+    match node.kind {
+        .expr_identifier: {
+            tok :: tokens[usize(node.main_token)]
+            try arraylist.append(&renderer.tokens, tok)
+        }
+        else: unreachable
+    }
+}
+
+@hide:file
+render_lit proc(
+    renderer @mut Renderer,
+    node_id NodeId,
+    parse_state @ParseState,
+    tokens []Token,
+) TokenId ! mem.AllocError {
+    node :: parse_state.nodes.items[usize(node_id)]
+    tok :: tokens[usize(node.main_token)]
+    try arraylist.append(&renderer.tokens, tok)
+    return node.main_token
+}

source/lexer/lexer.hon

@@ -3,13 +3,18 @@ import "@std/mem"
 import "@std/strmap"
 import "@std/enums/enummap"
 import "@std/arraylist"
-import "@std/debug"
 
 import "@source/strpool"
 
+ScanError :: struct {
+    code ErrorCode
+    end usize
+}
+
 ErrorCode :: enum {
     invalid_character,
     float_must_end_with_digit,
+    unterminated_string,
 }
 
 ErrorDetails :: struct {
@@ -37,11 +42,9 @@ keywords std.StringMap(TokenKind) :: strmap.init([
     { "while", .while },
 ])
 
-TokenId :: distinct u32
-
 Diagnostic :: struct {
-    code ErrorCode
     token TokenId
+    code ErrorCode
 }
 
 State :: struct {
@@ -61,17 +64,16 @@ deinit proc(state @mut State) void {
     arraylist.deinit(&state.diagnostics)
 }
 
-scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternError) {
-    tokens :: &state.tokens
+scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.InternError) {
     diagnostics :: &state.diagnostics
 
-    cursor usize = 0
-    while cursor < input.len {
-        char :: input[cursor]
+    cursor := 0
+    while cursor < program.len {
+        char :: program[cursor]
 
         # whitespace
         if char == '\n' {
-            try add_token(tokens, Token{ kind = .newline, start = cursor })
+            try add_token(state, Token{ start = cursor, kind = .newline })
             cursor += 1
             continue
         } else if is_whitespace(char) {
@@ -81,7 +83,7 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE
 
         # comments
         if char == '#' {
-            while (cursor < input.len and input[cursor] != '\n') cursor += 1
+            while (cursor < program.len and program[cursor] != '\n') cursor += 1
             cursor += 1 # also skip newline
             continue
         }
@@ -89,199 +91,252 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE
         # identifiers and keywords
         if is_alpha(char) or char == '_' {
             start :: cursor
-            cursor += 1
-
-            # scan whole identifier
-            while (cursor < input.len and (
-                is_alpha(input[cursor]) or
-                is_digit(input[cursor]) or
-                input[cursor] == '_'
-            )) cursor += 1
-
-            kind :: strmap.get(&keywords, input[start..cursor]) orelse .ident
-
-            # don't intern keywords (already O(1) lookup via token kind)
-            str_id :: if (kind == .ident)
-                try strpool.intern(&strpool.strings, input[start..cursor])
-            else
-                strpool.NO_ID
-
-            try add_token(tokens, Token{
-                kind = kind,
+            result :: scan_ident(start, program)
+            try add_token(state, Token{
                 start = start,
-                str_id = str_id
+                kind = result.kind,
+                str_id = result.str_id,
             })
+            cursor = result.end
             continue
         }
 
         # numeric literals
         if is_digit(char) {
             start :: cursor
-            has_decimal bool = false
-
-            # scan integer part
-            while (cursor < input.len and is_digit(input[cursor])) cursor += 1
-
-            # check for decimal point
-            if cursor < input.len and input[cursor] == '.' {
-                has_decimal = true
-                cursor += 1
-            }
-
-            # assert that decimals follow the decimal point
-            if has_decimal and (cursor >= input.len or !is_digit(input[cursor])) {
-                token :: token_id(tokens.items.len)
-                try arraylist.append(diagnostics, Diagnostic{ token = token, code = .float_must_end_with_digit })
-                try add_token(tokens, Token{ kind = .invalid, start = start })
+            result :: scan_number(start, program) catch |err| {
+                token :: token_id(state.tokens.items.len)
+                try add_token(state, Token{ start = start, kind = .invalid })
+                try arraylist.append(diagnostics, Diagnostic{ token = token, code = err.code })
+                cursor = err.end
                 continue
             }
 
-            # scan decimal part
-            while (cursor < input.len and is_digit(input[cursor])) cursor += 1
-
-            kind :: if (has_decimal) .float else .int
-            try add_token(tokens, Token{ kind = kind, start = start })
-
+            kind :: if (result.has_decimal) .float else .int
+            try add_token(state, Token{ start = start, kind = kind })
+            cursor = result.end
             continue
         }
 
         # string literals
         if char == '"' {
             start :: cursor
-            cursor += 1
-
-            while cursor < input.len and input[cursor] != '"' and input[cursor] != '\n' : cursor += 1 {
-                # ignore escaped characters
-                if (input[cursor] == '\\' and cursor + 1 < input.len) cursor += 1
-            }
-
-            if cursor < input.len and input[cursor] == '"' {
-                cursor += 1
-                try add_token(tokens, Token{ kind = .string, start = start })
-            } else {
-                try add_token(tokens, Token{ kind = .invalid, start = start })
-            }
-
+            result :: scan_string(start, program) catch |err| {
+                token :: token_id(state.tokens.items.len)
+                try add_token(state, Token{ start = start, kind = .invalid })
+                try arraylist.append(diagnostics, Diagnostic{ token = token, code = err.code })
+                cursor = err.end
                 continue
             }
 
-        # mutable assignment
-        if char == '=' {
-            try add_token(tokens, Token{ kind = .equal, start = cursor })
-            cursor += 1
+            try add_token(state, Token{ start = start, kind = .string })
+            cursor = result.end
             continue
         }
 
-        # immutable assignment or single colon
+        # immutable decl, mutable decl, or single colon
         if char == ':' {
-            if cursor + 1 < input.len and input[cursor + 1] == ':' {
-                try add_token(tokens, Token{ kind = .double_colon, start = cursor })
+            if cursor + 1 < program.len and program[cursor + 1] == ':' {
+                try add_token(state, Token{ start = cursor, kind = .double_colon })
                 cursor += 2
                 continue
             }
 
-            try add_token(tokens, Token{ kind = .colon, start = cursor })
+            if cursor + 1 < program.len and program[cursor + 1] == '=' {
+                try add_token(state, Token{ start = cursor, kind = .colon_equal })
+                cursor += 2
+                continue
+            }
+
+            try add_token(state, Token{ start = cursor, kind = .colon })
             cursor += 1
             continue
         }
 
-        # parentheses
-        if char == '(' {
-            try add_token(tokens, Token{ kind = .open_paren, start = cursor })
+        # compound arithmetic assignment
+        if char == '+' {
+            if cursor + 1 < program.len and program[cursor + 1] == '=' {
+                try add_token(state, Token{ start = cursor, kind = .plus_equal })
+                cursor += 2
+                continue
+            }
+
+            try add_token(state, Token{ start = cursor, kind = .plus })
             cursor += 1
             continue
-        } else if char == ')' {
-            try add_token(tokens, Token{ kind = .close_paren, start = cursor })
+        }
+        if char == '-' {
+            if cursor + 1 < program.len and program[cursor + 1] == '=' {
+                try add_token(state, Token{ start = cursor, kind = .minus_equal })
+                cursor += 2
+                continue
+            }
+
+            try add_token(state, Token{ start = cursor, kind = .minus })
+            cursor += 1
+            continue
+        }
+        if char == '*' {
+            if cursor + 1 < program.len and program[cursor + 1] == '=' {
+                try add_token(state, Token{ start = cursor, kind = .star_equal })
+                cursor += 2
+                continue
+            }
+
+            try add_token(state, Token{ start = cursor, kind = .star })
+            cursor += 1
+            continue
+        }
+        if char == '/' {
+            if cursor + 1 < program.len and program[cursor + 1] == '=' {
+                try add_token(state, Token{ start = cursor, kind = .slash_equal })
+                cursor += 2
+                continue
+            }
+
+            try add_token(state, Token{ start = cursor, kind = .slash })
             cursor += 1
             continue
         }
 
-        # curly braces
-        if char == '{' {
-            try add_token(tokens, Token{ kind = .open_curly, start = cursor })
-            cursor += 1
-            continue
-        } else if char == '}' {
-            try add_token(tokens, Token{ kind = .close_curly, start = cursor })
+        # single character tokens
+        kind ?TokenKind :: match char {
+            '=': .equal
+            ',': .comma
+            '(': .open_paren
+            ')': .close_paren
+            '{': .open_curly
+            '}': .close_curly
+            else: null
+        }
+
+        if (kind) |k| {
+            try add_token(state, Token{ start = cursor, kind = k })
             cursor += 1
             continue
         }
 
         # invalid character
-        token :: token_id(tokens.items.len)
+        token :: token_id(state.tokens.items.len)
+        try add_token(state, Token{ start = cursor, kind = .invalid })
         try arraylist.append(diagnostics, Diagnostic{ token = token, code = .invalid_character })
-        try add_token(tokens, Token{ kind = .invalid, start = cursor })
         cursor += 1
     }
 
-    try add_token(tokens, Token{ kind = .eof, start = cursor })
+    try add_token(state, Token{ start = cursor, kind = .eof })
 }
 
-#! returns the cursor position after scanning a number.
-scan_number proc(start usize, input []u8) usize {
-    debug.assert(is_digit(input[start]))
+ScanIdentResult :: struct {
+    end usize
+    kind TokenKind
+    str_id strpool.StringId
+}
 
-    has_decimal bool = false
-    cursor usize = start
+scan_ident proc(start usize, program []u8) ScanIdentResult {
+    cursor := start + 1
+
+    # scan whole identifier
+    while (cursor < program.len and (
+        is_alpha(program[cursor]) or
+        is_digit(program[cursor]) or
+        program[cursor] == '_'
+    )) cursor += 1
+
+    kind :: strmap.get(&keywords, program[start..cursor]) orelse .ident
+
+    # don't intern keywords (already O(1) lookup via token kind)
+    str_id :: if (kind == .ident)
+        strpool.intern(&strpool.STRINGS, program[start..cursor]) catch strpool.NO_STR
+    else
+        strpool.NO_STR
+
+    return ScanIdentResult{
+        end = cursor,
+        kind = kind,
+        str_id = str_id,
+    }
+}
+
+ScanNumResult :: struct {
+    end usize
+    has_decimal bool
+}
+
+scan_number proc(start usize, program []u8) ScanNumResult ! ScanError {
+    cursor := start
+    has_decimal := false
 
     # scan integer part
-    while cursor < input.len and is_digit(input[cursor]) : cursor += 1 {}
+    while (cursor < program.len and is_digit(program[cursor])) cursor += 1
 
-    # check for decimal point
-    if cursor < input.len and input[cursor] == '.' {
+    # check for decimal
+    if cursor < program.len and program[cursor] == '.' {
         has_decimal = true
         cursor += 1
     }
 
-    # assert that decimals follow the decimal point
-    debug.assert(!(has_decimal and (cursor >= input.len or !is_digit(input[cursor]))))
-
-    # scan decimal part
-    while cursor < input.len and is_digit(input[cursor]) : cursor += 1 {}
-
-    return cursor
-}
-
-#! returns the cursor position after scanning a string.
-scan_string proc(start usize, input []u8) usize {
-    debug.assert(input[start] == '"')
-
-    cursor usize = start + 1
-    while cursor < input.len and input[cursor] != '"' and input[cursor] != '\n' : cursor += 1 {
-        # ignore escaped characters
-        if (input[cursor] == '\\' and cursor + 1 < input.len) cursor += 1
+    # assert non-terminating decimal
+    if has_decimal and (cursor >= program.len or !is_digit(program[cursor])) {
+        return ScanError{
+            code = .float_must_end_with_digit,
+            end = cursor,
+        }
     }
 
-    debug.assert(cursor < input.len and input[cursor] == '"')
-    cursor += 1
+    # scan fractional part
+    while (cursor < program.len and is_digit(program[cursor])) cursor += 1
 
-    return cursor
+    return ScanNumResult{
+        end = cursor,
+        has_decimal = has_decimal,
+    }
 }
 
-hide is_whitespace proc(char u8) bool {
+ScanStrResult :: struct { end usize }
+
+scan_string proc(start usize, program []u8) ScanStrResult ! ScanError {
+    cursor := start + 1 # skip first `"`
+
+    # scan entire string
+    while cursor < program.len and program[cursor] != '"' and program[cursor] != '\n' : cursor += 1 {
+        # ignore escaped characters
+        if (program[cursor] == '\\' and cursor + 1 < program.len) cursor += 1
+    }
+
+    # assert string terminal
+    if (cursor >= program.len or program[cursor] != '"') return ScanError{
+        code = .unterminated_string,
+        end = cursor,
+    }
+
+    return ScanStrResult{
+        end = cursor + 1, # skip last `"`
+    }
+}
+
+@hide
+is_whitespace proc(char u8) bool {
     return char == ' ' or char == '\t' or char == '\n' or char == '\r'
 }
 
-hide is_alpha proc(char u8) bool {
+@hide
+is_alpha proc(char u8) bool {
     return match char {
         'a'..='z', 'A'..='Z': true
         else: false
     }
 }
 
-hide is_digit proc(char u8) bool {
+@hide
+is_digit proc(char u8) bool {
     return match char {
         '0'..='9': true
         else: false
     }
 }
 
-hide add_token proc(tokens @mut std.ArrayList(Token), token Token) void ! mem.AllocError {
-    _ = token_id(tokens.items.len)
-    try arraylist.append(tokens, token)
-}
-
-hide token_id proc(idx uint) TokenId {
-    debug.assert(u64(idx) < u64(maxval!(TokenId)))
-    return TokenId(idx)
+@hide
+add_token proc(state @mut State, token Token) void ! mem.AllocError {
+    _ = token_id(state.tokens.items.len)
+    try arraylist.append(&state.tokens, token)
 }

source/lexer/lexer.test.hon

@@ -3,19 +3,22 @@ import "@std/mem"
 import "@std/testing"
 
 handles_keywords_identifiers_and_error_progress test {
-    strpool.strings = strpool.init(mem.c_allocator)
-    defer strpool.deinit(&strpool.strings)
+    strpool.STRINGS = strpool.init(mem.c_allocator)
+    defer strpool.deinit(&strpool.STRINGS)
 
-    state State = init(mem.c_allocator)
+    state State := init(mem.c_allocator)
     defer deinit(&state)
-    try scan(&state, "if name # comment\n@1.")
+    try scan(&state, "if name # comment\n@1. 2 3.5 \"hi\"")
 
-    try testing.expect_equal(5, state.tokens.items.len)
+    try testing.expect_equal(8, state.tokens.items.len)
     try testing.expect_equal(TokenKind.if, state.tokens.items[0].kind)
     try testing.expect_equal(TokenKind.ident, state.tokens.items[1].kind)
     try testing.expect_equal(TokenKind.invalid, state.tokens.items[2].kind)
     try testing.expect_equal(TokenKind.invalid, state.tokens.items[3].kind)
-    try testing.expect_equal(TokenKind.eof, state.tokens.items[4].kind)
+    try testing.expect_equal(TokenKind.int, state.tokens.items[4].kind)
+    try testing.expect_equal(TokenKind.float, state.tokens.items[5].kind)
+    try testing.expect_equal(TokenKind.string, state.tokens.items[6].kind)
+    try testing.expect_equal(TokenKind.eof, state.tokens.items[7].kind)
     try testing.expect_equal(2, state.diagnostics.items.len)
-    try testing.expect_equal("name", strpool.get_str(&strpool.strings, state.tokens.items[1].str_id)?)
+    try testing.expect_equal("name", strpool.get_str(&strpool.STRINGS, state.tokens.items[1].str_id)?)
 }

source/lexer/token.hon

@@ -1,9 +1,14 @@
+import "@std/debug"
 import "@source/strpool"
 
+TokenId :: distinct u32
+
+NO_TOKEN :: maxval!(TokenId)
+
 Token :: struct {
+    start uint
     kind TokenKind
-    start int
-    str_id strpool.StringId = strpool.NO_ID
+    str_id strpool.StringId = strpool.NO_STR
 }
 
 TokenKind :: enum {
@@ -22,13 +27,23 @@ TokenKind :: enum {
     greater, greater_equal
 
     # assignment
-    equal, double_colon
+    double_colon, colon_equal
+    equal
+
+    # arithmetic
+    plus, minus
+    star, slash
+
+    # assignment & arithmetic
+    plus_equal, minus_equal
+    star_equal, slash_equal
 
     # delimiters
     open_paren, close_paren
     open_bracket, close_bracket
     open_curly, close_curly
 
+    comma
     colon
     newline
 
@@ -36,3 +51,36 @@ TokenKind :: enum {
     eof
     invalid
 }
+
+render_token proc(tok Token, program []u8) void {
+    match tok.kind {
+        .ident: {
+            if (strpool.get_str(&strpool.STRINGS, tok.str_id)) |str| {
+                debug.print("{}({})\n", {tok.kind, str})
+            } else {
+                res :: scan_ident(tok.start, program)
+                debug.print("{}({})\n", {tok.kind, program[tok.start..res.end]})
+            }
+        }
+        .string: {
+            res :: scan_string(tok.start, program) catch |_| {
+                debug.print("{}({})\n", {tok.kind, "null"})
+                return
+            }
+            debug.print("{}({})\n", {tok.kind, program[tok.start..res.end]})
+        }
+        .int, .float: {
+            res :: scan_number(tok.start, program) catch |_| {
+                debug.print("{}({})\n", {tok.kind, "null"})
+                return
+            }
+            debug.print("{}({})\n", {tok.kind, program[tok.start..res.end]})
+        }
+        else: debug.print("{}\n", {tok.kind})
+    }
+}
+
+token_id proc(idx uint) TokenId {
+    debug.assert(u64(idx) < u64(NO_TOKEN))
+    return TokenId(idx)
+}

source/main.hon

@@ -1,75 +1,111 @@
 import "@std/debug"
 import "@std/mem"
-import "@std/enums/enummap"
 
 test import "@std/enums/enummap"
 test import "@std/arraylist"
 test import "@std/hashmap"
 test import "@std/strmap"
+test import "@std/meta"
 
 import "@source/strpool"
 import "@source/lexer"
 import "@source/parser"
 
+ast_renderer :: import "@source/ast/renderer"
+
 test import "@source/strpool"
 test import "@source/lexer"
+test import "@source/parser"
 
 program ::
     `# literals
-    `x int :: 123
+    `x float :: 123.9
+    `y int := 43
+    `z :: "hello"
+    `
+    `a := 1 + 2 * 3
+    `b :: -3 / 2 + 1
+    `
+    `empty proc() void {}
+    `grouped proc(a, b T, c U) void {
+    `    value int :: 1
+    `}
 
-main proc() void {
-    strpool.strings = strpool.init(mem.c_allocator)
-    defer strpool.deinit(&strpool.strings)
+main proc() void! {
+    strpool.STRINGS = strpool.init(mem.c_allocator)
+    defer strpool.deinit(&strpool.STRINGS)
 
     debug.print("PROGRAM::[[\n{}\n]]\n\n", {program})
 
-    scan_state lexer.State = lexer.init(mem.c_allocator)
+    scan_state := lexer.init(mem.c_allocator)
     defer lexer.deinit(&scan_state)
-
     lexer.scan(&scan_state, program) catch |err| {
         debug.print("failed to scan: {}\n", {err})
         return
     }
 
     debug.print("TOKENS::[[\n", {})
-    for scan_state.tokens.items |token| {
-        debug.print("{}\n", {token.kind})
-    }
+    for (scan_state.tokens.items) |tok| lexer.render_token(tok, program)
     debug.print("]]\n\n", {})
 
-    debug.print("TOKENS::DIAGNOSTICS::[[\n", {})
-    for scan_state.diagnostics.items |diagnostic| {
-        details :: enummap.get(&lexer.error_msg_map, diagnostic.code)
-        debug.print("{}: {}\n", {details?.name, details?.message})
-    }
-    debug.print("]]\n\n", {})
-
-    parse_state parser.State = parser.init(mem.c_allocator)
+    parse_state := parser.init(mem.c_allocator)
     defer parser.deinit(&parse_state)
-    parser.parse(&parse_state, scan_state.tokens.items) catch |err| {
+    root :: parser.parse(&parse_state, scan_state.tokens.items) catch |err| {
         debug.print("failed to parse: {}\n", {err})
         return
     }
 
     debug.print("AST::[[\n", {})
-    for parse_state.nodes.items |node, id| {
-        token :: scan_state.tokens.items[node.main_token]
-
-        end :: if (token.kind == .int or token.kind == .float)
-            lexer.scan_number(token.start, program)
-        else if (token.kind == .string)
-            lexer.scan_string(token.start, program)
-        else
-            token.start
-
-        debug.print("{} (id = {}): {}\n", {
-            node.kind,
-            id,
-            program[token.start..end],
-            #node.data0,
-            #node.data1,
-        })
+    for parse_state.nodes.items |node, i| {
+        match node.kind {
+            .expr_identifier: {
+                ident :: scan_state.tokens.items[usize(node.main_token)]
+                res :: lexer.scan_ident(ident.start, program)
+                debug.print("{}: {}({})\n", { i, node.kind, program[ident.start..res.end] })
             }
+            .literal_int, .literal_float: {
+                lit :: scan_state.tokens.items[usize(node.main_token)]
+                res :: lexer.scan_number(lit.start, program) catch |_| {
+                    debug.print("{}: {}\n", { i, node.kind })
+                    continue
+                }
+                debug.print("{}: {}({})\n", { i, node.kind, program[lit.start..res.end] })
+            }
+            .literal_string: {
+                str :: scan_state.tokens.items[usize(node.main_token)]
+                res :: lexer.scan_string(str.start, program) catch |_| {
+                    debug.print("{}: {}\n", { i, node.kind })
+                    continue
+                }
+                debug.print("{}: {}({})\n", { i, node.kind, program[str.start..res.end] })
+            }
+            .expr_unary: {
+                operator :: scan_state.tokens.items[usize(node.main_token)]
+                operand :: node.data0.node_id
+                debug.print("{}: {}({}, operand={})\n", { i, node.kind, operator.kind, operand })
+            }
+            .expr_binary: {
+                operator :: scan_state.tokens.items[usize(node.main_token)]
+                lhs :: node.data0.node_id
+                rhs :: node.data1.node_id
+                debug.print("{}: {}({}, lhs={}, rhs={})\n", { i, node.kind, operator.kind, lhs, rhs })
+            }
+            .stmt_decl: {
+                ident :: scan_state.tokens.items[usize(node.main_token)]
+                init :: parse_state.extra.items[usize(node.data0.extra_id) + 1]
+                res :: lexer.scan_ident(ident.start, program)
+                debug.print("{}: {}({}, init={})\n", { i, node.kind, program[ident.start..res.end], init })
+            }
+            else: debug.print("{}: {}\n", { i, node.kind })
+        }
+    }
+    debug.print("]]\n\n", {})
+
+    renderer := ast_renderer.init(mem.c_allocator)
+    defer ast_renderer.deinit(&renderer)
+    try ast_renderer.render_token_stream(&renderer, root, &parse_state, scan_state.tokens.items)
+
+    debug.print("AST Render::[[\n", {})
+    for (renderer.tokens.items) |tok| lexer.render_token(tok, program)
     debug.print("]]\n", {})
 }

source/parser/parser.hon

@@ -1,8 +1,8 @@
 import "@std"
 import "@std/mem"
 import "@std/arraylist"
-import "@std/debug"
 import "@std/enums/enummap"
+import "@std/debug"
 
 import "@source/lexer"
 import "@source/ast"
@@ -32,16 +32,45 @@ error_msg_map std.EnumMap(ErrorCode, ErrorDetails) :: enummap.init({
     },
 })
 
-hide Token :: alias lexer.Token
-hide TokenId :: alias lexer.TokenId
-hide TokenKind :: alias lexer.TokenKind
-hide Node :: alias ast.Node
-hide NodeId :: alias ast.NodeId
-hide ExtraId :: alias ast.ExtraId
-hide NodeData :: alias ast.NodeData
+@hide:file Token :: alias lexer.Token
+@hide:file TokenId :: alias lexer.TokenId
+@hide:file TokenKind :: alias lexer.TokenKind
+@hide:file Node :: alias ast.Node
+@hide:file NodeId :: alias ast.NodeId
+@hide:file ExtraId :: alias ast.ExtraId
+@hide:file NodeData :: alias ast.NodeData
+@hide:file NO_NODE :: alias ast.NO_NODE
+@hide:file NO_EXTRA :: alias ast.NO_EXTRA
+@hide:file token_id :: alias lexer.token_id
 
-hide NO_ID_NODE :: alias ast.NO_ID_NODE
-hide NO_ID_EXTRA :: alias ast.NO_ID_EXTRA
+BindingPowerScalar :: distinct u32
+
+BindingPower :: struct {
+    left BindingPowerScalar
+    right BindingPowerScalar
+}
+
+prefix_binding_power proc(token_kind TokenKind) ?BindingPowerScalar {
+    return match token_kind {
+        .minus: BindingPowerScalar(30)
+        else: null
+    }
+}
+
+infix_binding_power proc(token_kind TokenKind) ?BindingPower {
+    return match token_kind {
+        .plus, .minus: BindingPower{ left = BindingPowerScalar(10), right = BindingPowerScalar(11) }
+        .star, .slash: BindingPower{ left = BindingPowerScalar(20), right = BindingPowerScalar(21) }
+        else: null
+    }
+}
+
+postfix_binding_power proc(token_kind TokenKind) ?BindingPowerScalar {
+    return match token_kind {
+        .open_paren, .open_bracket: BindingPowerScalar(40)
+        else: null
+    }
+}
 
 State :: struct {
     nodes std.ArrayList(Node)
@@ -50,7 +79,7 @@ State :: struct {
     extra std.ArrayList(u32)
 
     tokens []Token = &[]
-    next_token TokenId = TokenId(0)
+    next_token usize = 0
 }
 
 init proc(allocator mem.Allocator) State {
@@ -65,112 +94,308 @@ deinit proc(state @mut State) void {
     arraylist.deinit(&state.extra)
 }
 
-parse proc(state @mut State, tokens []Token) void ! ParseError {
+parse proc(state @mut State, tokens []Token) NodeId ! ParseError {
     state.tokens = tokens
-    _ = try parse_decl(state)
+    return try parse_pkg(state)
+}
+
+parse_pkg proc(state @mut State) NodeId ! ParseError {
+    if (state.next_token < state.tokens.len
+        and state.tokens[state.next_token].kind == .eof
+    ) return NO_NODE
+
+    stmt_ids std.ArrayList(NodeId) := arraylist.init(state.nodes.allocator)
+    defer arraylist.deinit(&stmt_ids)
+
+    while (state.next_token < state.tokens.len
+        and state.tokens[state.next_token].kind != .eof
+    ) {
+        stmt_id :: try parse_decl(state)
+        try arraylist.append(&stmt_ids, stmt_id)
+    }
+
+    pkg_start :: ast.extra_id(state.extra.items.len)
+    for (stmt_ids.items) |stmt_id| _ = try add_extra(state, u32(stmt_id))
+    pkg_end :: ast.extra_id(state.extra.items.len)
+
+    return try add_node(state, Node{
+        kind = .package,
+        data0 = NodeData{ extra_id = pkg_start },
+        data1 = NodeData{ extra_id = pkg_end },
+    })
 }
 
 parse_decl proc(state @mut State) NodeId ! ParseError {
-    # expect identifier
+    if next_n_is(state, 1, .proc)
+        return try parse_func_decl(state)
+    else
+        return try parse_stmt_decl(state)
+}
+
+parse_func_decl proc(state @mut State) NodeId ! ParseError {
+    ident_tok_id :: try consume(state, .ident)
+    _ = try consume(state, .proc)
+    _ = try consume(state, .open_paren)
+
+    param_decls std.ArrayList(NodeId) := arraylist.init(state.nodes.allocator)
+    defer arraylist.deinit(&param_decls)
+
+    ident_ids std.ArrayList(TokenId) := arraylist.init(state.nodes.allocator)
+    defer arraylist.deinit(&ident_ids)
+
+    while !next_is(state, .close_paren) {
+        arraylist.clear(&ident_ids)
+
+        # collect param group
+        while true {
+            param_tok_id :: try consume(state, .ident)
+            try arraylist.append(&ident_ids, param_tok_id)
+
+            if next_is(state, .ident) break
+            _ = try consume(state, .comma)
+        }
+
+        # collect type
         try expect(state, .ident)
-    ident_token :: state.next_token
-    state.next_token += 1
+        type_id :: try parse_primary(state)
+        for (ident_ids.items) |id| {
+            param_decl :: try add_node(state, Node{
+                kind = .param_decl,
+                main_token = id,
+                data0 = NodeData{ node_id = type_id },
+            })
+            try arraylist.append(&param_decls, param_decl)
+        }
+
+        if next_is(state, .comma) _ = try consume(state, .comma)
+    }
+
+    _ = try consume(state, .close_paren)
+
+    try expect(state, .ident)
+    return_type_id :: try parse_primary(state)
+    block_id :: try parse_block(state)
+
+    while next_is(state, .newline) _ = try consume(state, .newline)
+
+    extra_id :: try add_extra(state, u32(return_type_id))
+    _ = try add_extra(state, u32(param_decls.items.len))
+    for (param_decls.items) |param| _ = try add_extra(state, u32(param))
+
+    return try add_node(state, Node{
+        kind = .func_decl,
+        main_token = ident_tok_id,
+        data0 = NodeData{ node_id = block_id },
+        data1 = NodeData{ extra_id = extra_id },
+    })
+}
+
+parse_block proc(state @mut State) NodeId ! ParseError {
+    stmt_ids std.ArrayList(NodeId) := arraylist.init(state.nodes.allocator)
+    defer arraylist.deinit(&stmt_ids)
+
+    _ = try consume(state, .open_curly)
+    while next_is(state, .newline) _ = try consume(state, .newline)
+
+    while !next_is(state, .close_curly) {
+        stmt_id :: try parse_stmt(state)
+        try arraylist.append(&stmt_ids, stmt_id)
+        while next_is(state, .newline) _ = try consume(state, .newline)
+    }
+
+    _ = try consume(state, .close_curly)
+
+    block_start :: ast.extra_id(state.extra.items.len)
+    for (stmt_ids.items) |id| _ = try add_extra(state, u32(id))
+    block_end :: ast.extra_id(state.extra.items.len)
+
+    return try add_node(state, Node{
+        kind = .block,
+        data0 = NodeData{ extra_id = block_start },
+        data1 = NodeData{ extra_id = block_end },
+    })
+}
+
+parse_stmt proc(state @mut State) NodeId ! ParseError {
+    if next_is(state, .ident) return try parse_stmt_decl(state)
+    return .unexpected_token
+}
+
+parse_stmt_decl proc(state @mut State) NodeId ! ParseError {
+    # expect identifier
+    ident_tok_id :: try consume(state, .ident)
 
     # check for type
     # todo: make a parse_type proc for this
-    type_id :: if (state.tokens[state.next_token].kind == .ident)
+    type_id :: if (next_is(state, .ident))
         try parse_primary(state)
     else
-        NO_ID_NODE
+        NO_NODE
 
-    # expect `::` (immutable assignment)
-    mutability_tok_id :: try consume(state, .double_colon)
+    # expect `::` or `:=` (immutable or mutable assignment)
+    mutability_tok_id :: try consume_either(state, &[.double_colon, .colon_equal])
 
     # expect expression
-    expr_id :: try parse_primary(state)
+    expr_id :: try parse_expr(state)
 
     # expect statement terminator (newline)
-    _ = try consume_either(state, &[.newline, .eof])
+    while (next_is(state, .newline)) _ = try consume(state, .newline)
 
-    extra_start :: try add_extra(&state.extra, u32(type_id))
-    _ = try add_extra(&state.extra, u32(expr_id))
+    extra_start_id :: try add_extra(state, u32(type_id))
+    _ = try add_extra(state, u32(expr_id))
 
-    decl :: try add_node(&state.nodes, Node{
+    return try add_node(state, Node{
         kind = .stmt_decl,
-        main_token = ident_token,
-        data0 = NodeData{ extra_id = extra_start },
+        main_token = ident_tok_id,
+        data0 = NodeData{ extra_id = extra_start_id },
         data1 = NodeData{ token_id = mutability_tok_id },
     })
+}
 
-    return decl
+parse_expr proc(state @mut State) NodeId ! mem.AllocError {
+    return try parse_expr_bp(state, BindingPowerScalar(0))
+}
+
+parse_expr_bp proc(state @mut State, min_bp BindingPowerScalar) NodeId ! mem.AllocError {
+    # parse left-hand side
+    kind := state.tokens[state.next_token].kind
+
+    lhs := if (prefix_binding_power(kind)) |right_bp| {
+        operator :: token_id(state.next_token)
+        state.next_token += 1
+
+        operand :: try parse_expr_bp(state, right_bp)
+
+        yield try add_node(state, Node{
+            kind = .expr_unary,
+            main_token = operator,
+            data0 = NodeData{ node_id = operand },
+        })
+    } else {
+        yield try parse_primary(state)
+    }
+
+    # parse right-hand side
+    while true {
+        kind = state.tokens[state.next_token].kind
+
+        # try postfix (higher precedence)
+        if postfix_binding_power(kind) |left_bp| {
+            if (left_bp < min_bp) break
+
+            lhs = match state.tokens[state.next_token].kind {
+                .open_paren: debug.unimplemented() # todo: parse call
+                .open_bracket: debug.unimplemented() # todo: parse index
+                else: unreachable
+            }
+            continue
+        }
+
+        # try infex (lower precedence)
+        if infix_binding_power(kind) |bp| {
+            if (bp.left < min_bp) break
+
+            operator :: token_id(state.next_token)
+            state.next_token += 1
+
+            rhs :: try parse_expr_bp(state, bp.right)
+            lhs = try add_node(state, Node{
+                kind = .expr_binary,
+                main_token = operator,
+                data0 = NodeData{ node_id = lhs },
+                data1 = NodeData{ node_id = rhs },
+            })
+
+            continue
+        }
+
+        break
+    }
+
+    return lhs
 }
 
 parse_primary proc(state @mut State) NodeId ! mem.AllocError {
-    start_token :: state.next_token
+    start_idx :: state.next_token
+    start_tok_id :: TokenId(start_idx)
     state.next_token += 1
-    return match state.tokens[start_token].kind {
-        .int: try add_node(&state.nodes, Node{
+    return match state.tokens[start_idx].kind {
+        .int: try add_node(state, Node{
             kind = .literal_int,
-            main_token = start_token,
+            main_token = start_tok_id,
         })
-        .float: try add_node(&state.nodes, Node{
+        .float: try add_node(state, Node{
             kind = .literal_float,
-            main_token = start_token,
+            main_token = start_tok_id,
         })
-        .string: try add_node(&state.nodes, Node{
+        .string: try add_node(state, Node{
             kind = .literal_string,
-            main_token = start_token,
+            main_token = start_tok_id,
         })
-        .ident: try add_node(&state.nodes, Node{
+        .ident: try add_node(state, Node{
             kind = .expr_identifier,
-            main_token = start_token,
+            main_token = start_tok_id,
         })
-        else: try add_node(&state.nodes, Node{
+        else: try add_node(state, Node{
             kind = .invalid,
-            main_token = start_token,
+            main_token = start_tok_id,
         })
     }
 }
 
-hide expect proc(state @mut State, token_kind TokenKind) void ! ErrorCode {
+@hide
+next_is proc(state @mut State, token_kind TokenKind) bool {
+    return state.tokens[state.next_token].kind == token_kind
+}
+
+@hide
+next_is_either proc(state @mut State, token_kinds []TokenKind) bool {
+    for (token_kinds) |kind| if next_is(state, kind) return true
+    return false
+}
+
+@hide
+next_n_is proc(state @mut State, n usize, token_kind TokenKind) bool {
+    return state.tokens[state.next_token + n].kind == token_kind
+}
+
+@hide
+expect proc(state @mut State, token_kind TokenKind) void ! ErrorCode {
     if (state.tokens[state.next_token].kind != token_kind) return .unexpected_token
 }
 
-hide expect_either proc(state @mut State, token_kinds []TokenKind) void ! ErrorCode {
+@hide
+expect_either proc(state @mut State, token_kinds []TokenKind) void ! ErrorCode {
     for (token_kinds) |kind| if (state.tokens[state.next_token].kind == kind) return
     return .unexpected_token
 }
 
-hide consume proc(state @mut State, token_kind TokenKind) TokenId ! ErrorCode {
+@hide
+consume proc(state @mut State, token_kind TokenKind) TokenId ! ErrorCode {
     try expect(state, token_kind)
+    id :: TokenId(state.next_token)
     state.next_token += 1
-    return state.next_token - 1
+    return id
 }
 
-hide consume_either proc(state @mut State, token_kinds []TokenKind) TokenId ! ErrorCode {
+@hide
+consume_either proc(state @mut State, token_kinds []TokenKind) TokenId ! ErrorCode {
     try expect_either(state, token_kinds)
+    id :: TokenId(state.next_token)
     state.next_token += 1
-    return state.next_token - 1
-}
-
-hide add_node proc(nodes @mut std.ArrayList(Node), node Node) NodeId ! mem.AllocError {
-    id :: node_id(nodes.items.len)
-    try arraylist.append(nodes, node)
     return id
 }
 
-hide node_id proc(idx uint) NodeId {
-    debug.assert(u64(idx) < u64(NO_ID_NODE))
-    return NodeId(idx)
-}
-
-hide add_extra proc(extra @mut std.ArrayList(u32), idx u32) ExtraId ! mem.AllocError {
-    id :: extra_id(extra.items.len)
-    try arraylist.append(extra, idx)
+@hide
+add_node proc(state @mut State, node Node) NodeId ! mem.AllocError {
+    id :: ast.node_id(state.nodes.items.len)
+    try arraylist.append(&state.nodes, node)
     return id
 }
 
-hide extra_id proc(idx uint) ExtraId {
-    debug.assert(u64(idx) < u64(NO_ID_EXTRA))
-    return ExtraId(idx)
+@hide
+add_extra proc(state @mut State, data u32) ExtraId ! mem.AllocError {
+    id :: ast.extra_id(state.extra.items.len)
+    try arraylist.append(&state.extra, data)
+    return id
 }

source/parser/parser.test.hon

@@ -0,0 +1,202 @@
+import "@std/mem"
+import "@std/testing"
+
+import "@source/strpool"
+import "@source/lexer"
+import "@source/ast"
+
+ast_renderer :: import "@source/ast/renderer"
+
+handles_package_declarations test {
+    strpool.STRINGS = strpool.init(mem.c_allocator)
+    defer strpool.deinit(&strpool.STRINGS)
+
+    source ::
+        `first int :: 42
+        `second float :: 2.15 + 3
+        `message :: "hello"
+
+    scan_state := lexer.init(mem.c_allocator)
+    defer lexer.deinit(&scan_state)
+    try lexer.scan(&scan_state, source)
+
+    parse_state := init(mem.c_allocator)
+    defer deinit(&parse_state)
+    root :: try parse(&parse_state, scan_state.tokens.items)
+
+    renderer := ast_renderer.init(mem.c_allocator)
+    defer ast_renderer.deinit(&renderer)
+    try ast_renderer.render_token_stream(&renderer, root, &parse_state, scan_state.tokens.items)
+
+    try testing.expect_equal(scan_state.tokens.items.len, renderer.tokens.items.len)
+    for scan_state.tokens.items |expected, i| {
+        actual :: renderer.tokens.items[i]
+        try testing.expect_equal(expected.kind, actual.kind)
+        try testing.expect_equal(expected.start, actual.start)
+        try testing.expect_equal(expected.str_id, actual.str_id)
+    }
+}
+
+handles_unary_arithmetic_prefix_expression test {
+    strpool.STRINGS = strpool.init(mem.c_allocator)
+    defer strpool.deinit(&strpool.STRINGS)
+
+    source ::
+        `first :: -42 + 1
+
+    scan_state := lexer.init(mem.c_allocator)
+    defer lexer.deinit(&scan_state)
+    try lexer.scan(&scan_state, source)
+
+    parse_state := init(mem.c_allocator)
+    defer deinit(&parse_state)
+    root :: try parse(&parse_state, scan_state.tokens.items)
+
+    renderer := ast_renderer.init(mem.c_allocator)
+    defer ast_renderer.deinit(&renderer)
+    try ast_renderer.render_token_stream(&renderer, root, &parse_state, scan_state.tokens.items)
+
+    try testing.expect_equal(scan_state.tokens.items.len, renderer.tokens.items.len)
+    for scan_state.tokens.items |expected, i| {
+        actual :: renderer.tokens.items[i]
+        try testing.expect_equal(expected.kind, actual.kind)
+        try testing.expect_equal(expected.start, actual.start)
+        try testing.expect_equal(expected.str_id, actual.str_id)
+    }
+}
+
+handles_binary_arithmetic_expression test {
+    strpool.STRINGS = strpool.init(mem.c_allocator)
+    defer strpool.deinit(&strpool.STRINGS)
+
+    source ::
+        `first int :: 42 + 3
+        `second :: 134 + 5
+        `third float :: 1.35 + 2
+        `fourth :: first + second
+
+    scan_state := lexer.init(mem.c_allocator)
+    defer lexer.deinit(&scan_state)
+    try lexer.scan(&scan_state, source)
+
+    parse_state := init(mem.c_allocator)
+    defer deinit(&parse_state)
+    root :: try parse(&parse_state, scan_state.tokens.items)
+
+    renderer := ast_renderer.init(mem.c_allocator)
+    defer ast_renderer.deinit(&renderer)
+    try ast_renderer.render_token_stream(&renderer, root, &parse_state, scan_state.tokens.items)
+
+    try testing.expect_equal(scan_state.tokens.items.len, renderer.tokens.items.len)
+    for scan_state.tokens.items |expected, i| {
+        actual :: renderer.tokens.items[i]
+        try testing.expect_equal(expected.kind, actual.kind)
+        try testing.expect_equal(expected.start, actual.start)
+        try testing.expect_equal(expected.str_id, actual.str_id)
+    }
+}
+
+handles_statement_declaration test {
+    strpool.STRINGS = strpool.init(mem.c_allocator)
+    defer strpool.deinit(&strpool.STRINGS)
+
+    source ::
+        `value int :: 42
+
+    scan_state := lexer.init(mem.c_allocator)
+    defer lexer.deinit(&scan_state)
+    try lexer.scan(&scan_state, source)
+
+    parse_state := init(mem.c_allocator)
+    defer deinit(&parse_state)
+    root :: try parse(&parse_state, scan_state.tokens.items)
+
+    renderer := ast_renderer.init(mem.c_allocator)
+    defer ast_renderer.deinit(&renderer)
+    try ast_renderer.render_token_stream(&renderer, root, &parse_state, scan_state.tokens.items)
+
+    try testing.expect_equal(scan_state.tokens.items.len, renderer.tokens.items.len)
+    for scan_state.tokens.items |expected, i| {
+        actual :: renderer.tokens.items[i]
+        try testing.expect_equal(expected.kind, actual.kind)
+        try testing.expect_equal(expected.start, actual.start)
+        try testing.expect_equal(expected.str_id, actual.str_id)
+    }
+}
+
+handles_function_declarations test {
+    strpool.STRINGS = strpool.init(mem.c_allocator)
+    defer strpool.deinit(&strpool.STRINGS)
+
+    source ::
+        `empty proc() void {}
+        `grouped proc(a, b T, c U) void {
+        `    value int :: 1
+        `}
+
+    scan_state := lexer.init(mem.c_allocator)
+    defer lexer.deinit(&scan_state)
+    try lexer.scan(&scan_state, source)
+
+    parse_state := init(mem.c_allocator)
+    defer deinit(&parse_state)
+    root_id :: try parse(&parse_state, scan_state.tokens.items)
+    root :: parse_state.nodes.items[usize(root_id)]
+
+    package_start :: usize(root.data0.extra_id)
+    package_end :: usize(root.data1.extra_id)
+    try testing.expect_equal(usize(2), package_end - package_start)
+
+    empty_id :: ast.node_id(parse_state.extra.items[package_start])
+    empty :: parse_state.nodes.items[usize(empty_id)]
+    empty_payload :: usize(empty.data1.extra_id)
+    empty_block :: parse_state.nodes.items[usize(empty.data0.node_id)]
+    try testing.expect_equal(ast.NodeKind.func_decl, empty.kind)
+    try testing.expect_equal(u32(0), parse_state.extra.items[empty_payload + 1])
+    try testing.expect_equal(
+        usize(empty_block.data0.extra_id),
+        usize(empty_block.data1.extra_id),
+    )
+
+    grouped_id :: ast.node_id(parse_state.extra.items[package_start + 1])
+    grouped :: parse_state.nodes.items[usize(grouped_id)]
+    grouped_payload :: usize(grouped.data1.extra_id)
+    try testing.expect_equal(ast.NodeKind.func_decl, grouped.kind)
+    try testing.expect_equal(u32(3), parse_state.extra.items[grouped_payload + 1])
+
+    param_a_id :: ast.node_id(parse_state.extra.items[grouped_payload + 2])
+    param_b_id :: ast.node_id(parse_state.extra.items[grouped_payload + 3])
+    param_c_id :: ast.node_id(parse_state.extra.items[grouped_payload + 4])
+    param_a :: parse_state.nodes.items[usize(param_a_id)]
+    param_b :: parse_state.nodes.items[usize(param_b_id)]
+    param_c :: parse_state.nodes.items[usize(param_c_id)]
+    try testing.expect_equal(ast.NodeKind.param_decl, param_a.kind)
+    try testing.expect_equal(ast.NodeKind.param_decl, param_b.kind)
+    try testing.expect_equal(ast.NodeKind.param_decl, param_c.kind)
+    try testing.expect(param_a.data0.node_id == param_b.data0.node_id)
+    try testing.expect(param_a.data0.node_id != param_c.data0.node_id)
+
+    grouped_block :: parse_state.nodes.items[usize(grouped.data0.node_id)]
+    try testing.expect_equal(ast.NodeKind.block, grouped_block.kind)
+    try testing.expect_equal(
+        usize(1),
+        usize(grouped_block.data1.extra_id) - usize(grouped_block.data0.extra_id),
+    )
+
+    renderer := ast_renderer.init(mem.c_allocator)
+    defer ast_renderer.deinit(&renderer)
+    try ast_renderer.render_token_stream(
+        &renderer,
+        root_id,
+        &parse_state,
+        scan_state.tokens.items,
+    )
+
+    try testing.expect_equal(scan_state.tokens.items.len, renderer.tokens.items.len)
+    for scan_state.tokens.items |expected, i| {
+        actual :: renderer.tokens.items[i]
+        try testing.expect_equal(expected.kind, actual.kind)
+        try testing.expect_equal(expected.start, actual.start)
+        try testing.expect_equal(expected.str_id, actual.str_id)
+    }
+}

source/strpool/strpool.hon

@@ -2,14 +2,14 @@ import "@std/mem"
 import "@std/arraylist"
 import "@std/hashmap"
 
-# cross-cutting concern, hence global singleton (owned by main.hon)
-strings StringPool = undefined
-
 InternError :: enum { out_of_space }
 
 StringId :: alias u32
 
-NO_ID :: maxval!(StringId)
+NO_STR :: maxval!(StringId)
+
+# cross-cutting concern, hence global singleton (owned by main.hon)
+STRINGS StringPool := undefined
 
 StringPool :: struct {
     ids hashmap.StringHashMap(StringId) # string → id
@@ -36,7 +36,7 @@ deinit proc(pool @mut StringPool) void {
 intern proc(pool @mut StringPool, str []u8) StringId ! (mem.AllocError | InternError) {
     if hashmap.get(&pool.ids, str) |id| return id
 
-    if (pool.strings.items.len >= usize(NO_ID)) return .out_of_space
+    if (pool.strings.items.len >= usize(NO_STR)) return .out_of_space
     id StringId :: StringId(pool.strings.items.len)
 
     owned_str []mut u8 :: try mem.alloc(u8, pool.allocator, str.len)

source/strpool/strpool.test.hon

@@ -1,11 +1,11 @@
 import "@std/mem"
 import "@std/testing"
 
-handles_intern test {
-    pool StringPool = init(mem.c_allocator)
+handles_interning test {
+    pool StringPool := init(mem.c_allocator)
     defer deinit(&pool)
 
-    input [5]mut u8 = ['h', 'e', 'l', 'l', 'o']
+    input [5]mut u8 := ['h', 'e', 'l', 'l', 'o']
     id :: try intern(&pool, input[..])
     duplicate :: try intern(&pool, "hello")
     input[0] = 'j'

std/arraylist/arraylist.hon

@@ -17,20 +17,20 @@ init proc($T type, allocator mem.Allocator) ArrayList(T) {
 }
 
 deinit proc($T type, list @mut ArrayList(T)) void {
-    allocation []mut T :: list.items.ptr[..list.capacity]
+    allocation :: list.items.ptr[..list.capacity]
     mem.free(list.allocator, allocation)
     list.items = mem.empty(T)
     list.capacity = 0
 }
 
-hide reserve proc($T type, list @mut ArrayList(T), min_capacity usize) void ! mem.AllocError {
+reserve proc($T type, list @mut ArrayList(T), min_capacity usize) void ! mem.AllocError {
     if min_capacity <= list.capacity {
         return
     }
 
-    new_capacity usize = 8
+    new_capacity := 8
     if list.capacity >= 8 {
-        half usize :: divtrunc!(list.capacity, 2)
+        half :: divtrunc!(list.capacity, 2)
         if list.capacity > maxval!(usize) - half {
             new_capacity = min_capacity
         } else {

std/arraylist/arraylist.test.hon

@@ -2,7 +2,7 @@ import "@std/mem"
 import "@std/testing"
 
 handles_append test {
-    list ArrayList(i32) = init(mem.c_allocator)
+    list ArrayList(i32) := init(mem.c_allocator)
     defer deinit(&list)
 
     try append(&list, 42)
@@ -12,7 +12,7 @@ handles_append test {
 }
 
 handles_clear test {
-    list ArrayList(i32) = init(mem.c_allocator)
+    list ArrayList(i32) := init(mem.c_allocator)
     defer deinit(&list)
 
     try append(&list, 42)
@@ -22,7 +22,7 @@ handles_clear test {
 }
 
 handles_reserve test {
-    list ArrayList(i32) = init(mem.c_allocator)
+    list ArrayList(i32) := init(mem.c_allocator)
     defer deinit(&list)
 
     try reserve(&list, 10)

std/debug/debug.hon

@@ -5,8 +5,14 @@ assert proc(ok bool) void {
     if (!ok) unreachable
 }
 
+unimplemented proc() noreturn {
+    print("not yet implemented\n", {})
+    c.abort()
+    unreachable
+}
+
 print proc($format []u8, $Args type, args Args) void {
-    writer io.Writer :: io.Writer{
+    writer :: io.Writer{
         context = null,
         handle = io.Handle{ file_desc = c_int(io.Stream.stderr) },
         write = write,
@@ -14,14 +20,15 @@ print proc($format []u8, $Args type, args Args) void {
     io.print(writer, format, Args, args) catch |_| {}
 }
 
-hide write proc(_ ?@mut anyopaque, handle io.Handle, bytes []u8) usize ! io.WriteError {
-    request usize = bytes.len
-    maximum usize :: usize(maxval!(c_long))
+@hide
+write proc(_ ?@mut anyopaque, handle io.Handle, bytes []u8) usize ! io.WriteError {
+    request := bytes.len
+    maximum :: usize(maxval!(c_long))
     if request > maximum {
         request = maximum
     }
     while true {
-        count c_long :: c.write(handle.file_desc, bytes.ptr, c_ulong(request))
+        count :: c.write(handle.file_desc, bytes.ptr, c_ulong(request))
         if count >= 0 {
             return usize(count)
         }

std/enums/enummap/enummap.hon

@@ -14,7 +14,7 @@ init proc(
     $E, $V type,
     values meta.EnumFieldStruct(E, ?V, some!(null)),
 ) EnumMap(E, V) {
-    map EnumMap(E, V) = undefined
+    map EnumMap(E, V) := undefined
 
     match typeinfo!(E) {
         .enum |info|: inline for info.fields |field, i| {

std/enums/enummap/enummap.test.hon

@@ -8,7 +8,7 @@ TestEnum :: enum(u8) {
 }
 
 handles_sparse_enum_get test {
-    names EnumMap(TestEnum, []u8) = init({
+    names EnumMap(TestEnum, []u8) := init({
         ident = "identifier",
         int = "integer",
     })

std/hashmap/hashmap.hon

@@ -59,7 +59,7 @@ get proc(
     if (map.count == 0) return null
 
     hash :: normalize(hash_key(key))
-    idx usize = hash & (map.entries.len - 1)
+    idx := hash & (map.entries.len - 1)
 
     while true {
         entry :: map.entries[idx]
@@ -96,7 +96,7 @@ put proc(
             if (entry.hash == 0) continue
 
             # find an empty slot
-            idx usize = entry.hash & (new_entries.len - 1)
+            idx := entry.hash & (new_entries.len - 1)
             while new_entries[idx].hash != 0 {
                 idx = (idx + 1) & (new_entries.len - 1)
             }
@@ -110,7 +110,7 @@ put proc(
 
     # put new entry
     hash :: normalize(hash_key(key))
-    idx usize = hash & (map.entries.len - 1)
+    idx := hash & (map.entries.len - 1)
 
     while true {
         entry :: map.entries[idx]
@@ -132,7 +132,8 @@ put proc(
     }
 }
 
-hide normalize proc(hash usize) usize {
+@hide
+normalize proc(hash usize) usize {
     # mapping both 0 and 1 to 1 is safe because equality resolves
     # collisions (since hash and key must both be equal).
     if (hash == 0) return 1
@@ -141,9 +142,10 @@ hide normalize proc(hash usize) usize {
 
 #! FNV-1a hash implementation.
 #! note: vulnerable to collision attacks.
-hide str_hash proc(key []u8) usize {
-    hash u32 = 2166136261 # offset basis
-    prime u32 = 16777619
+@hide
+str_hash proc(key []u8) usize {
+    hash u32 := 2166136261 # offset basis
+    prime u32 := 16777619
 
     for key |byte| {
         product u64 :: u64(hash xor u32(byte)) * prime
@@ -153,4 +155,5 @@ hide str_hash proc(key []u8) usize {
     return usize(hash)
 }
 
-hide str_eql proc(a, b []u8) bool { return mem.eql(a, b) }
+@hide
+str_eql proc(a, b []u8) bool { return mem.eql(a, b) }

std/hashmap/hashmap.test.hon

@@ -2,7 +2,7 @@ import "@std/mem"
 import "@std/testing"
 
 handles_put_and_get test {
-    map StringHashMap(u32) = init(mem.c_allocator)
+    map StringHashMap(u32) := init(mem.c_allocator)
     defer deinit(&map)
 
     try put(&map, "key", 42)

std/io/file.hon

@@ -44,8 +44,8 @@ writer proc(file File) Writer {
     }
 }
 
-hide system_read proc(_ ?@mut anyopaque, handle Handle, buffer []mut u8) usize ! ReadError {
-    request usize = buffer.len
+@hide system_read proc(_ ?@mut anyopaque, handle Handle, buffer []mut u8) usize ! ReadError {
+    request := buffer.len
     maximum usize :: usize(maxval!(c_long))
     if (request > maximum) request = maximum
 
@@ -60,8 +60,8 @@ hide system_read proc(_ ?@mut anyopaque, handle Handle, buffer []mut u8) usize !
     }
 }
 
-hide system_write proc(_ ?@mut anyopaque, handle Handle, bytes []u8) usize ! WriteError {
-    request usize = bytes.len
+@hide system_write proc(_ ?@mut anyopaque, handle Handle, bytes []u8) usize ! WriteError {
+    request := bytes.len
     maximum usize :: usize(maxval!(c_long))
     if (request > maximum) request = maximum
 
@@ -76,8 +76,8 @@ hide system_write proc(_ ?@mut anyopaque, handle Handle, bytes []u8) usize ! Wri
     }
 }
 
-hide system_open proc(_ ?@mut anyopaque, path [;0]u8, mode FileMode) Handle ! OpenError {
-    flags c_int = c.O_RDONLY
+@hide system_open proc(_ ?@mut anyopaque, path [;0]u8, mode FileMode) Handle ! OpenError {
+    flags := c.O_RDONLY
     match mode {
         .read_only: flags = c.O_RDONLY
         .write_only: flags = c.O_WRONLY
@@ -90,23 +90,23 @@ hide system_open proc(_ ?@mut anyopaque, path [;0]u8, mode FileMode) Handle ! Op
     }
 }
 
-hide system_close proc(_ ?@mut anyopaque, handle Handle) void ! CloseError {
+@hide system_close proc(_ ?@mut anyopaque, handle Handle) void ! CloseError {
     if (c.close(handle.file_desc) != 0) return .close_failed
 }
 
-hide system_stdin proc(_ ?@mut anyopaque) Handle {
+@hide system_stdin proc(_ ?@mut anyopaque) Handle {
     return Handle{ file_desc = c_int(Stream.stdin) }
 }
 
-hide system_stdout proc(_ ?@mut anyopaque) Handle {
+@hide system_stdout proc(_ ?@mut anyopaque) Handle {
     return Handle{ file_desc = c_int(Stream.stdout) }
 }
 
-hide system_stderr proc(_ ?@mut anyopaque) Handle {
+@hide system_stderr proc(_ ?@mut anyopaque) Handle {
     return Handle{ file_desc = c_int(Stream.stderr) }
 }
 
-hide system_vtable IoVTable :: IoVTable {
+@hide system_vtable IoVTable :: IoVTable {
     read = system_read,
     write = system_write,
     open = system_open,
@@ -116,7 +116,7 @@ hide system_vtable IoVTable :: IoVTable {
     stderr = system_stderr,
 }
 
-hide system proc() Io {
+@hide system proc() Io {
     return Io {
         context = null,
         vtable = &system_vtable,

std/io/io.hon

@@ -69,7 +69,7 @@ write proc(output Writer, bytes []u8) usize ! WriteError {
 }
 
 write_all proc(output Writer, bytes []u8) void ! WriteError {
-    offset usize = 0
+    offset usize := 0
     while offset < bytes.len {
         count usize :: try write(output, bytes[offset..])
         if (count == 0) return .no_progress
@@ -119,14 +119,15 @@ print proc(output Writer, $format []u8, $Args type, args Args) void ! WriteError
     }
 }
 
-hide write_integer_signed proc(output Writer, value i64, base u64, uppercase bool) void ! WriteError {
-    buffer [65]mut u8 = undefined
-    end usize = buffer.len
-    current i64 = value
+@hide:file
+write_integer_signed proc(output Writer, value i64, base u64, uppercase bool) void ! WriteError {
+    buffer [65]mut u8 := undefined
+    end := buffer.len
+    current := value
 
     while true {
         digit_value i64 :: rem!(current, i64(base))
-        digit u8 = if (digit_value < 0)
+        digit := if (digit_value < 0)
             u8(-digit_value)
         else
             u8(digit_value)
@@ -150,10 +151,11 @@ hide write_integer_signed proc(output Writer, value i64, base u64, uppercase boo
     try write_all(output, buffer[end..])
 }
 
-hide write_integer_unsigned proc(output Writer, value u64, base u64, uppercase bool) void ! WriteError {
-    buffer [65]mut u8 = undefined
-    end usize = buffer.len
-    current u64 = value
+@hide:file
+write_integer_unsigned proc(output Writer, value u64, base u64, uppercase bool) void ! WriteError {
+    buffer [65]mut u8 := undefined
+    end := buffer.len
+    current := value
 
     while true {
         digit u8 :: u8(rem!(current, base))
@@ -173,7 +175,8 @@ hide write_integer_unsigned proc(output Writer, value u64, base u64, uppercase b
     try write_all(output, buffer[end..])
 }
 
-hide FormatTokenKind :: enum {
+@hide:file
+FormatTokenKind :: enum {
     unused
     literal
     default
@@ -187,20 +190,22 @@ hide FormatTokenKind :: enum {
     scientific
 }
 
-hide FormatToken :: struct {
+@hide:file
+FormatToken :: struct {
     kind FormatTokenKind
     start usize
     end usize
     field []u8
 }
 
-hide parse_format proc($N usize, $format []u8, $Args type) [N]mut FormatToken {
-    tokens [N]mut FormatToken = undefined
+@hide:file
+parse_format proc($N usize, $format []u8, $Args type) [N]mut FormatToken {
+    tokens [N]mut FormatToken := undefined
     for (usize(0))..format.len |index| {
         tokens[index] = FormatToken{ kind = .unused, start = 0, end = 0, field = "" }
     }
 
-    field_count usize = 0
+    field_count := 0
     match typeinfo!(Args) {
         .record |r|: {
             if (!r.is_tuple) compile_error!("io.print arguments must be a tuple")
@@ -209,10 +214,10 @@ hide parse_format proc($N usize, $format []u8, $Args type) [N]mut FormatToken {
         else: compile_error!("io.print arguments must be a tuple")
     }
 
-    token_count usize = 0
-    argument_count usize = 0
-    literal_start usize = 0
-    cursor usize = 0
+    token_count := 0
+    argument_count := 0
+    literal_start := 0
+    cursor := 0
     while cursor < format.len {
         byte :: format[cursor]
         if byte == '{' {
@@ -242,8 +247,8 @@ hide parse_format proc($N usize, $format []u8, $Args type) [N]mut FormatToken {
                 continue
             }
 
-            kind FormatTokenKind = .default
-            width usize = 2
+            kind FormatTokenKind := .default
+            width := 2
             if next != '}' {
                 if cursor + 2 >= format.len or format[cursor + 2] != '}' {
                     compile_error!("io.print format expects a one-character specifier")
@@ -321,18 +326,21 @@ hide parse_format proc($N usize, $format []u8, $Args type) [N]mut FormatToken {
     return tokens
 }
 
-hide format_field_name proc($T type, index usize) []u8 {
+@hide:file
+format_field_name proc($T type, index usize) []u8 {
     match typeinfo!(T) {
         .record |r|: return r.fields[index].name
         else: compile_error!("io.print arguments must be a tuple")
     }
 }
 
-hide distinct_value proc($Backing, $Distinct type, value Distinct) Backing {
+@hide:file
+distinct_value proc($Backing, $Distinct type, value Distinct) Backing {
     return ptrcast!(Backing, &value)^
 }
 
-hide scalar_or_distinct_type proc($T type) bool {
+@hide:file
+scalar_or_distinct_type proc($T type) bool {
     match typeinfo!(T) {
         .bool: return true
         .integer: return true
@@ -342,7 +350,8 @@ hide scalar_or_distinct_type proc($T type) bool {
     }
 }
 
-hide write_integer proc(output Writer, $T type, value T, base u64, uppercase bool) void ! WriteError {
+@hide:file
+write_integer proc(output Writer, $T type, value T, base u64, uppercase bool) void ! WriteError {
     match typeinfo!(T) {
         .integer: if minval!(T) < 0 {
             try write_integer_signed(output, i64(value), base, uppercase)
@@ -359,11 +368,12 @@ hide write_integer proc(output Writer, $T type, value T, base u64, uppercase boo
 }
 
 # note: libc keeps float formatting small; replace it with a native shortest-roundtrip writer if locale independence matters.
-hide write_float proc(output Writer, $T type, value T, scientific bool) void ! WriteError {
+@hide:file
+write_float proc(output Writer, $T type, value T, scientific bool) void ! WriteError {
     match typeinfo!(T) {
         .float: {
-            buffer [64]mut u8 = undefined
-            count c_int = 0
+            buffer [64]mut u8 := undefined
+            count := 0
             if sizeof!(T) == 4 {
                 if (scientific) count = c.snprintf(ptrcast!(c_char, (&buffer).ptr), c_ulong(buffer.len), "%.8e", value)
                 else count = c.snprintf(ptrcast!(c_char, (&buffer).ptr), c_ulong(buffer.len), "%.9g", value)
@@ -384,7 +394,8 @@ hide write_float proc(output Writer, $T type, value T, scientific bool) void ! W
     }
 }
 
-hide write_decimal proc(output Writer, $T type, value T) void ! WriteError {
+@hide:file
+write_decimal proc(output Writer, $T type, value T) void ! WriteError {
     match typeinfo!(T) {
         .integer: try write_integer(output, value, 10, false)
         .float: try write_float(output, value, false)
@@ -397,13 +408,14 @@ hide write_decimal proc(output Writer, $T type, value T) void ! WriteError {
     }
 }
 
-hide write_character proc(output Writer, $T type, value T) void ! WriteError {
+@hide:file
+write_character proc(output Writer, $T type, value T) void ! WriteError {
     match typeinfo!(T) {
         .integer: {
             if minval!(T) < 0 or maxval!(T) > 255 {
                 compile_error!("io.print '{c}' requires an unsigned integer that fits in u8")
             }
-            buffer [1]u8 = [u8(value)]
+            buffer [1]u8 := [u8(value)]
             try write_all(output, buffer[..])
         }
         .distinct |backing|: if scalar_or_distinct_type(backing) {
@@ -415,7 +427,8 @@ hide write_character proc(output Writer, $T type, value T) void ! WriteError {
     }
 }
 
-hide write_default proc(output Writer, $T type, value T) void ! WriteError {
+@hide:file
+write_default proc(output Writer, $T type, value T) void ! WriteError {
     match typeinfo!(T) {
         .bool: if value {
             try write_all(output, "true")

std/mem/mem.hon

@@ -46,7 +46,7 @@ alloc proc($T type, allocator Allocator, count usize) []mut T ! AllocError {
         return .out_of_memory
     }
 
-    memory ?*mut u8 = raw_alloc(allocator, count * element_size, alignof!(T))
+    memory := raw_alloc(allocator, count * element_size, alignof!(T))
     if memory |bytes| {
         pointer *mut T :: ptrcast!(T, bytes)
         return pointer[..count]
@@ -72,19 +72,21 @@ realloc proc($T type, allocator Allocator, memory []mut T, new_count usize) []mu
     if (element_size == 0) return empty_slice(T, new_count)
     if (new_count > divtrunc!(maxval!(usize), element_size)) return .out_of_memory
 
-    old_memory ?*mut u8 = null
-    old_size usize = 0
+    old_memory ?*mut u8 := null
+    old_size := 0
     if memory.len != 0 {
         old_memory = ptrcast!(u8, memory.ptr)
         old_size = memory.len * element_size
     }
-    resized ?*mut u8 = raw_realloc(
+
+    resized := raw_realloc(
         allocator,
         old_memory,
         old_size,
         new_count * element_size,
         alignof!(T),
     )
+
     if resized |bytes| {
         pointer *mut T :: ptrcast!(T, bytes)
         return pointer[..new_count]
@@ -105,7 +107,10 @@ free proc($T type, allocator Allocator, memory []T) void {
     )
 }
 
-#! get an empty slice of type `T` with `count` elements.
+#! create a slice backed by `empty_storage` without allocating.
+#! precondition: `count` must be 0 or `sizeof!(T)` must be 0; otherwise the slice
+#! exceeds its backing storage.
+@hide:file
 empty_slice proc($T type, count usize) []mut T {
     pointer *mut T :: ptrcast!(T, (&empty_storage).ptr)
     return pointer[..count]
@@ -116,33 +121,38 @@ empty proc($T type) []mut T {
     return empty_slice(T, 0)
 }
 
-hide empty_storage [1]mut u64 = [0]
+@hide
+empty_storage [1]mut u64 := [0]
 
-hide malloc_alignment usize :: 16 # note: aarch64-macos libc malloc alignment assumption.
+@hide
+malloc_alignment usize :: 16 # note: aarch64-macos libc malloc alignment assumption.
 
-hide power_of_two proc(value usize) bool {
+@hide:file
+power_of_two proc(value usize) bool {
     if (value == 0) return false
-    current usize = value
+    current := value
     while current > 1 {
-        half usize = divtrunc!(current, 2)
+        half := divtrunc!(current, 2)
         if (half * 2 != current) return false
         current = half
     }
     return true
 }
 
-hide c_alloc proc(_ ?@mut anyopaque, size usize, alignment usize) ?*mut u8 {
+@hide:file
+c_alloc proc(_ ?@mut anyopaque, size usize, alignment usize) ?*mut u8 {
     if (power_of_two(alignment) == false) return null
     if (alignment <= malloc_alignment) return ptrcast!(u8, c.malloc(c_ulong(size)))
 
-    memory [1]mut ?*mut anyopaque = [null]
-    status c_int = c.posix_memalign((&memory).ptr, c_ulong(alignment), c_ulong(size))
+    memory [1]mut ?*mut anyopaque := [null]
+    status := c.posix_memalign((&memory).ptr, c_ulong(alignment), c_ulong(size))
     if (status != 0) return null
 
     return ptrcast!(u8, memory[0])
 }
 
-hide c_realloc proc(
+@hide:file
+c_realloc proc(
     _ ?@mut anyopaque,
     memory ?*mut u8,
     old_size usize,
@@ -161,9 +171,9 @@ hide c_realloc proc(
             return ptrcast!(u8, c.realloc(old_memory, c_ulong(new_size)))
         }
 
-        new_memory ?*mut u8 = c_alloc(null, new_size, alignment)
+        new_memory := c_alloc(null, new_size, alignment)
         if new_memory |new_bytes| {
-            copy_size usize = old_size
+            copy_size := old_size
             if (new_size < copy_size) copy_size = new_size
             memcopy!(new_bytes[..copy_size], old_memory[..copy_size])
             c.free(old_memory)
@@ -174,11 +184,13 @@ hide c_realloc proc(
     return c_alloc(null, new_size, alignment)
 }
 
-hide c_free proc(_ ?@mut anyopaque, memory ?*mut u8, _ usize, _ usize) void {
+@hide:file
+c_free proc(_ ?@mut anyopaque, memory ?*mut u8, _ usize, _ usize) void {
     c.free(memory)
 }
 
-hide c_vtable AllocatorVTable :: AllocatorVTable {
+@hide:file
+c_vtable AllocatorVTable :: AllocatorVTable {
     alloc = c_alloc,
     realloc = c_realloc,
     free = c_free,

std/meta/meta.hon

@@ -46,9 +46,9 @@ TypeInfo :: union(enum) {
 EnumFieldStruct proc($E, $Field type, $default ?Field) type {
     match typeinfo!(E) {
         .enum |info|: {
-            names [info.fields.len]mut []u8 = undefined
-            field_types [info.fields.len]mut type = undefined
-            defaults [info.fields.len]mut ?Field = undefined
+            names [info.fields.len]mut []u8 := undefined
+            field_types [info.fields.len]mut type := undefined
+            defaults [info.fields.len]mut ?Field := undefined
             inline for info.fields |field, index| {
                 names[index] = field.name
                 field_types[index] = Field

std/meta/meta.test.hon

@@ -1,4 +1,4 @@
-testing :: import "@std/testing"
+import "@std/testing"
 
 TestTokenKind :: enum(u8) {
     ident = 3
@@ -12,20 +12,21 @@ TestInner :: distinct u16
 TestOuter :: distinct TestInner
 TestOuterAlias :: alias TestOuter
 
-
-hide array_info_matches proc($Array, $Child type, $len usize) bool {
-    match typeinfo!(Array) {
-        .array |info|: return info.child == Child and info.len == len
-        else: return false
-    }
-}
-hide distinct_info_matches proc($Distinct, $Backing type) bool {
-    match typeinfo!(Distinct) {
-        .distinct |backing|: return backing == Backing
-        else: return false
+@hide
+array_info_matches proc($Array, $Child type, $len usize) bool {
+    return match typeinfo!(Array) {
+        .array |info|: info.child == Child and info.len == len
+        else: false
     }
 }
 
+@hide
+distinct_info_matches proc($Distinct, $Backing type) bool {
+    return match typeinfo!(Distinct) {
+        .distinct |backing|: backing == Backing
+        else: false
+    }
+}
 
 array_reflection_exposes_child_and_logical_length test {
     try testing.expect($(array_info_matches([4]i32, i32, 4)))
@@ -34,34 +35,31 @@ array_reflection_exposes_child_and_logical_length test {
     try testing.expect($(array_info_matches([2]mut i64, i64, 2)))
     try testing.expect($(array_info_matches([2;0]u8, u8, 2)))
 }
+
 distinct_reflection_exposes_immediate_backing test {
     try testing.expect($(distinct_info_matches(TestInner, u16)))
     try testing.expect($(distinct_info_matches(TestOuter, TestInner)))
     try testing.expect($(distinct_info_matches(TestOuterAlias, TestInner)))
 }
 
-
 enum_field_struct_defaults test {
-    names TestNames = {
+    names TestNames := {
         ident = "identifier",
         int = "integer",
     }
-    if field!(names, "ident") |value| {
-        try testing.expect(value.len == 10)
-    } else {
-        try testing.expect(false)
-    }
-    if field!(names, "int") |value| {
-        try testing.expect(value.len == 7)
-    } else {
-        try testing.expect(false)
-    }
-    if field!(names, "eof") |_| {
-        try testing.expect(false)
-    }
 
-    empty TestNames = {}
-    if field!(empty, "ident") |_| {
+    if (field!(names, "ident")) |value|
+        try testing.expect(value.len == 10)
+    else
         try testing.expect(false)
-    }
+
+    if (field!(names, "int")) |value|
+        try testing.expect(value.len == 7)
+    else
+        try testing.expect(false)
+
+    if (field!(names, "eof")) |_| try testing.expect(false)
+
+    empty TestNames := {}
+    if (field!(empty, "ident")) |_| try testing.expect(false)
 }

std/strmap/strmap.hon

@@ -10,7 +10,8 @@ StringMap proc($V type) type {
     }
 }
 
-hide Pair proc($V type) type {
+@hide:file
+Pair proc($V type) type {
     return struct { []u8, V }
 }
 
@@ -20,8 +21,8 @@ init proc($V type, $N usize, $entries [N]Pair(V)) StringMap(V) {
         compile_error!("static string map has too many entries")
     }
 
-    keys [N]mut []u8 = undefined
-    values [N]mut V = undefined
+    keys [N]mut []u8 := undefined
+    values [N]mut V := undefined
 
     # assert no duplicate keys
     for entries |entry, i| {
@@ -38,7 +39,7 @@ init proc($V type, $N usize, $entries [N]Pair(V)) StringMap(V) {
     }
 
     if N == 0 {
-        len_indexes [0]u32 = undefined
+        len_indexes [0]u32 := undefined
         return StringMap(V){
             keys = keys[..],
             values = values[..],
@@ -52,7 +53,7 @@ init proc($V type, $N usize, $entries [N]Pair(V)) StringMap(V) {
     for 1..N |i| {
         key :: keys[i]
         value :: values[i]
-        j usize = i
+        j := i
         while j > 0 and keys[j - 1].len > key.len : j -= 1 {
             keys[j] = keys[j - 1]
             values[j] = values[j - 1]
@@ -63,8 +64,8 @@ init proc($V type, $N usize, $entries [N]Pair(V)) StringMap(V) {
 
     min_len u32 :: u32(keys[0].len)
     max_len u32 :: u32(keys[N - 1].len)
-    len_indexes [usize(max_len) + 1]mut u32 = undefined
-    entry_index usize = 0
+    len_indexes [usize(max_len) + 1]mut u32 := undefined
+    entry_index usize := 0
     for 0..=usize(max_len) |length| {
         while entry_index < N and keys[entry_index].len < length : entry_index += 1 {}
         len_indexes[length] = u32(entry_index)
@@ -82,10 +83,10 @@ init proc($V type, $N usize, $entries [N]Pair(V)) StringMap(V) {
 get proc($V type, map @StringMap(V), key []u8) ?V {
     if (map.keys.len == 0 or key.len > maxval!(u32)) return null
 
-    length u32 = u32(key.len)
+    length := u32(key.len)
     if (length < map.min_len or length > map.max_len) return null
 
-    idx usize = usize(map.len_indexes[usize(length)])
+    idx := usize(map.len_indexes[usize(length)])
     while idx < map.keys.len : idx += 1 {
         candidate :: map.keys[idx]
         if (candidate.len != key.len) return null # key not found

std/testing/testing.hon

@@ -73,10 +73,10 @@ expect_type  proc($Expected,  $Actual  type,  _  Actual,  location  SourceLocati
 
 run proc(name []u8, callback *proc() void ! Error) bool {
     callback() catch |_| {
-        debug.print("{s}...[failed]\n",  {name,})
+        debug.print("{s} ... [\x1b[91mfailed\x1b[0m]\n", {name})
         return false
     }
-    debug.print("{s}...[ok]\n",  {name,})
+    debug.print("{s} ... [\x1b[92mok\x1b[0m]\n", {name})
     return true
 }