diff --git a/source/ast/ast.hon b/source/ast/ast.hon index 6aebb56..d87488e 100644 --- a/source/ast/ast.hon +++ b/source/ast/ast.hon @@ -7,106 +7,478 @@ #! * stmt ..... -> decl import "@std/debug" -import "@source/lexer" +import "@std/mem" +import "@std/arraylist" +import "@source/diagnostics" -@hide:file -TokenId :: alias lexer.TokenId - -@hide:file -NO_TOKEN :: alias lexer.NO_TOKEN +Span :: alias diagnostics.Span +DiagnosticId :: alias diagnostics.DiagnosticId +NO_DIAGNOSTIC :: alias diagnostics.NO_DIAGNOSTIC NodeId :: distinct u32 -ExtraId :: distinct u32 +@hide:file NO_NODE :: maxval!(NodeId) -NO_EXTRA :: maxval!(ExtraId) -# unsafe access on its own; NodeKind serves as a the tag -# that determines the variant. -NodeData :: union { - token_id TokenId - node_id NodeId - extra_id ExtraId +LiteralKind :: enum { int, float, string } +ExprKind :: enum { literal_int, literal_float, literal_string, identifier, unary, binary, invalid } +DeclKind :: enum { statement, function, invalid } +StmtKind :: enum { declaration, invalid } +Mutability :: enum { constant, variable } +UnaryOperator :: enum { negate } +BinaryOperator :: enum { add, subtract, multiply, divide } + +@hide:file +NodeKind :: enum { + literal_int, + literal_float, + literal_string, + expr_identifier, + expr_unary, + expr_binary, + stmt_decl, + func_decl, + param_decl, + block, + package, + invalid_expr, + invalid_stmt, + invalid_decl, } +@hide:file +NodeData :: union { + node_id NodeId + extra u32 + diagnostic DiagnosticId + mutability Mutability + unary_operator UnaryOperator + binary_operator BinaryOperator +} + +@hide:file Node :: struct { kind NodeKind - main_token TokenId = NO_TOKEN - + main NodeData = NodeData{ node_id = NO_NODE } data0 NodeData = NodeData{ node_id = NO_NODE } data1 NodeData = NodeData{ node_id = NO_NODE } } -NodeKind :: enum { - # LITERALS: - # * main_token: token literal - literal_int - literal_float - literal_string - - # IDENTIFIER EXPR: - # * main_token: token literal - expr_identifier - - # UNARY: - # * main_token: operator - # * data0: NodeId - expression - expr_unary - - # BINARY: - # * main_token: operator - # * data0: NodeId - left-hand side - # * data1: NodeId - right-hand side - expr_binary - - # STATEMENT DECLS: - # * main_token: symbol name (identifier) - # * data0: ExtraId - # + extra[data0]: NodeId - type expr or NO_NODE - # + extra[data0 + 1]: NodeId - initializer expression - # * data1: TokenId of either `::` or `:=` indicating mutability - stmt_decl - - # FUNC DECLS: - # * main_token: function name (identifier) - # * data0: NodeId - body block (or NO_NODE for extern) - # * data1: ExtraId - # + extra[data1]: NodeId - return type - # + extra[data1 + 1]: u32 - param count - # + extra[data1 + 2..]: NodeId - param decls - func_decl - - # PARAM DECLS: - # * main_token: param name (identifier) - # * data0: NodeId - param type - param_decl - - # BLOCK: - # * data0: ExtraId - statement range start - # * data1: ExtraId - statement range end - block - - # PACKAGE: - # * data0: ExtraId - # * data1: ExtraId - # - # extra[data0..data1]: NodeId - all statement node ids in package - package - - invalid +@hide:file +Storage :: struct { + nodes arraylist.ArrayList(Node) + extra arraylist.ArrayList(u32) + spans arraylist.ArrayList(Span) } -node_id proc(idx uint) NodeId { - debug.assert(u64(idx) < u64(NO_NODE)) - return NodeId(idx) +Builder :: struct { + storage Storage + source []u8 } -maybe_node_id proc(idx uint) ?NodeId { - if (u64(idx) >= u64(NO_NODE)) return null - return NodeId(idx) +Tree :: struct { + storage Storage + source []u8 + root NodeId } -extra_id proc(idx uint) ExtraId { - debug.assert(u64(idx) < u64(NO_EXTRA)) - return ExtraId(idx) +StatementDeclView :: struct { + span Span + name Span + type ?NodeId + mutability Mutability + initializer NodeId +} + +FunctionDeclView :: struct { + span Span + name Span + return_type NodeId + body NodeId +} + +ParamView :: struct { + span Span + name Span + type NodeId +} + +UnaryExprView :: struct { + span Span + operator UnaryOperator + operand NodeId +} + +BinaryExprView :: struct { + span Span + operator BinaryOperator + left NodeId + right NodeId +} + +InvalidView :: struct { + span Span + diagnostic DiagnosticId +} + +init_builder proc(allocator mem.Allocator, source []u8) Builder { + return Builder{ + storage = Storage{ + nodes = arraylist.init(allocator), + extra = arraylist.init(allocator), + spans = arraylist.init(allocator), + }, + source = source, + } +} + +deinit_builder proc(builder @mut Builder) void { + arraylist.deinit(&builder.storage.nodes) + arraylist.deinit(&builder.storage.extra) + arraylist.deinit(&builder.storage.spans) +} + +finish proc(builder @mut Builder, root NodeId) Tree { + tree :: Tree{ storage = builder.storage, source = builder.source, root = root } + allocator :: builder.storage.nodes.allocator + builder.storage = Storage{ + nodes = arraylist.init(allocator), + extra = arraylist.init(allocator), + spans = arraylist.init(allocator), + } + builder.source = &[] + return tree +} + +deinit proc(tree @mut Tree) void { + arraylist.deinit(&tree.storage.nodes) + arraylist.deinit(&tree.storage.extra) + arraylist.deinit(&tree.storage.spans) +} + +add_literal proc(builder @mut Builder, kind LiteralKind, span Span) NodeId ! mem.AllocError { + node_kind :: match kind { + .int: .literal_int + .float: .literal_float + .string: .literal_string + } + return try add_node(builder, Node{ kind = node_kind }, span) +} + +add_identifier proc(builder @mut Builder, span Span) NodeId ! mem.AllocError { + return try add_node(builder, Node{ kind = .expr_identifier }, span) +} + +add_unary proc( + builder @mut Builder, + span Span, + operator UnaryOperator, + operand NodeId, +) NodeId ! mem.AllocError { + return try add_node(builder, Node{ + kind = .expr_unary, + main = NodeData{ unary_operator = operator }, + data0 = NodeData{ node_id = operand }, + }, span) +} + +add_binary proc( + builder @mut Builder, + span Span, + operator BinaryOperator, + left NodeId, + right NodeId, +) NodeId ! mem.AllocError { + return try add_node(builder, Node{ + kind = .expr_binary, + main = NodeData{ binary_operator = operator }, + data0 = NodeData{ node_id = left }, + data1 = NodeData{ node_id = right }, + }, span) +} + +add_statement_decl proc( + builder @mut Builder, + span Span, + name Span, + type ?NodeId, + mutability Mutability, + initializer NodeId, +) NodeId ! mem.AllocError { + raw_type :: if (type) |id| u32(id) else u32(NO_NODE) + extra :: try add_extra(builder, name.start) + _ = try add_extra(builder, name.end) + _ = try add_extra(builder, raw_type) + _ = try add_extra(builder, u32(initializer)) + return try add_node(builder, Node{ + kind = .stmt_decl, + main = NodeData{ extra = extra }, + data0 = NodeData{ mutability = mutability }, + }, span) +} + +add_param proc(builder @mut Builder, span, name Span, type NodeId) NodeId ! mem.AllocError { + extra :: try add_extra(builder, name.start) + _ = try add_extra(builder, name.end) + return try add_node(builder, Node{ + kind = .param_decl, + main = NodeData{ extra = extra }, + data0 = NodeData{ node_id = type }, + }, span) +} + +add_function_decl proc( + builder @mut Builder, + span Span, + name Span, + return_type NodeId, + params []NodeId, + body NodeId, +) NodeId ! mem.AllocError { + start :: builder.storage.extra.items.len + _ = try add_extra(builder, name.start) + _ = try add_extra(builder, name.end) + _ = try add_extra(builder, u32(return_type)) + for (params) |parameter| _ = try add_extra(builder, u32(parameter)) + return try add_node(builder, Node{ + kind = .func_decl, + main = NodeData{ extra = u32(start) }, + data0 = NodeData{ node_id = body }, + data1 = NodeData{ extra = u32(builder.storage.extra.items.len) }, + }, span) +} + +add_block proc(builder @mut Builder, span Span, statements []NodeId) NodeId ! mem.AllocError { + start :: builder.storage.extra.items.len + for (statements) |statement| _ = try add_extra(builder, u32(statement)) + return try add_node(builder, Node{ + kind = .block, + data0 = NodeData{ extra = u32(start) }, + data1 = NodeData{ extra = u32(builder.storage.extra.items.len) }, + }, span) +} + +add_package proc(builder @mut Builder, span Span, declarations []NodeId) NodeId ! mem.AllocError { + start :: builder.storage.extra.items.len + for (declarations) |declaration| _ = try add_extra(builder, u32(declaration)) + return try add_node(builder, Node{ + kind = .package, + data0 = NodeData{ extra = u32(start) }, + data1 = NodeData{ extra = u32(builder.storage.extra.items.len) }, + }, span) +} + +add_invalid_expr proc(builder @mut Builder, span Span, diagnostic DiagnosticId) NodeId ! mem.AllocError { + return try add_node(builder, Node{ + kind = .invalid_expr, + main = NodeData{ diagnostic = diagnostic }, + }, span) +} + +add_invalid_stmt proc(builder @mut Builder, span Span, diagnostic DiagnosticId) NodeId ! mem.AllocError { + return try add_node(builder, Node{ + kind = .invalid_stmt, + main = NodeData{ diagnostic = diagnostic }, + }, span) +} + +add_invalid_decl proc(builder @mut Builder, span Span, diagnostic DiagnosticId) NodeId ! mem.AllocError { + return try add_node(builder, Node{ + kind = .invalid_decl, + main = NodeData{ diagnostic = diagnostic }, + }, span) +} + +package_decl_count proc(tree @Tree) usize { + root :: get_node(&tree.storage, tree.root) + debug.assert(root.kind == .package) + return usize(root.data1.extra - root.data0.extra) +} + +package_decl proc(tree @Tree, index usize) NodeId { + root :: get_node(&tree.storage, tree.root) + debug.assert(root.kind == .package) + debug.assert(index < usize(root.data1.extra - root.data0.extra)) + return NodeId(tree.storage.extra.items[usize(root.data0.extra) + index]) +} + +decl_kind proc(tree @Tree, id NodeId) DeclKind { + return match get_node(&tree.storage, id).kind { + .stmt_decl: .statement + .func_decl: .function + .invalid_decl: .invalid + else: unreachable + } +} + +stmt_kind proc(tree @Tree, id NodeId) StmtKind { + return match get_node(&tree.storage, id).kind { + .stmt_decl: .declaration + .invalid_stmt: .invalid + else: unreachable + } +} + +statement_decl proc(tree @Tree, id NodeId) StatementDeclView { + node :: get_node(&tree.storage, id) + debug.assert(node.kind == .stmt_decl) + extra :: usize(node.main.extra) + raw_type :: tree.storage.extra.items[extra + 2] + return StatementDeclView{ + span = get_span(&tree.storage, id), + name = span_from_extra(&tree.storage, extra), + type = maybe_node_id(raw_type), + initializer = NodeId(tree.storage.extra.items[extra + 3]), + mutability = node.data0.mutability, + } +} + +function_decl proc(tree @Tree, id NodeId) FunctionDeclView { + node :: get_node(&tree.storage, id) + debug.assert(node.kind == .func_decl) + extra :: usize(node.main.extra) + return FunctionDeclView{ + span = get_span(&tree.storage, id), + name = span_from_extra(&tree.storage, extra), + return_type = NodeId(tree.storage.extra.items[extra + 2]), + body = node.data0.node_id, + } +} + +function_param_count proc(tree @Tree, id NodeId) usize { + node :: get_node(&tree.storage, id) + debug.assert(node.kind == .func_decl) + return usize(node.data1.extra - node.main.extra - 3) +} + +function_param proc(tree @Tree, id NodeId, index usize) NodeId { + node :: get_node(&tree.storage, id) + debug.assert(node.kind == .func_decl) + debug.assert(index < usize(node.data1.extra - node.main.extra - 3)) + return NodeId(tree.storage.extra.items[usize(node.main.extra) + 3 + index]) +} + +param proc(tree @Tree, id NodeId) ParamView { + node :: get_node(&tree.storage, id) + debug.assert(node.kind == .param_decl) + return ParamView{ + span = get_span(&tree.storage, id), + name = span_from_extra(&tree.storage, usize(node.main.extra)), + type = node.data0.node_id, + } +} + +block_statement_count proc(tree @Tree, id NodeId) usize { + node :: get_node(&tree.storage, id) + debug.assert(node.kind == .block) + return usize(node.data1.extra - node.data0.extra) +} + +block_statement proc(tree @Tree, id NodeId, index usize) NodeId { + node :: get_node(&tree.storage, id) + debug.assert(node.kind == .block) + debug.assert(index < usize(node.data1.extra - node.data0.extra)) + return NodeId(tree.storage.extra.items[usize(node.data0.extra) + index]) +} + +expr_kind proc(tree @Tree, id NodeId) ExprKind { + return match get_node(&tree.storage, id).kind { + .literal_int: .literal_int + .literal_float: .literal_float + .literal_string: .literal_string + .expr_identifier: .identifier + .expr_unary: .unary + .expr_binary: .binary + .invalid_expr: .invalid + else: unreachable + } +} + +unary_expr proc(tree @Tree, id NodeId) UnaryExprView { + node :: get_node(&tree.storage, id) + debug.assert(node.kind == .expr_unary) + return UnaryExprView{ + span = get_span(&tree.storage, id), + operator = node.main.unary_operator, + operand = node.data0.node_id, + } +} + +binary_expr proc(tree @Tree, id NodeId) BinaryExprView { + node :: get_node(&tree.storage, id) + debug.assert(node.kind == .expr_binary) + return BinaryExprView{ + span = get_span(&tree.storage, id), + operator = node.main.binary_operator, + left = node.data0.node_id, + right = node.data1.node_id, + } +} + +node_span proc(tree @Tree, id NodeId) Span { return get_span(&tree.storage, id) } + +invalid proc(tree @Tree, id NodeId) InvalidView { + node :: get_node(&tree.storage, id) + debug.assert(node.kind == .invalid_expr or node.kind == .invalid_stmt or node.kind == .invalid_decl) + return InvalidView{ span = get_span(&tree.storage, id), diagnostic = node.main.diagnostic } +} + +source_text proc(tree @Tree, span Span) []u8 { + debug.assert(span.start <= span.end and usize(span.end) <= tree.source.len) + return tree.source[usize(span.start)..usize(span.end)] +} + +builder_span proc(builder @Builder, id NodeId) Span { + return get_span(&builder.storage, id) +} + +@hide:file +span_from_extra proc(storage @Storage, start usize) Span { + return Span{ + start = storage.extra.items[start], + end = storage.extra.items[start + 1], + } +} + +@hide:file +add_node proc(builder @mut Builder, node Node, span Span) NodeId ! mem.AllocError { + debug.assert(builder.storage.nodes.items.len == builder.storage.spans.items.len) + id :: node_id(builder.storage.nodes.items.len) + try arraylist.reserve(&builder.storage.nodes, builder.storage.nodes.items.len + 1) + try arraylist.reserve(&builder.storage.spans, builder.storage.spans.items.len + 1) + try arraylist.append(&builder.storage.nodes, node) + try arraylist.append(&builder.storage.spans, span) + return id +} + +@hide:file +add_extra proc(builder @mut Builder, value u32) u32 ! mem.AllocError { + debug.assert(u64(builder.storage.extra.items.len) < u64(maxval!(u32))) + id :: u32(builder.storage.extra.items.len) + try arraylist.append(&builder.storage.extra, value) + return id +} + +@hide:file +get_node proc(storage @Storage, id NodeId) Node { + debug.assert(usize(id) < storage.nodes.items.len) + return storage.nodes.items[usize(id)] +} + +@hide:file +get_span proc(storage @Storage, id NodeId) Span { + debug.assert(usize(id) < storage.spans.items.len) + return storage.spans.items[usize(id)] +} + +@hide:file +node_id proc(index usize) NodeId { + debug.assert(u64(index) < u64(NO_NODE)) + return NodeId(index) +} + +@hide:file +maybe_node_id proc(value u32) ?NodeId { + if (value == u32(NO_NODE)) return null + return NodeId(value) } diff --git a/source/ast/ast.test.hon b/source/ast/ast.test.hon new file mode 100644 index 0000000..a725d98 --- /dev/null +++ b/source/ast/ast.test.hon @@ -0,0 +1,34 @@ +import "@std/mem" +import "@std/testing" + +exposes_typed_declaration_views test { + builder := init_builder(mem.c_allocator, "value :: 42") + defer deinit_builder(&builder) + + value_name :: Span{ start = 0, end = 5 } + value :: try add_literal(&builder, .int, Span{ start = 9, end = 11 }) + declaration :: try add_statement_decl( + &builder, + Span{ start = 0, end = 11 }, + value_name, + null, + .constant, + value, + ) + declarations [1]NodeId := [declaration] + package :: try add_package(&builder, Span{ start = 0, end = 11 }, declarations[..]) + tree := finish(&builder, package) + defer deinit(&tree) + + try testing.expect_equal(1, package_decl_count(&tree)) + decl :: package_decl(&tree, 0) + try testing.expect_equal(DeclKind.statement, decl_kind(&tree, decl)) + + statement :: statement_decl(&tree, decl) + try testing.expect_equal(value_name.start, statement.name.start) + try testing.expect_equal(value_name.end, statement.name.end) + try testing.expect_equal(Mutability.constant, statement.mutability) + try testing.expect_equal(null, statement.type) + try testing.expect_equal(ExprKind.literal_int, expr_kind(&tree, statement.initializer)) + try testing.expect_equal("value", source_text(&tree, statement.name)) +} diff --git a/source/ast/renderer/renderer.hon b/source/ast/renderer/renderer.hon index 9af71a5..3ce24b4 100644 --- a/source/ast/renderer/renderer.hon +++ b/source/ast/renderer/renderer.hon @@ -1,319 +1,27 @@ -import "@std" import "@std/mem" import "@std/arraylist" - -import "@source/lexer" -import "@source/parser" import "@source/ast" -@hide:file TokenId :: alias lexer.TokenId -@hide:file Token :: alias lexer.Token -@hide:file Node :: alias ast.Node -@hide:file NodeId :: alias ast.NodeId -@hide:file ParseState :: alias parser.State - -Renderer :: struct { tokens std.ArrayList(Token) } +Renderer :: struct { + bytes arraylist.ArrayList(u8) +} init proc(allocator mem.Allocator) Renderer { - return Renderer{ tokens = arraylist.init(allocator) } + return Renderer{ bytes = arraylist.init(allocator) } } deinit proc(renderer @mut Renderer) void { - arraylist.deinit(&renderer.tokens) + arraylist.deinit(&renderer.bytes) } -render_token_stream proc( - renderer @mut Renderer, - node_id NodeId, - parse_state @ParseState, - tokens []Token -) void ! mem.AllocError { - node :: parse_state.nodes.items[usize(node_id)] - match node.kind { - .literal_int, .literal_float, .literal_string: - _ = try render_lit(renderer, node_id, parse_state, tokens) - .stmt_decl: - try render_stmt_decl(renderer, node_id, parse_state, tokens) - .func_decl: - try render_func_decl(renderer, node_id, parse_state, tokens) - .package: - try render_pkg(renderer, node_id, parse_state, tokens) - else: unreachable - } -} - -@hide:file -render_pkg proc( - renderer @mut Renderer, - node_id NodeId, - parse_state @ParseState, - tokens []Token, -) void ! mem.AllocError { - node :: parse_state.nodes.items[usize(node_id)] - extra_start :: usize(node.data0.extra_id) - extra_end :: usize(node.data1.extra_id) - for extra_start..extra_end |i| { - stmt_node_id :: ast.node_id(parse_state.extra.items[i]) - stmt_node :: parse_state.nodes.items[usize(stmt_node_id)] - match stmt_node.kind { - .stmt_decl: - try render_stmt_decl(renderer, stmt_node_id, parse_state, tokens) - .func_decl: - try render_func_decl(renderer, stmt_node_id, parse_state, tokens) - else: unreachable - } - } -} - -@hide:file -render_stmt_decl proc( - renderer @mut Renderer, - node_id NodeId, - parse_state @ParseState, - tokens []Token, -) void ! mem.AllocError { - node :: parse_state.nodes.items[usize(node_id)] - tok_ident :: tokens[usize(node.main_token)] - try arraylist.append(&renderer.tokens, tok_ident) - - raw_node_id_type :: parse_state.extra.items[usize(node.data0.extra_id)] - node_id_type :: ast.maybe_node_id(raw_node_id_type) - if (node_id_type) |id| try render_type(renderer, id, parse_state, tokens) - - tok_assign :: tokens[usize(node.data1.token_id)] - try arraylist.append(&renderer.tokens, tok_assign) - - node_id_expr :: ast.node_id(parse_state.extra.items[usize(node.data0.extra_id + 1)]) - last_expr_token :: try render_expr(renderer, node_id_expr, parse_state, tokens) - - terminator_idx :: usize(last_expr_token) + 1 - tok_terminator :: if (terminator_idx < tokens.len) tokens[terminator_idx] else return - if (tok_terminator.kind != .newline and tok_terminator.kind != .eof) return - try arraylist.append(&renderer.tokens, tok_terminator) - - eof_idx :: terminator_idx + 1 - if (tok_terminator.kind == .newline and - eof_idx < tokens.len and - tokens[eof_idx].kind == .eof - ) try arraylist.append(&renderer.tokens, tokens[eof_idx]) -} - -@hide:file -render_func_decl proc( - renderer @mut Renderer, - node_id NodeId, - parse_state @ParseState, - tokens []Token, -) void ! mem.AllocError { - node :: parse_state.nodes.items[usize(node_id)] - ident_idx :: usize(node.main_token) - try arraylist.append(&renderer.tokens, tokens[ident_idx]) - try arraylist.append(&renderer.tokens, tokens[ident_idx + 1]) - try arraylist.append(&renderer.tokens, tokens[ident_idx + 2]) - - payload_start :: usize(node.data1.extra_id) - param_count :: usize(parse_state.extra.items[payload_start + 1]) - first_param :: payload_start + 2 - for 0..param_count |i| { - param_id :: ast.node_id(parse_state.extra.items[first_param + i]) - param_node :: parse_state.nodes.items[usize(param_id)] - type_id :: param_node.data0.node_id - type_node :: parse_state.nodes.items[usize(type_id)] - try arraylist.append(&renderer.tokens, tokens[usize(param_node.main_token)]) - - has_next :: i + 1 < param_count - if has_next { - next_param_id :: ast.node_id(parse_state.extra.items[first_param + i + 1]) - next_param :: parse_state.nodes.items[usize(next_param_id)] - if (next_param.data0.node_id == type_id) { - try arraylist.append( - &renderer.tokens, - tokens[usize(param_node.main_token) + 1], - ) - continue - } - } - - try render_type(renderer, type_id, parse_state, tokens) - if has_next { - try arraylist.append( - &renderer.tokens, - tokens[usize(type_node.main_token) + 1], - ) - } - } - - return_type_id :: ast.node_id(parse_state.extra.items[payload_start]) - return_type_node :: parse_state.nodes.items[usize(return_type_id)] - try arraylist.append( - &renderer.tokens, - tokens[usize(return_type_node.main_token) - 1], - ) - try render_type(renderer, return_type_id, parse_state, tokens) - - open_curly_id :: TokenId(usize(return_type_node.main_token) + 1) - close_curly_id :: try render_block( - renderer, - node.data0.node_id, - open_curly_id, - parse_state, - tokens, - ) - - terminator_idx :: usize(close_curly_id) + 1 - if (terminator_idx >= tokens.len) return - tok_terminator :: tokens[terminator_idx] - if (tok_terminator.kind != .newline and tok_terminator.kind != .eof) return - try arraylist.append(&renderer.tokens, tok_terminator) - - eof_idx :: terminator_idx + 1 - if (tok_terminator.kind == .newline and - eof_idx < tokens.len and - tokens[eof_idx].kind == .eof - ) try arraylist.append(&renderer.tokens, tokens[eof_idx]) -} - -@hide:file -render_block proc( - renderer @mut Renderer, - node_id NodeId, - open_curly_id TokenId, - parse_state @ParseState, - tokens []Token, -) TokenId ! mem.AllocError { - node :: parse_state.nodes.items[usize(node_id)] - open_curly_idx :: usize(open_curly_id) - try arraylist.append(&renderer.tokens, tokens[open_curly_idx]) - - leading_token := open_curly_idx + 1 - while (leading_token < tokens.len and tokens[leading_token].kind == .newline) { - try arraylist.append(&renderer.tokens, tokens[leading_token]) - leading_token += 1 - } - - extra_start :: usize(node.data0.extra_id) - extra_end :: usize(node.data1.extra_id) - for extra_start..extra_end |i| { - stmt_id :: ast.node_id(parse_state.extra.items[i]) - stmt :: parse_state.nodes.items[usize(stmt_id)] - match stmt.kind { - .stmt_decl: - try render_stmt_decl(renderer, stmt_id, parse_state, tokens) - else: unreachable - } - } - - # ponytail: linear delimiter scan; store brace tokens if nested blocks make rendering hot. - depth usize := 1 - close_curly_idx := open_curly_idx + 1 - while close_curly_idx < tokens.len { - match tokens[close_curly_idx].kind { - .open_curly: depth += 1 - .close_curly: { - depth -= 1 - if (depth == 0) { - try arraylist.append(&renderer.tokens, tokens[close_curly_idx]) - return TokenId(close_curly_idx) - } - } - else: {} - } - close_curly_idx += 1 - } - unreachable +render proc(renderer @mut Renderer, tree @ast.Tree) void ! mem.AllocError { + _ = tree + arraylist.clear(&renderer.bytes) + try append_text(renderer, "package\n") } @hide:file -render_expr proc( - renderer @mut Renderer, - node_id NodeId, - parse_state @ParseState, - tokens []Token, -) TokenId ! mem.AllocError { - # todo: support unary expressions - node :: parse_state.nodes.items[usize(node_id)] - return match node.kind { - .expr_identifier: { - tok :: tokens[usize(node.main_token)] - try arraylist.append(&renderer.tokens, tok) - yield node.main_token - } - .expr_unary: - try render_expr_unary(renderer, node_id, parse_state, tokens) - .expr_binary: - try render_expr_binary(renderer, node_id, parse_state, tokens) - .literal_int, .literal_float, .literal_string: - try render_lit(renderer, node_id, parse_state, tokens) - else: unreachable - } -} - -@hide:file -render_expr_unary proc( - renderer @mut Renderer, - node_id NodeId, - parse_state @ParseState, - tokens []Token, -) TokenId ! mem.AllocError { - node :: parse_state.nodes.items[usize(node_id)] - operator :: node.main_token - operand :: node.data0.node_id - - # render operator operand - try arraylist.append(&renderer.tokens, tokens[usize(operator)]) - last_token :: try render_expr(renderer, operand, parse_state, tokens) - - return last_token -} - -@hide:file -render_expr_binary proc( - renderer @mut Renderer, - node_id NodeId, - parse_state @ParseState, - tokens []Token, -) TokenId ! mem.AllocError { - node :: parse_state.nodes.items[usize(node_id)] - operator :: node.main_token - lhs :: node.data0.node_id - rhs :: node.data1.node_id - - # render lhs operator rhs - _ = try render_expr(renderer, lhs, parse_state, tokens) - try arraylist.append(&renderer.tokens, tokens[usize(operator)]) - last_token :: try render_expr(renderer, rhs, parse_state, tokens) - - return last_token -} - -@hide:file -render_type proc( - renderer @mut Renderer, - node_id NodeId, - parse_state @ParseState, - tokens []Token, -) void ! mem.AllocError { - # todo: support more expressive type expressions... - node :: parse_state.nodes.items[usize(node_id)] - match node.kind { - .expr_identifier: { - tok :: tokens[usize(node.main_token)] - try arraylist.append(&renderer.tokens, tok) - } - else: unreachable - } -} - -@hide:file -render_lit proc( - renderer @mut Renderer, - node_id NodeId, - parse_state @ParseState, - tokens []Token, -) TokenId ! mem.AllocError { - node :: parse_state.nodes.items[usize(node_id)] - tok :: tokens[usize(node.main_token)] - try arraylist.append(&renderer.tokens, tok) - return node.main_token +append_text proc(renderer @mut Renderer, text []u8) void ! mem.AllocError { + for (text) |byte| try arraylist.append(&renderer.bytes, byte) } diff --git a/source/lexer/lexer.hon b/source/lexer/lexer.hon index 5d7ceb7..710d95b 100644 --- a/source/lexer/lexer.hon +++ b/source/lexer/lexer.hon @@ -1,9 +1,8 @@ -import "@std" import "@std/mem" import "@std/strmap" -import "@std/enums/enummap" import "@std/arraylist" +import "@source/diagnostics" import "@source/strpool" ScanError :: struct { @@ -11,29 +10,15 @@ ScanError :: struct { end usize } +@hide:file ErrorCode :: enum { - invalid_character, float_must_end_with_digit, unterminated_string, } +SourceError :: enum { source_too_large } -ErrorDetails :: struct { - name []u8 - message []u8 -} -error_msg_map std.EnumMap(ErrorCode, ErrorDetails) :: enummap.init({ - invalid_character = ErrorDetails { - name = "L0", - message = "invalid character", - }, - float_must_end_with_digit = ErrorDetails { - name = "L1", - message = "float must end with a digit", - }, -}) - -keywords std.StringMap(TokenKind) :: strmap.init([ +keywords strmap.StringMap(TokenKind) :: strmap.init([ { "proc", .proc }, { "return", .return }, { "if", .if }, @@ -42,38 +27,31 @@ keywords std.StringMap(TokenKind) :: strmap.init([ { "while", .while }, ]) -Diagnostic :: struct { - token TokenId - code ErrorCode -} - State :: struct { - tokens std.ArrayList(Token) - diagnostics std.ArrayList(Diagnostic) + tokens arraylist.ArrayList(Token) } init proc(allocator mem.Allocator) State { - return State { - tokens = arraylist.init(allocator), - diagnostics = arraylist.init(allocator), - } + return State{ tokens = arraylist.init(allocator) } } deinit proc(state @mut State) void { arraylist.deinit(&state.tokens) - arraylist.deinit(&state.diagnostics) } -scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.InternError) { - diagnostics :: &state.diagnostics - +scan proc( + state @mut State, + program []u8, + diagnostic_store @mut diagnostics.Store, +) void ! (mem.AllocError | strpool.InternError | diagnostics.CapacityError | SourceError) { + if (program.len > usize(maxval!(u32))) return .source_too_large cursor := 0 while cursor < program.len { char :: program[cursor] # whitespace if char == '\n' { - try add_token(state, Token{ start = cursor, kind = .newline }) + try add_token(state, Token{ span = make_span(cursor, cursor + 1), kind = .newline }) cursor += 1 continue } else if is_whitespace(char) { @@ -84,7 +62,7 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter # comments if char == '#' { while (cursor < program.len and program[cursor] != '\n') cursor += 1 - cursor += 1 # also skip newline + if (cursor < program.len) cursor += 1 continue } @@ -93,9 +71,9 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter start :: cursor result :: scan_ident(start, program) try add_token(state, Token{ - start = start, + span = make_span(start, result.end), kind = result.kind, - str_id = result.str_id, + data = TokenData{ str_id = result.str_id }, }) cursor = result.end continue @@ -105,15 +83,21 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter if is_digit(char) { start :: cursor result :: scan_number(start, program) catch |err| { - token :: token_id(state.tokens.items.len) - try add_token(state, Token{ start = start, kind = .invalid }) - try arraylist.append(diagnostics, Diagnostic{ token = token, code = err.code }) + id :: try diagnostics.add( + diagnostic_store, + diagnostic_code(err.code), + make_span(start, err.end), + ) + try add_token(state, Token{ + span = make_span(start, err.end), + kind = .invalid, + data = TokenData{ diagnostic = id }, + }) cursor = err.end continue } - kind :: if (result.has_decimal) .float else .int - try add_token(state, Token{ start = start, kind = kind }) + try add_token(state, Token{ span = make_span(start, result.end), kind = kind }) cursor = result.end continue } @@ -122,14 +106,20 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter if char == '"' { start :: cursor result :: scan_string(start, program) catch |err| { - token :: token_id(state.tokens.items.len) - try add_token(state, Token{ start = start, kind = .invalid }) - try arraylist.append(diagnostics, Diagnostic{ token = token, code = err.code }) + id :: try diagnostics.add( + diagnostic_store, + diagnostic_code(err.code), + make_span(start, err.end), + ) + try add_token(state, Token{ + span = make_span(start, err.end), + kind = .invalid, + data = TokenData{ diagnostic = id }, + }) cursor = err.end continue } - - try add_token(state, Token{ start = start, kind = .string }) + try add_token(state, Token{ span = make_span(start, result.end), kind = .string }) cursor = result.end continue } @@ -137,18 +127,16 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter # immutable decl, mutable decl, or single colon if char == ':' { if cursor + 1 < program.len and program[cursor + 1] == ':' { - try add_token(state, Token{ start = cursor, kind = .double_colon }) + try add_token(state, Token{ span = make_span(cursor, cursor + 2), kind = .double_colon }) cursor += 2 continue } - if cursor + 1 < program.len and program[cursor + 1] == '=' { - try add_token(state, Token{ start = cursor, kind = .colon_equal }) + try add_token(state, Token{ span = make_span(cursor, cursor + 2), kind = .colon_equal }) cursor += 2 continue } - - try add_token(state, Token{ start = cursor, kind = .colon }) + try add_token(state, Token{ span = make_span(cursor, cursor + 1), kind = .colon }) cursor += 1 continue } @@ -156,45 +144,41 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter # compound arithmetic assignment if char == '+' { if cursor + 1 < program.len and program[cursor + 1] == '=' { - try add_token(state, Token{ start = cursor, kind = .plus_equal }) + try add_token(state, Token{ span = make_span(cursor, cursor + 2), kind = .plus_equal }) cursor += 2 continue } - - try add_token(state, Token{ start = cursor, kind = .plus }) + try add_token(state, Token{ span = make_span(cursor, cursor + 1), kind = .plus }) cursor += 1 continue } if char == '-' { if cursor + 1 < program.len and program[cursor + 1] == '=' { - try add_token(state, Token{ start = cursor, kind = .minus_equal }) + try add_token(state, Token{ span = make_span(cursor, cursor + 2), kind = .minus_equal }) cursor += 2 continue } - - try add_token(state, Token{ start = cursor, kind = .minus }) + try add_token(state, Token{ span = make_span(cursor, cursor + 1), kind = .minus }) cursor += 1 continue } if char == '*' { if cursor + 1 < program.len and program[cursor + 1] == '=' { - try add_token(state, Token{ start = cursor, kind = .star_equal }) + try add_token(state, Token{ span = make_span(cursor, cursor + 2), kind = .star_equal }) cursor += 2 continue } - - try add_token(state, Token{ start = cursor, kind = .star }) + try add_token(state, Token{ span = make_span(cursor, cursor + 1), kind = .star }) cursor += 1 continue } if char == '/' { if cursor + 1 < program.len and program[cursor + 1] == '=' { - try add_token(state, Token{ start = cursor, kind = .slash_equal }) + try add_token(state, Token{ span = make_span(cursor, cursor + 2), kind = .slash_equal }) cursor += 2 continue } - - try add_token(state, Token{ start = cursor, kind = .slash }) + try add_token(state, Token{ span = make_span(cursor, cursor + 1), kind = .slash }) cursor += 1 continue } @@ -205,25 +189,33 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter ',': .comma '(': .open_paren ')': .close_paren + '[': .open_bracket + ']': .close_bracket '{': .open_curly '}': .close_curly else: null } - - if (kind) |k| { - try add_token(state, Token{ start = cursor, kind = k }) + if (kind) |value| { + try add_token(state, Token{ span = make_span(cursor, cursor + 1), kind = value }) cursor += 1 continue } # invalid character - token :: token_id(state.tokens.items.len) - try add_token(state, Token{ start = cursor, kind = .invalid }) - try arraylist.append(diagnostics, Diagnostic{ token = token, code = .invalid_character }) + id :: try diagnostics.add( + diagnostic_store, + .invalid_character, + make_span(cursor, cursor + 1), + ) + try add_token(state, Token{ + span = make_span(cursor, cursor + 1), + kind = .invalid, + data = TokenData{ diagnostic = id }, + }) cursor += 1 } - try add_token(state, Token{ start = cursor, kind = .eof }) + try add_token(state, Token{ span = make_span(cursor, cursor), kind = .eof }) } ScanIdentResult :: struct { @@ -234,7 +226,6 @@ ScanIdentResult :: struct { scan_ident proc(start usize, program []u8) ScanIdentResult { cursor := start + 1 - # scan whole identifier while (cursor < program.len and ( is_alpha(program[cursor]) or @@ -242,19 +233,13 @@ scan_ident proc(start usize, program []u8) ScanIdentResult { program[cursor] == '_' )) cursor += 1 - kind :: strmap.get(&keywords, program[start..cursor]) orelse .ident - # don't intern keywords (already O(1) lookup via token kind) + kind :: strmap.get(&keywords, program[start..cursor]) orelse .ident str_id :: if (kind == .ident) strpool.intern(&strpool.STRINGS, program[start..cursor]) catch strpool.NO_STR else strpool.NO_STR - - return ScanIdentResult{ - end = cursor, - kind = kind, - str_id = str_id, - } + return ScanIdentResult{ end = cursor, kind = kind, str_id = str_id } } ScanNumResult :: struct { @@ -265,53 +250,36 @@ ScanNumResult :: struct { scan_number proc(start usize, program []u8) ScanNumResult ! ScanError { cursor := start has_decimal := false - # scan integer part while (cursor < program.len and is_digit(program[cursor])) cursor += 1 - # check for decimal if cursor < program.len and program[cursor] == '.' { has_decimal = true cursor += 1 } - # assert non-terminating decimal if has_decimal and (cursor >= program.len or !is_digit(program[cursor])) { - return ScanError{ - code = .float_must_end_with_digit, - end = cursor, - } + return ScanError{ code = .float_must_end_with_digit, end = cursor } } - # scan fractional part while (cursor < program.len and is_digit(program[cursor])) cursor += 1 - - return ScanNumResult{ - end = cursor, - has_decimal = has_decimal, - } + return ScanNumResult{ end = cursor, has_decimal = has_decimal } } ScanStrResult :: struct { end usize } scan_string proc(start usize, program []u8) ScanStrResult ! ScanError { cursor := start + 1 # skip first `"` - # scan entire string while cursor < program.len and program[cursor] != '"' and program[cursor] != '\n' : cursor += 1 { # ignore escaped characters if (program[cursor] == '\\' and cursor + 1 < program.len) cursor += 1 } - # assert string terminal - if (cursor >= program.len or program[cursor] != '"') return ScanError{ - code = .unterminated_string, - end = cursor, - } - - return ScanStrResult{ - end = cursor + 1, # skip last `"` + if (cursor >= program.len or program[cursor] != '"') { + return ScanError{ code = .unterminated_string, end = cursor } } + return ScanStrResult{ end = cursor + 1 } # skip last `"` } @hide @@ -335,6 +303,19 @@ is_digit proc(char u8) bool { } } +@hide:file +diagnostic_code proc(code ErrorCode) diagnostics.Code { + return match code { + .float_must_end_with_digit: .float_must_end_with_digit + .unterminated_string: .unterminated_string + } +} + +@hide:file +make_span proc(start, end usize) diagnostics.Span { + return diagnostics.Span{ start = u32(start), end = u32(end) } +} + @hide add_token proc(state @mut State, token Token) void ! mem.AllocError { _ = token_id(state.tokens.items.len) diff --git a/source/lexer/lexer.test.hon b/source/lexer/lexer.test.hon index e6813ac..bdc9379 100644 --- a/source/lexer/lexer.test.hon +++ b/source/lexer/lexer.test.hon @@ -1,14 +1,17 @@ -import "@source/strpool" import "@std/mem" import "@std/testing" +import "@source/diagnostics" +import "@source/strpool" -handles_keywords_identifiers_and_error_progress test { +reports_invalid_input_and_keeps_scanning test { strpool.STRINGS = strpool.init(mem.c_allocator) defer strpool.deinit(&strpool.STRINGS) - - state State := init(mem.c_allocator) + diagnostic_store := diagnostics.init(mem.c_allocator) + defer diagnostics.deinit(&diagnostic_store) + state := init(mem.c_allocator) defer deinit(&state) - try scan(&state, "if name # comment\n@1. 2 3.5 \"hi\"") + + try scan(&state, "if name # comment\n@1. 2 3.5 \"hi\"", &diagnostic_store) try testing.expect_equal(8, state.tokens.items.len) try testing.expect_equal(TokenKind.if, state.tokens.items[0].kind) @@ -19,6 +22,8 @@ handles_keywords_identifiers_and_error_progress test { try testing.expect_equal(TokenKind.float, state.tokens.items[5].kind) try testing.expect_equal(TokenKind.string, state.tokens.items[6].kind) try testing.expect_equal(TokenKind.eof, state.tokens.items[7].kind) - try testing.expect_equal(2, state.diagnostics.items.len) - try testing.expect_equal("name", strpool.get_str(&strpool.STRINGS, state.tokens.items[1].str_id)?) + try testing.expect_equal(2, diagnostics.len(&diagnostic_store)) + try testing.expect_equal("name", strpool.get_str(&strpool.STRINGS, state.tokens.items[1].data.str_id)?) + try testing.expect_equal(u32(3), state.tokens.items[1].span.start) + try testing.expect_equal(u32(7), state.tokens.items[1].span.end) } diff --git a/source/lexer/token.hon b/source/lexer/token.hon index 64644f4..3c99b55 100644 --- a/source/lexer/token.hon +++ b/source/lexer/token.hon @@ -1,14 +1,19 @@ import "@std/debug" +import "@source/diagnostics" import "@source/strpool" TokenId :: distinct u32 - NO_TOKEN :: maxval!(TokenId) +TokenData :: union { + str_id strpool.StringId + diagnostic diagnostics.DiagnosticId +} + Token :: struct { - start uint + span diagnostics.Span kind TokenKind - str_id strpool.StringId = strpool.NO_STR + data TokenData = TokenData{ str_id = strpool.NO_STR } } TokenKind :: enum { @@ -55,27 +60,14 @@ TokenKind :: enum { render_token proc(tok Token, program []u8) void { match tok.kind { .ident: { - if (strpool.get_str(&strpool.STRINGS, tok.str_id)) |str| { + if (strpool.get_str(&strpool.STRINGS, tok.data.str_id)) |str| { debug.print("{}({})\n", {tok.kind, str}) } else { - res :: scan_ident(tok.start, program) - debug.print("{}({})\n", {tok.kind, program[tok.start..res.end]}) + debug.print("{}({})\n", {tok.kind, program[tok.span.start..tok.span.end]}) } } - .string: { - res :: scan_string(tok.start, program) catch |_| { - debug.print("{}({})\n", {tok.kind, "null"}) - return - } - debug.print("{}({})\n", {tok.kind, program[tok.start..res.end]}) - } - .int, .float: { - res :: scan_number(tok.start, program) catch |_| { - debug.print("{}({})\n", {tok.kind, "null"}) - return - } - debug.print("{}({})\n", {tok.kind, program[tok.start..res.end]}) - } + .string, .int, .float: + debug.print("{}({})\n", {tok.kind, program[tok.span.start..tok.span.end]}) else: debug.print("{}\n", {tok.kind}) } } diff --git a/source/main.hon b/source/main.hon index cc77504..17d425b 100644 --- a/source/main.hon +++ b/source/main.hon @@ -1,20 +1,24 @@ import "@std/debug" import "@std/mem" +import "@source/diagnostics" +import "@source/strpool" +import "@source/lexer" +import "@source/ast" +import "@source/parser" + +ast_renderer :: import "@source/ast/renderer" + test import "@std/enums/enummap" test import "@std/arraylist" test import "@std/hashmap" test import "@std/strmap" test import "@std/meta" - -import "@source/strpool" -import "@source/lexer" -import "@source/parser" - -ast_renderer :: import "@source/ast/renderer" - +test import "@source/diagnostics" test import "@source/strpool" test import "@source/lexer" +test import "@source/ast" +test import "@source/ast/renderer" test import "@source/parser" program :: @@ -35,77 +39,25 @@ main proc() void! { strpool.STRINGS = strpool.init(mem.c_allocator) defer strpool.deinit(&strpool.STRINGS) - debug.print("PROGRAM::[[\n{}\n]]\n\n", {program}) + diagnostic_store := diagnostics.init(mem.c_allocator) + defer diagnostics.deinit(&diagnostic_store) scan_state := lexer.init(mem.c_allocator) defer lexer.deinit(&scan_state) - lexer.scan(&scan_state, program) catch |err| { - debug.print("failed to scan: {}\n", {err}) - return - } + try lexer.scan(&scan_state, program, &diagnostic_store) - debug.print("TOKENS::[[\n", {}) - for (scan_state.tokens.items) |tok| lexer.render_token(tok, program) - debug.print("]]\n\n", {}) - - parse_state := parser.init(mem.c_allocator) - defer parser.deinit(&parse_state) - root :: parser.parse(&parse_state, scan_state.tokens.items) catch |err| { - debug.print("failed to parse: {}\n", {err}) - return - } - - debug.print("AST::[[\n", {}) - for parse_state.nodes.items |node, i| { - match node.kind { - .expr_identifier: { - ident :: scan_state.tokens.items[usize(node.main_token)] - res :: lexer.scan_ident(ident.start, program) - debug.print("{}: {}({})\n", { i, node.kind, program[ident.start..res.end] }) - } - .literal_int, .literal_float: { - lit :: scan_state.tokens.items[usize(node.main_token)] - res :: lexer.scan_number(lit.start, program) catch |_| { - debug.print("{}: {}\n", { i, node.kind }) - continue - } - debug.print("{}: {}({})\n", { i, node.kind, program[lit.start..res.end] }) - } - .literal_string: { - str :: scan_state.tokens.items[usize(node.main_token)] - res :: lexer.scan_string(str.start, program) catch |_| { - debug.print("{}: {}\n", { i, node.kind }) - continue - } - debug.print("{}: {}({})\n", { i, node.kind, program[str.start..res.end] }) - } - .expr_unary: { - operator :: scan_state.tokens.items[usize(node.main_token)] - operand :: node.data0.node_id - debug.print("{}: {}({}, operand={})\n", { i, node.kind, operator.kind, operand }) - } - .expr_binary: { - operator :: scan_state.tokens.items[usize(node.main_token)] - lhs :: node.data0.node_id - rhs :: node.data1.node_id - debug.print("{}: {}({}, lhs={}, rhs={})\n", { i, node.kind, operator.kind, lhs, rhs }) - } - .stmt_decl: { - ident :: scan_state.tokens.items[usize(node.main_token)] - init :: parse_state.extra.items[usize(node.data0.extra_id) + 1] - res :: lexer.scan_ident(ident.start, program) - debug.print("{}: {}({}, init={})\n", { i, node.kind, program[ident.start..res.end], init }) - } - else: debug.print("{}: {}\n", { i, node.kind }) - } - } - debug.print("]]\n\n", {}) + tree := try parser.parse( + scan_state.tokens.items, + program, + &diagnostic_store, + mem.c_allocator, + ) + defer ast.deinit(&tree) renderer := ast_renderer.init(mem.c_allocator) defer ast_renderer.deinit(&renderer) - try ast_renderer.render_token_stream(&renderer, root, &parse_state, scan_state.tokens.items) + try ast_renderer.render(&renderer, &tree) - debug.print("AST Render::[[\n", {}) - for (renderer.tokens.items) |tok| lexer.render_token(tok, program) - debug.print("]]\n", {}) + debug.print("{s}", {renderer.bytes.items}) + debug.print("diagnostics: {}\n", {diagnostics.len(&diagnostic_store)}) } diff --git a/source/parser/parser.hon b/source/parser/parser.hon index e88cb23..ee6aa1e 100644 --- a/source/parser/parser.hon +++ b/source/parser/parser.hon @@ -1,401 +1,317 @@ -import "@std" import "@std/mem" import "@std/arraylist" -import "@std/enums/enummap" -import "@std/debug" +import "@source/diagnostics" import "@source/lexer" import "@source/ast" -# todo: move error stuff into a separate file +@hide:file +SyntaxError :: enum { unexpected_token } -Diagnostic :: struct { - code ErrorCode - node NodeId -} - -ParseError :: alias ErrorCode | mem.AllocError - -ErrorCode :: enum { - unexpected_token, -} - -ErrorDetails :: struct { - name []u8 - message []u8 -} - -error_msg_map std.EnumMap(ErrorCode, ErrorDetails) :: enummap.init({ - unexpected_token = ErrorDetails { - name = "P0", - message = "unexpected token", - }, -}) - -@hide:file Token :: alias lexer.Token -@hide:file TokenId :: alias lexer.TokenId -@hide:file TokenKind :: alias lexer.TokenKind -@hide:file Node :: alias ast.Node -@hide:file NodeId :: alias ast.NodeId -@hide:file ExtraId :: alias ast.ExtraId -@hide:file NodeData :: alias ast.NodeData -@hide:file NO_NODE :: alias ast.NO_NODE -@hide:file NO_EXTRA :: alias ast.NO_EXTRA -@hide:file token_id :: alias lexer.token_id +@hide:file +ParseError :: alias SyntaxError | mem.AllocError | diagnostics.CapacityError +@hide:file BindingPowerScalar :: distinct u32 +@hide:file BindingPower :: struct { left BindingPowerScalar right BindingPowerScalar } -prefix_binding_power proc(token_kind TokenKind) ?BindingPowerScalar { - return match token_kind { +@hide:file +State :: struct { + builder ast.Builder + diagnostics @mut diagnostics.Store + tokens []lexer.Token + allocator mem.Allocator + next_token usize +} + +parse proc( + tokens []lexer.Token, + source []u8, + diagnostic_store @mut diagnostics.Store, + allocator mem.Allocator, +) ast.Tree ! ParseError { + state := State{ + builder = ast.init_builder(allocator, source), + diagnostics = diagnostic_store, + tokens = tokens, + allocator = allocator, + next_token = 0, + } + defer ast.deinit_builder(&state.builder) + root :: try parse_package(&state) + return ast.finish(&state.builder, root) +} + +@hide:file +parse_package proc(state @mut State) ast.NodeId ! ParseError { + declarations arraylist.ArrayList(ast.NodeId) := arraylist.init(state.allocator) + defer arraylist.deinit(&declarations) + + skip_newlines(state) + while !next_is(state, .eof) { + declaration :: try parse_decl(state) + try arraylist.append(&declarations, declaration) + skip_newlines(state) + } + + return try ast.add_package( + &state.builder, + ast.Span{ start = 0, end = u32(state.builder.source.len) }, + declarations.items, + ) +} + +@hide:file +parse_decl proc(state @mut State) ast.NodeId ! ParseError { + if next_is(state, .ident) and next_n_is(state, 1, .proc) { + return try parse_func_decl(state) + } + return try parse_statement_decl(state) +} + +@hide:file +parse_func_decl proc(state @mut State) ast.NodeId ! ParseError { + name :: try consume(state, .ident) + _ = try consume(state, .proc) + _ = try consume(state, .open_paren) + + params arraylist.ArrayList(ast.NodeId) := arraylist.init(state.allocator) + defer arraylist.deinit(¶ms) + names arraylist.ArrayList(ast.Span) := arraylist.init(state.allocator) + defer arraylist.deinit(&names) + + while !next_is(state, .close_paren) { + arraylist.clear(&names) + while true { + token :: try consume(state, .ident) + try arraylist.append(&names, token.span) + if next_is(state, .ident) break + _ = try consume(state, .comma) + } + + type_id :: try parse_type(state) + type_span :: ast.builder_span(&state.builder, type_id) + for (names.items) |name_span| { + param_id :: try ast.add_param( + &state.builder, + ast.Span{ start = name_span.start, end = type_span.end }, + name_span, + type_id, + ) + try arraylist.append(¶ms, param_id) + } + if next_is(state, .comma) _ = try consume(state, .comma) + } + + _ = try consume(state, .close_paren) + return_type :: try parse_type(state) + body :: try parse_block(state) + body_span :: ast.builder_span(&state.builder, body) + return try ast.add_function_decl( + &state.builder, + ast.Span{ start = name.span.start, end = body_span.end }, + name.span, + return_type, + params.items, + body, + ) +} + +@hide:file +parse_block proc(state @mut State) ast.NodeId ! ParseError { + open :: try consume(state, .open_curly) + statements arraylist.ArrayList(ast.NodeId) := arraylist.init(state.allocator) + defer arraylist.deinit(&statements) + + skip_newlines(state) + while !next_is(state, .close_curly) and !next_is(state, .eof) { + statement :: try parse_statement(state) + try arraylist.append(&statements, statement) + skip_newlines(state) + } + + close :: try consume(state, .close_curly) + return try ast.add_block( + &state.builder, + ast.Span{ start = open.span.start, end = close.span.end }, + statements.items, + ) +} + +@hide:file +parse_statement proc(state @mut State) ast.NodeId ! ParseError { + if next_is(state, .ident) return try parse_statement_decl(state) + return .unexpected_token +} + +@hide:file +parse_statement_decl proc(state @mut State) ast.NodeId ! ParseError { + name :: try consume(state, .ident) + type_id ?ast.NodeId := if next_is(state, .ident) + try parse_type(state) + else + null + + assignment :: try consume_either(state, &[.double_colon, .colon_equal]) + initializer :: try parse_expr(state) + end :: ast.builder_span(&state.builder, initializer).end + mutability :: if (assignment.kind == .double_colon) .constant else .variable + return try ast.add_statement_decl( + &state.builder, + ast.Span{ start = name.span.start, end = end }, + name.span, + type_id, + mutability, + initializer, + ) +} + +@hide:file +parse_type proc(state @mut State) ast.NodeId ! ParseError { + if (!next_is(state, .ident)) return .unexpected_token + token :: current(state) + state.next_token += 1 + return try ast.add_identifier(&state.builder, token.span) +} + +@hide:file +parse_expr proc(state @mut State) ast.NodeId ! ParseError { + return try parse_expr_bp(state, BindingPowerScalar(0)) +} + +@hide:file +parse_expr_bp proc(state @mut State, min_bp BindingPowerScalar) ast.NodeId ! ParseError { + token := current(state) + left := if (prefix_binding_power(token.kind)) |right_bp| { + state.next_token += 1 + operand :: try parse_expr_bp(state, right_bp) + operand_span :: ast.builder_span(&state.builder, operand) + yield try ast.add_unary( + &state.builder, + ast.Span{ start = token.span.start, end = operand_span.end }, + .negate, + operand, + ) + } else { + yield try parse_primary(state) + } + + while true { + token = current(state) + if (infix_binding_power(token.kind)) |power| { + if (power.left < min_bp) break + state.next_token += 1 + right :: try parse_expr_bp(state, power.right) + left_span :: ast.builder_span(&state.builder, left) + right_span :: ast.builder_span(&state.builder, right) + left = try ast.add_binary( + &state.builder, + ast.Span{ start = left_span.start, end = right_span.end }, + binary_operator(token.kind), + left, + right, + ) + continue + } + break + } + return left +} + +@hide:file +parse_primary proc(state @mut State) ast.NodeId ! ParseError { + token :: current(state) + match token.kind { + .int: { + state.next_token += 1 + return try ast.add_literal(&state.builder, .int, token.span) + } + .float: { + state.next_token += 1 + return try ast.add_literal(&state.builder, .float, token.span) + } + .string: { + state.next_token += 1 + return try ast.add_literal(&state.builder, .string, token.span) + } + .ident: { + state.next_token += 1 + return try ast.add_identifier(&state.builder, token.span) + } + else: return .unexpected_token + } +} + + +@hide:file +skip_newlines proc(state @mut State) void { + while next_is(state, .newline) state.next_token += 1 +} + +@hide:file +current proc(state @State) lexer.Token { + if (state.next_token >= state.tokens.len) return state.tokens[state.tokens.len - 1] + return state.tokens[state.next_token] +} + +@hide:file +next_is proc(state @State, kind lexer.TokenKind) bool { + return current(state).kind == kind +} + +@hide:file +next_n_is proc(state @State, count usize, kind lexer.TokenKind) bool { + index :: state.next_token + count + return index < state.tokens.len and state.tokens[index].kind == kind +} + +@hide:file +consume proc(state @mut State, kind lexer.TokenKind) lexer.Token ! SyntaxError { + token :: current(state) + if (token.kind != kind) return .unexpected_token + state.next_token += 1 + return token +} + +@hide:file +consume_either proc(state @mut State, kinds []lexer.TokenKind) lexer.Token ! SyntaxError { + token :: current(state) + for (kinds) |kind| if token.kind == kind { + state.next_token += 1 + return token + } + return .unexpected_token +} + +@hide:file +prefix_binding_power proc(kind lexer.TokenKind) ?BindingPowerScalar { + return match kind { .minus: BindingPowerScalar(30) else: null } } -infix_binding_power proc(token_kind TokenKind) ?BindingPower { - return match token_kind { +@hide:file +infix_binding_power proc(kind lexer.TokenKind) ?BindingPower { + return match kind { .plus, .minus: BindingPower{ left = BindingPowerScalar(10), right = BindingPowerScalar(11) } .star, .slash: BindingPower{ left = BindingPowerScalar(20), right = BindingPowerScalar(21) } else: null } } -postfix_binding_power proc(token_kind TokenKind) ?BindingPowerScalar { - return match token_kind { - .open_paren, .open_bracket: BindingPowerScalar(40) - else: null +@hide:file +binary_operator proc(kind lexer.TokenKind) ast.BinaryOperator { + return match kind { + .plus: .add + .minus: .subtract + .star: .multiply + .slash: .divide + else: unreachable } } - -State :: struct { - nodes std.ArrayList(Node) - - #! array of raw u32's indexing nodes, tokens, etc. - extra std.ArrayList(u32) - - tokens []Token = &[] - next_token usize = 0 -} - -init proc(allocator mem.Allocator) State { - return State{ - nodes = arraylist.init(allocator), - extra = arraylist.init(allocator), - } -} - -deinit proc(state @mut State) void { - arraylist.deinit(&state.nodes) - arraylist.deinit(&state.extra) -} - -parse proc(state @mut State, tokens []Token) NodeId ! ParseError { - state.tokens = tokens - return try parse_pkg(state) -} - -parse_pkg proc(state @mut State) NodeId ! ParseError { - if (state.next_token < state.tokens.len - and state.tokens[state.next_token].kind == .eof - ) return NO_NODE - - stmt_ids std.ArrayList(NodeId) := arraylist.init(state.nodes.allocator) - defer arraylist.deinit(&stmt_ids) - - while (state.next_token < state.tokens.len - and state.tokens[state.next_token].kind != .eof - ) { - stmt_id :: try parse_decl(state) - try arraylist.append(&stmt_ids, stmt_id) - } - - pkg_start :: ast.extra_id(state.extra.items.len) - for (stmt_ids.items) |stmt_id| _ = try add_extra(state, u32(stmt_id)) - pkg_end :: ast.extra_id(state.extra.items.len) - - return try add_node(state, Node{ - kind = .package, - data0 = NodeData{ extra_id = pkg_start }, - data1 = NodeData{ extra_id = pkg_end }, - }) -} - -parse_decl proc(state @mut State) NodeId ! ParseError { - if next_n_is(state, 1, .proc) - return try parse_func_decl(state) - else - return try parse_stmt_decl(state) -} - -parse_func_decl proc(state @mut State) NodeId ! ParseError { - ident_tok_id :: try consume(state, .ident) - _ = try consume(state, .proc) - _ = try consume(state, .open_paren) - - param_decls std.ArrayList(NodeId) := arraylist.init(state.nodes.allocator) - defer arraylist.deinit(¶m_decls) - - ident_ids std.ArrayList(TokenId) := arraylist.init(state.nodes.allocator) - defer arraylist.deinit(&ident_ids) - - while !next_is(state, .close_paren) { - arraylist.clear(&ident_ids) - - # collect param group - while true { - param_tok_id :: try consume(state, .ident) - try arraylist.append(&ident_ids, param_tok_id) - - if next_is(state, .ident) break - _ = try consume(state, .comma) - } - - # collect type - try expect(state, .ident) - type_id :: try parse_primary(state) - for (ident_ids.items) |id| { - param_decl :: try add_node(state, Node{ - kind = .param_decl, - main_token = id, - data0 = NodeData{ node_id = type_id }, - }) - try arraylist.append(¶m_decls, param_decl) - } - - if next_is(state, .comma) _ = try consume(state, .comma) - } - - _ = try consume(state, .close_paren) - - try expect(state, .ident) - return_type_id :: try parse_primary(state) - block_id :: try parse_block(state) - - while next_is(state, .newline) _ = try consume(state, .newline) - - extra_id :: try add_extra(state, u32(return_type_id)) - _ = try add_extra(state, u32(param_decls.items.len)) - for (param_decls.items) |param| _ = try add_extra(state, u32(param)) - - return try add_node(state, Node{ - kind = .func_decl, - main_token = ident_tok_id, - data0 = NodeData{ node_id = block_id }, - data1 = NodeData{ extra_id = extra_id }, - }) -} - -parse_block proc(state @mut State) NodeId ! ParseError { - stmt_ids std.ArrayList(NodeId) := arraylist.init(state.nodes.allocator) - defer arraylist.deinit(&stmt_ids) - - _ = try consume(state, .open_curly) - while next_is(state, .newline) _ = try consume(state, .newline) - - while !next_is(state, .close_curly) { - stmt_id :: try parse_stmt(state) - try arraylist.append(&stmt_ids, stmt_id) - while next_is(state, .newline) _ = try consume(state, .newline) - } - - _ = try consume(state, .close_curly) - - block_start :: ast.extra_id(state.extra.items.len) - for (stmt_ids.items) |id| _ = try add_extra(state, u32(id)) - block_end :: ast.extra_id(state.extra.items.len) - - return try add_node(state, Node{ - kind = .block, - data0 = NodeData{ extra_id = block_start }, - data1 = NodeData{ extra_id = block_end }, - }) -} - -parse_stmt proc(state @mut State) NodeId ! ParseError { - if next_is(state, .ident) return try parse_stmt_decl(state) - return .unexpected_token -} - -parse_stmt_decl proc(state @mut State) NodeId ! ParseError { - # expect identifier - ident_tok_id :: try consume(state, .ident) - - # check for type - # todo: make a parse_type proc for this - type_id :: if (next_is(state, .ident)) - try parse_primary(state) - else - NO_NODE - - # expect `::` or `:=` (immutable or mutable assignment) - mutability_tok_id :: try consume_either(state, &[.double_colon, .colon_equal]) - - # expect expression - expr_id :: try parse_expr(state) - - # expect statement terminator (newline) - while (next_is(state, .newline)) _ = try consume(state, .newline) - - extra_start_id :: try add_extra(state, u32(type_id)) - _ = try add_extra(state, u32(expr_id)) - - return try add_node(state, Node{ - kind = .stmt_decl, - main_token = ident_tok_id, - data0 = NodeData{ extra_id = extra_start_id }, - data1 = NodeData{ token_id = mutability_tok_id }, - }) -} - -parse_expr proc(state @mut State) NodeId ! mem.AllocError { - return try parse_expr_bp(state, BindingPowerScalar(0)) -} - -parse_expr_bp proc(state @mut State, min_bp BindingPowerScalar) NodeId ! mem.AllocError { - # parse left-hand side - kind := state.tokens[state.next_token].kind - - lhs := if (prefix_binding_power(kind)) |right_bp| { - operator :: token_id(state.next_token) - state.next_token += 1 - - operand :: try parse_expr_bp(state, right_bp) - - yield try add_node(state, Node{ - kind = .expr_unary, - main_token = operator, - data0 = NodeData{ node_id = operand }, - }) - } else { - yield try parse_primary(state) - } - - # parse right-hand side - while true { - kind = state.tokens[state.next_token].kind - - # try postfix (higher precedence) - if postfix_binding_power(kind) |left_bp| { - if (left_bp < min_bp) break - - lhs = match state.tokens[state.next_token].kind { - .open_paren: debug.unimplemented() # todo: parse call - .open_bracket: debug.unimplemented() # todo: parse index - else: unreachable - } - continue - } - - # try infex (lower precedence) - if infix_binding_power(kind) |bp| { - if (bp.left < min_bp) break - - operator :: token_id(state.next_token) - state.next_token += 1 - - rhs :: try parse_expr_bp(state, bp.right) - lhs = try add_node(state, Node{ - kind = .expr_binary, - main_token = operator, - data0 = NodeData{ node_id = lhs }, - data1 = NodeData{ node_id = rhs }, - }) - - continue - } - - break - } - - return lhs -} - -parse_primary proc(state @mut State) NodeId ! mem.AllocError { - start_idx :: state.next_token - start_tok_id :: TokenId(start_idx) - state.next_token += 1 - return match state.tokens[start_idx].kind { - .int: try add_node(state, Node{ - kind = .literal_int, - main_token = start_tok_id, - }) - .float: try add_node(state, Node{ - kind = .literal_float, - main_token = start_tok_id, - }) - .string: try add_node(state, Node{ - kind = .literal_string, - main_token = start_tok_id, - }) - .ident: try add_node(state, Node{ - kind = .expr_identifier, - main_token = start_tok_id, - }) - else: try add_node(state, Node{ - kind = .invalid, - main_token = start_tok_id, - }) - } -} - -@hide -next_is proc(state @mut State, token_kind TokenKind) bool { - return state.tokens[state.next_token].kind == token_kind -} - -@hide -next_is_either proc(state @mut State, token_kinds []TokenKind) bool { - for (token_kinds) |kind| if next_is(state, kind) return true - return false -} - -@hide -next_n_is proc(state @mut State, n usize, token_kind TokenKind) bool { - return state.tokens[state.next_token + n].kind == token_kind -} - -@hide -expect proc(state @mut State, token_kind TokenKind) void ! ErrorCode { - if (state.tokens[state.next_token].kind != token_kind) return .unexpected_token -} - -@hide -expect_either proc(state @mut State, token_kinds []TokenKind) void ! ErrorCode { - for (token_kinds) |kind| if (state.tokens[state.next_token].kind == kind) return - return .unexpected_token -} - -@hide -consume proc(state @mut State, token_kind TokenKind) TokenId ! ErrorCode { - try expect(state, token_kind) - id :: TokenId(state.next_token) - state.next_token += 1 - return id -} - -@hide -consume_either proc(state @mut State, token_kinds []TokenKind) TokenId ! ErrorCode { - try expect_either(state, token_kinds) - id :: TokenId(state.next_token) - state.next_token += 1 - return id -} - -@hide -add_node proc(state @mut State, node Node) NodeId ! mem.AllocError { - id :: ast.node_id(state.nodes.items.len) - try arraylist.append(&state.nodes, node) - return id -} - -@hide -add_extra proc(state @mut State, data u32) ExtraId ! mem.AllocError { - id :: ast.extra_id(state.extra.items.len) - try arraylist.append(&state.extra, data) - return id -} diff --git a/source/parser/parser.test.hon b/source/parser/parser.test.hon index 60fe943..14929db 100644 --- a/source/parser/parser.test.hon +++ b/source/parser/parser.test.hon @@ -1,202 +1,68 @@ import "@std/mem" import "@std/testing" +import "@source/diagnostics" import "@source/strpool" import "@source/lexer" import "@source/ast" -ast_renderer :: import "@source/ast/renderer" - -handles_package_declarations test { +parses_expression_precedence_through_tree_views test { strpool.STRINGS = strpool.init(mem.c_allocator) defer strpool.deinit(&strpool.STRINGS) - - source :: - `first int :: 42 - `second float :: 2.15 + 3 - `message :: "hello" - + diagnostic_store := diagnostics.init(mem.c_allocator) + defer diagnostics.deinit(&diagnostic_store) + source :: "value :: 1 + 2 * 3\n" scan_state := lexer.init(mem.c_allocator) defer lexer.deinit(&scan_state) - try lexer.scan(&scan_state, source) + try lexer.scan(&scan_state, source, &diagnostic_store) - parse_state := init(mem.c_allocator) - defer deinit(&parse_state) - root :: try parse(&parse_state, scan_state.tokens.items) + tree := try parse(scan_state.tokens.items, source, &diagnostic_store, mem.c_allocator) + defer ast.deinit(&tree) - renderer := ast_renderer.init(mem.c_allocator) - defer ast_renderer.deinit(&renderer) - try ast_renderer.render_token_stream(&renderer, root, &parse_state, scan_state.tokens.items) - - try testing.expect_equal(scan_state.tokens.items.len, renderer.tokens.items.len) - for scan_state.tokens.items |expected, i| { - actual :: renderer.tokens.items[i] - try testing.expect_equal(expected.kind, actual.kind) - try testing.expect_equal(expected.start, actual.start) - try testing.expect_equal(expected.str_id, actual.str_id) - } + try testing.expect_equal(1, ast.package_decl_count(&tree)) + declaration :: ast.package_decl(&tree, 0) + statement :: ast.statement_decl(&tree, declaration) + try testing.expect_equal("value", ast.source_text(&tree, statement.name)) + try testing.expect_equal(ast.ExprKind.binary, ast.expr_kind(&tree, statement.initializer)) + add :: ast.binary_expr(&tree, statement.initializer) + try testing.expect_equal(ast.BinaryOperator.add, add.operator) + try testing.expect_equal(ast.ExprKind.literal_int, ast.expr_kind(&tree, add.left)) + try testing.expect_equal(ast.ExprKind.binary, ast.expr_kind(&tree, add.right)) + multiply :: ast.binary_expr(&tree, add.right) + try testing.expect_equal(ast.BinaryOperator.multiply, multiply.operator) + try testing.expect_equal(0, diagnostics.len(&diagnostic_store)) } -handles_unary_arithmetic_prefix_expression test { +parses_function_structure_through_tree_views test { strpool.STRINGS = strpool.init(mem.c_allocator) defer strpool.deinit(&strpool.STRINGS) - + diagnostic_store := diagnostics.init(mem.c_allocator) + defer diagnostics.deinit(&diagnostic_store) source :: - `first :: -42 + 1 - - scan_state := lexer.init(mem.c_allocator) - defer lexer.deinit(&scan_state) - try lexer.scan(&scan_state, source) - - parse_state := init(mem.c_allocator) - defer deinit(&parse_state) - root :: try parse(&parse_state, scan_state.tokens.items) - - renderer := ast_renderer.init(mem.c_allocator) - defer ast_renderer.deinit(&renderer) - try ast_renderer.render_token_stream(&renderer, root, &parse_state, scan_state.tokens.items) - - try testing.expect_equal(scan_state.tokens.items.len, renderer.tokens.items.len) - for scan_state.tokens.items |expected, i| { - actual :: renderer.tokens.items[i] - try testing.expect_equal(expected.kind, actual.kind) - try testing.expect_equal(expected.start, actual.start) - try testing.expect_equal(expected.str_id, actual.str_id) - } -} - -handles_binary_arithmetic_expression test { - strpool.STRINGS = strpool.init(mem.c_allocator) - defer strpool.deinit(&strpool.STRINGS) - - source :: - `first int :: 42 + 3 - `second :: 134 + 5 - `third float :: 1.35 + 2 - `fourth :: first + second - - scan_state := lexer.init(mem.c_allocator) - defer lexer.deinit(&scan_state) - try lexer.scan(&scan_state, source) - - parse_state := init(mem.c_allocator) - defer deinit(&parse_state) - root :: try parse(&parse_state, scan_state.tokens.items) - - renderer := ast_renderer.init(mem.c_allocator) - defer ast_renderer.deinit(&renderer) - try ast_renderer.render_token_stream(&renderer, root, &parse_state, scan_state.tokens.items) - - try testing.expect_equal(scan_state.tokens.items.len, renderer.tokens.items.len) - for scan_state.tokens.items |expected, i| { - actual :: renderer.tokens.items[i] - try testing.expect_equal(expected.kind, actual.kind) - try testing.expect_equal(expected.start, actual.start) - try testing.expect_equal(expected.str_id, actual.str_id) - } -} - -handles_statement_declaration test { - strpool.STRINGS = strpool.init(mem.c_allocator) - defer strpool.deinit(&strpool.STRINGS) - - source :: - `value int :: 42 - - scan_state := lexer.init(mem.c_allocator) - defer lexer.deinit(&scan_state) - try lexer.scan(&scan_state, source) - - parse_state := init(mem.c_allocator) - defer deinit(&parse_state) - root :: try parse(&parse_state, scan_state.tokens.items) - - renderer := ast_renderer.init(mem.c_allocator) - defer ast_renderer.deinit(&renderer) - try ast_renderer.render_token_stream(&renderer, root, &parse_state, scan_state.tokens.items) - - try testing.expect_equal(scan_state.tokens.items.len, renderer.tokens.items.len) - for scan_state.tokens.items |expected, i| { - actual :: renderer.tokens.items[i] - try testing.expect_equal(expected.kind, actual.kind) - try testing.expect_equal(expected.start, actual.start) - try testing.expect_equal(expected.str_id, actual.str_id) - } -} - -handles_function_declarations test { - strpool.STRINGS = strpool.init(mem.c_allocator) - defer strpool.deinit(&strpool.STRINGS) - - source :: - `empty proc() void {} `grouped proc(a, b T, c U) void { ` value int :: 1 `} - scan_state := lexer.init(mem.c_allocator) defer lexer.deinit(&scan_state) - try lexer.scan(&scan_state, source) + try lexer.scan(&scan_state, source, &diagnostic_store) - parse_state := init(mem.c_allocator) - defer deinit(&parse_state) - root_id :: try parse(&parse_state, scan_state.tokens.items) - root :: parse_state.nodes.items[usize(root_id)] + tree := try parse(scan_state.tokens.items, source, &diagnostic_store, mem.c_allocator) + defer ast.deinit(&tree) - package_start :: usize(root.data0.extra_id) - package_end :: usize(root.data1.extra_id) - try testing.expect_equal(usize(2), package_end - package_start) - - empty_id :: ast.node_id(parse_state.extra.items[package_start]) - empty :: parse_state.nodes.items[usize(empty_id)] - empty_payload :: usize(empty.data1.extra_id) - empty_block :: parse_state.nodes.items[usize(empty.data0.node_id)] - try testing.expect_equal(ast.NodeKind.func_decl, empty.kind) - try testing.expect_equal(u32(0), parse_state.extra.items[empty_payload + 1]) - try testing.expect_equal( - usize(empty_block.data0.extra_id), - usize(empty_block.data1.extra_id), - ) - - grouped_id :: ast.node_id(parse_state.extra.items[package_start + 1]) - grouped :: parse_state.nodes.items[usize(grouped_id)] - grouped_payload :: usize(grouped.data1.extra_id) - try testing.expect_equal(ast.NodeKind.func_decl, grouped.kind) - try testing.expect_equal(u32(3), parse_state.extra.items[grouped_payload + 1]) - - param_a_id :: ast.node_id(parse_state.extra.items[grouped_payload + 2]) - param_b_id :: ast.node_id(parse_state.extra.items[grouped_payload + 3]) - param_c_id :: ast.node_id(parse_state.extra.items[grouped_payload + 4]) - param_a :: parse_state.nodes.items[usize(param_a_id)] - param_b :: parse_state.nodes.items[usize(param_b_id)] - param_c :: parse_state.nodes.items[usize(param_c_id)] - try testing.expect_equal(ast.NodeKind.param_decl, param_a.kind) - try testing.expect_equal(ast.NodeKind.param_decl, param_b.kind) - try testing.expect_equal(ast.NodeKind.param_decl, param_c.kind) - try testing.expect(param_a.data0.node_id == param_b.data0.node_id) - try testing.expect(param_a.data0.node_id != param_c.data0.node_id) - - grouped_block :: parse_state.nodes.items[usize(grouped.data0.node_id)] - try testing.expect_equal(ast.NodeKind.block, grouped_block.kind) - try testing.expect_equal( - usize(1), - usize(grouped_block.data1.extra_id) - usize(grouped_block.data0.extra_id), - ) - - renderer := ast_renderer.init(mem.c_allocator) - defer ast_renderer.deinit(&renderer) - try ast_renderer.render_token_stream( - &renderer, - root_id, - &parse_state, - scan_state.tokens.items, - ) - - try testing.expect_equal(scan_state.tokens.items.len, renderer.tokens.items.len) - for scan_state.tokens.items |expected, i| { - actual :: renderer.tokens.items[i] - try testing.expect_equal(expected.kind, actual.kind) - try testing.expect_equal(expected.start, actual.start) - try testing.expect_equal(expected.str_id, actual.str_id) - } + declaration :: ast.package_decl(&tree, 0) + try testing.expect_equal(ast.DeclKind.function, ast.decl_kind(&tree, declaration)) + function :: ast.function_decl(&tree, declaration) + try testing.expect_equal("grouped", ast.source_text(&tree, function.name)) + try testing.expect_equal(3, ast.function_param_count(&tree, declaration)) + first :: ast.param(&tree, ast.function_param(&tree, declaration, 0)) + second :: ast.param(&tree, ast.function_param(&tree, declaration, 1)) + third :: ast.param(&tree, ast.function_param(&tree, declaration, 2)) + try testing.expect_equal("a", ast.source_text(&tree, first.name)) + try testing.expect_equal("b", ast.source_text(&tree, second.name)) + try testing.expect_equal("c", ast.source_text(&tree, third.name)) + try testing.expect_equal(first.type, second.type) + try testing.expect(first.type != third.type) + try testing.expect_equal(1, ast.block_statement_count(&tree, function.body)) + try testing.expect_equal(0, diagnostics.len(&diagnostic_store)) } +