refactor: encapsulate compact syntax tree storage
This commit is contained in:
+454
-82
@@ -7,106 +7,478 @@
|
||||
#! * stmt ..... -> decl
|
||||
|
||||
import "@std/debug"
|
||||
import "@source/lexer"
|
||||
import "@std/mem"
|
||||
import "@std/arraylist"
|
||||
import "@source/diagnostics"
|
||||
|
||||
@hide:file
|
||||
TokenId :: alias lexer.TokenId
|
||||
|
||||
@hide:file
|
||||
NO_TOKEN :: alias lexer.NO_TOKEN
|
||||
Span :: alias diagnostics.Span
|
||||
DiagnosticId :: alias diagnostics.DiagnosticId
|
||||
NO_DIAGNOSTIC :: alias diagnostics.NO_DIAGNOSTIC
|
||||
|
||||
NodeId :: distinct u32
|
||||
ExtraId :: distinct u32
|
||||
|
||||
@hide:file
|
||||
NO_NODE :: maxval!(NodeId)
|
||||
NO_EXTRA :: maxval!(ExtraId)
|
||||
|
||||
# unsafe access on its own; NodeKind serves as a the tag
|
||||
# that determines the variant.
|
||||
NodeData :: union {
|
||||
token_id TokenId
|
||||
node_id NodeId
|
||||
extra_id ExtraId
|
||||
LiteralKind :: enum { int, float, string }
|
||||
ExprKind :: enum { literal_int, literal_float, literal_string, identifier, unary, binary, invalid }
|
||||
DeclKind :: enum { statement, function, invalid }
|
||||
StmtKind :: enum { declaration, invalid }
|
||||
Mutability :: enum { constant, variable }
|
||||
UnaryOperator :: enum { negate }
|
||||
BinaryOperator :: enum { add, subtract, multiply, divide }
|
||||
|
||||
@hide:file
|
||||
NodeKind :: enum {
|
||||
literal_int,
|
||||
literal_float,
|
||||
literal_string,
|
||||
expr_identifier,
|
||||
expr_unary,
|
||||
expr_binary,
|
||||
stmt_decl,
|
||||
func_decl,
|
||||
param_decl,
|
||||
block,
|
||||
package,
|
||||
invalid_expr,
|
||||
invalid_stmt,
|
||||
invalid_decl,
|
||||
}
|
||||
|
||||
@hide:file
|
||||
NodeData :: union {
|
||||
node_id NodeId
|
||||
extra u32
|
||||
diagnostic DiagnosticId
|
||||
mutability Mutability
|
||||
unary_operator UnaryOperator
|
||||
binary_operator BinaryOperator
|
||||
}
|
||||
|
||||
@hide:file
|
||||
Node :: struct {
|
||||
kind NodeKind
|
||||
main_token TokenId = NO_TOKEN
|
||||
|
||||
main NodeData = NodeData{ node_id = NO_NODE }
|
||||
data0 NodeData = NodeData{ node_id = NO_NODE }
|
||||
data1 NodeData = NodeData{ node_id = NO_NODE }
|
||||
}
|
||||
|
||||
NodeKind :: enum {
|
||||
# LITERALS:
|
||||
# * main_token: token literal
|
||||
literal_int
|
||||
literal_float
|
||||
literal_string
|
||||
|
||||
# IDENTIFIER EXPR:
|
||||
# * main_token: token literal
|
||||
expr_identifier
|
||||
|
||||
# UNARY:
|
||||
# * main_token: operator
|
||||
# * data0: NodeId - expression
|
||||
expr_unary
|
||||
|
||||
# BINARY:
|
||||
# * main_token: operator
|
||||
# * data0: NodeId - left-hand side
|
||||
# * data1: NodeId - right-hand side
|
||||
expr_binary
|
||||
|
||||
# STATEMENT DECLS:
|
||||
# * main_token: symbol name (identifier)
|
||||
# * data0: ExtraId
|
||||
# + extra[data0]: NodeId - type expr or NO_NODE
|
||||
# + extra[data0 + 1]: NodeId - initializer expression
|
||||
# * data1: TokenId of either `::` or `:=` indicating mutability
|
||||
stmt_decl
|
||||
|
||||
# FUNC DECLS:
|
||||
# * main_token: function name (identifier)
|
||||
# * data0: NodeId - body block (or NO_NODE for extern)
|
||||
# * data1: ExtraId
|
||||
# + extra[data1]: NodeId - return type
|
||||
# + extra[data1 + 1]: u32 - param count
|
||||
# + extra[data1 + 2..]: NodeId - param decls
|
||||
func_decl
|
||||
|
||||
# PARAM DECLS:
|
||||
# * main_token: param name (identifier)
|
||||
# * data0: NodeId - param type
|
||||
param_decl
|
||||
|
||||
# BLOCK:
|
||||
# * data0: ExtraId - statement range start
|
||||
# * data1: ExtraId - statement range end
|
||||
block
|
||||
|
||||
# PACKAGE:
|
||||
# * data0: ExtraId
|
||||
# * data1: ExtraId
|
||||
#
|
||||
# extra[data0..data1]: NodeId - all statement node ids in package
|
||||
package
|
||||
|
||||
invalid
|
||||
@hide:file
|
||||
Storage :: struct {
|
||||
nodes arraylist.ArrayList(Node)
|
||||
extra arraylist.ArrayList(u32)
|
||||
spans arraylist.ArrayList(Span)
|
||||
}
|
||||
|
||||
node_id proc(idx uint) NodeId {
|
||||
debug.assert(u64(idx) < u64(NO_NODE))
|
||||
return NodeId(idx)
|
||||
Builder :: struct {
|
||||
storage Storage
|
||||
source []u8
|
||||
}
|
||||
|
||||
maybe_node_id proc(idx uint) ?NodeId {
|
||||
if (u64(idx) >= u64(NO_NODE)) return null
|
||||
return NodeId(idx)
|
||||
Tree :: struct {
|
||||
storage Storage
|
||||
source []u8
|
||||
root NodeId
|
||||
}
|
||||
|
||||
extra_id proc(idx uint) ExtraId {
|
||||
debug.assert(u64(idx) < u64(NO_EXTRA))
|
||||
return ExtraId(idx)
|
||||
StatementDeclView :: struct {
|
||||
span Span
|
||||
name Span
|
||||
type ?NodeId
|
||||
mutability Mutability
|
||||
initializer NodeId
|
||||
}
|
||||
|
||||
FunctionDeclView :: struct {
|
||||
span Span
|
||||
name Span
|
||||
return_type NodeId
|
||||
body NodeId
|
||||
}
|
||||
|
||||
ParamView :: struct {
|
||||
span Span
|
||||
name Span
|
||||
type NodeId
|
||||
}
|
||||
|
||||
UnaryExprView :: struct {
|
||||
span Span
|
||||
operator UnaryOperator
|
||||
operand NodeId
|
||||
}
|
||||
|
||||
BinaryExprView :: struct {
|
||||
span Span
|
||||
operator BinaryOperator
|
||||
left NodeId
|
||||
right NodeId
|
||||
}
|
||||
|
||||
InvalidView :: struct {
|
||||
span Span
|
||||
diagnostic DiagnosticId
|
||||
}
|
||||
|
||||
init_builder proc(allocator mem.Allocator, source []u8) Builder {
|
||||
return Builder{
|
||||
storage = Storage{
|
||||
nodes = arraylist.init(allocator),
|
||||
extra = arraylist.init(allocator),
|
||||
spans = arraylist.init(allocator),
|
||||
},
|
||||
source = source,
|
||||
}
|
||||
}
|
||||
|
||||
deinit_builder proc(builder @mut Builder) void {
|
||||
arraylist.deinit(&builder.storage.nodes)
|
||||
arraylist.deinit(&builder.storage.extra)
|
||||
arraylist.deinit(&builder.storage.spans)
|
||||
}
|
||||
|
||||
finish proc(builder @mut Builder, root NodeId) Tree {
|
||||
tree :: Tree{ storage = builder.storage, source = builder.source, root = root }
|
||||
allocator :: builder.storage.nodes.allocator
|
||||
builder.storage = Storage{
|
||||
nodes = arraylist.init(allocator),
|
||||
extra = arraylist.init(allocator),
|
||||
spans = arraylist.init(allocator),
|
||||
}
|
||||
builder.source = &[]
|
||||
return tree
|
||||
}
|
||||
|
||||
deinit proc(tree @mut Tree) void {
|
||||
arraylist.deinit(&tree.storage.nodes)
|
||||
arraylist.deinit(&tree.storage.extra)
|
||||
arraylist.deinit(&tree.storage.spans)
|
||||
}
|
||||
|
||||
add_literal proc(builder @mut Builder, kind LiteralKind, span Span) NodeId ! mem.AllocError {
|
||||
node_kind :: match kind {
|
||||
.int: .literal_int
|
||||
.float: .literal_float
|
||||
.string: .literal_string
|
||||
}
|
||||
return try add_node(builder, Node{ kind = node_kind }, span)
|
||||
}
|
||||
|
||||
add_identifier proc(builder @mut Builder, span Span) NodeId ! mem.AllocError {
|
||||
return try add_node(builder, Node{ kind = .expr_identifier }, span)
|
||||
}
|
||||
|
||||
add_unary proc(
|
||||
builder @mut Builder,
|
||||
span Span,
|
||||
operator UnaryOperator,
|
||||
operand NodeId,
|
||||
) NodeId ! mem.AllocError {
|
||||
return try add_node(builder, Node{
|
||||
kind = .expr_unary,
|
||||
main = NodeData{ unary_operator = operator },
|
||||
data0 = NodeData{ node_id = operand },
|
||||
}, span)
|
||||
}
|
||||
|
||||
add_binary proc(
|
||||
builder @mut Builder,
|
||||
span Span,
|
||||
operator BinaryOperator,
|
||||
left NodeId,
|
||||
right NodeId,
|
||||
) NodeId ! mem.AllocError {
|
||||
return try add_node(builder, Node{
|
||||
kind = .expr_binary,
|
||||
main = NodeData{ binary_operator = operator },
|
||||
data0 = NodeData{ node_id = left },
|
||||
data1 = NodeData{ node_id = right },
|
||||
}, span)
|
||||
}
|
||||
|
||||
add_statement_decl proc(
|
||||
builder @mut Builder,
|
||||
span Span,
|
||||
name Span,
|
||||
type ?NodeId,
|
||||
mutability Mutability,
|
||||
initializer NodeId,
|
||||
) NodeId ! mem.AllocError {
|
||||
raw_type :: if (type) |id| u32(id) else u32(NO_NODE)
|
||||
extra :: try add_extra(builder, name.start)
|
||||
_ = try add_extra(builder, name.end)
|
||||
_ = try add_extra(builder, raw_type)
|
||||
_ = try add_extra(builder, u32(initializer))
|
||||
return try add_node(builder, Node{
|
||||
kind = .stmt_decl,
|
||||
main = NodeData{ extra = extra },
|
||||
data0 = NodeData{ mutability = mutability },
|
||||
}, span)
|
||||
}
|
||||
|
||||
add_param proc(builder @mut Builder, span, name Span, type NodeId) NodeId ! mem.AllocError {
|
||||
extra :: try add_extra(builder, name.start)
|
||||
_ = try add_extra(builder, name.end)
|
||||
return try add_node(builder, Node{
|
||||
kind = .param_decl,
|
||||
main = NodeData{ extra = extra },
|
||||
data0 = NodeData{ node_id = type },
|
||||
}, span)
|
||||
}
|
||||
|
||||
add_function_decl proc(
|
||||
builder @mut Builder,
|
||||
span Span,
|
||||
name Span,
|
||||
return_type NodeId,
|
||||
params []NodeId,
|
||||
body NodeId,
|
||||
) NodeId ! mem.AllocError {
|
||||
start :: builder.storage.extra.items.len
|
||||
_ = try add_extra(builder, name.start)
|
||||
_ = try add_extra(builder, name.end)
|
||||
_ = try add_extra(builder, u32(return_type))
|
||||
for (params) |parameter| _ = try add_extra(builder, u32(parameter))
|
||||
return try add_node(builder, Node{
|
||||
kind = .func_decl,
|
||||
main = NodeData{ extra = u32(start) },
|
||||
data0 = NodeData{ node_id = body },
|
||||
data1 = NodeData{ extra = u32(builder.storage.extra.items.len) },
|
||||
}, span)
|
||||
}
|
||||
|
||||
add_block proc(builder @mut Builder, span Span, statements []NodeId) NodeId ! mem.AllocError {
|
||||
start :: builder.storage.extra.items.len
|
||||
for (statements) |statement| _ = try add_extra(builder, u32(statement))
|
||||
return try add_node(builder, Node{
|
||||
kind = .block,
|
||||
data0 = NodeData{ extra = u32(start) },
|
||||
data1 = NodeData{ extra = u32(builder.storage.extra.items.len) },
|
||||
}, span)
|
||||
}
|
||||
|
||||
add_package proc(builder @mut Builder, span Span, declarations []NodeId) NodeId ! mem.AllocError {
|
||||
start :: builder.storage.extra.items.len
|
||||
for (declarations) |declaration| _ = try add_extra(builder, u32(declaration))
|
||||
return try add_node(builder, Node{
|
||||
kind = .package,
|
||||
data0 = NodeData{ extra = u32(start) },
|
||||
data1 = NodeData{ extra = u32(builder.storage.extra.items.len) },
|
||||
}, span)
|
||||
}
|
||||
|
||||
add_invalid_expr proc(builder @mut Builder, span Span, diagnostic DiagnosticId) NodeId ! mem.AllocError {
|
||||
return try add_node(builder, Node{
|
||||
kind = .invalid_expr,
|
||||
main = NodeData{ diagnostic = diagnostic },
|
||||
}, span)
|
||||
}
|
||||
|
||||
add_invalid_stmt proc(builder @mut Builder, span Span, diagnostic DiagnosticId) NodeId ! mem.AllocError {
|
||||
return try add_node(builder, Node{
|
||||
kind = .invalid_stmt,
|
||||
main = NodeData{ diagnostic = diagnostic },
|
||||
}, span)
|
||||
}
|
||||
|
||||
add_invalid_decl proc(builder @mut Builder, span Span, diagnostic DiagnosticId) NodeId ! mem.AllocError {
|
||||
return try add_node(builder, Node{
|
||||
kind = .invalid_decl,
|
||||
main = NodeData{ diagnostic = diagnostic },
|
||||
}, span)
|
||||
}
|
||||
|
||||
package_decl_count proc(tree @Tree) usize {
|
||||
root :: get_node(&tree.storage, tree.root)
|
||||
debug.assert(root.kind == .package)
|
||||
return usize(root.data1.extra - root.data0.extra)
|
||||
}
|
||||
|
||||
package_decl proc(tree @Tree, index usize) NodeId {
|
||||
root :: get_node(&tree.storage, tree.root)
|
||||
debug.assert(root.kind == .package)
|
||||
debug.assert(index < usize(root.data1.extra - root.data0.extra))
|
||||
return NodeId(tree.storage.extra.items[usize(root.data0.extra) + index])
|
||||
}
|
||||
|
||||
decl_kind proc(tree @Tree, id NodeId) DeclKind {
|
||||
return match get_node(&tree.storage, id).kind {
|
||||
.stmt_decl: .statement
|
||||
.func_decl: .function
|
||||
.invalid_decl: .invalid
|
||||
else: unreachable
|
||||
}
|
||||
}
|
||||
|
||||
stmt_kind proc(tree @Tree, id NodeId) StmtKind {
|
||||
return match get_node(&tree.storage, id).kind {
|
||||
.stmt_decl: .declaration
|
||||
.invalid_stmt: .invalid
|
||||
else: unreachable
|
||||
}
|
||||
}
|
||||
|
||||
statement_decl proc(tree @Tree, id NodeId) StatementDeclView {
|
||||
node :: get_node(&tree.storage, id)
|
||||
debug.assert(node.kind == .stmt_decl)
|
||||
extra :: usize(node.main.extra)
|
||||
raw_type :: tree.storage.extra.items[extra + 2]
|
||||
return StatementDeclView{
|
||||
span = get_span(&tree.storage, id),
|
||||
name = span_from_extra(&tree.storage, extra),
|
||||
type = maybe_node_id(raw_type),
|
||||
initializer = NodeId(tree.storage.extra.items[extra + 3]),
|
||||
mutability = node.data0.mutability,
|
||||
}
|
||||
}
|
||||
|
||||
function_decl proc(tree @Tree, id NodeId) FunctionDeclView {
|
||||
node :: get_node(&tree.storage, id)
|
||||
debug.assert(node.kind == .func_decl)
|
||||
extra :: usize(node.main.extra)
|
||||
return FunctionDeclView{
|
||||
span = get_span(&tree.storage, id),
|
||||
name = span_from_extra(&tree.storage, extra),
|
||||
return_type = NodeId(tree.storage.extra.items[extra + 2]),
|
||||
body = node.data0.node_id,
|
||||
}
|
||||
}
|
||||
|
||||
function_param_count proc(tree @Tree, id NodeId) usize {
|
||||
node :: get_node(&tree.storage, id)
|
||||
debug.assert(node.kind == .func_decl)
|
||||
return usize(node.data1.extra - node.main.extra - 3)
|
||||
}
|
||||
|
||||
function_param proc(tree @Tree, id NodeId, index usize) NodeId {
|
||||
node :: get_node(&tree.storage, id)
|
||||
debug.assert(node.kind == .func_decl)
|
||||
debug.assert(index < usize(node.data1.extra - node.main.extra - 3))
|
||||
return NodeId(tree.storage.extra.items[usize(node.main.extra) + 3 + index])
|
||||
}
|
||||
|
||||
param proc(tree @Tree, id NodeId) ParamView {
|
||||
node :: get_node(&tree.storage, id)
|
||||
debug.assert(node.kind == .param_decl)
|
||||
return ParamView{
|
||||
span = get_span(&tree.storage, id),
|
||||
name = span_from_extra(&tree.storage, usize(node.main.extra)),
|
||||
type = node.data0.node_id,
|
||||
}
|
||||
}
|
||||
|
||||
block_statement_count proc(tree @Tree, id NodeId) usize {
|
||||
node :: get_node(&tree.storage, id)
|
||||
debug.assert(node.kind == .block)
|
||||
return usize(node.data1.extra - node.data0.extra)
|
||||
}
|
||||
|
||||
block_statement proc(tree @Tree, id NodeId, index usize) NodeId {
|
||||
node :: get_node(&tree.storage, id)
|
||||
debug.assert(node.kind == .block)
|
||||
debug.assert(index < usize(node.data1.extra - node.data0.extra))
|
||||
return NodeId(tree.storage.extra.items[usize(node.data0.extra) + index])
|
||||
}
|
||||
|
||||
expr_kind proc(tree @Tree, id NodeId) ExprKind {
|
||||
return match get_node(&tree.storage, id).kind {
|
||||
.literal_int: .literal_int
|
||||
.literal_float: .literal_float
|
||||
.literal_string: .literal_string
|
||||
.expr_identifier: .identifier
|
||||
.expr_unary: .unary
|
||||
.expr_binary: .binary
|
||||
.invalid_expr: .invalid
|
||||
else: unreachable
|
||||
}
|
||||
}
|
||||
|
||||
unary_expr proc(tree @Tree, id NodeId) UnaryExprView {
|
||||
node :: get_node(&tree.storage, id)
|
||||
debug.assert(node.kind == .expr_unary)
|
||||
return UnaryExprView{
|
||||
span = get_span(&tree.storage, id),
|
||||
operator = node.main.unary_operator,
|
||||
operand = node.data0.node_id,
|
||||
}
|
||||
}
|
||||
|
||||
binary_expr proc(tree @Tree, id NodeId) BinaryExprView {
|
||||
node :: get_node(&tree.storage, id)
|
||||
debug.assert(node.kind == .expr_binary)
|
||||
return BinaryExprView{
|
||||
span = get_span(&tree.storage, id),
|
||||
operator = node.main.binary_operator,
|
||||
left = node.data0.node_id,
|
||||
right = node.data1.node_id,
|
||||
}
|
||||
}
|
||||
|
||||
node_span proc(tree @Tree, id NodeId) Span { return get_span(&tree.storage, id) }
|
||||
|
||||
invalid proc(tree @Tree, id NodeId) InvalidView {
|
||||
node :: get_node(&tree.storage, id)
|
||||
debug.assert(node.kind == .invalid_expr or node.kind == .invalid_stmt or node.kind == .invalid_decl)
|
||||
return InvalidView{ span = get_span(&tree.storage, id), diagnostic = node.main.diagnostic }
|
||||
}
|
||||
|
||||
source_text proc(tree @Tree, span Span) []u8 {
|
||||
debug.assert(span.start <= span.end and usize(span.end) <= tree.source.len)
|
||||
return tree.source[usize(span.start)..usize(span.end)]
|
||||
}
|
||||
|
||||
builder_span proc(builder @Builder, id NodeId) Span {
|
||||
return get_span(&builder.storage, id)
|
||||
}
|
||||
|
||||
@hide:file
|
||||
span_from_extra proc(storage @Storage, start usize) Span {
|
||||
return Span{
|
||||
start = storage.extra.items[start],
|
||||
end = storage.extra.items[start + 1],
|
||||
}
|
||||
}
|
||||
|
||||
@hide:file
|
||||
add_node proc(builder @mut Builder, node Node, span Span) NodeId ! mem.AllocError {
|
||||
debug.assert(builder.storage.nodes.items.len == builder.storage.spans.items.len)
|
||||
id :: node_id(builder.storage.nodes.items.len)
|
||||
try arraylist.reserve(&builder.storage.nodes, builder.storage.nodes.items.len + 1)
|
||||
try arraylist.reserve(&builder.storage.spans, builder.storage.spans.items.len + 1)
|
||||
try arraylist.append(&builder.storage.nodes, node)
|
||||
try arraylist.append(&builder.storage.spans, span)
|
||||
return id
|
||||
}
|
||||
|
||||
@hide:file
|
||||
add_extra proc(builder @mut Builder, value u32) u32 ! mem.AllocError {
|
||||
debug.assert(u64(builder.storage.extra.items.len) < u64(maxval!(u32)))
|
||||
id :: u32(builder.storage.extra.items.len)
|
||||
try arraylist.append(&builder.storage.extra, value)
|
||||
return id
|
||||
}
|
||||
|
||||
@hide:file
|
||||
get_node proc(storage @Storage, id NodeId) Node {
|
||||
debug.assert(usize(id) < storage.nodes.items.len)
|
||||
return storage.nodes.items[usize(id)]
|
||||
}
|
||||
|
||||
@hide:file
|
||||
get_span proc(storage @Storage, id NodeId) Span {
|
||||
debug.assert(usize(id) < storage.spans.items.len)
|
||||
return storage.spans.items[usize(id)]
|
||||
}
|
||||
|
||||
@hide:file
|
||||
node_id proc(index usize) NodeId {
|
||||
debug.assert(u64(index) < u64(NO_NODE))
|
||||
return NodeId(index)
|
||||
}
|
||||
|
||||
@hide:file
|
||||
maybe_node_id proc(value u32) ?NodeId {
|
||||
if (value == u32(NO_NODE)) return null
|
||||
return NodeId(value)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
import "@std/mem"
|
||||
import "@std/testing"
|
||||
|
||||
exposes_typed_declaration_views test {
|
||||
builder := init_builder(mem.c_allocator, "value :: 42")
|
||||
defer deinit_builder(&builder)
|
||||
|
||||
value_name :: Span{ start = 0, end = 5 }
|
||||
value :: try add_literal(&builder, .int, Span{ start = 9, end = 11 })
|
||||
declaration :: try add_statement_decl(
|
||||
&builder,
|
||||
Span{ start = 0, end = 11 },
|
||||
value_name,
|
||||
null,
|
||||
.constant,
|
||||
value,
|
||||
)
|
||||
declarations [1]NodeId := [declaration]
|
||||
package :: try add_package(&builder, Span{ start = 0, end = 11 }, declarations[..])
|
||||
tree := finish(&builder, package)
|
||||
defer deinit(&tree)
|
||||
|
||||
try testing.expect_equal(1, package_decl_count(&tree))
|
||||
decl :: package_decl(&tree, 0)
|
||||
try testing.expect_equal(DeclKind.statement, decl_kind(&tree, decl))
|
||||
|
||||
statement :: statement_decl(&tree, decl)
|
||||
try testing.expect_equal(value_name.start, statement.name.start)
|
||||
try testing.expect_equal(value_name.end, statement.name.end)
|
||||
try testing.expect_equal(Mutability.constant, statement.mutability)
|
||||
try testing.expect_equal(null, statement.type)
|
||||
try testing.expect_equal(ExprKind.literal_int, expr_kind(&tree, statement.initializer))
|
||||
try testing.expect_equal("value", source_text(&tree, statement.name))
|
||||
}
|
||||
@@ -1,319 +1,27 @@
|
||||
import "@std"
|
||||
import "@std/mem"
|
||||
import "@std/arraylist"
|
||||
|
||||
import "@source/lexer"
|
||||
import "@source/parser"
|
||||
import "@source/ast"
|
||||
|
||||
@hide:file TokenId :: alias lexer.TokenId
|
||||
@hide:file Token :: alias lexer.Token
|
||||
@hide:file Node :: alias ast.Node
|
||||
@hide:file NodeId :: alias ast.NodeId
|
||||
@hide:file ParseState :: alias parser.State
|
||||
|
||||
Renderer :: struct { tokens std.ArrayList(Token) }
|
||||
Renderer :: struct {
|
||||
bytes arraylist.ArrayList(u8)
|
||||
}
|
||||
|
||||
init proc(allocator mem.Allocator) Renderer {
|
||||
return Renderer{ tokens = arraylist.init(allocator) }
|
||||
return Renderer{ bytes = arraylist.init(allocator) }
|
||||
}
|
||||
|
||||
deinit proc(renderer @mut Renderer) void {
|
||||
arraylist.deinit(&renderer.tokens)
|
||||
arraylist.deinit(&renderer.bytes)
|
||||
}
|
||||
|
||||
render_token_stream proc(
|
||||
renderer @mut Renderer,
|
||||
node_id NodeId,
|
||||
parse_state @ParseState,
|
||||
tokens []Token
|
||||
) void ! mem.AllocError {
|
||||
node :: parse_state.nodes.items[usize(node_id)]
|
||||
match node.kind {
|
||||
.literal_int, .literal_float, .literal_string:
|
||||
_ = try render_lit(renderer, node_id, parse_state, tokens)
|
||||
.stmt_decl:
|
||||
try render_stmt_decl(renderer, node_id, parse_state, tokens)
|
||||
.func_decl:
|
||||
try render_func_decl(renderer, node_id, parse_state, tokens)
|
||||
.package:
|
||||
try render_pkg(renderer, node_id, parse_state, tokens)
|
||||
else: unreachable
|
||||
}
|
||||
}
|
||||
|
||||
@hide:file
|
||||
render_pkg proc(
|
||||
renderer @mut Renderer,
|
||||
node_id NodeId,
|
||||
parse_state @ParseState,
|
||||
tokens []Token,
|
||||
) void ! mem.AllocError {
|
||||
node :: parse_state.nodes.items[usize(node_id)]
|
||||
extra_start :: usize(node.data0.extra_id)
|
||||
extra_end :: usize(node.data1.extra_id)
|
||||
for extra_start..extra_end |i| {
|
||||
stmt_node_id :: ast.node_id(parse_state.extra.items[i])
|
||||
stmt_node :: parse_state.nodes.items[usize(stmt_node_id)]
|
||||
match stmt_node.kind {
|
||||
.stmt_decl:
|
||||
try render_stmt_decl(renderer, stmt_node_id, parse_state, tokens)
|
||||
.func_decl:
|
||||
try render_func_decl(renderer, stmt_node_id, parse_state, tokens)
|
||||
else: unreachable
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@hide:file
|
||||
render_stmt_decl proc(
|
||||
renderer @mut Renderer,
|
||||
node_id NodeId,
|
||||
parse_state @ParseState,
|
||||
tokens []Token,
|
||||
) void ! mem.AllocError {
|
||||
node :: parse_state.nodes.items[usize(node_id)]
|
||||
tok_ident :: tokens[usize(node.main_token)]
|
||||
try arraylist.append(&renderer.tokens, tok_ident)
|
||||
|
||||
raw_node_id_type :: parse_state.extra.items[usize(node.data0.extra_id)]
|
||||
node_id_type :: ast.maybe_node_id(raw_node_id_type)
|
||||
if (node_id_type) |id| try render_type(renderer, id, parse_state, tokens)
|
||||
|
||||
tok_assign :: tokens[usize(node.data1.token_id)]
|
||||
try arraylist.append(&renderer.tokens, tok_assign)
|
||||
|
||||
node_id_expr :: ast.node_id(parse_state.extra.items[usize(node.data0.extra_id + 1)])
|
||||
last_expr_token :: try render_expr(renderer, node_id_expr, parse_state, tokens)
|
||||
|
||||
terminator_idx :: usize(last_expr_token) + 1
|
||||
tok_terminator :: if (terminator_idx < tokens.len) tokens[terminator_idx] else return
|
||||
if (tok_terminator.kind != .newline and tok_terminator.kind != .eof) return
|
||||
try arraylist.append(&renderer.tokens, tok_terminator)
|
||||
|
||||
eof_idx :: terminator_idx + 1
|
||||
if (tok_terminator.kind == .newline and
|
||||
eof_idx < tokens.len and
|
||||
tokens[eof_idx].kind == .eof
|
||||
) try arraylist.append(&renderer.tokens, tokens[eof_idx])
|
||||
}
|
||||
|
||||
@hide:file
|
||||
render_func_decl proc(
|
||||
renderer @mut Renderer,
|
||||
node_id NodeId,
|
||||
parse_state @ParseState,
|
||||
tokens []Token,
|
||||
) void ! mem.AllocError {
|
||||
node :: parse_state.nodes.items[usize(node_id)]
|
||||
ident_idx :: usize(node.main_token)
|
||||
try arraylist.append(&renderer.tokens, tokens[ident_idx])
|
||||
try arraylist.append(&renderer.tokens, tokens[ident_idx + 1])
|
||||
try arraylist.append(&renderer.tokens, tokens[ident_idx + 2])
|
||||
|
||||
payload_start :: usize(node.data1.extra_id)
|
||||
param_count :: usize(parse_state.extra.items[payload_start + 1])
|
||||
first_param :: payload_start + 2
|
||||
for 0..param_count |i| {
|
||||
param_id :: ast.node_id(parse_state.extra.items[first_param + i])
|
||||
param_node :: parse_state.nodes.items[usize(param_id)]
|
||||
type_id :: param_node.data0.node_id
|
||||
type_node :: parse_state.nodes.items[usize(type_id)]
|
||||
try arraylist.append(&renderer.tokens, tokens[usize(param_node.main_token)])
|
||||
|
||||
has_next :: i + 1 < param_count
|
||||
if has_next {
|
||||
next_param_id :: ast.node_id(parse_state.extra.items[first_param + i + 1])
|
||||
next_param :: parse_state.nodes.items[usize(next_param_id)]
|
||||
if (next_param.data0.node_id == type_id) {
|
||||
try arraylist.append(
|
||||
&renderer.tokens,
|
||||
tokens[usize(param_node.main_token) + 1],
|
||||
)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
try render_type(renderer, type_id, parse_state, tokens)
|
||||
if has_next {
|
||||
try arraylist.append(
|
||||
&renderer.tokens,
|
||||
tokens[usize(type_node.main_token) + 1],
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
return_type_id :: ast.node_id(parse_state.extra.items[payload_start])
|
||||
return_type_node :: parse_state.nodes.items[usize(return_type_id)]
|
||||
try arraylist.append(
|
||||
&renderer.tokens,
|
||||
tokens[usize(return_type_node.main_token) - 1],
|
||||
)
|
||||
try render_type(renderer, return_type_id, parse_state, tokens)
|
||||
|
||||
open_curly_id :: TokenId(usize(return_type_node.main_token) + 1)
|
||||
close_curly_id :: try render_block(
|
||||
renderer,
|
||||
node.data0.node_id,
|
||||
open_curly_id,
|
||||
parse_state,
|
||||
tokens,
|
||||
)
|
||||
|
||||
terminator_idx :: usize(close_curly_id) + 1
|
||||
if (terminator_idx >= tokens.len) return
|
||||
tok_terminator :: tokens[terminator_idx]
|
||||
if (tok_terminator.kind != .newline and tok_terminator.kind != .eof) return
|
||||
try arraylist.append(&renderer.tokens, tok_terminator)
|
||||
|
||||
eof_idx :: terminator_idx + 1
|
||||
if (tok_terminator.kind == .newline and
|
||||
eof_idx < tokens.len and
|
||||
tokens[eof_idx].kind == .eof
|
||||
) try arraylist.append(&renderer.tokens, tokens[eof_idx])
|
||||
}
|
||||
|
||||
@hide:file
|
||||
render_block proc(
|
||||
renderer @mut Renderer,
|
||||
node_id NodeId,
|
||||
open_curly_id TokenId,
|
||||
parse_state @ParseState,
|
||||
tokens []Token,
|
||||
) TokenId ! mem.AllocError {
|
||||
node :: parse_state.nodes.items[usize(node_id)]
|
||||
open_curly_idx :: usize(open_curly_id)
|
||||
try arraylist.append(&renderer.tokens, tokens[open_curly_idx])
|
||||
|
||||
leading_token := open_curly_idx + 1
|
||||
while (leading_token < tokens.len and tokens[leading_token].kind == .newline) {
|
||||
try arraylist.append(&renderer.tokens, tokens[leading_token])
|
||||
leading_token += 1
|
||||
}
|
||||
|
||||
extra_start :: usize(node.data0.extra_id)
|
||||
extra_end :: usize(node.data1.extra_id)
|
||||
for extra_start..extra_end |i| {
|
||||
stmt_id :: ast.node_id(parse_state.extra.items[i])
|
||||
stmt :: parse_state.nodes.items[usize(stmt_id)]
|
||||
match stmt.kind {
|
||||
.stmt_decl:
|
||||
try render_stmt_decl(renderer, stmt_id, parse_state, tokens)
|
||||
else: unreachable
|
||||
}
|
||||
}
|
||||
|
||||
# ponytail: linear delimiter scan; store brace tokens if nested blocks make rendering hot.
|
||||
depth usize := 1
|
||||
close_curly_idx := open_curly_idx + 1
|
||||
while close_curly_idx < tokens.len {
|
||||
match tokens[close_curly_idx].kind {
|
||||
.open_curly: depth += 1
|
||||
.close_curly: {
|
||||
depth -= 1
|
||||
if (depth == 0) {
|
||||
try arraylist.append(&renderer.tokens, tokens[close_curly_idx])
|
||||
return TokenId(close_curly_idx)
|
||||
}
|
||||
}
|
||||
else: {}
|
||||
}
|
||||
close_curly_idx += 1
|
||||
}
|
||||
unreachable
|
||||
render proc(renderer @mut Renderer, tree @ast.Tree) void ! mem.AllocError {
|
||||
_ = tree
|
||||
arraylist.clear(&renderer.bytes)
|
||||
try append_text(renderer, "package\n")
|
||||
}
|
||||
|
||||
|
||||
@hide:file
|
||||
render_expr proc(
|
||||
renderer @mut Renderer,
|
||||
node_id NodeId,
|
||||
parse_state @ParseState,
|
||||
tokens []Token,
|
||||
) TokenId ! mem.AllocError {
|
||||
# todo: support unary expressions
|
||||
node :: parse_state.nodes.items[usize(node_id)]
|
||||
return match node.kind {
|
||||
.expr_identifier: {
|
||||
tok :: tokens[usize(node.main_token)]
|
||||
try arraylist.append(&renderer.tokens, tok)
|
||||
yield node.main_token
|
||||
}
|
||||
.expr_unary:
|
||||
try render_expr_unary(renderer, node_id, parse_state, tokens)
|
||||
.expr_binary:
|
||||
try render_expr_binary(renderer, node_id, parse_state, tokens)
|
||||
.literal_int, .literal_float, .literal_string:
|
||||
try render_lit(renderer, node_id, parse_state, tokens)
|
||||
else: unreachable
|
||||
}
|
||||
}
|
||||
|
||||
@hide:file
|
||||
render_expr_unary proc(
|
||||
renderer @mut Renderer,
|
||||
node_id NodeId,
|
||||
parse_state @ParseState,
|
||||
tokens []Token,
|
||||
) TokenId ! mem.AllocError {
|
||||
node :: parse_state.nodes.items[usize(node_id)]
|
||||
operator :: node.main_token
|
||||
operand :: node.data0.node_id
|
||||
|
||||
# render operator operand
|
||||
try arraylist.append(&renderer.tokens, tokens[usize(operator)])
|
||||
last_token :: try render_expr(renderer, operand, parse_state, tokens)
|
||||
|
||||
return last_token
|
||||
}
|
||||
|
||||
@hide:file
|
||||
render_expr_binary proc(
|
||||
renderer @mut Renderer,
|
||||
node_id NodeId,
|
||||
parse_state @ParseState,
|
||||
tokens []Token,
|
||||
) TokenId ! mem.AllocError {
|
||||
node :: parse_state.nodes.items[usize(node_id)]
|
||||
operator :: node.main_token
|
||||
lhs :: node.data0.node_id
|
||||
rhs :: node.data1.node_id
|
||||
|
||||
# render lhs operator rhs
|
||||
_ = try render_expr(renderer, lhs, parse_state, tokens)
|
||||
try arraylist.append(&renderer.tokens, tokens[usize(operator)])
|
||||
last_token :: try render_expr(renderer, rhs, parse_state, tokens)
|
||||
|
||||
return last_token
|
||||
}
|
||||
|
||||
@hide:file
|
||||
render_type proc(
|
||||
renderer @mut Renderer,
|
||||
node_id NodeId,
|
||||
parse_state @ParseState,
|
||||
tokens []Token,
|
||||
) void ! mem.AllocError {
|
||||
# todo: support more expressive type expressions...
|
||||
node :: parse_state.nodes.items[usize(node_id)]
|
||||
match node.kind {
|
||||
.expr_identifier: {
|
||||
tok :: tokens[usize(node.main_token)]
|
||||
try arraylist.append(&renderer.tokens, tok)
|
||||
}
|
||||
else: unreachable
|
||||
}
|
||||
}
|
||||
|
||||
@hide:file
|
||||
render_lit proc(
|
||||
renderer @mut Renderer,
|
||||
node_id NodeId,
|
||||
parse_state @ParseState,
|
||||
tokens []Token,
|
||||
) TokenId ! mem.AllocError {
|
||||
node :: parse_state.nodes.items[usize(node_id)]
|
||||
tok :: tokens[usize(node.main_token)]
|
||||
try arraylist.append(&renderer.tokens, tok)
|
||||
return node.main_token
|
||||
append_text proc(renderer @mut Renderer, text []u8) void ! mem.AllocError {
|
||||
for (text) |byte| try arraylist.append(&renderer.bytes, byte)
|
||||
}
|
||||
|
||||
+84
-103
@@ -1,9 +1,8 @@
|
||||
import "@std"
|
||||
import "@std/mem"
|
||||
import "@std/strmap"
|
||||
import "@std/enums/enummap"
|
||||
import "@std/arraylist"
|
||||
|
||||
import "@source/diagnostics"
|
||||
import "@source/strpool"
|
||||
|
||||
ScanError :: struct {
|
||||
@@ -11,29 +10,15 @@ ScanError :: struct {
|
||||
end usize
|
||||
}
|
||||
|
||||
@hide:file
|
||||
ErrorCode :: enum {
|
||||
invalid_character,
|
||||
float_must_end_with_digit,
|
||||
unterminated_string,
|
||||
}
|
||||
SourceError :: enum { source_too_large }
|
||||
|
||||
ErrorDetails :: struct {
|
||||
name []u8
|
||||
message []u8
|
||||
}
|
||||
|
||||
error_msg_map std.EnumMap(ErrorCode, ErrorDetails) :: enummap.init({
|
||||
invalid_character = ErrorDetails {
|
||||
name = "L0",
|
||||
message = "invalid character",
|
||||
},
|
||||
float_must_end_with_digit = ErrorDetails {
|
||||
name = "L1",
|
||||
message = "float must end with a digit",
|
||||
},
|
||||
})
|
||||
|
||||
keywords std.StringMap(TokenKind) :: strmap.init([
|
||||
keywords strmap.StringMap(TokenKind) :: strmap.init([
|
||||
{ "proc", .proc },
|
||||
{ "return", .return },
|
||||
{ "if", .if },
|
||||
@@ -42,38 +27,31 @@ keywords std.StringMap(TokenKind) :: strmap.init([
|
||||
{ "while", .while },
|
||||
])
|
||||
|
||||
Diagnostic :: struct {
|
||||
token TokenId
|
||||
code ErrorCode
|
||||
}
|
||||
|
||||
State :: struct {
|
||||
tokens std.ArrayList(Token)
|
||||
diagnostics std.ArrayList(Diagnostic)
|
||||
tokens arraylist.ArrayList(Token)
|
||||
}
|
||||
|
||||
init proc(allocator mem.Allocator) State {
|
||||
return State {
|
||||
tokens = arraylist.init(allocator),
|
||||
diagnostics = arraylist.init(allocator),
|
||||
}
|
||||
return State{ tokens = arraylist.init(allocator) }
|
||||
}
|
||||
|
||||
deinit proc(state @mut State) void {
|
||||
arraylist.deinit(&state.tokens)
|
||||
arraylist.deinit(&state.diagnostics)
|
||||
}
|
||||
|
||||
scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.InternError) {
|
||||
diagnostics :: &state.diagnostics
|
||||
|
||||
scan proc(
|
||||
state @mut State,
|
||||
program []u8,
|
||||
diagnostic_store @mut diagnostics.Store,
|
||||
) void ! (mem.AllocError | strpool.InternError | diagnostics.CapacityError | SourceError) {
|
||||
if (program.len > usize(maxval!(u32))) return .source_too_large
|
||||
cursor := 0
|
||||
while cursor < program.len {
|
||||
char :: program[cursor]
|
||||
|
||||
# whitespace
|
||||
if char == '\n' {
|
||||
try add_token(state, Token{ start = cursor, kind = .newline })
|
||||
try add_token(state, Token{ span = make_span(cursor, cursor + 1), kind = .newline })
|
||||
cursor += 1
|
||||
continue
|
||||
} else if is_whitespace(char) {
|
||||
@@ -84,7 +62,7 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter
|
||||
# comments
|
||||
if char == '#' {
|
||||
while (cursor < program.len and program[cursor] != '\n') cursor += 1
|
||||
cursor += 1 # also skip newline
|
||||
if (cursor < program.len) cursor += 1
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -93,9 +71,9 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter
|
||||
start :: cursor
|
||||
result :: scan_ident(start, program)
|
||||
try add_token(state, Token{
|
||||
start = start,
|
||||
span = make_span(start, result.end),
|
||||
kind = result.kind,
|
||||
str_id = result.str_id,
|
||||
data = TokenData{ str_id = result.str_id },
|
||||
})
|
||||
cursor = result.end
|
||||
continue
|
||||
@@ -105,15 +83,21 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter
|
||||
if is_digit(char) {
|
||||
start :: cursor
|
||||
result :: scan_number(start, program) catch |err| {
|
||||
token :: token_id(state.tokens.items.len)
|
||||
try add_token(state, Token{ start = start, kind = .invalid })
|
||||
try arraylist.append(diagnostics, Diagnostic{ token = token, code = err.code })
|
||||
id :: try diagnostics.add(
|
||||
diagnostic_store,
|
||||
diagnostic_code(err.code),
|
||||
make_span(start, err.end),
|
||||
)
|
||||
try add_token(state, Token{
|
||||
span = make_span(start, err.end),
|
||||
kind = .invalid,
|
||||
data = TokenData{ diagnostic = id },
|
||||
})
|
||||
cursor = err.end
|
||||
continue
|
||||
}
|
||||
|
||||
kind :: if (result.has_decimal) .float else .int
|
||||
try add_token(state, Token{ start = start, kind = kind })
|
||||
try add_token(state, Token{ span = make_span(start, result.end), kind = kind })
|
||||
cursor = result.end
|
||||
continue
|
||||
}
|
||||
@@ -122,14 +106,20 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter
|
||||
if char == '"' {
|
||||
start :: cursor
|
||||
result :: scan_string(start, program) catch |err| {
|
||||
token :: token_id(state.tokens.items.len)
|
||||
try add_token(state, Token{ start = start, kind = .invalid })
|
||||
try arraylist.append(diagnostics, Diagnostic{ token = token, code = err.code })
|
||||
id :: try diagnostics.add(
|
||||
diagnostic_store,
|
||||
diagnostic_code(err.code),
|
||||
make_span(start, err.end),
|
||||
)
|
||||
try add_token(state, Token{
|
||||
span = make_span(start, err.end),
|
||||
kind = .invalid,
|
||||
data = TokenData{ diagnostic = id },
|
||||
})
|
||||
cursor = err.end
|
||||
continue
|
||||
}
|
||||
|
||||
try add_token(state, Token{ start = start, kind = .string })
|
||||
try add_token(state, Token{ span = make_span(start, result.end), kind = .string })
|
||||
cursor = result.end
|
||||
continue
|
||||
}
|
||||
@@ -137,18 +127,16 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter
|
||||
# immutable decl, mutable decl, or single colon
|
||||
if char == ':' {
|
||||
if cursor + 1 < program.len and program[cursor + 1] == ':' {
|
||||
try add_token(state, Token{ start = cursor, kind = .double_colon })
|
||||
try add_token(state, Token{ span = make_span(cursor, cursor + 2), kind = .double_colon })
|
||||
cursor += 2
|
||||
continue
|
||||
}
|
||||
|
||||
if cursor + 1 < program.len and program[cursor + 1] == '=' {
|
||||
try add_token(state, Token{ start = cursor, kind = .colon_equal })
|
||||
try add_token(state, Token{ span = make_span(cursor, cursor + 2), kind = .colon_equal })
|
||||
cursor += 2
|
||||
continue
|
||||
}
|
||||
|
||||
try add_token(state, Token{ start = cursor, kind = .colon })
|
||||
try add_token(state, Token{ span = make_span(cursor, cursor + 1), kind = .colon })
|
||||
cursor += 1
|
||||
continue
|
||||
}
|
||||
@@ -156,45 +144,41 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter
|
||||
# compound arithmetic assignment
|
||||
if char == '+' {
|
||||
if cursor + 1 < program.len and program[cursor + 1] == '=' {
|
||||
try add_token(state, Token{ start = cursor, kind = .plus_equal })
|
||||
try add_token(state, Token{ span = make_span(cursor, cursor + 2), kind = .plus_equal })
|
||||
cursor += 2
|
||||
continue
|
||||
}
|
||||
|
||||
try add_token(state, Token{ start = cursor, kind = .plus })
|
||||
try add_token(state, Token{ span = make_span(cursor, cursor + 1), kind = .plus })
|
||||
cursor += 1
|
||||
continue
|
||||
}
|
||||
if char == '-' {
|
||||
if cursor + 1 < program.len and program[cursor + 1] == '=' {
|
||||
try add_token(state, Token{ start = cursor, kind = .minus_equal })
|
||||
try add_token(state, Token{ span = make_span(cursor, cursor + 2), kind = .minus_equal })
|
||||
cursor += 2
|
||||
continue
|
||||
}
|
||||
|
||||
try add_token(state, Token{ start = cursor, kind = .minus })
|
||||
try add_token(state, Token{ span = make_span(cursor, cursor + 1), kind = .minus })
|
||||
cursor += 1
|
||||
continue
|
||||
}
|
||||
if char == '*' {
|
||||
if cursor + 1 < program.len and program[cursor + 1] == '=' {
|
||||
try add_token(state, Token{ start = cursor, kind = .star_equal })
|
||||
try add_token(state, Token{ span = make_span(cursor, cursor + 2), kind = .star_equal })
|
||||
cursor += 2
|
||||
continue
|
||||
}
|
||||
|
||||
try add_token(state, Token{ start = cursor, kind = .star })
|
||||
try add_token(state, Token{ span = make_span(cursor, cursor + 1), kind = .star })
|
||||
cursor += 1
|
||||
continue
|
||||
}
|
||||
if char == '/' {
|
||||
if cursor + 1 < program.len and program[cursor + 1] == '=' {
|
||||
try add_token(state, Token{ start = cursor, kind = .slash_equal })
|
||||
try add_token(state, Token{ span = make_span(cursor, cursor + 2), kind = .slash_equal })
|
||||
cursor += 2
|
||||
continue
|
||||
}
|
||||
|
||||
try add_token(state, Token{ start = cursor, kind = .slash })
|
||||
try add_token(state, Token{ span = make_span(cursor, cursor + 1), kind = .slash })
|
||||
cursor += 1
|
||||
continue
|
||||
}
|
||||
@@ -205,25 +189,33 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter
|
||||
',': .comma
|
||||
'(': .open_paren
|
||||
')': .close_paren
|
||||
'[': .open_bracket
|
||||
']': .close_bracket
|
||||
'{': .open_curly
|
||||
'}': .close_curly
|
||||
else: null
|
||||
}
|
||||
|
||||
if (kind) |k| {
|
||||
try add_token(state, Token{ start = cursor, kind = k })
|
||||
if (kind) |value| {
|
||||
try add_token(state, Token{ span = make_span(cursor, cursor + 1), kind = value })
|
||||
cursor += 1
|
||||
continue
|
||||
}
|
||||
|
||||
# invalid character
|
||||
token :: token_id(state.tokens.items.len)
|
||||
try add_token(state, Token{ start = cursor, kind = .invalid })
|
||||
try arraylist.append(diagnostics, Diagnostic{ token = token, code = .invalid_character })
|
||||
id :: try diagnostics.add(
|
||||
diagnostic_store,
|
||||
.invalid_character,
|
||||
make_span(cursor, cursor + 1),
|
||||
)
|
||||
try add_token(state, Token{
|
||||
span = make_span(cursor, cursor + 1),
|
||||
kind = .invalid,
|
||||
data = TokenData{ diagnostic = id },
|
||||
})
|
||||
cursor += 1
|
||||
}
|
||||
|
||||
try add_token(state, Token{ start = cursor, kind = .eof })
|
||||
try add_token(state, Token{ span = make_span(cursor, cursor), kind = .eof })
|
||||
}
|
||||
|
||||
ScanIdentResult :: struct {
|
||||
@@ -234,7 +226,6 @@ ScanIdentResult :: struct {
|
||||
|
||||
scan_ident proc(start usize, program []u8) ScanIdentResult {
|
||||
cursor := start + 1
|
||||
|
||||
# scan whole identifier
|
||||
while (cursor < program.len and (
|
||||
is_alpha(program[cursor]) or
|
||||
@@ -242,19 +233,13 @@ scan_ident proc(start usize, program []u8) ScanIdentResult {
|
||||
program[cursor] == '_'
|
||||
)) cursor += 1
|
||||
|
||||
kind :: strmap.get(&keywords, program[start..cursor]) orelse .ident
|
||||
|
||||
# don't intern keywords (already O(1) lookup via token kind)
|
||||
kind :: strmap.get(&keywords, program[start..cursor]) orelse .ident
|
||||
str_id :: if (kind == .ident)
|
||||
strpool.intern(&strpool.STRINGS, program[start..cursor]) catch strpool.NO_STR
|
||||
else
|
||||
strpool.NO_STR
|
||||
|
||||
return ScanIdentResult{
|
||||
end = cursor,
|
||||
kind = kind,
|
||||
str_id = str_id,
|
||||
}
|
||||
return ScanIdentResult{ end = cursor, kind = kind, str_id = str_id }
|
||||
}
|
||||
|
||||
ScanNumResult :: struct {
|
||||
@@ -265,53 +250,36 @@ ScanNumResult :: struct {
|
||||
scan_number proc(start usize, program []u8) ScanNumResult ! ScanError {
|
||||
cursor := start
|
||||
has_decimal := false
|
||||
|
||||
# scan integer part
|
||||
while (cursor < program.len and is_digit(program[cursor])) cursor += 1
|
||||
|
||||
# check for decimal
|
||||
if cursor < program.len and program[cursor] == '.' {
|
||||
has_decimal = true
|
||||
cursor += 1
|
||||
}
|
||||
|
||||
# assert non-terminating decimal
|
||||
if has_decimal and (cursor >= program.len or !is_digit(program[cursor])) {
|
||||
return ScanError{
|
||||
code = .float_must_end_with_digit,
|
||||
end = cursor,
|
||||
return ScanError{ code = .float_must_end_with_digit, end = cursor }
|
||||
}
|
||||
}
|
||||
|
||||
# scan fractional part
|
||||
while (cursor < program.len and is_digit(program[cursor])) cursor += 1
|
||||
|
||||
return ScanNumResult{
|
||||
end = cursor,
|
||||
has_decimal = has_decimal,
|
||||
}
|
||||
return ScanNumResult{ end = cursor, has_decimal = has_decimal }
|
||||
}
|
||||
|
||||
ScanStrResult :: struct { end usize }
|
||||
|
||||
scan_string proc(start usize, program []u8) ScanStrResult ! ScanError {
|
||||
cursor := start + 1 # skip first `"`
|
||||
|
||||
# scan entire string
|
||||
while cursor < program.len and program[cursor] != '"' and program[cursor] != '\n' : cursor += 1 {
|
||||
# ignore escaped characters
|
||||
if (program[cursor] == '\\' and cursor + 1 < program.len) cursor += 1
|
||||
}
|
||||
|
||||
# assert string terminal
|
||||
if (cursor >= program.len or program[cursor] != '"') return ScanError{
|
||||
code = .unterminated_string,
|
||||
end = cursor,
|
||||
}
|
||||
|
||||
return ScanStrResult{
|
||||
end = cursor + 1, # skip last `"`
|
||||
if (cursor >= program.len or program[cursor] != '"') {
|
||||
return ScanError{ code = .unterminated_string, end = cursor }
|
||||
}
|
||||
return ScanStrResult{ end = cursor + 1 } # skip last `"`
|
||||
}
|
||||
|
||||
@hide
|
||||
@@ -335,6 +303,19 @@ is_digit proc(char u8) bool {
|
||||
}
|
||||
}
|
||||
|
||||
@hide:file
|
||||
diagnostic_code proc(code ErrorCode) diagnostics.Code {
|
||||
return match code {
|
||||
.float_must_end_with_digit: .float_must_end_with_digit
|
||||
.unterminated_string: .unterminated_string
|
||||
}
|
||||
}
|
||||
|
||||
@hide:file
|
||||
make_span proc(start, end usize) diagnostics.Span {
|
||||
return diagnostics.Span{ start = u32(start), end = u32(end) }
|
||||
}
|
||||
|
||||
@hide
|
||||
add_token proc(state @mut State, token Token) void ! mem.AllocError {
|
||||
_ = token_id(state.tokens.items.len)
|
||||
|
||||
@@ -1,14 +1,17 @@
|
||||
import "@source/strpool"
|
||||
import "@std/mem"
|
||||
import "@std/testing"
|
||||
import "@source/diagnostics"
|
||||
import "@source/strpool"
|
||||
|
||||
handles_keywords_identifiers_and_error_progress test {
|
||||
reports_invalid_input_and_keeps_scanning test {
|
||||
strpool.STRINGS = strpool.init(mem.c_allocator)
|
||||
defer strpool.deinit(&strpool.STRINGS)
|
||||
|
||||
state State := init(mem.c_allocator)
|
||||
diagnostic_store := diagnostics.init(mem.c_allocator)
|
||||
defer diagnostics.deinit(&diagnostic_store)
|
||||
state := init(mem.c_allocator)
|
||||
defer deinit(&state)
|
||||
try scan(&state, "if name # comment\n@1. 2 3.5 \"hi\"")
|
||||
|
||||
try scan(&state, "if name # comment\n@1. 2 3.5 \"hi\"", &diagnostic_store)
|
||||
|
||||
try testing.expect_equal(8, state.tokens.items.len)
|
||||
try testing.expect_equal(TokenKind.if, state.tokens.items[0].kind)
|
||||
@@ -19,6 +22,8 @@ handles_keywords_identifiers_and_error_progress test {
|
||||
try testing.expect_equal(TokenKind.float, state.tokens.items[5].kind)
|
||||
try testing.expect_equal(TokenKind.string, state.tokens.items[6].kind)
|
||||
try testing.expect_equal(TokenKind.eof, state.tokens.items[7].kind)
|
||||
try testing.expect_equal(2, state.diagnostics.items.len)
|
||||
try testing.expect_equal("name", strpool.get_str(&strpool.STRINGS, state.tokens.items[1].str_id)?)
|
||||
try testing.expect_equal(2, diagnostics.len(&diagnostic_store))
|
||||
try testing.expect_equal("name", strpool.get_str(&strpool.STRINGS, state.tokens.items[1].data.str_id)?)
|
||||
try testing.expect_equal(u32(3), state.tokens.items[1].span.start)
|
||||
try testing.expect_equal(u32(7), state.tokens.items[1].span.end)
|
||||
}
|
||||
|
||||
+12
-20
@@ -1,14 +1,19 @@
|
||||
import "@std/debug"
|
||||
import "@source/diagnostics"
|
||||
import "@source/strpool"
|
||||
|
||||
TokenId :: distinct u32
|
||||
|
||||
NO_TOKEN :: maxval!(TokenId)
|
||||
|
||||
TokenData :: union {
|
||||
str_id strpool.StringId
|
||||
diagnostic diagnostics.DiagnosticId
|
||||
}
|
||||
|
||||
Token :: struct {
|
||||
start uint
|
||||
span diagnostics.Span
|
||||
kind TokenKind
|
||||
str_id strpool.StringId = strpool.NO_STR
|
||||
data TokenData = TokenData{ str_id = strpool.NO_STR }
|
||||
}
|
||||
|
||||
TokenKind :: enum {
|
||||
@@ -55,27 +60,14 @@ TokenKind :: enum {
|
||||
render_token proc(tok Token, program []u8) void {
|
||||
match tok.kind {
|
||||
.ident: {
|
||||
if (strpool.get_str(&strpool.STRINGS, tok.str_id)) |str| {
|
||||
if (strpool.get_str(&strpool.STRINGS, tok.data.str_id)) |str| {
|
||||
debug.print("{}({})\n", {tok.kind, str})
|
||||
} else {
|
||||
res :: scan_ident(tok.start, program)
|
||||
debug.print("{}({})\n", {tok.kind, program[tok.start..res.end]})
|
||||
debug.print("{}({})\n", {tok.kind, program[tok.span.start..tok.span.end]})
|
||||
}
|
||||
}
|
||||
.string: {
|
||||
res :: scan_string(tok.start, program) catch |_| {
|
||||
debug.print("{}({})\n", {tok.kind, "null"})
|
||||
return
|
||||
}
|
||||
debug.print("{}({})\n", {tok.kind, program[tok.start..res.end]})
|
||||
}
|
||||
.int, .float: {
|
||||
res :: scan_number(tok.start, program) catch |_| {
|
||||
debug.print("{}({})\n", {tok.kind, "null"})
|
||||
return
|
||||
}
|
||||
debug.print("{}({})\n", {tok.kind, program[tok.start..res.end]})
|
||||
}
|
||||
.string, .int, .float:
|
||||
debug.print("{}({})\n", {tok.kind, program[tok.span.start..tok.span.end]})
|
||||
else: debug.print("{}\n", {tok.kind})
|
||||
}
|
||||
}
|
||||
|
||||
+24
-72
@@ -1,20 +1,24 @@
|
||||
import "@std/debug"
|
||||
import "@std/mem"
|
||||
|
||||
import "@source/diagnostics"
|
||||
import "@source/strpool"
|
||||
import "@source/lexer"
|
||||
import "@source/ast"
|
||||
import "@source/parser"
|
||||
|
||||
ast_renderer :: import "@source/ast/renderer"
|
||||
|
||||
test import "@std/enums/enummap"
|
||||
test import "@std/arraylist"
|
||||
test import "@std/hashmap"
|
||||
test import "@std/strmap"
|
||||
test import "@std/meta"
|
||||
|
||||
import "@source/strpool"
|
||||
import "@source/lexer"
|
||||
import "@source/parser"
|
||||
|
||||
ast_renderer :: import "@source/ast/renderer"
|
||||
|
||||
test import "@source/diagnostics"
|
||||
test import "@source/strpool"
|
||||
test import "@source/lexer"
|
||||
test import "@source/ast"
|
||||
test import "@source/ast/renderer"
|
||||
test import "@source/parser"
|
||||
|
||||
program ::
|
||||
@@ -35,77 +39,25 @@ main proc() void! {
|
||||
strpool.STRINGS = strpool.init(mem.c_allocator)
|
||||
defer strpool.deinit(&strpool.STRINGS)
|
||||
|
||||
debug.print("PROGRAM::[[\n{}\n]]\n\n", {program})
|
||||
diagnostic_store := diagnostics.init(mem.c_allocator)
|
||||
defer diagnostics.deinit(&diagnostic_store)
|
||||
|
||||
scan_state := lexer.init(mem.c_allocator)
|
||||
defer lexer.deinit(&scan_state)
|
||||
lexer.scan(&scan_state, program) catch |err| {
|
||||
debug.print("failed to scan: {}\n", {err})
|
||||
return
|
||||
}
|
||||
try lexer.scan(&scan_state, program, &diagnostic_store)
|
||||
|
||||
debug.print("TOKENS::[[\n", {})
|
||||
for (scan_state.tokens.items) |tok| lexer.render_token(tok, program)
|
||||
debug.print("]]\n\n", {})
|
||||
|
||||
parse_state := parser.init(mem.c_allocator)
|
||||
defer parser.deinit(&parse_state)
|
||||
root :: parser.parse(&parse_state, scan_state.tokens.items) catch |err| {
|
||||
debug.print("failed to parse: {}\n", {err})
|
||||
return
|
||||
}
|
||||
|
||||
debug.print("AST::[[\n", {})
|
||||
for parse_state.nodes.items |node, i| {
|
||||
match node.kind {
|
||||
.expr_identifier: {
|
||||
ident :: scan_state.tokens.items[usize(node.main_token)]
|
||||
res :: lexer.scan_ident(ident.start, program)
|
||||
debug.print("{}: {}({})\n", { i, node.kind, program[ident.start..res.end] })
|
||||
}
|
||||
.literal_int, .literal_float: {
|
||||
lit :: scan_state.tokens.items[usize(node.main_token)]
|
||||
res :: lexer.scan_number(lit.start, program) catch |_| {
|
||||
debug.print("{}: {}\n", { i, node.kind })
|
||||
continue
|
||||
}
|
||||
debug.print("{}: {}({})\n", { i, node.kind, program[lit.start..res.end] })
|
||||
}
|
||||
.literal_string: {
|
||||
str :: scan_state.tokens.items[usize(node.main_token)]
|
||||
res :: lexer.scan_string(str.start, program) catch |_| {
|
||||
debug.print("{}: {}\n", { i, node.kind })
|
||||
continue
|
||||
}
|
||||
debug.print("{}: {}({})\n", { i, node.kind, program[str.start..res.end] })
|
||||
}
|
||||
.expr_unary: {
|
||||
operator :: scan_state.tokens.items[usize(node.main_token)]
|
||||
operand :: node.data0.node_id
|
||||
debug.print("{}: {}({}, operand={})\n", { i, node.kind, operator.kind, operand })
|
||||
}
|
||||
.expr_binary: {
|
||||
operator :: scan_state.tokens.items[usize(node.main_token)]
|
||||
lhs :: node.data0.node_id
|
||||
rhs :: node.data1.node_id
|
||||
debug.print("{}: {}({}, lhs={}, rhs={})\n", { i, node.kind, operator.kind, lhs, rhs })
|
||||
}
|
||||
.stmt_decl: {
|
||||
ident :: scan_state.tokens.items[usize(node.main_token)]
|
||||
init :: parse_state.extra.items[usize(node.data0.extra_id) + 1]
|
||||
res :: lexer.scan_ident(ident.start, program)
|
||||
debug.print("{}: {}({}, init={})\n", { i, node.kind, program[ident.start..res.end], init })
|
||||
}
|
||||
else: debug.print("{}: {}\n", { i, node.kind })
|
||||
}
|
||||
}
|
||||
debug.print("]]\n\n", {})
|
||||
tree := try parser.parse(
|
||||
scan_state.tokens.items,
|
||||
program,
|
||||
&diagnostic_store,
|
||||
mem.c_allocator,
|
||||
)
|
||||
defer ast.deinit(&tree)
|
||||
|
||||
renderer := ast_renderer.init(mem.c_allocator)
|
||||
defer ast_renderer.deinit(&renderer)
|
||||
try ast_renderer.render_token_stream(&renderer, root, &parse_state, scan_state.tokens.items)
|
||||
try ast_renderer.render(&renderer, &tree)
|
||||
|
||||
debug.print("AST Render::[[\n", {})
|
||||
for (renderer.tokens.items) |tok| lexer.render_token(tok, program)
|
||||
debug.print("]]\n", {})
|
||||
debug.print("{s}", {renderer.bytes.items})
|
||||
debug.print("diagnostics: {}\n", {diagnostics.len(&diagnostic_store)})
|
||||
}
|
||||
|
||||
+288
-372
@@ -1,401 +1,317 @@
|
||||
import "@std"
|
||||
import "@std/mem"
|
||||
import "@std/arraylist"
|
||||
import "@std/enums/enummap"
|
||||
import "@std/debug"
|
||||
|
||||
import "@source/diagnostics"
|
||||
import "@source/lexer"
|
||||
import "@source/ast"
|
||||
|
||||
# todo: move error stuff into a separate file
|
||||
@hide:file
|
||||
SyntaxError :: enum { unexpected_token }
|
||||
|
||||
Diagnostic :: struct {
|
||||
code ErrorCode
|
||||
node NodeId
|
||||
}
|
||||
|
||||
ParseError :: alias ErrorCode | mem.AllocError
|
||||
|
||||
ErrorCode :: enum {
|
||||
unexpected_token,
|
||||
}
|
||||
|
||||
ErrorDetails :: struct {
|
||||
name []u8
|
||||
message []u8
|
||||
}
|
||||
|
||||
error_msg_map std.EnumMap(ErrorCode, ErrorDetails) :: enummap.init({
|
||||
unexpected_token = ErrorDetails {
|
||||
name = "P0",
|
||||
message = "unexpected token",
|
||||
},
|
||||
})
|
||||
|
||||
@hide:file Token :: alias lexer.Token
|
||||
@hide:file TokenId :: alias lexer.TokenId
|
||||
@hide:file TokenKind :: alias lexer.TokenKind
|
||||
@hide:file Node :: alias ast.Node
|
||||
@hide:file NodeId :: alias ast.NodeId
|
||||
@hide:file ExtraId :: alias ast.ExtraId
|
||||
@hide:file NodeData :: alias ast.NodeData
|
||||
@hide:file NO_NODE :: alias ast.NO_NODE
|
||||
@hide:file NO_EXTRA :: alias ast.NO_EXTRA
|
||||
@hide:file token_id :: alias lexer.token_id
|
||||
@hide:file
|
||||
ParseError :: alias SyntaxError | mem.AllocError | diagnostics.CapacityError
|
||||
|
||||
@hide:file
|
||||
BindingPowerScalar :: distinct u32
|
||||
|
||||
@hide:file
|
||||
BindingPower :: struct {
|
||||
left BindingPowerScalar
|
||||
right BindingPowerScalar
|
||||
}
|
||||
|
||||
prefix_binding_power proc(token_kind TokenKind) ?BindingPowerScalar {
|
||||
return match token_kind {
|
||||
@hide:file
|
||||
State :: struct {
|
||||
builder ast.Builder
|
||||
diagnostics @mut diagnostics.Store
|
||||
tokens []lexer.Token
|
||||
allocator mem.Allocator
|
||||
next_token usize
|
||||
}
|
||||
|
||||
parse proc(
|
||||
tokens []lexer.Token,
|
||||
source []u8,
|
||||
diagnostic_store @mut diagnostics.Store,
|
||||
allocator mem.Allocator,
|
||||
) ast.Tree ! ParseError {
|
||||
state := State{
|
||||
builder = ast.init_builder(allocator, source),
|
||||
diagnostics = diagnostic_store,
|
||||
tokens = tokens,
|
||||
allocator = allocator,
|
||||
next_token = 0,
|
||||
}
|
||||
defer ast.deinit_builder(&state.builder)
|
||||
root :: try parse_package(&state)
|
||||
return ast.finish(&state.builder, root)
|
||||
}
|
||||
|
||||
@hide:file
|
||||
parse_package proc(state @mut State) ast.NodeId ! ParseError {
|
||||
declarations arraylist.ArrayList(ast.NodeId) := arraylist.init(state.allocator)
|
||||
defer arraylist.deinit(&declarations)
|
||||
|
||||
skip_newlines(state)
|
||||
while !next_is(state, .eof) {
|
||||
declaration :: try parse_decl(state)
|
||||
try arraylist.append(&declarations, declaration)
|
||||
skip_newlines(state)
|
||||
}
|
||||
|
||||
return try ast.add_package(
|
||||
&state.builder,
|
||||
ast.Span{ start = 0, end = u32(state.builder.source.len) },
|
||||
declarations.items,
|
||||
)
|
||||
}
|
||||
|
||||
@hide:file
|
||||
parse_decl proc(state @mut State) ast.NodeId ! ParseError {
|
||||
if next_is(state, .ident) and next_n_is(state, 1, .proc) {
|
||||
return try parse_func_decl(state)
|
||||
}
|
||||
return try parse_statement_decl(state)
|
||||
}
|
||||
|
||||
@hide:file
|
||||
parse_func_decl proc(state @mut State) ast.NodeId ! ParseError {
|
||||
name :: try consume(state, .ident)
|
||||
_ = try consume(state, .proc)
|
||||
_ = try consume(state, .open_paren)
|
||||
|
||||
params arraylist.ArrayList(ast.NodeId) := arraylist.init(state.allocator)
|
||||
defer arraylist.deinit(¶ms)
|
||||
names arraylist.ArrayList(ast.Span) := arraylist.init(state.allocator)
|
||||
defer arraylist.deinit(&names)
|
||||
|
||||
while !next_is(state, .close_paren) {
|
||||
arraylist.clear(&names)
|
||||
while true {
|
||||
token :: try consume(state, .ident)
|
||||
try arraylist.append(&names, token.span)
|
||||
if next_is(state, .ident) break
|
||||
_ = try consume(state, .comma)
|
||||
}
|
||||
|
||||
type_id :: try parse_type(state)
|
||||
type_span :: ast.builder_span(&state.builder, type_id)
|
||||
for (names.items) |name_span| {
|
||||
param_id :: try ast.add_param(
|
||||
&state.builder,
|
||||
ast.Span{ start = name_span.start, end = type_span.end },
|
||||
name_span,
|
||||
type_id,
|
||||
)
|
||||
try arraylist.append(¶ms, param_id)
|
||||
}
|
||||
if next_is(state, .comma) _ = try consume(state, .comma)
|
||||
}
|
||||
|
||||
_ = try consume(state, .close_paren)
|
||||
return_type :: try parse_type(state)
|
||||
body :: try parse_block(state)
|
||||
body_span :: ast.builder_span(&state.builder, body)
|
||||
return try ast.add_function_decl(
|
||||
&state.builder,
|
||||
ast.Span{ start = name.span.start, end = body_span.end },
|
||||
name.span,
|
||||
return_type,
|
||||
params.items,
|
||||
body,
|
||||
)
|
||||
}
|
||||
|
||||
@hide:file
|
||||
parse_block proc(state @mut State) ast.NodeId ! ParseError {
|
||||
open :: try consume(state, .open_curly)
|
||||
statements arraylist.ArrayList(ast.NodeId) := arraylist.init(state.allocator)
|
||||
defer arraylist.deinit(&statements)
|
||||
|
||||
skip_newlines(state)
|
||||
while !next_is(state, .close_curly) and !next_is(state, .eof) {
|
||||
statement :: try parse_statement(state)
|
||||
try arraylist.append(&statements, statement)
|
||||
skip_newlines(state)
|
||||
}
|
||||
|
||||
close :: try consume(state, .close_curly)
|
||||
return try ast.add_block(
|
||||
&state.builder,
|
||||
ast.Span{ start = open.span.start, end = close.span.end },
|
||||
statements.items,
|
||||
)
|
||||
}
|
||||
|
||||
@hide:file
|
||||
parse_statement proc(state @mut State) ast.NodeId ! ParseError {
|
||||
if next_is(state, .ident) return try parse_statement_decl(state)
|
||||
return .unexpected_token
|
||||
}
|
||||
|
||||
@hide:file
|
||||
parse_statement_decl proc(state @mut State) ast.NodeId ! ParseError {
|
||||
name :: try consume(state, .ident)
|
||||
type_id ?ast.NodeId := if next_is(state, .ident)
|
||||
try parse_type(state)
|
||||
else
|
||||
null
|
||||
|
||||
assignment :: try consume_either(state, &[.double_colon, .colon_equal])
|
||||
initializer :: try parse_expr(state)
|
||||
end :: ast.builder_span(&state.builder, initializer).end
|
||||
mutability :: if (assignment.kind == .double_colon) .constant else .variable
|
||||
return try ast.add_statement_decl(
|
||||
&state.builder,
|
||||
ast.Span{ start = name.span.start, end = end },
|
||||
name.span,
|
||||
type_id,
|
||||
mutability,
|
||||
initializer,
|
||||
)
|
||||
}
|
||||
|
||||
@hide:file
|
||||
parse_type proc(state @mut State) ast.NodeId ! ParseError {
|
||||
if (!next_is(state, .ident)) return .unexpected_token
|
||||
token :: current(state)
|
||||
state.next_token += 1
|
||||
return try ast.add_identifier(&state.builder, token.span)
|
||||
}
|
||||
|
||||
@hide:file
|
||||
parse_expr proc(state @mut State) ast.NodeId ! ParseError {
|
||||
return try parse_expr_bp(state, BindingPowerScalar(0))
|
||||
}
|
||||
|
||||
@hide:file
|
||||
parse_expr_bp proc(state @mut State, min_bp BindingPowerScalar) ast.NodeId ! ParseError {
|
||||
token := current(state)
|
||||
left := if (prefix_binding_power(token.kind)) |right_bp| {
|
||||
state.next_token += 1
|
||||
operand :: try parse_expr_bp(state, right_bp)
|
||||
operand_span :: ast.builder_span(&state.builder, operand)
|
||||
yield try ast.add_unary(
|
||||
&state.builder,
|
||||
ast.Span{ start = token.span.start, end = operand_span.end },
|
||||
.negate,
|
||||
operand,
|
||||
)
|
||||
} else {
|
||||
yield try parse_primary(state)
|
||||
}
|
||||
|
||||
while true {
|
||||
token = current(state)
|
||||
if (infix_binding_power(token.kind)) |power| {
|
||||
if (power.left < min_bp) break
|
||||
state.next_token += 1
|
||||
right :: try parse_expr_bp(state, power.right)
|
||||
left_span :: ast.builder_span(&state.builder, left)
|
||||
right_span :: ast.builder_span(&state.builder, right)
|
||||
left = try ast.add_binary(
|
||||
&state.builder,
|
||||
ast.Span{ start = left_span.start, end = right_span.end },
|
||||
binary_operator(token.kind),
|
||||
left,
|
||||
right,
|
||||
)
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
return left
|
||||
}
|
||||
|
||||
@hide:file
|
||||
parse_primary proc(state @mut State) ast.NodeId ! ParseError {
|
||||
token :: current(state)
|
||||
match token.kind {
|
||||
.int: {
|
||||
state.next_token += 1
|
||||
return try ast.add_literal(&state.builder, .int, token.span)
|
||||
}
|
||||
.float: {
|
||||
state.next_token += 1
|
||||
return try ast.add_literal(&state.builder, .float, token.span)
|
||||
}
|
||||
.string: {
|
||||
state.next_token += 1
|
||||
return try ast.add_literal(&state.builder, .string, token.span)
|
||||
}
|
||||
.ident: {
|
||||
state.next_token += 1
|
||||
return try ast.add_identifier(&state.builder, token.span)
|
||||
}
|
||||
else: return .unexpected_token
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@hide:file
|
||||
skip_newlines proc(state @mut State) void {
|
||||
while next_is(state, .newline) state.next_token += 1
|
||||
}
|
||||
|
||||
@hide:file
|
||||
current proc(state @State) lexer.Token {
|
||||
if (state.next_token >= state.tokens.len) return state.tokens[state.tokens.len - 1]
|
||||
return state.tokens[state.next_token]
|
||||
}
|
||||
|
||||
@hide:file
|
||||
next_is proc(state @State, kind lexer.TokenKind) bool {
|
||||
return current(state).kind == kind
|
||||
}
|
||||
|
||||
@hide:file
|
||||
next_n_is proc(state @State, count usize, kind lexer.TokenKind) bool {
|
||||
index :: state.next_token + count
|
||||
return index < state.tokens.len and state.tokens[index].kind == kind
|
||||
}
|
||||
|
||||
@hide:file
|
||||
consume proc(state @mut State, kind lexer.TokenKind) lexer.Token ! SyntaxError {
|
||||
token :: current(state)
|
||||
if (token.kind != kind) return .unexpected_token
|
||||
state.next_token += 1
|
||||
return token
|
||||
}
|
||||
|
||||
@hide:file
|
||||
consume_either proc(state @mut State, kinds []lexer.TokenKind) lexer.Token ! SyntaxError {
|
||||
token :: current(state)
|
||||
for (kinds) |kind| if token.kind == kind {
|
||||
state.next_token += 1
|
||||
return token
|
||||
}
|
||||
return .unexpected_token
|
||||
}
|
||||
|
||||
@hide:file
|
||||
prefix_binding_power proc(kind lexer.TokenKind) ?BindingPowerScalar {
|
||||
return match kind {
|
||||
.minus: BindingPowerScalar(30)
|
||||
else: null
|
||||
}
|
||||
}
|
||||
|
||||
infix_binding_power proc(token_kind TokenKind) ?BindingPower {
|
||||
return match token_kind {
|
||||
@hide:file
|
||||
infix_binding_power proc(kind lexer.TokenKind) ?BindingPower {
|
||||
return match kind {
|
||||
.plus, .minus: BindingPower{ left = BindingPowerScalar(10), right = BindingPowerScalar(11) }
|
||||
.star, .slash: BindingPower{ left = BindingPowerScalar(20), right = BindingPowerScalar(21) }
|
||||
else: null
|
||||
}
|
||||
}
|
||||
|
||||
postfix_binding_power proc(token_kind TokenKind) ?BindingPowerScalar {
|
||||
return match token_kind {
|
||||
.open_paren, .open_bracket: BindingPowerScalar(40)
|
||||
else: null
|
||||
}
|
||||
}
|
||||
|
||||
State :: struct {
|
||||
nodes std.ArrayList(Node)
|
||||
|
||||
#! array of raw u32's indexing nodes, tokens, etc.
|
||||
extra std.ArrayList(u32)
|
||||
|
||||
tokens []Token = &[]
|
||||
next_token usize = 0
|
||||
}
|
||||
|
||||
init proc(allocator mem.Allocator) State {
|
||||
return State{
|
||||
nodes = arraylist.init(allocator),
|
||||
extra = arraylist.init(allocator),
|
||||
}
|
||||
}
|
||||
|
||||
deinit proc(state @mut State) void {
|
||||
arraylist.deinit(&state.nodes)
|
||||
arraylist.deinit(&state.extra)
|
||||
}
|
||||
|
||||
parse proc(state @mut State, tokens []Token) NodeId ! ParseError {
|
||||
state.tokens = tokens
|
||||
return try parse_pkg(state)
|
||||
}
|
||||
|
||||
parse_pkg proc(state @mut State) NodeId ! ParseError {
|
||||
if (state.next_token < state.tokens.len
|
||||
and state.tokens[state.next_token].kind == .eof
|
||||
) return NO_NODE
|
||||
|
||||
stmt_ids std.ArrayList(NodeId) := arraylist.init(state.nodes.allocator)
|
||||
defer arraylist.deinit(&stmt_ids)
|
||||
|
||||
while (state.next_token < state.tokens.len
|
||||
and state.tokens[state.next_token].kind != .eof
|
||||
) {
|
||||
stmt_id :: try parse_decl(state)
|
||||
try arraylist.append(&stmt_ids, stmt_id)
|
||||
}
|
||||
|
||||
pkg_start :: ast.extra_id(state.extra.items.len)
|
||||
for (stmt_ids.items) |stmt_id| _ = try add_extra(state, u32(stmt_id))
|
||||
pkg_end :: ast.extra_id(state.extra.items.len)
|
||||
|
||||
return try add_node(state, Node{
|
||||
kind = .package,
|
||||
data0 = NodeData{ extra_id = pkg_start },
|
||||
data1 = NodeData{ extra_id = pkg_end },
|
||||
})
|
||||
}
|
||||
|
||||
parse_decl proc(state @mut State) NodeId ! ParseError {
|
||||
if next_n_is(state, 1, .proc)
|
||||
return try parse_func_decl(state)
|
||||
else
|
||||
return try parse_stmt_decl(state)
|
||||
}
|
||||
|
||||
parse_func_decl proc(state @mut State) NodeId ! ParseError {
|
||||
ident_tok_id :: try consume(state, .ident)
|
||||
_ = try consume(state, .proc)
|
||||
_ = try consume(state, .open_paren)
|
||||
|
||||
param_decls std.ArrayList(NodeId) := arraylist.init(state.nodes.allocator)
|
||||
defer arraylist.deinit(¶m_decls)
|
||||
|
||||
ident_ids std.ArrayList(TokenId) := arraylist.init(state.nodes.allocator)
|
||||
defer arraylist.deinit(&ident_ids)
|
||||
|
||||
while !next_is(state, .close_paren) {
|
||||
arraylist.clear(&ident_ids)
|
||||
|
||||
# collect param group
|
||||
while true {
|
||||
param_tok_id :: try consume(state, .ident)
|
||||
try arraylist.append(&ident_ids, param_tok_id)
|
||||
|
||||
if next_is(state, .ident) break
|
||||
_ = try consume(state, .comma)
|
||||
}
|
||||
|
||||
# collect type
|
||||
try expect(state, .ident)
|
||||
type_id :: try parse_primary(state)
|
||||
for (ident_ids.items) |id| {
|
||||
param_decl :: try add_node(state, Node{
|
||||
kind = .param_decl,
|
||||
main_token = id,
|
||||
data0 = NodeData{ node_id = type_id },
|
||||
})
|
||||
try arraylist.append(¶m_decls, param_decl)
|
||||
}
|
||||
|
||||
if next_is(state, .comma) _ = try consume(state, .comma)
|
||||
}
|
||||
|
||||
_ = try consume(state, .close_paren)
|
||||
|
||||
try expect(state, .ident)
|
||||
return_type_id :: try parse_primary(state)
|
||||
block_id :: try parse_block(state)
|
||||
|
||||
while next_is(state, .newline) _ = try consume(state, .newline)
|
||||
|
||||
extra_id :: try add_extra(state, u32(return_type_id))
|
||||
_ = try add_extra(state, u32(param_decls.items.len))
|
||||
for (param_decls.items) |param| _ = try add_extra(state, u32(param))
|
||||
|
||||
return try add_node(state, Node{
|
||||
kind = .func_decl,
|
||||
main_token = ident_tok_id,
|
||||
data0 = NodeData{ node_id = block_id },
|
||||
data1 = NodeData{ extra_id = extra_id },
|
||||
})
|
||||
}
|
||||
|
||||
parse_block proc(state @mut State) NodeId ! ParseError {
|
||||
stmt_ids std.ArrayList(NodeId) := arraylist.init(state.nodes.allocator)
|
||||
defer arraylist.deinit(&stmt_ids)
|
||||
|
||||
_ = try consume(state, .open_curly)
|
||||
while next_is(state, .newline) _ = try consume(state, .newline)
|
||||
|
||||
while !next_is(state, .close_curly) {
|
||||
stmt_id :: try parse_stmt(state)
|
||||
try arraylist.append(&stmt_ids, stmt_id)
|
||||
while next_is(state, .newline) _ = try consume(state, .newline)
|
||||
}
|
||||
|
||||
_ = try consume(state, .close_curly)
|
||||
|
||||
block_start :: ast.extra_id(state.extra.items.len)
|
||||
for (stmt_ids.items) |id| _ = try add_extra(state, u32(id))
|
||||
block_end :: ast.extra_id(state.extra.items.len)
|
||||
|
||||
return try add_node(state, Node{
|
||||
kind = .block,
|
||||
data0 = NodeData{ extra_id = block_start },
|
||||
data1 = NodeData{ extra_id = block_end },
|
||||
})
|
||||
}
|
||||
|
||||
parse_stmt proc(state @mut State) NodeId ! ParseError {
|
||||
if next_is(state, .ident) return try parse_stmt_decl(state)
|
||||
return .unexpected_token
|
||||
}
|
||||
|
||||
parse_stmt_decl proc(state @mut State) NodeId ! ParseError {
|
||||
# expect identifier
|
||||
ident_tok_id :: try consume(state, .ident)
|
||||
|
||||
# check for type
|
||||
# todo: make a parse_type proc for this
|
||||
type_id :: if (next_is(state, .ident))
|
||||
try parse_primary(state)
|
||||
else
|
||||
NO_NODE
|
||||
|
||||
# expect `::` or `:=` (immutable or mutable assignment)
|
||||
mutability_tok_id :: try consume_either(state, &[.double_colon, .colon_equal])
|
||||
|
||||
# expect expression
|
||||
expr_id :: try parse_expr(state)
|
||||
|
||||
# expect statement terminator (newline)
|
||||
while (next_is(state, .newline)) _ = try consume(state, .newline)
|
||||
|
||||
extra_start_id :: try add_extra(state, u32(type_id))
|
||||
_ = try add_extra(state, u32(expr_id))
|
||||
|
||||
return try add_node(state, Node{
|
||||
kind = .stmt_decl,
|
||||
main_token = ident_tok_id,
|
||||
data0 = NodeData{ extra_id = extra_start_id },
|
||||
data1 = NodeData{ token_id = mutability_tok_id },
|
||||
})
|
||||
}
|
||||
|
||||
parse_expr proc(state @mut State) NodeId ! mem.AllocError {
|
||||
return try parse_expr_bp(state, BindingPowerScalar(0))
|
||||
}
|
||||
|
||||
parse_expr_bp proc(state @mut State, min_bp BindingPowerScalar) NodeId ! mem.AllocError {
|
||||
# parse left-hand side
|
||||
kind := state.tokens[state.next_token].kind
|
||||
|
||||
lhs := if (prefix_binding_power(kind)) |right_bp| {
|
||||
operator :: token_id(state.next_token)
|
||||
state.next_token += 1
|
||||
|
||||
operand :: try parse_expr_bp(state, right_bp)
|
||||
|
||||
yield try add_node(state, Node{
|
||||
kind = .expr_unary,
|
||||
main_token = operator,
|
||||
data0 = NodeData{ node_id = operand },
|
||||
})
|
||||
} else {
|
||||
yield try parse_primary(state)
|
||||
}
|
||||
|
||||
# parse right-hand side
|
||||
while true {
|
||||
kind = state.tokens[state.next_token].kind
|
||||
|
||||
# try postfix (higher precedence)
|
||||
if postfix_binding_power(kind) |left_bp| {
|
||||
if (left_bp < min_bp) break
|
||||
|
||||
lhs = match state.tokens[state.next_token].kind {
|
||||
.open_paren: debug.unimplemented() # todo: parse call
|
||||
.open_bracket: debug.unimplemented() # todo: parse index
|
||||
@hide:file
|
||||
binary_operator proc(kind lexer.TokenKind) ast.BinaryOperator {
|
||||
return match kind {
|
||||
.plus: .add
|
||||
.minus: .subtract
|
||||
.star: .multiply
|
||||
.slash: .divide
|
||||
else: unreachable
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
# try infex (lower precedence)
|
||||
if infix_binding_power(kind) |bp| {
|
||||
if (bp.left < min_bp) break
|
||||
|
||||
operator :: token_id(state.next_token)
|
||||
state.next_token += 1
|
||||
|
||||
rhs :: try parse_expr_bp(state, bp.right)
|
||||
lhs = try add_node(state, Node{
|
||||
kind = .expr_binary,
|
||||
main_token = operator,
|
||||
data0 = NodeData{ node_id = lhs },
|
||||
data1 = NodeData{ node_id = rhs },
|
||||
})
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
break
|
||||
}
|
||||
|
||||
return lhs
|
||||
}
|
||||
|
||||
parse_primary proc(state @mut State) NodeId ! mem.AllocError {
|
||||
start_idx :: state.next_token
|
||||
start_tok_id :: TokenId(start_idx)
|
||||
state.next_token += 1
|
||||
return match state.tokens[start_idx].kind {
|
||||
.int: try add_node(state, Node{
|
||||
kind = .literal_int,
|
||||
main_token = start_tok_id,
|
||||
})
|
||||
.float: try add_node(state, Node{
|
||||
kind = .literal_float,
|
||||
main_token = start_tok_id,
|
||||
})
|
||||
.string: try add_node(state, Node{
|
||||
kind = .literal_string,
|
||||
main_token = start_tok_id,
|
||||
})
|
||||
.ident: try add_node(state, Node{
|
||||
kind = .expr_identifier,
|
||||
main_token = start_tok_id,
|
||||
})
|
||||
else: try add_node(state, Node{
|
||||
kind = .invalid,
|
||||
main_token = start_tok_id,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@hide
|
||||
next_is proc(state @mut State, token_kind TokenKind) bool {
|
||||
return state.tokens[state.next_token].kind == token_kind
|
||||
}
|
||||
|
||||
@hide
|
||||
next_is_either proc(state @mut State, token_kinds []TokenKind) bool {
|
||||
for (token_kinds) |kind| if next_is(state, kind) return true
|
||||
return false
|
||||
}
|
||||
|
||||
@hide
|
||||
next_n_is proc(state @mut State, n usize, token_kind TokenKind) bool {
|
||||
return state.tokens[state.next_token + n].kind == token_kind
|
||||
}
|
||||
|
||||
@hide
|
||||
expect proc(state @mut State, token_kind TokenKind) void ! ErrorCode {
|
||||
if (state.tokens[state.next_token].kind != token_kind) return .unexpected_token
|
||||
}
|
||||
|
||||
@hide
|
||||
expect_either proc(state @mut State, token_kinds []TokenKind) void ! ErrorCode {
|
||||
for (token_kinds) |kind| if (state.tokens[state.next_token].kind == kind) return
|
||||
return .unexpected_token
|
||||
}
|
||||
|
||||
@hide
|
||||
consume proc(state @mut State, token_kind TokenKind) TokenId ! ErrorCode {
|
||||
try expect(state, token_kind)
|
||||
id :: TokenId(state.next_token)
|
||||
state.next_token += 1
|
||||
return id
|
||||
}
|
||||
|
||||
@hide
|
||||
consume_either proc(state @mut State, token_kinds []TokenKind) TokenId ! ErrorCode {
|
||||
try expect_either(state, token_kinds)
|
||||
id :: TokenId(state.next_token)
|
||||
state.next_token += 1
|
||||
return id
|
||||
}
|
||||
|
||||
@hide
|
||||
add_node proc(state @mut State, node Node) NodeId ! mem.AllocError {
|
||||
id :: ast.node_id(state.nodes.items.len)
|
||||
try arraylist.append(&state.nodes, node)
|
||||
return id
|
||||
}
|
||||
|
||||
@hide
|
||||
add_extra proc(state @mut State, data u32) ExtraId ! mem.AllocError {
|
||||
id :: ast.extra_id(state.extra.items.len)
|
||||
try arraylist.append(&state.extra, data)
|
||||
return id
|
||||
}
|
||||
|
||||
+42
-176
@@ -1,202 +1,68 @@
|
||||
import "@std/mem"
|
||||
import "@std/testing"
|
||||
|
||||
import "@source/diagnostics"
|
||||
import "@source/strpool"
|
||||
import "@source/lexer"
|
||||
import "@source/ast"
|
||||
|
||||
ast_renderer :: import "@source/ast/renderer"
|
||||
|
||||
handles_package_declarations test {
|
||||
parses_expression_precedence_through_tree_views test {
|
||||
strpool.STRINGS = strpool.init(mem.c_allocator)
|
||||
defer strpool.deinit(&strpool.STRINGS)
|
||||
|
||||
source ::
|
||||
`first int :: 42
|
||||
`second float :: 2.15 + 3
|
||||
`message :: "hello"
|
||||
|
||||
diagnostic_store := diagnostics.init(mem.c_allocator)
|
||||
defer diagnostics.deinit(&diagnostic_store)
|
||||
source :: "value :: 1 + 2 * 3\n"
|
||||
scan_state := lexer.init(mem.c_allocator)
|
||||
defer lexer.deinit(&scan_state)
|
||||
try lexer.scan(&scan_state, source)
|
||||
try lexer.scan(&scan_state, source, &diagnostic_store)
|
||||
|
||||
parse_state := init(mem.c_allocator)
|
||||
defer deinit(&parse_state)
|
||||
root :: try parse(&parse_state, scan_state.tokens.items)
|
||||
tree := try parse(scan_state.tokens.items, source, &diagnostic_store, mem.c_allocator)
|
||||
defer ast.deinit(&tree)
|
||||
|
||||
renderer := ast_renderer.init(mem.c_allocator)
|
||||
defer ast_renderer.deinit(&renderer)
|
||||
try ast_renderer.render_token_stream(&renderer, root, &parse_state, scan_state.tokens.items)
|
||||
|
||||
try testing.expect_equal(scan_state.tokens.items.len, renderer.tokens.items.len)
|
||||
for scan_state.tokens.items |expected, i| {
|
||||
actual :: renderer.tokens.items[i]
|
||||
try testing.expect_equal(expected.kind, actual.kind)
|
||||
try testing.expect_equal(expected.start, actual.start)
|
||||
try testing.expect_equal(expected.str_id, actual.str_id)
|
||||
}
|
||||
try testing.expect_equal(1, ast.package_decl_count(&tree))
|
||||
declaration :: ast.package_decl(&tree, 0)
|
||||
statement :: ast.statement_decl(&tree, declaration)
|
||||
try testing.expect_equal("value", ast.source_text(&tree, statement.name))
|
||||
try testing.expect_equal(ast.ExprKind.binary, ast.expr_kind(&tree, statement.initializer))
|
||||
add :: ast.binary_expr(&tree, statement.initializer)
|
||||
try testing.expect_equal(ast.BinaryOperator.add, add.operator)
|
||||
try testing.expect_equal(ast.ExprKind.literal_int, ast.expr_kind(&tree, add.left))
|
||||
try testing.expect_equal(ast.ExprKind.binary, ast.expr_kind(&tree, add.right))
|
||||
multiply :: ast.binary_expr(&tree, add.right)
|
||||
try testing.expect_equal(ast.BinaryOperator.multiply, multiply.operator)
|
||||
try testing.expect_equal(0, diagnostics.len(&diagnostic_store))
|
||||
}
|
||||
|
||||
handles_unary_arithmetic_prefix_expression test {
|
||||
parses_function_structure_through_tree_views test {
|
||||
strpool.STRINGS = strpool.init(mem.c_allocator)
|
||||
defer strpool.deinit(&strpool.STRINGS)
|
||||
|
||||
diagnostic_store := diagnostics.init(mem.c_allocator)
|
||||
defer diagnostics.deinit(&diagnostic_store)
|
||||
source ::
|
||||
`first :: -42 + 1
|
||||
|
||||
scan_state := lexer.init(mem.c_allocator)
|
||||
defer lexer.deinit(&scan_state)
|
||||
try lexer.scan(&scan_state, source)
|
||||
|
||||
parse_state := init(mem.c_allocator)
|
||||
defer deinit(&parse_state)
|
||||
root :: try parse(&parse_state, scan_state.tokens.items)
|
||||
|
||||
renderer := ast_renderer.init(mem.c_allocator)
|
||||
defer ast_renderer.deinit(&renderer)
|
||||
try ast_renderer.render_token_stream(&renderer, root, &parse_state, scan_state.tokens.items)
|
||||
|
||||
try testing.expect_equal(scan_state.tokens.items.len, renderer.tokens.items.len)
|
||||
for scan_state.tokens.items |expected, i| {
|
||||
actual :: renderer.tokens.items[i]
|
||||
try testing.expect_equal(expected.kind, actual.kind)
|
||||
try testing.expect_equal(expected.start, actual.start)
|
||||
try testing.expect_equal(expected.str_id, actual.str_id)
|
||||
}
|
||||
}
|
||||
|
||||
handles_binary_arithmetic_expression test {
|
||||
strpool.STRINGS = strpool.init(mem.c_allocator)
|
||||
defer strpool.deinit(&strpool.STRINGS)
|
||||
|
||||
source ::
|
||||
`first int :: 42 + 3
|
||||
`second :: 134 + 5
|
||||
`third float :: 1.35 + 2
|
||||
`fourth :: first + second
|
||||
|
||||
scan_state := lexer.init(mem.c_allocator)
|
||||
defer lexer.deinit(&scan_state)
|
||||
try lexer.scan(&scan_state, source)
|
||||
|
||||
parse_state := init(mem.c_allocator)
|
||||
defer deinit(&parse_state)
|
||||
root :: try parse(&parse_state, scan_state.tokens.items)
|
||||
|
||||
renderer := ast_renderer.init(mem.c_allocator)
|
||||
defer ast_renderer.deinit(&renderer)
|
||||
try ast_renderer.render_token_stream(&renderer, root, &parse_state, scan_state.tokens.items)
|
||||
|
||||
try testing.expect_equal(scan_state.tokens.items.len, renderer.tokens.items.len)
|
||||
for scan_state.tokens.items |expected, i| {
|
||||
actual :: renderer.tokens.items[i]
|
||||
try testing.expect_equal(expected.kind, actual.kind)
|
||||
try testing.expect_equal(expected.start, actual.start)
|
||||
try testing.expect_equal(expected.str_id, actual.str_id)
|
||||
}
|
||||
}
|
||||
|
||||
handles_statement_declaration test {
|
||||
strpool.STRINGS = strpool.init(mem.c_allocator)
|
||||
defer strpool.deinit(&strpool.STRINGS)
|
||||
|
||||
source ::
|
||||
`value int :: 42
|
||||
|
||||
scan_state := lexer.init(mem.c_allocator)
|
||||
defer lexer.deinit(&scan_state)
|
||||
try lexer.scan(&scan_state, source)
|
||||
|
||||
parse_state := init(mem.c_allocator)
|
||||
defer deinit(&parse_state)
|
||||
root :: try parse(&parse_state, scan_state.tokens.items)
|
||||
|
||||
renderer := ast_renderer.init(mem.c_allocator)
|
||||
defer ast_renderer.deinit(&renderer)
|
||||
try ast_renderer.render_token_stream(&renderer, root, &parse_state, scan_state.tokens.items)
|
||||
|
||||
try testing.expect_equal(scan_state.tokens.items.len, renderer.tokens.items.len)
|
||||
for scan_state.tokens.items |expected, i| {
|
||||
actual :: renderer.tokens.items[i]
|
||||
try testing.expect_equal(expected.kind, actual.kind)
|
||||
try testing.expect_equal(expected.start, actual.start)
|
||||
try testing.expect_equal(expected.str_id, actual.str_id)
|
||||
}
|
||||
}
|
||||
|
||||
handles_function_declarations test {
|
||||
strpool.STRINGS = strpool.init(mem.c_allocator)
|
||||
defer strpool.deinit(&strpool.STRINGS)
|
||||
|
||||
source ::
|
||||
`empty proc() void {}
|
||||
`grouped proc(a, b T, c U) void {
|
||||
` value int :: 1
|
||||
`}
|
||||
|
||||
scan_state := lexer.init(mem.c_allocator)
|
||||
defer lexer.deinit(&scan_state)
|
||||
try lexer.scan(&scan_state, source)
|
||||
try lexer.scan(&scan_state, source, &diagnostic_store)
|
||||
|
||||
parse_state := init(mem.c_allocator)
|
||||
defer deinit(&parse_state)
|
||||
root_id :: try parse(&parse_state, scan_state.tokens.items)
|
||||
root :: parse_state.nodes.items[usize(root_id)]
|
||||
tree := try parse(scan_state.tokens.items, source, &diagnostic_store, mem.c_allocator)
|
||||
defer ast.deinit(&tree)
|
||||
|
||||
package_start :: usize(root.data0.extra_id)
|
||||
package_end :: usize(root.data1.extra_id)
|
||||
try testing.expect_equal(usize(2), package_end - package_start)
|
||||
|
||||
empty_id :: ast.node_id(parse_state.extra.items[package_start])
|
||||
empty :: parse_state.nodes.items[usize(empty_id)]
|
||||
empty_payload :: usize(empty.data1.extra_id)
|
||||
empty_block :: parse_state.nodes.items[usize(empty.data0.node_id)]
|
||||
try testing.expect_equal(ast.NodeKind.func_decl, empty.kind)
|
||||
try testing.expect_equal(u32(0), parse_state.extra.items[empty_payload + 1])
|
||||
try testing.expect_equal(
|
||||
usize(empty_block.data0.extra_id),
|
||||
usize(empty_block.data1.extra_id),
|
||||
)
|
||||
|
||||
grouped_id :: ast.node_id(parse_state.extra.items[package_start + 1])
|
||||
grouped :: parse_state.nodes.items[usize(grouped_id)]
|
||||
grouped_payload :: usize(grouped.data1.extra_id)
|
||||
try testing.expect_equal(ast.NodeKind.func_decl, grouped.kind)
|
||||
try testing.expect_equal(u32(3), parse_state.extra.items[grouped_payload + 1])
|
||||
|
||||
param_a_id :: ast.node_id(parse_state.extra.items[grouped_payload + 2])
|
||||
param_b_id :: ast.node_id(parse_state.extra.items[grouped_payload + 3])
|
||||
param_c_id :: ast.node_id(parse_state.extra.items[grouped_payload + 4])
|
||||
param_a :: parse_state.nodes.items[usize(param_a_id)]
|
||||
param_b :: parse_state.nodes.items[usize(param_b_id)]
|
||||
param_c :: parse_state.nodes.items[usize(param_c_id)]
|
||||
try testing.expect_equal(ast.NodeKind.param_decl, param_a.kind)
|
||||
try testing.expect_equal(ast.NodeKind.param_decl, param_b.kind)
|
||||
try testing.expect_equal(ast.NodeKind.param_decl, param_c.kind)
|
||||
try testing.expect(param_a.data0.node_id == param_b.data0.node_id)
|
||||
try testing.expect(param_a.data0.node_id != param_c.data0.node_id)
|
||||
|
||||
grouped_block :: parse_state.nodes.items[usize(grouped.data0.node_id)]
|
||||
try testing.expect_equal(ast.NodeKind.block, grouped_block.kind)
|
||||
try testing.expect_equal(
|
||||
usize(1),
|
||||
usize(grouped_block.data1.extra_id) - usize(grouped_block.data0.extra_id),
|
||||
)
|
||||
|
||||
renderer := ast_renderer.init(mem.c_allocator)
|
||||
defer ast_renderer.deinit(&renderer)
|
||||
try ast_renderer.render_token_stream(
|
||||
&renderer,
|
||||
root_id,
|
||||
&parse_state,
|
||||
scan_state.tokens.items,
|
||||
)
|
||||
|
||||
try testing.expect_equal(scan_state.tokens.items.len, renderer.tokens.items.len)
|
||||
for scan_state.tokens.items |expected, i| {
|
||||
actual :: renderer.tokens.items[i]
|
||||
try testing.expect_equal(expected.kind, actual.kind)
|
||||
try testing.expect_equal(expected.start, actual.start)
|
||||
try testing.expect_equal(expected.str_id, actual.str_id)
|
||||
}
|
||||
declaration :: ast.package_decl(&tree, 0)
|
||||
try testing.expect_equal(ast.DeclKind.function, ast.decl_kind(&tree, declaration))
|
||||
function :: ast.function_decl(&tree, declaration)
|
||||
try testing.expect_equal("grouped", ast.source_text(&tree, function.name))
|
||||
try testing.expect_equal(3, ast.function_param_count(&tree, declaration))
|
||||
first :: ast.param(&tree, ast.function_param(&tree, declaration, 0))
|
||||
second :: ast.param(&tree, ast.function_param(&tree, declaration, 1))
|
||||
third :: ast.param(&tree, ast.function_param(&tree, declaration, 2))
|
||||
try testing.expect_equal("a", ast.source_text(&tree, first.name))
|
||||
try testing.expect_equal("b", ast.source_text(&tree, second.name))
|
||||
try testing.expect_equal("c", ast.source_text(&tree, third.name))
|
||||
try testing.expect_equal(first.type, second.type)
|
||||
try testing.expect(first.type != third.type)
|
||||
try testing.expect_equal(1, ast.block_statement_count(&tree, function.body))
|
||||
try testing.expect_equal(0, diagnostics.len(&diagnostic_store))
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user