hl-valdemar
9932750e21
Use typed singleton arrays for braceless arm blocks: &[statement] compiled but trapped under the Bro bootstrap.
615 lines
19 KiB
Plaintext
615 lines
19 KiB
Plaintext
import "@std/mem"
|
|
import "@std/arraylist"
|
|
|
|
import "@source/diagnostics"
|
|
import "@source/lexer"
|
|
import "@source/ast"
|
|
|
|
@hide:file
|
|
SyntaxError :: enum { unexpected_token }
|
|
|
|
@hide:file
|
|
ParseError :: alias SyntaxError | mem.AllocError | diagnostics.CapacityError
|
|
|
|
@hide:file
|
|
BindingPowerScalar :: distinct u32
|
|
|
|
@hide:file
|
|
BindingPower :: struct {
|
|
left BindingPowerScalar
|
|
right BindingPowerScalar
|
|
}
|
|
|
|
@hide:file
|
|
State :: struct {
|
|
builder ast.Builder
|
|
diagnostics @mut diagnostics.Store
|
|
tokens []lexer.Token
|
|
allocator mem.Allocator
|
|
next_token usize
|
|
}
|
|
|
|
#! borrows `tokens` and `source` while parsing; the returned Tree borrows `source` until ast.deinit.
|
|
#! `allocator` supplies returned node storage and temporary lists released before return.
|
|
#! the caller owns the Tree and must release it with ast.deinit.
|
|
parse proc(
|
|
allocator mem.Allocator,
|
|
tokens []lexer.Token,
|
|
source []u8,
|
|
diagnostic_store @mut diagnostics.Store,
|
|
) ast.Tree ! (mem.AllocError | diagnostics.CapacityError) {
|
|
state := State{
|
|
builder = ast.init_builder(allocator, source),
|
|
diagnostics = diagnostic_store,
|
|
tokens = tokens,
|
|
allocator = allocator,
|
|
next_token = 0,
|
|
}
|
|
defer ast.deinit_builder(&state.builder)
|
|
root :: try parse_package(&state)
|
|
return ast.finish(&state.builder, root)
|
|
}
|
|
|
|
@hide:file
|
|
parse_package proc(state @mut State) ast.NodeId ! (mem.AllocError | diagnostics.CapacityError) {
|
|
declarations arraylist.ArrayList(ast.NodeId) := arraylist.init(state.allocator)
|
|
defer arraylist.deinit(&declarations)
|
|
|
|
skip_newlines(state)
|
|
while !next_is(state, .eof) {
|
|
start :: state.next_token
|
|
declaration :: parse_decl(state) catch |err| {
|
|
match err {
|
|
.unexpected_token: {
|
|
recovered :: try recover_decl(state, start)
|
|
try arraylist.append(&declarations, recovered)
|
|
skip_newlines(state)
|
|
continue
|
|
}
|
|
else: return err
|
|
}
|
|
}
|
|
try arraylist.append(&declarations, declaration)
|
|
skip_newlines(state)
|
|
}
|
|
|
|
return try ast.add_package(
|
|
&state.builder,
|
|
ast.Span{ start = 0, end = u32(state.builder.source.len) },
|
|
declarations.items,
|
|
)
|
|
}
|
|
|
|
@hide:file
|
|
parse_decl proc(state @mut State) ast.NodeId ! ParseError {
|
|
if (next_is(state, .ident) and next_n_is(state, 1, .proc)) return try parse_func_decl(state)
|
|
return try parse_statement_decl(state)
|
|
}
|
|
|
|
@hide:file
|
|
parse_func_decl proc(state @mut State) ast.NodeId ! ParseError {
|
|
name :: try consume(state, .ident)
|
|
_ = try consume(state, .proc)
|
|
_ = try consume(state, .open_paren)
|
|
|
|
params arraylist.ArrayList(ast.NodeId) := arraylist.init(state.allocator)
|
|
defer arraylist.deinit(¶ms)
|
|
names arraylist.ArrayList(ast.Span) := arraylist.init(state.allocator)
|
|
defer arraylist.deinit(&names)
|
|
|
|
while !next_is(state, .close_paren) {
|
|
arraylist.clear(&names)
|
|
while true {
|
|
token :: try consume(state, .ident)
|
|
try arraylist.append(&names, token.span)
|
|
if next_is(state, .ident) break
|
|
_ = try consume(state, .comma)
|
|
}
|
|
|
|
type_id :: try parse_type(state)
|
|
type_span :: ast.builder_span(&state.builder, type_id)
|
|
for (names.items) |name_span| {
|
|
param_id :: try ast.add_param(
|
|
&state.builder,
|
|
ast.Span{ start = name_span.start, end = type_span.end },
|
|
name_span,
|
|
type_id,
|
|
)
|
|
try arraylist.append(¶ms, param_id)
|
|
}
|
|
if next_is(state, .comma) _ = try consume(state, .comma)
|
|
}
|
|
|
|
_ = try consume(state, .close_paren)
|
|
return_type :: try parse_type(state)
|
|
body :: try parse_block(state)
|
|
body_span :: ast.builder_span(&state.builder, body)
|
|
return try ast.add_function_decl(
|
|
&state.builder,
|
|
ast.Span{ start = name.span.start, end = body_span.end },
|
|
name.span,
|
|
return_type,
|
|
params.items,
|
|
body,
|
|
)
|
|
}
|
|
|
|
@hide:file
|
|
parse_block proc(state @mut State) ast.NodeId ! ParseError {
|
|
open :: try consume(state, .open_curly)
|
|
statements arraylist.ArrayList(ast.NodeId) := arraylist.init(state.allocator)
|
|
defer arraylist.deinit(&statements)
|
|
|
|
skip_newlines(state)
|
|
while !next_is(state, .close_curly) and !next_is(state, .eof) {
|
|
start :: state.next_token
|
|
statement :: parse_statement(state, false) catch |err| {
|
|
match err {
|
|
.unexpected_token: {
|
|
recovered :: try recover_statement(state, start)
|
|
try arraylist.append(&statements, recovered)
|
|
skip_newlines(state)
|
|
continue
|
|
}
|
|
else: return err
|
|
}
|
|
}
|
|
try arraylist.append(&statements, statement)
|
|
skip_newlines(state)
|
|
}
|
|
|
|
close :: try consume(state, .close_curly)
|
|
return try ast.add_block(
|
|
&state.builder,
|
|
ast.Span{ start = open.span.start, end = close.span.end },
|
|
statements.items,
|
|
)
|
|
}
|
|
|
|
@hide:file
|
|
parse_statement proc(state @mut State, allow_else bool) ast.NodeId ! ParseError {
|
|
if (
|
|
next_is(state, .ident)
|
|
and (
|
|
next_n_is(state, 1, .ident)
|
|
or next_n_is(state, 1, .double_colon)
|
|
or next_n_is(state, 1, .colon_equal)
|
|
)
|
|
) {
|
|
return try parse_statement_decl(state)
|
|
}
|
|
match current(state).kind {
|
|
.if: return try parse_if(state)
|
|
.return: return try parse_return(state, allow_else)
|
|
.ident, .int, .float, .string, .true, .false, .minus, .open_paren: {
|
|
expression :: try parse_expr(state)
|
|
return try ast.add_expression_stmt(&state.builder, expression)
|
|
}
|
|
else: return .unexpected_token
|
|
}
|
|
}
|
|
|
|
@hide:file
|
|
parse_return proc(state @mut State, allow_else bool) ast.NodeId ! ParseError {
|
|
keyword :: try consume(state, .return)
|
|
if (
|
|
next_is(state, .newline) or next_is(state, .close_curly) or next_is(state, .eof)
|
|
or (allow_else and next_is(state, .else))
|
|
) {
|
|
return try ast.add_return_stmt(&state.builder, keyword.span, null)
|
|
}
|
|
value :: try parse_expr(state)
|
|
if (
|
|
!next_is(state, .newline) and !next_is(state, .close_curly) and !next_is(state, .eof)
|
|
and !(allow_else and next_is(state, .else))
|
|
) {
|
|
return .unexpected_token
|
|
}
|
|
# include consumed grouping punctuation, which has no separate expression node.
|
|
end :: state.tokens[state.next_token - 1].span.end
|
|
return try ast.add_return_stmt(
|
|
&state.builder,
|
|
ast.Span{ start = keyword.span.start, end = end },
|
|
value,
|
|
)
|
|
}
|
|
|
|
@hide:file
|
|
parse_if proc(state @mut State) ast.NodeId ! ParseError {
|
|
keyword :: try consume(state, .if)
|
|
condition_start :: state.next_token
|
|
condition := parse_expr(state) catch |err| {
|
|
match err {
|
|
.unexpected_token: {}
|
|
else: return err
|
|
}
|
|
diagnostic :: try diagnostics.add(state.diagnostics, .unexpected_token, current(state).span)
|
|
yield try ast.add_invalid_expr(&state.builder, current(state).span, diagnostic)
|
|
}
|
|
if (ast.builder_expr_kind(&state.builder, condition) == .invalid) {
|
|
# discard a malformed guard's tail, but leave its arm and enclosing block intact.
|
|
while (
|
|
!next_is(state, .open_curly) and !next_is(state, .close_curly)
|
|
and !next_is(state, .newline) and !next_is(state, .eof)
|
|
and !next_is(state, .else) and !next_is(state, .return) and !next_is(state, .if)
|
|
) {
|
|
state.next_token += 1
|
|
}
|
|
}
|
|
if (
|
|
!next_is(state, .open_curly)
|
|
and !parenthesized_guard(state, condition_start)
|
|
and ast.builder_expr_kind(&state.builder, condition) != .call
|
|
and ast.builder_expr_kind(&state.builder, condition) != .invalid
|
|
) {
|
|
diagnostic :: try diagnostics.add(state.diagnostics, .unexpected_token, current(state).span)
|
|
condition = try ast.add_invalid_expr(
|
|
&state.builder,
|
|
ast.builder_span(&state.builder, condition),
|
|
diagnostic,
|
|
)
|
|
}
|
|
braced :: next_is(state, .open_curly)
|
|
then_block :: try parse_arm(state)
|
|
# only a braced arm permits its else on a later line.
|
|
after_then :: state.next_token
|
|
if (braced) skip_newlines(state)
|
|
alternative ?ast.NodeId := null
|
|
if next_is(state, .else) {
|
|
state.next_token += 1
|
|
alternative = try parse_arm(state)
|
|
} else {
|
|
state.next_token = after_then
|
|
}
|
|
end :: if (alternative) |block|
|
|
ast.builder_span(&state.builder, block).end
|
|
else
|
|
ast.builder_span(&state.builder, then_block).end
|
|
return try ast.add_if_stmt(
|
|
&state.builder,
|
|
ast.Span{ start = keyword.span.start, end = end },
|
|
condition,
|
|
then_block,
|
|
alternative,
|
|
)
|
|
}
|
|
|
|
@hide:file
|
|
parenthesized_guard proc(state @State, start usize) bool {
|
|
if (state.tokens[start].kind != .open_paren) return false
|
|
depth usize := 0
|
|
for start..state.next_token |index| {
|
|
kind :: state.tokens[index].kind
|
|
if (kind == .open_paren) depth += 1
|
|
if (kind == .close_paren) {
|
|
depth -= 1
|
|
if (depth == 0) return index + 1 == state.next_token
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
@hide:file
|
|
parse_arm proc(state @mut State) ast.NodeId ! ParseError {
|
|
if next_is(state, .open_curly) return try parse_block(state)
|
|
start :: state.next_token
|
|
statement :: parse_statement(state, true) catch |err| {
|
|
match err {
|
|
.unexpected_token: {}
|
|
else: return err
|
|
}
|
|
diagnostic :: try diagnostics.add(state.diagnostics, .unexpected_token, current(state).span)
|
|
span :: synchronize_arm(state, start)
|
|
yield try ast.add_invalid_stmt(&state.builder, span, diagnostic)
|
|
}
|
|
statements [1]ast.NodeId :: [statement]
|
|
return try ast.add_block(
|
|
&state.builder,
|
|
ast.builder_span(&state.builder, statement),
|
|
statements[..],
|
|
)
|
|
}
|
|
|
|
@hide:file
|
|
synchronize_arm proc(state @mut State, start usize) ast.Span {
|
|
first :: state.tokens[start].span
|
|
end := first.end
|
|
while (
|
|
!next_is(state, .eof) and !next_is(state, .newline)
|
|
and !next_is(state, .close_curly) and !next_is(state, .else)
|
|
) {
|
|
end = current(state).span.end
|
|
state.next_token += 1
|
|
}
|
|
return ast.Span{ start = first.start, end = end }
|
|
}
|
|
|
|
@hide:file
|
|
parse_statement_decl proc(state @mut State) ast.NodeId ! ParseError {
|
|
name :: try consume(state, .ident)
|
|
type_id ?ast.NodeId :: if next_is(state, .ident)
|
|
try parse_type(state)
|
|
else
|
|
null
|
|
|
|
assignment :: try consume_either(state, &[.double_colon, .colon_equal])
|
|
initializer :: try parse_expr(state)
|
|
end :: ast.builder_span(&state.builder, initializer).end
|
|
mutability :: if (assignment.kind == .double_colon) .constant else .variable
|
|
return try ast.add_statement_decl(
|
|
&state.builder,
|
|
ast.Span{ start = name.span.start, end = end },
|
|
name.span,
|
|
type_id,
|
|
mutability,
|
|
initializer,
|
|
)
|
|
}
|
|
|
|
@hide:file
|
|
parse_type proc(state @mut State) ast.NodeId ! ParseError {
|
|
if (!next_is(state, .ident)) return .unexpected_token
|
|
token :: current(state)
|
|
state.next_token += 1
|
|
return try ast.add_identifier(&state.builder, token.span)
|
|
}
|
|
|
|
@hide:file
|
|
parse_expr proc(state @mut State) ast.NodeId ! ParseError {
|
|
return try parse_expr_bp(state, BindingPowerScalar(0))
|
|
}
|
|
|
|
@hide:file
|
|
parse_expr_bp proc(state @mut State, min_bp BindingPowerScalar) ast.NodeId ! ParseError {
|
|
token := current(state)
|
|
left := if (prefix_binding_power(token.kind)) |right_bp| {
|
|
state.next_token += 1
|
|
operand :: try parse_expr_bp(state, right_bp)
|
|
operand_span :: ast.builder_span(&state.builder, operand)
|
|
yield try ast.add_unary(
|
|
&state.builder,
|
|
ast.Span{ start = token.span.start, end = operand_span.end },
|
|
.negate,
|
|
operand,
|
|
)
|
|
} else {
|
|
yield try parse_primary(state)
|
|
}
|
|
|
|
while true {
|
|
token = current(state)
|
|
if (token.kind == .open_paren) {
|
|
left = try parse_call(state, left)
|
|
continue
|
|
}
|
|
if (infix_binding_power(token.kind)) |power| {
|
|
if (power.left < min_bp) break
|
|
state.next_token += 1
|
|
right :: try parse_expr_bp(state, power.right)
|
|
left_span :: ast.builder_span(&state.builder, left)
|
|
right_span :: ast.builder_span(&state.builder, right)
|
|
left = try ast.add_binary(
|
|
&state.builder,
|
|
ast.Span{ start = left_span.start, end = right_span.end },
|
|
binary_operator(token.kind),
|
|
left,
|
|
right,
|
|
)
|
|
continue
|
|
}
|
|
break
|
|
}
|
|
return left
|
|
}
|
|
|
|
@hide:file
|
|
parse_call proc(state @mut State, callee ast.NodeId) ast.NodeId ! ParseError {
|
|
_ = try consume(state, .open_paren)
|
|
arguments arraylist.ArrayList(ast.NodeId) := arraylist.init(state.allocator)
|
|
defer arraylist.deinit(&arguments)
|
|
|
|
while (
|
|
!next_is(state, .close_paren)
|
|
and !next_is(state, .newline)
|
|
and !next_is(state, .close_curly)
|
|
and !next_is(state, .open_curly)
|
|
and !next_is(state, .else)
|
|
and !next_is(state, .return)
|
|
and !next_is(state, .if)
|
|
and !next_is(state, .eof)
|
|
) {
|
|
argument :: try parse_expr(state)
|
|
try arraylist.append(&arguments, argument)
|
|
if (!next_is(state, .comma)) break
|
|
state.next_token += 1
|
|
}
|
|
close :: try consume(state, .close_paren)
|
|
return try ast.add_call(
|
|
&state.builder,
|
|
ast.Span{
|
|
start = ast.builder_span(&state.builder, callee).start,
|
|
end = close.span.end,
|
|
},
|
|
callee,
|
|
arguments.items,
|
|
)
|
|
}
|
|
|
|
@hide:file
|
|
parse_primary proc(state @mut State) ast.NodeId ! ParseError {
|
|
token :: current(state)
|
|
match token.kind {
|
|
.open_paren: {
|
|
state.next_token += 1
|
|
expression :: try parse_expr(state)
|
|
if next_is(state, .close_paren) {
|
|
state.next_token += 1
|
|
return expression
|
|
}
|
|
diagnostic :: try diagnostics.add(
|
|
state.diagnostics,
|
|
.unexpected_token,
|
|
current(state).span,
|
|
)
|
|
return try ast.add_invalid_expr(&state.builder, ast.Span{
|
|
start = token.span.start,
|
|
end = ast.builder_span(&state.builder, expression).end,
|
|
}, diagnostic)
|
|
}
|
|
.int: {
|
|
state.next_token += 1
|
|
return try ast.add_literal(&state.builder, .int, token.span)
|
|
}
|
|
.float: {
|
|
state.next_token += 1
|
|
return try ast.add_literal(&state.builder, .float, token.span)
|
|
}
|
|
.string: {
|
|
state.next_token += 1
|
|
return try ast.add_literal(&state.builder, .string, token.span)
|
|
}
|
|
.true, .false: {
|
|
state.next_token += 1
|
|
return try ast.add_literal(&state.builder, .bool, token.span)
|
|
}
|
|
.ident: {
|
|
state.next_token += 1
|
|
return try ast.add_identifier(&state.builder, token.span)
|
|
}
|
|
else: {
|
|
diagnostic :: if (token.data.diagnostic != diagnostics.no_diagnostic)
|
|
token.data.diagnostic
|
|
else
|
|
try diagnostics.add(state.diagnostics, .unexpected_token, token.span)
|
|
|
|
if (
|
|
token.kind != .newline
|
|
and token.kind != .comma
|
|
and token.kind != .close_paren
|
|
and token.kind != .close_curly
|
|
and token.kind != .open_curly
|
|
and token.kind != .else
|
|
and token.kind != .return
|
|
and token.kind != .if
|
|
and token.kind != .eof
|
|
) {
|
|
state.next_token += 1
|
|
}
|
|
|
|
return try ast.add_invalid_expr(&state.builder, token.span, diagnostic)
|
|
}
|
|
}
|
|
}
|
|
|
|
@hide:file
|
|
recover_decl proc(
|
|
state @mut State,
|
|
start usize,
|
|
) ast.NodeId ! (mem.AllocError | diagnostics.CapacityError) {
|
|
diagnostic :: try diagnostics.add(state.diagnostics, .unexpected_token, current(state).span)
|
|
span :: synchronize(state, start, false)
|
|
return try ast.add_invalid_decl(&state.builder, span, diagnostic)
|
|
}
|
|
|
|
@hide:file
|
|
recover_statement proc(
|
|
state @mut State,
|
|
start usize,
|
|
) ast.NodeId ! (mem.AllocError | diagnostics.CapacityError) {
|
|
diagnostic :: try diagnostics.add(state.diagnostics, .unexpected_token, current(state).span)
|
|
span :: synchronize(state, start, true)
|
|
return try ast.add_invalid_stmt(&state.builder, span, diagnostic)
|
|
}
|
|
|
|
@hide:file
|
|
synchronize proc(state @mut State, start usize, stop_at_curly bool) ast.Span {
|
|
first :: state.tokens[start].span
|
|
end := first.end
|
|
while !next_is(state, .eof) and !next_is(state, .newline) {
|
|
if (stop_at_curly and next_is(state, .close_curly)) break
|
|
end = current(state).span.end
|
|
state.next_token += 1
|
|
}
|
|
if next_is(state, .newline) {
|
|
end = current(state).span.end
|
|
state.next_token += 1
|
|
}
|
|
return ast.Span{ start = first.start, end = end }
|
|
}
|
|
|
|
@hide:file
|
|
skip_newlines proc(state @mut State) void {
|
|
while next_is(state, .newline) state.next_token += 1
|
|
}
|
|
|
|
@hide:file
|
|
current proc(state @State) lexer.Token {
|
|
if (state.next_token >= state.tokens.len) return state.tokens[state.tokens.len - 1]
|
|
return state.tokens[state.next_token]
|
|
}
|
|
|
|
@hide:file
|
|
next_is proc(state @State, kind lexer.TokenKind) bool {
|
|
return current(state).kind == kind
|
|
}
|
|
|
|
@hide:file
|
|
next_n_is proc(state @State, count usize, kind lexer.TokenKind) bool {
|
|
index :: state.next_token + count
|
|
return index < state.tokens.len and state.tokens[index].kind == kind
|
|
}
|
|
|
|
@hide:file
|
|
consume proc(state @mut State, kind lexer.TokenKind) lexer.Token ! SyntaxError {
|
|
token :: current(state)
|
|
if (token.kind != kind) return .unexpected_token
|
|
state.next_token += 1
|
|
return token
|
|
}
|
|
|
|
@hide:file
|
|
consume_either proc(state @mut State, kinds []lexer.TokenKind) lexer.Token ! SyntaxError {
|
|
token :: current(state)
|
|
for (kinds) |kind| if token.kind == kind {
|
|
state.next_token += 1
|
|
return token
|
|
}
|
|
return .unexpected_token
|
|
}
|
|
|
|
@hide:file
|
|
prefix_binding_power proc(kind lexer.TokenKind) ?BindingPowerScalar {
|
|
return match kind {
|
|
.minus: BindingPowerScalar(50)
|
|
else: null
|
|
}
|
|
}
|
|
|
|
@hide:file
|
|
infix_binding_power proc(kind lexer.TokenKind) ?BindingPower {
|
|
return match kind {
|
|
.equal_equal, .not_equal: BindingPower{ left = BindingPowerScalar(10), right = BindingPowerScalar(11) }
|
|
.less, .less_equal, .greater, .greater_equal: BindingPower{ left = BindingPowerScalar(20), right = BindingPowerScalar(21) }
|
|
.plus, .minus: BindingPower{ left = BindingPowerScalar(30), right = BindingPowerScalar(31) }
|
|
.star, .slash: BindingPower{ left = BindingPowerScalar(40), right = BindingPowerScalar(41) }
|
|
else: null
|
|
}
|
|
}
|
|
|
|
@hide:file
|
|
binary_operator proc(kind lexer.TokenKind) ast.BinaryOperator {
|
|
return match kind {
|
|
.plus: .add
|
|
.minus: .subtract
|
|
.star: .multiply
|
|
.slash: .divide
|
|
.equal_equal: .equal
|
|
.not_equal: .not_equal
|
|
.less: .less
|
|
.less_equal: .less_equal
|
|
.greater: .greater
|
|
.greater_equal: .greater_equal
|
|
else: unreachable
|
|
}
|
|
}
|