parse arithmetic binary expressions
This commit is contained in:
+23
-1
@@ -12,6 +12,9 @@ import "@source/lexer"
|
||||
@hide:file
|
||||
TokenId :: alias lexer.TokenId
|
||||
|
||||
@hide:file
|
||||
NO_ID_TOKEN :: alias lexer.NO_ID_TOKEN
|
||||
|
||||
NodeId :: distinct u32
|
||||
ExtraId :: distinct u32
|
||||
|
||||
@@ -28,7 +31,7 @@ NodeData :: union {
|
||||
|
||||
Node :: struct {
|
||||
kind NodeKind
|
||||
main_token TokenId
|
||||
main_token TokenId = NO_ID_TOKEN
|
||||
|
||||
data0 NodeData = NodeData{ node_id = NO_ID_NODE }
|
||||
data1 NodeData = NodeData{ node_id = NO_ID_NODE }
|
||||
@@ -45,7 +48,14 @@ NodeKind :: enum {
|
||||
# * main_token: token literal
|
||||
expr_identifier
|
||||
|
||||
# UNARY:
|
||||
# todo: document
|
||||
expr_unary
|
||||
|
||||
# BINARY:
|
||||
# * main_token: operator
|
||||
# * data0: NodeId - left-hand side
|
||||
# * data1: NodeId - right-hand side
|
||||
expr_binary
|
||||
|
||||
# DECL:
|
||||
@@ -56,6 +66,13 @@ NodeKind :: enum {
|
||||
# * data1: TokenId of either `::` or `:=` indicating mutability
|
||||
stmt_decl
|
||||
|
||||
# PACKAGE:
|
||||
# * data0: ExtraId
|
||||
# * data1: ExtraId
|
||||
#
|
||||
# extra[data0..data1]: NodeId - all statement node ids in package
|
||||
package
|
||||
|
||||
invalid
|
||||
}
|
||||
|
||||
@@ -64,6 +81,11 @@ node_id proc(idx uint) NodeId {
|
||||
return NodeId(idx)
|
||||
}
|
||||
|
||||
maybe_node_id proc(idx uint) ?NodeId {
|
||||
if (u64(idx) >= u64(NO_ID_NODE)) return null
|
||||
return NodeId(idx)
|
||||
}
|
||||
|
||||
extra_id proc(idx uint) ExtraId {
|
||||
debug.assert(u64(idx) < u64(NO_ID_EXTRA))
|
||||
return ExtraId(idx)
|
||||
|
||||
@@ -43,10 +43,33 @@ render proc(
|
||||
_ = try render_lit(renderer, node_id, parse_state, tokens)
|
||||
.stmt_decl:
|
||||
try render_decl(renderer, node_id, parse_state, tokens)
|
||||
.package:
|
||||
try render_pkg(renderer, node_id, parse_state, tokens)
|
||||
else: unreachable
|
||||
}
|
||||
}
|
||||
|
||||
@hide:file
|
||||
render_pkg proc(
|
||||
renderer @mut Renderer,
|
||||
node_id NodeId,
|
||||
parse_state @ParseState,
|
||||
tokens []Token,
|
||||
) void ! mem.AllocError {
|
||||
node :: parse_state.nodes.items[usize(node_id)]
|
||||
extra_start :: usize(node.data0.extra_id)
|
||||
extra_end :: usize(node.data1.extra_id)
|
||||
for extra_start..extra_end |i| {
|
||||
stmt_node_id :: ast.node_id(parse_state.extra.items[i])
|
||||
stmt_node :: parse_state.nodes.items[usize(stmt_node_id)]
|
||||
match stmt_node.kind {
|
||||
.stmt_decl:
|
||||
try render_decl(renderer, stmt_node_id, parse_state, tokens)
|
||||
else: unreachable
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@hide:file
|
||||
render_decl proc(
|
||||
renderer @mut Renderer,
|
||||
@@ -58,8 +81,9 @@ render_decl proc(
|
||||
tok_ident :: tokens[usize(node.main_token)]
|
||||
try arraylist.append(&renderer.tokens, tok_ident)
|
||||
|
||||
node_id_type :: ast.node_id(parse_state.extra.items[usize(node.data0.extra_id)])
|
||||
try render_type(renderer, node_id_type, parse_state, tokens)
|
||||
raw_node_id_type :: parse_state.extra.items[usize(node.data0.extra_id)]
|
||||
node_id_type :: ast.maybe_node_id(raw_node_id_type)
|
||||
if (node_id_type) |id| try render_type(renderer, id, parse_state, tokens)
|
||||
|
||||
tok_assign :: tokens[usize(node.data1.token_id)]
|
||||
try arraylist.append(&renderer.tokens, tok_assign)
|
||||
@@ -84,20 +108,42 @@ render_expr proc(renderer @mut Renderer,
|
||||
parse_state @ParseState,
|
||||
tokens []Token,
|
||||
) TokenId ! mem.AllocError {
|
||||
# todo: support unary and binary expressions
|
||||
# todo: support unary expressions
|
||||
node :: parse_state.nodes.items[usize(node_id)]
|
||||
match node.kind {
|
||||
return match node.kind {
|
||||
.expr_identifier: {
|
||||
tok :: tokens[usize(node.main_token)]
|
||||
try arraylist.append(&renderer.tokens, tok)
|
||||
return node.main_token
|
||||
yield node.main_token
|
||||
}
|
||||
.expr_binary:
|
||||
try render_expr_binary(renderer, node_id, parse_state, tokens)
|
||||
.literal_int, .literal_float, .literal_string:
|
||||
return try render_lit(renderer, node_id, parse_state, tokens)
|
||||
try render_lit(renderer, node_id, parse_state, tokens)
|
||||
else: unreachable
|
||||
}
|
||||
}
|
||||
|
||||
@hide:file
|
||||
render_expr_binary proc(
|
||||
renderer @mut Renderer,
|
||||
node_id NodeId,
|
||||
parse_state @ParseState,
|
||||
tokens []Token,
|
||||
) TokenId ! mem.AllocError {
|
||||
node :: parse_state.nodes.items[usize(node_id)]
|
||||
operator :: node.main_token
|
||||
lhs :: node.data0.node_id
|
||||
rhs :: node.data1.node_id
|
||||
|
||||
# render lhs operator rhs
|
||||
_ = try render_expr(renderer, lhs, parse_state, tokens)
|
||||
try arraylist.append(&renderer.tokens, tokens[usize(operator)])
|
||||
last_token :: try render_expr(renderer, rhs, parse_state, tokens)
|
||||
|
||||
return last_token
|
||||
}
|
||||
|
||||
@hide:file
|
||||
render_type proc(
|
||||
renderer @mut Renderer,
|
||||
@@ -124,7 +170,7 @@ render_lit proc(
|
||||
tokens []Token,
|
||||
) TokenId ! mem.AllocError {
|
||||
node :: parse_state.nodes.items[usize(node_id)]
|
||||
tok :: tokens[node.main_token]
|
||||
tok :: tokens[usize(node.main_token)]
|
||||
try arraylist.append(&renderer.tokens, tok)
|
||||
return node.main_token
|
||||
}
|
||||
|
||||
+79
-50
@@ -3,7 +3,6 @@ import "@std/mem"
|
||||
import "@std/strmap"
|
||||
import "@std/enums/enummap"
|
||||
import "@std/arraylist"
|
||||
import "@std/debug"
|
||||
|
||||
import "@source/strpool"
|
||||
|
||||
@@ -43,8 +42,6 @@ keywords std.StringMap(TokenKind) :: strmap.init([
|
||||
{ "while", .while },
|
||||
])
|
||||
|
||||
TokenId :: distinct u32
|
||||
|
||||
Diagnostic :: struct {
|
||||
token TokenId
|
||||
code ErrorCode
|
||||
@@ -68,7 +65,6 @@ deinit proc(state @mut State) void {
|
||||
}
|
||||
|
||||
scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.InternError) {
|
||||
tokens :: &state.tokens
|
||||
diagnostics :: &state.diagnostics
|
||||
|
||||
cursor := 0
|
||||
@@ -77,7 +73,7 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter
|
||||
|
||||
# whitespace
|
||||
if char == '\n' {
|
||||
try add_token(tokens, Token{ kind = .newline, start = cursor })
|
||||
try add_token(state, Token{ start = cursor, kind = .newline })
|
||||
cursor += 1
|
||||
continue
|
||||
} else if is_whitespace(char) {
|
||||
@@ -96,9 +92,9 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter
|
||||
if is_alpha(char) or char == '_' {
|
||||
start :: cursor
|
||||
result :: scan_ident(start, program)
|
||||
try add_token(tokens, Token{
|
||||
kind = result.kind,
|
||||
try add_token(state, Token{
|
||||
start = start,
|
||||
kind = result.kind,
|
||||
str_id = result.str_id,
|
||||
})
|
||||
cursor = result.end
|
||||
@@ -109,15 +105,15 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter
|
||||
if is_digit(char) {
|
||||
start :: cursor
|
||||
result :: scan_number(start, program) catch |err| {
|
||||
token :: token_id(tokens.items.len)
|
||||
try add_token(tokens, Token{ kind = .invalid, start = start })
|
||||
token :: token_id(state.tokens.items.len)
|
||||
try add_token(state, Token{ start = start, kind = .invalid })
|
||||
try arraylist.append(diagnostics, Diagnostic{ token = token, code = err.code })
|
||||
cursor = err.end
|
||||
continue
|
||||
}
|
||||
|
||||
kind :: if (result.has_decimal) .float else .int
|
||||
try add_token(tokens, Token{ kind = kind, start = start })
|
||||
try add_token(state, Token{ start = start, kind = kind })
|
||||
cursor = result.end
|
||||
continue
|
||||
}
|
||||
@@ -126,68 +122,107 @@ scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.Inter
|
||||
if char == '"' {
|
||||
start :: cursor
|
||||
result :: scan_string(start, program) catch |err| {
|
||||
token :: token_id(tokens.items.len)
|
||||
try add_token(tokens, Token{ kind = .invalid, start = start })
|
||||
token :: token_id(state.tokens.items.len)
|
||||
try add_token(state, Token{ start = start, kind = .invalid })
|
||||
try arraylist.append(diagnostics, Diagnostic{ token = token, code = err.code })
|
||||
cursor = err.end
|
||||
continue
|
||||
}
|
||||
|
||||
try add_token(tokens, Token{ kind = .string, start = start })
|
||||
try add_token(state, Token{ start = start, kind = .string })
|
||||
cursor = result.end
|
||||
continue
|
||||
}
|
||||
|
||||
# immutable assignment or single colon
|
||||
# immutable decl, mutable decl, or single colon
|
||||
if char == ':' {
|
||||
if cursor + 1 < program.len and program[cursor + 1] == ':' {
|
||||
try add_token(tokens, Token{ kind = .double_colon, start = cursor })
|
||||
try add_token(state, Token{ start = cursor, kind = .double_colon })
|
||||
cursor += 2
|
||||
continue
|
||||
}
|
||||
|
||||
try add_token(tokens, Token{ kind = .colon, start = cursor })
|
||||
if cursor + 1 < program.len and program[cursor + 1] == '=' {
|
||||
try add_token(state, Token{ start = cursor, kind = .colon_equal })
|
||||
cursor += 2
|
||||
continue
|
||||
}
|
||||
|
||||
try add_token(state, Token{ start = cursor, kind = .colon })
|
||||
cursor += 1
|
||||
continue
|
||||
}
|
||||
|
||||
# mutable reassignment
|
||||
if char == '=' {
|
||||
try add_token(tokens, Token{ kind = .equal, start = cursor })
|
||||
# compound arithmetic assignment
|
||||
if char == '+' {
|
||||
if cursor + 1 < program.len and program[cursor + 1] == '=' {
|
||||
try add_token(state, Token{ start = cursor, kind = .plus_equal })
|
||||
cursor += 2
|
||||
continue
|
||||
}
|
||||
|
||||
try add_token(state, Token{ start = cursor, kind = .plus })
|
||||
cursor += 1
|
||||
continue
|
||||
}
|
||||
if char == '-' {
|
||||
if cursor + 1 < program.len and program[cursor + 1] == '=' {
|
||||
try add_token(state, Token{ start = cursor, kind = .minus_equal })
|
||||
cursor += 2
|
||||
continue
|
||||
}
|
||||
|
||||
try add_token(state, Token{ start = cursor, kind = .minus })
|
||||
cursor += 1
|
||||
continue
|
||||
}
|
||||
if char == '*' {
|
||||
if cursor + 1 < program.len and program[cursor + 1] == '=' {
|
||||
try add_token(state, Token{ start = cursor, kind = .star_equal })
|
||||
cursor += 2
|
||||
continue
|
||||
}
|
||||
|
||||
try add_token(state, Token{ start = cursor, kind = .star })
|
||||
cursor += 1
|
||||
continue
|
||||
}
|
||||
if char == '/' {
|
||||
if cursor + 1 < program.len and program[cursor + 1] == '=' {
|
||||
try add_token(state, Token{ start = cursor, kind = .slash_equal })
|
||||
cursor += 2
|
||||
continue
|
||||
}
|
||||
|
||||
try add_token(state, Token{ start = cursor, kind = .slash })
|
||||
cursor += 1
|
||||
continue
|
||||
}
|
||||
|
||||
# parentheses
|
||||
if char == '(' {
|
||||
try add_token(tokens, Token{ kind = .open_paren, start = cursor })
|
||||
cursor += 1
|
||||
continue
|
||||
} else if char == ')' {
|
||||
try add_token(tokens, Token{ kind = .close_paren, start = cursor })
|
||||
cursor += 1
|
||||
continue
|
||||
# single character tokens
|
||||
kind ?TokenKind :: match char {
|
||||
'=': .equal
|
||||
'(': .open_paren
|
||||
')': .close_paren
|
||||
'{': .open_curly
|
||||
'}': .close_curly
|
||||
else: null
|
||||
}
|
||||
|
||||
# curly braces
|
||||
if char == '{' {
|
||||
try add_token(tokens, Token{ kind = .open_curly, start = cursor })
|
||||
cursor += 1
|
||||
continue
|
||||
} else if char == '}' {
|
||||
try add_token(tokens, Token{ kind = .close_curly, start = cursor })
|
||||
if (kind) |k| {
|
||||
try add_token(state, Token{ start = cursor, kind = k })
|
||||
cursor += 1
|
||||
continue
|
||||
}
|
||||
|
||||
# invalid character
|
||||
token :: token_id(tokens.items.len)
|
||||
token :: token_id(state.tokens.items.len)
|
||||
try add_token(state, Token{ start = cursor, kind = .invalid })
|
||||
try arraylist.append(diagnostics, Diagnostic{ token = token, code = .invalid_character })
|
||||
try add_token(tokens, Token{ kind = .invalid, start = cursor })
|
||||
cursor += 1
|
||||
}
|
||||
|
||||
try add_token(tokens, Token{ kind = .eof, start = cursor })
|
||||
try add_token(state, Token{ start = cursor, kind = .eof })
|
||||
}
|
||||
|
||||
ScanIdentResult :: struct {
|
||||
@@ -210,9 +245,9 @@ scan_ident proc(start usize, program []u8) ScanIdentResult {
|
||||
|
||||
# don't intern keywords (already O(1) lookup via token kind)
|
||||
str_id :: if (kind == .ident)
|
||||
strpool.intern(&strpool.strings, program[start..cursor]) catch strpool.NO_ID
|
||||
strpool.intern(&strpool.strings, program[start..cursor]) catch strpool.NO_ID_STR
|
||||
else
|
||||
strpool.NO_ID
|
||||
strpool.NO_ID_STR
|
||||
|
||||
return ScanIdentResult{
|
||||
end = cursor,
|
||||
@@ -243,7 +278,7 @@ scan_number proc(start usize, program []u8) ScanNumResult ! ScanError {
|
||||
if has_decimal and (cursor >= program.len or !is_digit(program[cursor])) {
|
||||
return ScanError{
|
||||
code = .float_must_end_with_digit,
|
||||
end = cursor + 1,
|
||||
end = cursor,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -300,13 +335,7 @@ is_digit proc(char u8) bool {
|
||||
}
|
||||
|
||||
@hide
|
||||
add_token proc(tokens @mut std.ArrayList(Token), token Token) void ! mem.AllocError {
|
||||
_ = token_id(tokens.items.len)
|
||||
try arraylist.append(tokens, token)
|
||||
}
|
||||
|
||||
@hide
|
||||
token_id proc(idx uint) TokenId {
|
||||
debug.assert(u64(idx) < u64(maxval!(TokenId)))
|
||||
return TokenId(idx)
|
||||
add_token proc(state @mut State, token Token) void ! mem.AllocError {
|
||||
_ = token_id(state.tokens.items.len)
|
||||
try arraylist.append(&state.tokens, token)
|
||||
}
|
||||
|
||||
+23
-3
@@ -1,9 +1,14 @@
|
||||
import "@std/debug"
|
||||
import "@source/strpool"
|
||||
|
||||
TokenId :: distinct u32
|
||||
|
||||
NO_ID_TOKEN :: maxval!(TokenId)
|
||||
|
||||
Token :: struct {
|
||||
kind TokenKind
|
||||
start uint
|
||||
str_id strpool.StringId = strpool.NO_ID
|
||||
kind TokenKind
|
||||
str_id strpool.StringId = strpool.NO_ID_STR
|
||||
}
|
||||
|
||||
TokenKind :: enum {
|
||||
@@ -22,7 +27,16 @@ TokenKind :: enum {
|
||||
greater, greater_equal
|
||||
|
||||
# assignment
|
||||
equal, double_colon
|
||||
double_colon, colon_equal
|
||||
equal
|
||||
|
||||
# arithmetic
|
||||
plus, minus
|
||||
star, slash
|
||||
|
||||
# assignment & arithmetic
|
||||
plus_equal, minus_equal
|
||||
star_equal, slash_equal
|
||||
|
||||
# delimiters
|
||||
open_paren, close_paren
|
||||
@@ -36,3 +50,9 @@ TokenKind :: enum {
|
||||
eof
|
||||
invalid
|
||||
}
|
||||
|
||||
@hide
|
||||
token_id proc(idx uint) TokenId {
|
||||
debug.assert(u64(idx) < u64(NO_ID_TOKEN))
|
||||
return TokenId(idx)
|
||||
}
|
||||
|
||||
+6
-25
@@ -1,6 +1,5 @@
|
||||
import "@std/debug"
|
||||
import "@std/mem"
|
||||
import "@std/enums/enummap"
|
||||
|
||||
test import "@std/enums/enummap"
|
||||
test import "@std/arraylist"
|
||||
@@ -20,8 +19,11 @@ test import "@source/parser"
|
||||
|
||||
program ::
|
||||
`# literals
|
||||
`x int :: 123.9
|
||||
`y :: "hello"
|
||||
`x float :: 123.9
|
||||
`y int := 43
|
||||
`z :: "hello"
|
||||
`
|
||||
`a := 1 + 2 * 3
|
||||
|
||||
main proc() void! {
|
||||
strpool.strings = strpool.init(mem.c_allocator)
|
||||
@@ -42,13 +44,6 @@ main proc() void! {
|
||||
}
|
||||
debug.print("]]\n\n", {})
|
||||
|
||||
debug.print("TOKENS::DIAGNOSTICS::[[\n", {})
|
||||
for scan_state.diagnostics.items |diagnostic| {
|
||||
details :: enummap.get(&lexer.error_msg_map, diagnostic.code)
|
||||
debug.print("{}: {}\n", {details?.name, details?.message})
|
||||
}
|
||||
debug.print("]]\n\n", {})
|
||||
|
||||
parse_state := parser.init(mem.c_allocator)
|
||||
defer parser.deinit(&parse_state)
|
||||
root :: parser.parse(&parse_state, scan_state.tokens.items) catch |err| {
|
||||
@@ -58,21 +53,7 @@ main proc() void! {
|
||||
|
||||
debug.print("AST::[[\n", {})
|
||||
for parse_state.nodes.items |node, id| {
|
||||
token :: scan_state.tokens.items[node.main_token]
|
||||
|
||||
end :: if (token.kind == .int or token.kind == .float) lbl: {
|
||||
res :: lexer.scan_number(token.start, program) catch {
|
||||
yield :lbl token.start
|
||||
}
|
||||
yield :lbl res.end
|
||||
} else if (token.kind == .string) lbl: {
|
||||
res :: lexer.scan_string(token.start, program) catch {
|
||||
yield :lbl token.start
|
||||
}
|
||||
yield :lbl res.end
|
||||
} else token.start
|
||||
|
||||
debug.print("{} (id = {}): {}\n", { node.kind, id, program[token.start..end] })
|
||||
debug.print("{} (id = {})\n", { node.kind, id })
|
||||
}
|
||||
debug.print("]]\n\n", {})
|
||||
|
||||
|
||||
+108
-29
@@ -58,6 +58,21 @@ NO_ID_NODE :: alias ast.NO_ID_NODE
|
||||
@hide:file
|
||||
NO_ID_EXTRA :: alias ast.NO_ID_EXTRA
|
||||
|
||||
BindingPowerScalar :: distinct u32
|
||||
|
||||
BindingPower :: struct {
|
||||
left BindingPowerScalar
|
||||
right BindingPowerScalar
|
||||
}
|
||||
|
||||
infix_binding_power proc(token_kind TokenKind) ?BindingPower {
|
||||
return match token_kind {
|
||||
.plus, .minus: BindingPower{ left = BindingPowerScalar(10), right = BindingPowerScalar(11) }
|
||||
.star, .slash: BindingPower{ left = BindingPowerScalar(20), right = BindingPowerScalar(21) }
|
||||
else: null
|
||||
}
|
||||
}
|
||||
|
||||
State :: struct {
|
||||
nodes std.ArrayList(Node)
|
||||
|
||||
@@ -65,7 +80,7 @@ State :: struct {
|
||||
extra std.ArrayList(u32)
|
||||
|
||||
tokens []Token = &[]
|
||||
next_token TokenId = TokenId(0)
|
||||
next_token usize = 0
|
||||
}
|
||||
|
||||
init proc(allocator mem.Allocator) State {
|
||||
@@ -82,35 +97,65 @@ deinit proc(state @mut State) void {
|
||||
|
||||
parse proc(state @mut State, tokens []Token) NodeId ! ParseError {
|
||||
state.tokens = tokens
|
||||
return try parse_pkg(state)
|
||||
}
|
||||
|
||||
parse_pkg proc(state @mut State) NodeId ! ParseError {
|
||||
if (state.next_token < state.tokens.len
|
||||
and state.tokens[state.next_token].kind == .eof
|
||||
) return NO_ID_NODE
|
||||
|
||||
stmt_ids std.ArrayList(NodeId) := arraylist.init(state.nodes.allocator)
|
||||
defer arraylist.deinit(&stmt_ids)
|
||||
|
||||
while (state.next_token < state.tokens.len
|
||||
and state.tokens[state.next_token].kind != .eof
|
||||
) {
|
||||
stmt_id :: try parse_stmt(state)
|
||||
try arraylist.append(&stmt_ids, stmt_id)
|
||||
}
|
||||
|
||||
pkg_start :: ast.extra_id(state.extra.items.len)
|
||||
for (stmt_ids.items) |stmt_id| _ = try add_extra(state, u32(stmt_id))
|
||||
pkg_end :: ast.extra_id(state.extra.items.len)
|
||||
|
||||
package :: try add_node(state, Node{
|
||||
kind = .package,
|
||||
data0 = NodeData{ extra_id = pkg_start },
|
||||
data1 = NodeData{ extra_id = pkg_end },
|
||||
})
|
||||
|
||||
return package
|
||||
}
|
||||
|
||||
parse_stmt proc(state @mut State) NodeId ! ParseError {
|
||||
return try parse_decl(state)
|
||||
}
|
||||
|
||||
parse_decl proc(state @mut State) NodeId ! ParseError {
|
||||
# expect identifier
|
||||
try expect(state, .ident)
|
||||
ident_token :: state.next_token
|
||||
state.next_token += 1
|
||||
ident_token :: try consume(state, .ident)
|
||||
|
||||
# check for type
|
||||
# todo: make a parse_type proc for this
|
||||
type_id :: if (state.tokens[state.next_token].kind == .ident)
|
||||
type_id :: if (next_is(state, .ident))
|
||||
try parse_primary(state)
|
||||
else
|
||||
NO_ID_NODE
|
||||
|
||||
# expect `::` (immutable assignment)
|
||||
mutability_tok_id :: try consume(state, .double_colon)
|
||||
# expect `::` or `:=` (immutable or mutable assignment)
|
||||
mutability_tok_id :: try consume_either(state, &[.double_colon, .colon_equal])
|
||||
|
||||
# expect expression
|
||||
expr_id :: try parse_expr(state)
|
||||
|
||||
# expect statement terminator (newline)
|
||||
_ = try consume_either(state, &[.newline, .eof])
|
||||
while (next_is(state, .newline)) _ = try consume(state, .newline)
|
||||
|
||||
extra_start :: try add_extra(&state.extra, u32(type_id))
|
||||
_ = try add_extra(&state.extra, u32(expr_id))
|
||||
extra_start :: try add_extra(state, u32(type_id))
|
||||
_ = try add_extra(state, u32(expr_id))
|
||||
|
||||
decl :: try add_node(&state.nodes, Node{
|
||||
decl :: try add_node(state, Node{
|
||||
kind = .stmt_decl,
|
||||
main_token = ident_token,
|
||||
data0 = NodeData{ extra_id = extra_start },
|
||||
@@ -121,36 +166,68 @@ parse_decl proc(state @mut State) NodeId ! ParseError {
|
||||
}
|
||||
|
||||
parse_expr proc(state @mut State) NodeId ! mem.AllocError {
|
||||
return try parse_primary(state)
|
||||
return try parse_expr_bp(state, BindingPowerScalar(0))
|
||||
}
|
||||
|
||||
parse_expr_bp proc(state @mut State, min_bp BindingPowerScalar) NodeId ! mem.AllocError {
|
||||
lhs := try parse_primary(state)
|
||||
|
||||
while infix_binding_power(state.tokens[state.next_token].kind) |bp| {
|
||||
if (bp.left < min_bp) break
|
||||
|
||||
operator :: TokenId(state.next_token)
|
||||
state.next_token += 1
|
||||
|
||||
rhs :: try parse_expr_bp(state, bp.right)
|
||||
lhs = try add_node(state, Node{
|
||||
kind = .expr_binary,
|
||||
main_token = operator,
|
||||
data0 = NodeData{ node_id = lhs },
|
||||
data1 = NodeData{ node_id = rhs },
|
||||
})
|
||||
}
|
||||
|
||||
return lhs
|
||||
}
|
||||
|
||||
parse_primary proc(state @mut State) NodeId ! mem.AllocError {
|
||||
start_token :: state.next_token
|
||||
start_idx :: state.next_token
|
||||
start_token :: TokenId(start_idx)
|
||||
state.next_token += 1
|
||||
return match state.tokens[start_token].kind {
|
||||
.int: try add_node(&state.nodes, Node{
|
||||
return match state.tokens[start_idx].kind {
|
||||
.int: try add_node(state, Node{
|
||||
kind = .literal_int,
|
||||
main_token = start_token,
|
||||
})
|
||||
.float: try add_node(&state.nodes, Node{
|
||||
.float: try add_node(state, Node{
|
||||
kind = .literal_float,
|
||||
main_token = start_token,
|
||||
})
|
||||
.string: try add_node(&state.nodes, Node{
|
||||
.string: try add_node(state, Node{
|
||||
kind = .literal_string,
|
||||
main_token = start_token,
|
||||
})
|
||||
.ident: try add_node(&state.nodes, Node{
|
||||
.ident: try add_node(state, Node{
|
||||
kind = .expr_identifier,
|
||||
main_token = start_token,
|
||||
})
|
||||
else: try add_node(&state.nodes, Node{
|
||||
else: try add_node(state, Node{
|
||||
kind = .invalid,
|
||||
main_token = start_token,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
next_is proc(state @mut State, token_kind TokenKind) bool {
|
||||
return state.tokens[state.next_token].kind == token_kind
|
||||
}
|
||||
|
||||
next_is_either proc(state @mut State, token_kinds []TokenKind) bool {
|
||||
is_either := false
|
||||
for (token_kinds) |kind| is_either = is_either or next_is(state, kind)
|
||||
return is_either
|
||||
}
|
||||
|
||||
expect proc(state @mut State, token_kind TokenKind) void ! ErrorCode {
|
||||
if (state.tokens[state.next_token].kind != token_kind) return .unexpected_token
|
||||
}
|
||||
@@ -162,24 +239,26 @@ expect_either proc(state @mut State, token_kinds []TokenKind) void ! ErrorCode {
|
||||
|
||||
consume proc(state @mut State, token_kind TokenKind) TokenId ! ErrorCode {
|
||||
try expect(state, token_kind)
|
||||
id :: TokenId(state.next_token)
|
||||
state.next_token += 1
|
||||
return state.next_token - 1
|
||||
return id
|
||||
}
|
||||
|
||||
consume_either proc(state @mut State, token_kinds []TokenKind) TokenId ! ErrorCode {
|
||||
try expect_either(state, token_kinds)
|
||||
id :: TokenId(state.next_token)
|
||||
state.next_token += 1
|
||||
return state.next_token - 1
|
||||
}
|
||||
|
||||
add_node proc(nodes @mut std.ArrayList(Node), node Node) NodeId ! mem.AllocError {
|
||||
id :: ast.node_id(nodes.items.len)
|
||||
try arraylist.append(nodes, node)
|
||||
return id
|
||||
}
|
||||
|
||||
add_extra proc(extra @mut std.ArrayList(u32), data u32) ExtraId ! mem.AllocError {
|
||||
id :: ast.extra_id(extra.items.len)
|
||||
try arraylist.append(extra, data)
|
||||
add_node proc(state @mut State, node Node) NodeId ! mem.AllocError {
|
||||
id :: ast.node_id(state.nodes.items.len)
|
||||
try arraylist.append(&state.nodes, node)
|
||||
return id
|
||||
}
|
||||
|
||||
add_extra proc(state @mut State, data u32) ExtraId ! mem.AllocError {
|
||||
id :: ast.extra_id(state.extra.items.len)
|
||||
try arraylist.append(&state.extra, data)
|
||||
return id
|
||||
}
|
||||
|
||||
@@ -6,6 +6,67 @@ import "@source/lexer"
|
||||
|
||||
ast_renderer :: import "@source/ast/renderer"
|
||||
|
||||
handles_package_declarations test {
|
||||
strpool.strings = strpool.init(mem.c_allocator)
|
||||
defer strpool.deinit(&strpool.strings)
|
||||
|
||||
source ::
|
||||
`first int :: 42
|
||||
`second float :: 2.15 + 3
|
||||
`message :: "hello"
|
||||
|
||||
scan_state := lexer.init(mem.c_allocator)
|
||||
defer lexer.deinit(&scan_state)
|
||||
try lexer.scan(&scan_state, source)
|
||||
|
||||
parse_state := init(mem.c_allocator)
|
||||
defer deinit(&parse_state)
|
||||
root :: try parse(&parse_state, scan_state.tokens.items)
|
||||
|
||||
renderer := ast_renderer.init(mem.c_allocator)
|
||||
defer ast_renderer.deinit(&renderer)
|
||||
try ast_renderer.render(&renderer, root, &parse_state, scan_state.tokens.items)
|
||||
|
||||
try testing.expect_equal(scan_state.tokens.items.len, renderer.tokens.items.len)
|
||||
for scan_state.tokens.items |expected, i| {
|
||||
actual :: renderer.tokens.items[i]
|
||||
try testing.expect_equal(expected.kind, actual.kind)
|
||||
try testing.expect_equal(expected.start, actual.start)
|
||||
try testing.expect_equal(expected.str_id, actual.str_id)
|
||||
}
|
||||
}
|
||||
|
||||
handles_binary_arithmetic_expression test {
|
||||
strpool.strings = strpool.init(mem.c_allocator)
|
||||
defer strpool.deinit(&strpool.strings)
|
||||
|
||||
source ::
|
||||
`first int :: 42 + 3
|
||||
`second :: 134 + 5
|
||||
`third float :: 1.35 + 2
|
||||
`fourth :: first + second
|
||||
|
||||
scan_state := lexer.init(mem.c_allocator)
|
||||
defer lexer.deinit(&scan_state)
|
||||
try lexer.scan(&scan_state, source)
|
||||
|
||||
parse_state := init(mem.c_allocator)
|
||||
defer deinit(&parse_state)
|
||||
root :: try parse(&parse_state, scan_state.tokens.items)
|
||||
|
||||
renderer := ast_renderer.init(mem.c_allocator)
|
||||
defer ast_renderer.deinit(&renderer)
|
||||
try ast_renderer.render(&renderer, root, &parse_state, scan_state.tokens.items)
|
||||
|
||||
try testing.expect_equal(scan_state.tokens.items.len, renderer.tokens.items.len)
|
||||
for scan_state.tokens.items |expected, i| {
|
||||
actual :: renderer.tokens.items[i]
|
||||
try testing.expect_equal(expected.kind, actual.kind)
|
||||
try testing.expect_equal(expected.start, actual.start)
|
||||
try testing.expect_equal(expected.str_id, actual.str_id)
|
||||
}
|
||||
}
|
||||
|
||||
handles_statement_declaration test {
|
||||
strpool.strings = strpool.init(mem.c_allocator)
|
||||
defer strpool.deinit(&strpool.strings)
|
||||
|
||||
@@ -9,7 +9,7 @@ InternError :: enum { out_of_space }
|
||||
|
||||
StringId :: alias u32
|
||||
|
||||
NO_ID :: maxval!(StringId)
|
||||
NO_ID_STR :: maxval!(StringId)
|
||||
|
||||
StringPool :: struct {
|
||||
ids hashmap.StringHashMap(StringId) # string → id
|
||||
@@ -36,7 +36,7 @@ deinit proc(pool @mut StringPool) void {
|
||||
intern proc(pool @mut StringPool, str []u8) StringId ! (mem.AllocError | InternError) {
|
||||
if hashmap.get(&pool.ids, str) |id| return id
|
||||
|
||||
if (pool.strings.items.len >= usize(NO_ID)) return .out_of_space
|
||||
if (pool.strings.items.len >= usize(NO_ID_STR)) return .out_of_space
|
||||
id StringId :: StringId(pool.strings.items.len)
|
||||
|
||||
owned_str []mut u8 :: try mem.alloc(u8, pool.allocator, str.len)
|
||||
|
||||
+1
-1
@@ -217,7 +217,7 @@ parse_format proc($N usize, $format []u8, $Args type) [N]mut FormatToken {
|
||||
token_count := 0
|
||||
argument_count := 0
|
||||
literal_start := 0
|
||||
cursor usize := 0
|
||||
cursor := 0
|
||||
while cursor < format.len {
|
||||
byte :: format[cursor]
|
||||
if byte == '{' {
|
||||
|
||||
@@ -73,10 +73,10 @@ expect_type proc($Expected, $Actual type, _ Actual, location SourceLocation) voi
|
||||
|
||||
run proc(name []u8, callback *proc() void ! Error) bool {
|
||||
callback() catch |_| {
|
||||
debug.print("{s}...[failed]\n", {name,})
|
||||
debug.print("{s} ... [\x1b[91mfailed\x1b[0m]\n", {name})
|
||||
return false
|
||||
}
|
||||
debug.print("{s}...[ok]\n", {name,})
|
||||
debug.print("{s} ... [\x1b[92mok\x1b[0m]\n", {name})
|
||||
return true
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user