parse const decls
This commit is contained in:
@@ -0,0 +1 @@
|
|||||||
|
# todo: move ast stuff from parser module to here
|
||||||
+113
-43
@@ -1,10 +1,33 @@
|
|||||||
import "@std"
|
import "@std"
|
||||||
import "@std/mem"
|
import "@std/mem"
|
||||||
import "@std/arraylist"
|
|
||||||
import "@std/strmap"
|
import "@std/strmap"
|
||||||
|
import "@std/enums/enummap"
|
||||||
|
import "@std/arraylist"
|
||||||
|
import "@std/debug"
|
||||||
|
|
||||||
import "@source/strpool"
|
import "@source/strpool"
|
||||||
|
|
||||||
|
ErrorCode :: enum {
|
||||||
|
invalid_character,
|
||||||
|
float_must_end_with_digit,
|
||||||
|
}
|
||||||
|
|
||||||
|
ErrorDetails :: struct {
|
||||||
|
name []u8
|
||||||
|
message []u8
|
||||||
|
}
|
||||||
|
|
||||||
|
error_msg_map std.EnumMap(ErrorCode, ErrorDetails) :: enummap.init({
|
||||||
|
invalid_character = ErrorDetails {
|
||||||
|
name = "L0",
|
||||||
|
message = "invalid character",
|
||||||
|
},
|
||||||
|
float_must_end_with_digit = ErrorDetails {
|
||||||
|
name = "L1",
|
||||||
|
message = "float must end with a digit",
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
keywords std.StringMap(TokenKind) :: strmap.init([
|
keywords std.StringMap(TokenKind) :: strmap.init([
|
||||||
{ "proc", .proc },
|
{ "proc", .proc },
|
||||||
{ "return", .return },
|
{ "return", .return },
|
||||||
@@ -14,16 +37,16 @@ keywords std.StringMap(TokenKind) :: strmap.init([
|
|||||||
{ "while", .while },
|
{ "while", .while },
|
||||||
])
|
])
|
||||||
|
|
||||||
TokenIndex :: alias usize
|
TokenId :: distinct u32
|
||||||
|
|
||||||
ScanDiagnostic :: struct {
|
Diagnostic :: struct {
|
||||||
token TokenIndex
|
code ErrorCode
|
||||||
message []u8
|
token TokenId
|
||||||
}
|
}
|
||||||
|
|
||||||
State :: struct {
|
State :: struct {
|
||||||
tokens std.ArrayList(Token)
|
tokens std.ArrayList(Token)
|
||||||
diagnostics std.ArrayList(ScanDiagnostic)
|
diagnostics std.ArrayList(Diagnostic)
|
||||||
}
|
}
|
||||||
|
|
||||||
init proc(allocator mem.Allocator) State {
|
init proc(allocator mem.Allocator) State {
|
||||||
@@ -48,7 +71,7 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE
|
|||||||
|
|
||||||
# whitespace
|
# whitespace
|
||||||
if char == '\n' {
|
if char == '\n' {
|
||||||
try arraylist.append(tokens, Token{ kind = .newline, start = cursor })
|
try add_token(tokens, Token{ kind = .newline, start = cursor })
|
||||||
cursor += 1
|
cursor += 1
|
||||||
continue
|
continue
|
||||||
} else if is_whitespace(char) {
|
} else if is_whitespace(char) {
|
||||||
@@ -58,10 +81,8 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE
|
|||||||
|
|
||||||
# comments
|
# comments
|
||||||
if char == '#' {
|
if char == '#' {
|
||||||
# fixme(brolang): while currently requires curly braces but this should be legal
|
while (cursor < input.len and input[cursor] != '\n') cursor += 1
|
||||||
#while (cursor < input.len and input[cursor] != '\n') cursor += 1
|
cursor += 1 # also skip newline
|
||||||
|
|
||||||
while cursor < input.len and input[cursor] != '\n' : cursor += 1 {}
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -75,7 +96,7 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE
|
|||||||
is_alpha(input[cursor]) or
|
is_alpha(input[cursor]) or
|
||||||
is_digit(input[cursor]) or
|
is_digit(input[cursor]) or
|
||||||
input[cursor] == '_'
|
input[cursor] == '_'
|
||||||
)) : cursor += 1 {}
|
)) cursor += 1
|
||||||
|
|
||||||
kind :: strmap.get(&keywords, input[start..cursor]) orelse .ident
|
kind :: strmap.get(&keywords, input[start..cursor]) orelse .ident
|
||||||
|
|
||||||
@@ -83,9 +104,9 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE
|
|||||||
str_id :: if (kind == .ident)
|
str_id :: if (kind == .ident)
|
||||||
try strpool.intern(&strpool.strings, input[start..cursor])
|
try strpool.intern(&strpool.strings, input[start..cursor])
|
||||||
else
|
else
|
||||||
strpool.NoId
|
strpool.NO_ID
|
||||||
|
|
||||||
try arraylist.append(tokens, Token{
|
try add_token(tokens, Token{
|
||||||
kind = kind,
|
kind = kind,
|
||||||
start = start,
|
start = start,
|
||||||
str_id = str_id
|
str_id = str_id
|
||||||
@@ -99,7 +120,7 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE
|
|||||||
has_decimal bool = false
|
has_decimal bool = false
|
||||||
|
|
||||||
# scan integer part
|
# scan integer part
|
||||||
while cursor < input.len and is_digit(input[cursor]) : cursor += 1 {}
|
while (cursor < input.len and is_digit(input[cursor])) cursor += 1
|
||||||
|
|
||||||
# check for decimal point
|
# check for decimal point
|
||||||
if cursor < input.len and input[cursor] == '.' {
|
if cursor < input.len and input[cursor] == '.' {
|
||||||
@@ -109,22 +130,17 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE
|
|||||||
|
|
||||||
# assert that decimals follow the decimal point
|
# assert that decimals follow the decimal point
|
||||||
if has_decimal and (cursor >= input.len or !is_digit(input[cursor])) {
|
if has_decimal and (cursor >= input.len or !is_digit(input[cursor])) {
|
||||||
token :: tokens.items.len
|
token :: token_id(tokens.items.len)
|
||||||
try arraylist.append(tokens, Token{ kind = .invalid, start = start })
|
try arraylist.append(diagnostics, Diagnostic{ token = token, code = .float_must_end_with_digit })
|
||||||
try arraylist.append(diagnostics, ScanDiagnostic{
|
try add_token(tokens, Token{ kind = .invalid, start = start })
|
||||||
token = token,
|
|
||||||
message = "float must end with a digit",
|
|
||||||
})
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
# scan decimal part
|
# scan decimal part
|
||||||
while cursor < input.len and is_digit(input[cursor]) : cursor += 1 {}
|
while (cursor < input.len and is_digit(input[cursor])) cursor += 1
|
||||||
|
|
||||||
if (has_decimal)
|
kind :: if (has_decimal) .float else .int
|
||||||
try arraylist.append(tokens, Token{ kind = .float, start = start })
|
try add_token(tokens, Token{ kind = kind, start = start })
|
||||||
else
|
|
||||||
try arraylist.append(tokens, Token{ kind = .int, start = start })
|
|
||||||
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -141,9 +157,9 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE
|
|||||||
|
|
||||||
if cursor < input.len and input[cursor] == '"' {
|
if cursor < input.len and input[cursor] == '"' {
|
||||||
cursor += 1
|
cursor += 1
|
||||||
try arraylist.append(tokens, Token{ kind = .string, start = start })
|
try add_token(tokens, Token{ kind = .string, start = start })
|
||||||
} else {
|
} else {
|
||||||
try arraylist.append(tokens, Token{ kind = .invalid, start = start })
|
try add_token(tokens, Token{ kind = .invalid, start = start })
|
||||||
}
|
}
|
||||||
|
|
||||||
continue
|
continue
|
||||||
@@ -151,51 +167,95 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE
|
|||||||
|
|
||||||
# mutable assignment
|
# mutable assignment
|
||||||
if char == '=' {
|
if char == '=' {
|
||||||
try arraylist.append(tokens, Token{ kind = .equal, start = cursor })
|
try add_token(tokens, Token{ kind = .equal, start = cursor })
|
||||||
cursor += 1
|
cursor += 1
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
# immutable assignment
|
# immutable assignment or single colon
|
||||||
if cursor + 1 < input.len and char == ':' and input[cursor + 1] == ':' {
|
if char == ':' {
|
||||||
try arraylist.append(tokens, Token{ kind = .double_colon, start = cursor })
|
if cursor + 1 < input.len and input[cursor + 1] == ':' {
|
||||||
|
try add_token(tokens, Token{ kind = .double_colon, start = cursor })
|
||||||
cursor += 2
|
cursor += 2
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
try add_token(tokens, Token{ kind = .colon, start = cursor })
|
||||||
|
cursor += 1
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
# parentheses
|
# parentheses
|
||||||
if char == '(' {
|
if char == '(' {
|
||||||
try arraylist.append(tokens, Token{ kind = .open_paren, start = cursor })
|
try add_token(tokens, Token{ kind = .open_paren, start = cursor })
|
||||||
cursor += 1
|
cursor += 1
|
||||||
continue
|
continue
|
||||||
} else if char == ')' {
|
} else if char == ')' {
|
||||||
try arraylist.append(tokens, Token{ kind = .close_paren, start = cursor })
|
try add_token(tokens, Token{ kind = .close_paren, start = cursor })
|
||||||
cursor += 1
|
cursor += 1
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
# curly braces
|
# curly braces
|
||||||
if char == '{' {
|
if char == '{' {
|
||||||
try arraylist.append(tokens, Token{ kind = .open_curly, start = cursor })
|
try add_token(tokens, Token{ kind = .open_curly, start = cursor })
|
||||||
cursor += 1
|
cursor += 1
|
||||||
continue
|
continue
|
||||||
} else if char == '}' {
|
} else if char == '}' {
|
||||||
try arraylist.append(tokens, Token{ kind = .close_curly, start = cursor })
|
try add_token(tokens, Token{ kind = .close_curly, start = cursor })
|
||||||
cursor += 1
|
cursor += 1
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
# invalid character
|
# invalid character
|
||||||
token :: tokens.items.len
|
token :: token_id(tokens.items.len)
|
||||||
try arraylist.append(tokens, Token{ kind = .invalid, start = cursor })
|
try arraylist.append(diagnostics, Diagnostic{ token = token, code = .invalid_character })
|
||||||
try arraylist.append(diagnostics, ScanDiagnostic{
|
try add_token(tokens, Token{ kind = .invalid, start = cursor })
|
||||||
token = token,
|
|
||||||
message = "invalid character",
|
|
||||||
})
|
|
||||||
cursor += 1
|
cursor += 1
|
||||||
}
|
}
|
||||||
|
|
||||||
try arraylist.append(tokens, Token{ kind = .eof, start = cursor })
|
try add_token(tokens, Token{ kind = .eof, start = cursor })
|
||||||
|
}
|
||||||
|
|
||||||
|
#! returns the cursor position after scanning a number.
|
||||||
|
scan_number proc(start usize, input []u8) usize {
|
||||||
|
debug.assert(is_digit(input[start]))
|
||||||
|
|
||||||
|
has_decimal bool = false
|
||||||
|
cursor usize = start
|
||||||
|
|
||||||
|
# scan integer part
|
||||||
|
while cursor < input.len and is_digit(input[cursor]) : cursor += 1 {}
|
||||||
|
|
||||||
|
# check for decimal point
|
||||||
|
if cursor < input.len and input[cursor] == '.' {
|
||||||
|
has_decimal = true
|
||||||
|
cursor += 1
|
||||||
|
}
|
||||||
|
|
||||||
|
# assert that decimals follow the decimal point
|
||||||
|
debug.assert(!(has_decimal and (cursor >= input.len or !is_digit(input[cursor]))))
|
||||||
|
|
||||||
|
# scan decimal part
|
||||||
|
while cursor < input.len and is_digit(input[cursor]) : cursor += 1 {}
|
||||||
|
|
||||||
|
return cursor
|
||||||
|
}
|
||||||
|
|
||||||
|
#! returns the cursor position after scanning a string.
|
||||||
|
scan_string proc(start usize, input []u8) usize {
|
||||||
|
debug.assert(input[start] == '"')
|
||||||
|
|
||||||
|
cursor usize = start + 1
|
||||||
|
while cursor < input.len and input[cursor] != '"' and input[cursor] != '\n' : cursor += 1 {
|
||||||
|
# ignore escaped characters
|
||||||
|
if (input[cursor] == '\\' and cursor + 1 < input.len) cursor += 1
|
||||||
|
}
|
||||||
|
|
||||||
|
debug.assert(cursor < input.len and input[cursor] == '"')
|
||||||
|
cursor += 1
|
||||||
|
|
||||||
|
return cursor
|
||||||
}
|
}
|
||||||
|
|
||||||
hide is_whitespace proc(char u8) bool {
|
hide is_whitespace proc(char u8) bool {
|
||||||
@@ -215,3 +275,13 @@ hide is_digit proc(char u8) bool {
|
|||||||
else: false
|
else: false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
hide add_token proc(tokens @mut std.ArrayList(Token), token Token) void ! mem.AllocError {
|
||||||
|
_ = token_id(tokens.items.len)
|
||||||
|
try arraylist.append(tokens, token)
|
||||||
|
}
|
||||||
|
|
||||||
|
hide token_id proc(idx uint) TokenId {
|
||||||
|
debug.assert(u64(idx) < u64(maxval!(TokenId)))
|
||||||
|
return TokenId(idx)
|
||||||
|
}
|
||||||
|
|||||||
@@ -10,13 +10,12 @@ handles_keywords_identifiers_and_error_progress test {
|
|||||||
defer deinit(&state)
|
defer deinit(&state)
|
||||||
try scan(&state, "if name # comment\n@1.")
|
try scan(&state, "if name # comment\n@1.")
|
||||||
|
|
||||||
try testing.expect_equal(6, state.tokens.items.len)
|
try testing.expect_equal(5, state.tokens.items.len)
|
||||||
try testing.expect_equal(TokenKind.if, state.tokens.items[0].kind)
|
try testing.expect_equal(TokenKind.if, state.tokens.items[0].kind)
|
||||||
try testing.expect_equal(TokenKind.ident, state.tokens.items[1].kind)
|
try testing.expect_equal(TokenKind.ident, state.tokens.items[1].kind)
|
||||||
try testing.expect_equal(TokenKind.newline, state.tokens.items[2].kind)
|
try testing.expect_equal(TokenKind.invalid, state.tokens.items[2].kind)
|
||||||
try testing.expect_equal(TokenKind.invalid, state.tokens.items[3].kind)
|
try testing.expect_equal(TokenKind.invalid, state.tokens.items[3].kind)
|
||||||
try testing.expect_equal(TokenKind.invalid, state.tokens.items[4].kind)
|
try testing.expect_equal(TokenKind.eof, state.tokens.items[4].kind)
|
||||||
try testing.expect_equal(TokenKind.eof, state.tokens.items[5].kind)
|
|
||||||
try testing.expect_equal(2, state.diagnostics.items.len)
|
try testing.expect_equal(2, state.diagnostics.items.len)
|
||||||
try testing.expect_equal("name", strpool.get_str(&strpool.strings, state.tokens.items[1].str_id)?)
|
try testing.expect_equal("name", strpool.get_str(&strpool.strings, state.tokens.items[1].str_id)?)
|
||||||
}
|
}
|
||||||
|
|||||||
+32
-28
@@ -1,34 +1,38 @@
|
|||||||
import "@source/strpool"
|
import "@source/strpool"
|
||||||
|
|
||||||
TokenKind :: enum {
|
|
||||||
if
|
|
||||||
else
|
|
||||||
for
|
|
||||||
while
|
|
||||||
proc
|
|
||||||
return
|
|
||||||
|
|
||||||
ident
|
|
||||||
int
|
|
||||||
float
|
|
||||||
string
|
|
||||||
|
|
||||||
equal
|
|
||||||
double_colon
|
|
||||||
|
|
||||||
open_paren
|
|
||||||
close_paren
|
|
||||||
open_curly
|
|
||||||
close_curly
|
|
||||||
|
|
||||||
newline
|
|
||||||
|
|
||||||
invalid
|
|
||||||
eof
|
|
||||||
}
|
|
||||||
|
|
||||||
Token :: struct {
|
Token :: struct {
|
||||||
kind TokenKind
|
kind TokenKind
|
||||||
start int
|
start int
|
||||||
str_id strpool.StringId = strpool.NoId
|
str_id strpool.StringId = strpool.NO_ID
|
||||||
|
}
|
||||||
|
|
||||||
|
TokenKind :: enum {
|
||||||
|
# keywords
|
||||||
|
if, else
|
||||||
|
for, while
|
||||||
|
proc, return
|
||||||
|
|
||||||
|
# literals
|
||||||
|
ident, string
|
||||||
|
int, float
|
||||||
|
|
||||||
|
# comparison
|
||||||
|
equal_equal, not_equal
|
||||||
|
less, less_equal
|
||||||
|
greater, greater_equal
|
||||||
|
|
||||||
|
# assignment
|
||||||
|
equal, double_colon
|
||||||
|
|
||||||
|
# delimiters
|
||||||
|
open_paren, close_paren
|
||||||
|
open_bracket, close_bracket
|
||||||
|
open_curly, close_curly
|
||||||
|
|
||||||
|
colon
|
||||||
|
newline
|
||||||
|
|
||||||
|
# special
|
||||||
|
eof
|
||||||
|
invalid
|
||||||
}
|
}
|
||||||
|
|||||||
+42
-17
@@ -1,34 +1,24 @@
|
|||||||
import "@std/debug"
|
import "@std/debug"
|
||||||
import "@std/mem"
|
import "@std/mem"
|
||||||
|
import "@std/enums/enummap"
|
||||||
|
|
||||||
test import "@std/enums"
|
test import "@std/enums/enummap"
|
||||||
test import "@std/arraylist"
|
test import "@std/arraylist"
|
||||||
test import "@std/hashmap"
|
test import "@std/hashmap"
|
||||||
test import "@std/strmap"
|
test import "@std/strmap"
|
||||||
|
|
||||||
import "@source/strpool"
|
import "@source/strpool"
|
||||||
import "@source/lexer"
|
import "@source/lexer"
|
||||||
|
import "@source/parser"
|
||||||
|
|
||||||
test import "@source/strpool"
|
test import "@source/strpool"
|
||||||
test import "@source/lexer"
|
test import "@source/lexer"
|
||||||
|
|
||||||
program ::
|
program ::
|
||||||
`# these are immutable
|
`# literals
|
||||||
`x :: 32
|
`x :: 123
|
||||||
`y :: 3.2
|
|
||||||
`
|
# todo: consider making current aliased types (IDs) distinct instead
|
||||||
`# these are mutable
|
|
||||||
`z u32 = 54
|
|
||||||
`
|
|
||||||
`# keywords
|
|
||||||
`if
|
|
||||||
`else
|
|
||||||
`for
|
|
||||||
`while
|
|
||||||
`proc
|
|
||||||
`return
|
|
||||||
`
|
|
||||||
`main proc() void {}
|
|
||||||
|
|
||||||
main proc() void {
|
main proc() void {
|
||||||
strpool.strings = strpool.init(mem.c_allocator)
|
strpool.strings = strpool.init(mem.c_allocator)
|
||||||
@@ -48,5 +38,40 @@ main proc() void {
|
|||||||
for scan_state.tokens.items |token| {
|
for scan_state.tokens.items |token| {
|
||||||
debug.print("{}\n", {token.kind})
|
debug.print("{}\n", {token.kind})
|
||||||
}
|
}
|
||||||
|
debug.print("]]\n\n", {})
|
||||||
|
|
||||||
|
debug.print("TOKENS::DIAGNOSTICS::[[\n", {})
|
||||||
|
for scan_state.diagnostics.items |diagnostic| {
|
||||||
|
details :: enummap.get(&lexer.error_msg_map, diagnostic.code)
|
||||||
|
debug.print("{}: {}\n", {details?.name, details?.message})
|
||||||
|
}
|
||||||
|
debug.print("]]\n\n", {})
|
||||||
|
|
||||||
|
parse_state parser.State = parser.init(mem.c_allocator)
|
||||||
|
defer parser.deinit(&parse_state)
|
||||||
|
parser.parse(&parse_state, scan_state.tokens.items) catch |err| {
|
||||||
|
debug.print("failed to parse: {}\n", {err})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
debug.print("AST::[[\n", {})
|
||||||
|
for parse_state.nodes.items |node, id| {
|
||||||
|
token :: scan_state.tokens.items[node.main_token]
|
||||||
|
|
||||||
|
end :: if (token.kind == .int or token.kind == .float)
|
||||||
|
lexer.scan_number(token.start, program)
|
||||||
|
else if (token.kind == .string)
|
||||||
|
lexer.scan_string(token.start, program)
|
||||||
|
else
|
||||||
|
token.start
|
||||||
|
|
||||||
|
debug.print("{} (id = {}): {}\n", {
|
||||||
|
node.kind,
|
||||||
|
id,
|
||||||
|
program[token.start..end],
|
||||||
|
#node.data0,
|
||||||
|
#node.data1,
|
||||||
|
})
|
||||||
|
}
|
||||||
debug.print("]]\n", {})
|
debug.print("]]\n", {})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,156 @@
|
|||||||
|
import "@std"
|
||||||
|
import "@std/mem"
|
||||||
|
import "@std/arraylist"
|
||||||
|
import "@std/debug"
|
||||||
|
import "@std/enums/enummap"
|
||||||
|
|
||||||
|
import "@source/lexer"
|
||||||
|
|
||||||
|
NodeId :: distinct u32
|
||||||
|
ExtraId :: distinct NodeId
|
||||||
|
|
||||||
|
NO_ID :: maxval!(NodeId)
|
||||||
|
|
||||||
|
Diagnostic :: struct {
|
||||||
|
code ErrorCode
|
||||||
|
node NodeId
|
||||||
|
}
|
||||||
|
|
||||||
|
ParseError :: alias ErrorCode | mem.AllocError
|
||||||
|
|
||||||
|
ErrorCode :: enum {
|
||||||
|
unexpected_token,
|
||||||
|
}
|
||||||
|
|
||||||
|
ErrorDetails :: struct {
|
||||||
|
name []u8
|
||||||
|
message []u8
|
||||||
|
}
|
||||||
|
|
||||||
|
error_msg_map std.EnumMap(ErrorCode, ErrorDetails) :: enummap.init({
|
||||||
|
unexpected_token = ErrorDetails {
|
||||||
|
name = "P0",
|
||||||
|
message = "unexpected token",
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
hide Token :: alias lexer.Token
|
||||||
|
hide TokenId :: alias lexer.TokenId
|
||||||
|
hide TokenKind :: alias lexer.TokenKind
|
||||||
|
|
||||||
|
NodeData :: union {
|
||||||
|
node_id NodeId
|
||||||
|
extra_id ExtraId
|
||||||
|
}
|
||||||
|
|
||||||
|
Node :: struct {
|
||||||
|
kind NodeKind
|
||||||
|
main_token TokenId
|
||||||
|
|
||||||
|
data0 NodeData = NodeData{ node_id = NO_ID }
|
||||||
|
data1 NodeData = NodeData{ node_id = NO_ID }
|
||||||
|
}
|
||||||
|
|
||||||
|
NodeKind :: enum {
|
||||||
|
literal_int
|
||||||
|
literal_float
|
||||||
|
literal_string
|
||||||
|
expr_unary
|
||||||
|
expr_binary
|
||||||
|
decl
|
||||||
|
invalid
|
||||||
|
}
|
||||||
|
|
||||||
|
State :: struct {
|
||||||
|
nodes std.ArrayList(Node)
|
||||||
|
|
||||||
|
#! array of indexes into the nodes array.
|
||||||
|
extra std.ArrayList(NodeId)
|
||||||
|
|
||||||
|
next_token TokenId = TokenId(0)
|
||||||
|
}
|
||||||
|
|
||||||
|
init proc(allocator mem.Allocator) State {
|
||||||
|
return State{
|
||||||
|
nodes = arraylist.init(allocator),
|
||||||
|
extra = arraylist.init(allocator),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
deinit proc(state @mut State) void {
|
||||||
|
arraylist.deinit(&state.nodes)
|
||||||
|
arraylist.deinit(&state.extra)
|
||||||
|
}
|
||||||
|
|
||||||
|
parse proc(state @mut State, tokens []Token) void ! ParseError {
|
||||||
|
_ = try parse_decl(state, tokens)
|
||||||
|
}
|
||||||
|
|
||||||
|
parse_decl proc(state @mut State, tokens []Token) NodeId ! ParseError {
|
||||||
|
# expect identifier
|
||||||
|
try expect(state, tokens, .ident)
|
||||||
|
ident_token :: state.next_token
|
||||||
|
state.next_token += 1
|
||||||
|
|
||||||
|
# expect `::` (immutable assignment)
|
||||||
|
try expect(state, tokens, .double_colon)
|
||||||
|
state.next_token += 1
|
||||||
|
|
||||||
|
# expect expression
|
||||||
|
expr :: try parse_primary(state, tokens)
|
||||||
|
|
||||||
|
# expect statement terminator (newline)
|
||||||
|
try expect_either(state, tokens, &[.newline, .eof])
|
||||||
|
state.next_token += 1
|
||||||
|
|
||||||
|
decl :: try add_node(&state.nodes, Node{
|
||||||
|
kind = .decl,
|
||||||
|
main_token = ident_token,
|
||||||
|
data0 = NodeData{ node_id = expr },
|
||||||
|
})
|
||||||
|
|
||||||
|
return decl
|
||||||
|
}
|
||||||
|
|
||||||
|
parse_primary proc(state @mut State, tokens []Token) NodeId ! mem.AllocError {
|
||||||
|
start_token :: state.next_token
|
||||||
|
state.next_token += 1
|
||||||
|
return match tokens[start_token].kind {
|
||||||
|
.int: try add_node(&state.nodes, Node{
|
||||||
|
kind = .literal_int,
|
||||||
|
main_token = start_token,
|
||||||
|
})
|
||||||
|
.float: try add_node(&state.nodes, Node{
|
||||||
|
kind = .literal_float,
|
||||||
|
main_token = start_token,
|
||||||
|
})
|
||||||
|
.string: try add_node(&state.nodes, Node{
|
||||||
|
kind = .literal_string,
|
||||||
|
main_token = start_token,
|
||||||
|
})
|
||||||
|
else: try add_node(&state.nodes, Node{
|
||||||
|
kind = .invalid,
|
||||||
|
main_token = start_token,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
hide expect proc(state @mut State, tokens []Token, token_kind TokenKind) void ! ErrorCode {
|
||||||
|
if (tokens[state.next_token].kind != token_kind) return .unexpected_token
|
||||||
|
}
|
||||||
|
|
||||||
|
hide expect_either proc(state @mut State, tokens []Token, token_kinds []TokenKind) void ! ErrorCode {
|
||||||
|
for (token_kinds) |kind| if (tokens[state.next_token].kind == kind) return
|
||||||
|
return .unexpected_token
|
||||||
|
}
|
||||||
|
|
||||||
|
hide add_node proc(nodes @mut std.ArrayList(Node), node Node) NodeId ! mem.AllocError {
|
||||||
|
id :: node_id(nodes.items.len)
|
||||||
|
try arraylist.append(nodes, node)
|
||||||
|
return id
|
||||||
|
}
|
||||||
|
|
||||||
|
hide node_id proc(idx uint) NodeId {
|
||||||
|
debug.assert(u64(idx) < u64(NO_ID))
|
||||||
|
return NodeId(idx)
|
||||||
|
}
|
||||||
@@ -8,7 +8,8 @@ strings StringPool = undefined
|
|||||||
InternError :: enum { out_of_space }
|
InternError :: enum { out_of_space }
|
||||||
|
|
||||||
StringId :: alias u32
|
StringId :: alias u32
|
||||||
NoId :: maxval!(StringId)
|
|
||||||
|
NO_ID :: maxval!(StringId)
|
||||||
|
|
||||||
StringPool :: struct {
|
StringPool :: struct {
|
||||||
ids hashmap.StringHashMap(StringId) # string → id
|
ids hashmap.StringHashMap(StringId) # string → id
|
||||||
@@ -35,7 +36,7 @@ deinit proc(pool @mut StringPool) void {
|
|||||||
intern proc(pool @mut StringPool, str []u8) StringId ! (mem.AllocError | InternError) {
|
intern proc(pool @mut StringPool, str []u8) StringId ! (mem.AllocError | InternError) {
|
||||||
if hashmap.get(&pool.ids, str) |id| return id
|
if hashmap.get(&pool.ids, str) |id| return id
|
||||||
|
|
||||||
if (pool.strings.items.len >= usize(NoId)) return .out_of_space
|
if (pool.strings.items.len >= usize(NO_ID)) return .out_of_space
|
||||||
id StringId :: StringId(pool.strings.items.len)
|
id StringId :: StringId(pool.strings.items.len)
|
||||||
|
|
||||||
owned_str []mut u8 :: try mem.alloc(u8, pool.allocator, str.len)
|
owned_str []mut u8 :: try mem.alloc(u8, pool.allocator, str.len)
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ deinit proc($T type, list @mut ArrayList(T)) void {
|
|||||||
list.capacity = 0
|
list.capacity = 0
|
||||||
}
|
}
|
||||||
|
|
||||||
reserve proc($T type, list @mut ArrayList(T), min_capacity usize) void ! mem.AllocError {
|
hide reserve proc($T type, list @mut ArrayList(T), min_capacity usize) void ! mem.AllocError {
|
||||||
if min_capacity <= list.capacity {
|
if min_capacity <= list.capacity {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,10 @@
|
|||||||
import "@ffi/c"
|
import "@ffi/c"
|
||||||
import "@std/io"
|
import "@std/io"
|
||||||
|
|
||||||
|
assert proc(ok bool) void {
|
||||||
|
if (!ok) unreachable
|
||||||
|
}
|
||||||
|
|
||||||
print proc($format []u8, $Args type, args Args) void {
|
print proc($format []u8, $Args type, args Args) void {
|
||||||
writer io.Writer :: io.Writer{
|
writer io.Writer :: io.Writer{
|
||||||
context = null,
|
context = null,
|
||||||
|
|||||||
@@ -0,0 +1,45 @@
|
|||||||
|
import "@std/meta"
|
||||||
|
|
||||||
|
EnumMap proc($E, $V type) type {
|
||||||
|
match typeinfo!(E) {
|
||||||
|
.enum |info|: return struct {
|
||||||
|
present [info.fields.len]mut bool # fixme: replace with bitset
|
||||||
|
values [info.fields.len]mut V
|
||||||
|
}
|
||||||
|
else: compile_error!("EnumMap key must be an enum")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
init proc(
|
||||||
|
$E, $V type,
|
||||||
|
values meta.EnumFieldStruct(E, ?V, some!(null)),
|
||||||
|
) EnumMap(E, V) {
|
||||||
|
map EnumMap(E, V) = undefined
|
||||||
|
|
||||||
|
match typeinfo!(E) {
|
||||||
|
.enum |info|: expand for info.fields |field, i| {
|
||||||
|
map.present[i] = false
|
||||||
|
|
||||||
|
if field!(values, field.name) |value| {
|
||||||
|
map.present[i] = true
|
||||||
|
map.values[i] = value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else: compile_error!("EnumMap key must be an enum")
|
||||||
|
}
|
||||||
|
|
||||||
|
return map
|
||||||
|
}
|
||||||
|
|
||||||
|
get proc($E, $V type, map @EnumMap(E, V), key E) ?V {
|
||||||
|
# fixme: linear lookup; implement an enum index/discriminant map for O(1) lookup
|
||||||
|
match typeinfo!(E) {
|
||||||
|
.enum |info|: expand for info.fields |field, i| {
|
||||||
|
if key == field!(E, field.name) {
|
||||||
|
if (map.present[i]) return map.values[i]
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else: compile_error!("EnumMap key must be an enum")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
import "@std/mem"
|
||||||
|
import "@std/testing"
|
||||||
|
|
||||||
|
TestEnum :: enum(u8) {
|
||||||
|
ident = 3
|
||||||
|
int = 8
|
||||||
|
eof = 21
|
||||||
|
}
|
||||||
|
|
||||||
|
handles_sparse_enum_get test {
|
||||||
|
names EnumMap(TestEnum, []u8) = init({
|
||||||
|
ident = "identifier",
|
||||||
|
int = "integer",
|
||||||
|
})
|
||||||
|
|
||||||
|
ident :: get(&names, TestEnum.ident)
|
||||||
|
|
||||||
|
try testing.expect_type(?[]u8, ident)
|
||||||
|
try testing.expect(mem.eql("identifier", ident?))
|
||||||
|
|
||||||
|
eof :: get(&names, TestEnum.eof)
|
||||||
|
|
||||||
|
try testing.expect_type(?[]u8, eof)
|
||||||
|
try testing.expect_equal(null, eof)
|
||||||
|
}
|
||||||
@@ -1,45 +0,0 @@
|
|||||||
import "@std/meta"
|
|
||||||
|
|
||||||
EnumMap proc($E, $V type) type {
|
|
||||||
match typeinfo!(E) {
|
|
||||||
.enum |info|: return struct {
|
|
||||||
present [info.fields.len]mut bool # fixme: replace with bitset
|
|
||||||
values [info.fields.len]mut V
|
|
||||||
}
|
|
||||||
else: compile_error!("EnumMap key must be an enum")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
init proc(
|
|
||||||
$E, $V type,
|
|
||||||
values meta.EnumFieldStruct(E, ?V, some!(null)),
|
|
||||||
) EnumMap(E, V) {
|
|
||||||
map EnumMap(E, V) = undefined
|
|
||||||
|
|
||||||
match typeinfo!(E) {
|
|
||||||
.enum |info|: expand for info.fields |field, i| {
|
|
||||||
map.present[i] = false
|
|
||||||
|
|
||||||
if field!(values, field.name) |value| {
|
|
||||||
map.present[i] = true
|
|
||||||
map.values[i] = value
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else: compile_error!("EnumMap key must be an enum")
|
|
||||||
}
|
|
||||||
|
|
||||||
return map
|
|
||||||
}
|
|
||||||
|
|
||||||
get proc($E, $V type, map @EnumMap(E, V), key E) ?V {
|
|
||||||
# fixme: linear lookup; implement an enum index/discriminant map for O(1) lookup
|
|
||||||
match typeinfo!(E) {
|
|
||||||
.enum |info|: expand for info.fields |field, i| {
|
|
||||||
if key == field!(E, field.name) {
|
|
||||||
if (map.present[i]) return map.values[i]
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else: compile_error!("EnumMap key must be an enum")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,25 +0,0 @@
|
|||||||
import "@std/mem"
|
|
||||||
import "@std/testing"
|
|
||||||
|
|
||||||
TestEnum :: enum(u8) {
|
|
||||||
ident = 3
|
|
||||||
int = 8
|
|
||||||
eof = 21
|
|
||||||
}
|
|
||||||
|
|
||||||
handles_sparse_enum_get test {
|
|
||||||
names EnumMap(TestEnum, []u8) = init({
|
|
||||||
ident = "identifier",
|
|
||||||
int = "integer",
|
|
||||||
})
|
|
||||||
|
|
||||||
ident :: get(&names, TestEnum.ident)
|
|
||||||
|
|
||||||
try testing.expect_type(?[]u8, ident)
|
|
||||||
try testing.expect(mem.eql("identifier", ident?))
|
|
||||||
|
|
||||||
eof :: get(&names, TestEnum.eof)
|
|
||||||
|
|
||||||
try testing.expect_type(?[]u8, eof)
|
|
||||||
try testing.expect_equal(null, eof)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -327,6 +327,21 @@ hide format_field_name proc($T type, index usize) []u8 {
|
|||||||
else: compile_error!("io.print arguments must be a tuple")
|
else: compile_error!("io.print arguments must be a tuple")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
hide distinct_value proc($Backing, $Distinct type, value Distinct) Backing {
|
||||||
|
return ptrcast!(Backing, &value)^
|
||||||
|
}
|
||||||
|
|
||||||
|
hide scalar_or_distinct_type proc($T type) bool {
|
||||||
|
match typeinfo!(T) {
|
||||||
|
.bool: return true
|
||||||
|
.integer: return true
|
||||||
|
.float: return true
|
||||||
|
.distinct: return true
|
||||||
|
else: return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
hide write_integer proc(output Writer, $T type, value T, base u64, uppercase bool) void ! WriteError {
|
hide write_integer proc(output Writer, $T type, value T, base u64, uppercase bool) void ! WriteError {
|
||||||
match typeinfo!(T) {
|
match typeinfo!(T) {
|
||||||
@@ -335,6 +350,11 @@ hide write_integer proc(output Writer, $T type, value T, base u64, uppercase boo
|
|||||||
} else {
|
} else {
|
||||||
try write_integer_unsigned(output, u64(value), base, uppercase)
|
try write_integer_unsigned(output, u64(value), base, uppercase)
|
||||||
}
|
}
|
||||||
|
.distinct |backing|: if scalar_or_distinct_type(backing) {
|
||||||
|
try write_integer(output, distinct_value(backing, T, value), base, uppercase)
|
||||||
|
} else {
|
||||||
|
compile_error!("io.print integer format requires an integer argument")
|
||||||
|
}
|
||||||
else: compile_error!("io.print integer format requires an integer argument")
|
else: compile_error!("io.print integer format requires an integer argument")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -356,6 +376,11 @@ hide write_float proc(output Writer, $T type, value T, scientific bool) void ! W
|
|||||||
if (count < 0 or usize(count) >= buffer.len) return .write_failed
|
if (count < 0 or usize(count) >= buffer.len) return .write_failed
|
||||||
try write_all(output, buffer[0..usize(count)])
|
try write_all(output, buffer[0..usize(count)])
|
||||||
}
|
}
|
||||||
|
.distinct |backing|: if scalar_or_distinct_type(backing) {
|
||||||
|
try write_float(output, distinct_value(backing, T, value), scientific)
|
||||||
|
} else {
|
||||||
|
compile_error!("io.print float format requires a float argument")
|
||||||
|
}
|
||||||
else: compile_error!("io.print float format requires a float argument")
|
else: compile_error!("io.print float format requires a float argument")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -364,6 +389,11 @@ hide write_decimal proc(output Writer, $T type, value T) void ! WriteError {
|
|||||||
match typeinfo!(T) {
|
match typeinfo!(T) {
|
||||||
.integer: try write_integer(output, value, 10, false)
|
.integer: try write_integer(output, value, 10, false)
|
||||||
.float: try write_float(output, value, false)
|
.float: try write_float(output, value, false)
|
||||||
|
.distinct |backing|: if scalar_or_distinct_type(backing) {
|
||||||
|
try write_decimal(output, distinct_value(backing, T, value))
|
||||||
|
} else {
|
||||||
|
compile_error!("io.print '{d}' requires an integer or float argument")
|
||||||
|
}
|
||||||
else: compile_error!("io.print '{d}' requires an integer or float argument")
|
else: compile_error!("io.print '{d}' requires an integer or float argument")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -377,6 +407,11 @@ hide write_character proc(output Writer, $T type, value T) void ! WriteError {
|
|||||||
buffer [1]u8 = [u8(value)]
|
buffer [1]u8 = [u8(value)]
|
||||||
try write_all(output, buffer[..])
|
try write_all(output, buffer[..])
|
||||||
}
|
}
|
||||||
|
.distinct |backing|: if scalar_or_distinct_type(backing) {
|
||||||
|
try write_character(output, distinct_value(backing, T, value))
|
||||||
|
} else {
|
||||||
|
compile_error!("io.print '{c}' requires an unsigned integer that fits in u8")
|
||||||
|
}
|
||||||
else: compile_error!("io.print '{c}' requires an unsigned integer that fits in u8")
|
else: compile_error!("io.print '{c}' requires an unsigned integer that fits in u8")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -403,6 +438,11 @@ hide write_default proc(output Writer, $T type, value T) void ! WriteError {
|
|||||||
}
|
}
|
||||||
return .write_failed
|
return .write_failed
|
||||||
}
|
}
|
||||||
|
.distinct |backing|: if scalar_or_distinct_type(backing) {
|
||||||
|
try write_default(output, distinct_value(backing, T, value))
|
||||||
|
} else {
|
||||||
|
compile_error!("io.print '{}' does not support this argument type")
|
||||||
|
}
|
||||||
else: compile_error!("io.print '{}' does not support this argument type")
|
else: compile_error!("io.print '{}' does not support this argument type")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-1
@@ -40,7 +40,7 @@ TypeInfo :: union(enum) {
|
|||||||
record RecordInfo
|
record RecordInfo
|
||||||
union void
|
union void
|
||||||
fallible void
|
fallible void
|
||||||
distinct void
|
distinct type
|
||||||
}
|
}
|
||||||
|
|
||||||
EnumFieldStruct proc($E, $Field type, $default ?Field) type {
|
EnumFieldStruct proc($E, $Field type, $default ?Field) type {
|
||||||
|
|||||||
@@ -8,6 +8,10 @@ TestTokenKind :: enum(u8) {
|
|||||||
|
|
||||||
TestNames :: alias EnumFieldStruct(TestTokenKind, ?[]u8, some!(null))
|
TestNames :: alias EnumFieldStruct(TestTokenKind, ?[]u8, some!(null))
|
||||||
TestArrayAlias :: alias [3]u16
|
TestArrayAlias :: alias [3]u16
|
||||||
|
TestInner :: distinct u16
|
||||||
|
TestOuter :: distinct TestInner
|
||||||
|
TestOuterAlias :: alias TestOuter
|
||||||
|
|
||||||
|
|
||||||
hide array_info_matches proc($Array, $Child type, $len usize) bool {
|
hide array_info_matches proc($Array, $Child type, $len usize) bool {
|
||||||
match typeinfo!(Array) {
|
match typeinfo!(Array) {
|
||||||
@@ -15,6 +19,13 @@ hide array_info_matches proc($Array, $Child type, $len usize) bool {
|
|||||||
else: return false
|
else: return false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
hide distinct_info_matches proc($Distinct, $Backing type) bool {
|
||||||
|
match typeinfo!(Distinct) {
|
||||||
|
.distinct |backing|: return backing == Backing
|
||||||
|
else: return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
array_reflection_exposes_child_and_logical_length test {
|
array_reflection_exposes_child_and_logical_length test {
|
||||||
try testing.expect($(array_info_matches([4]i32, i32, 4)))
|
try testing.expect($(array_info_matches([4]i32, i32, 4)))
|
||||||
@@ -23,6 +34,12 @@ array_reflection_exposes_child_and_logical_length test {
|
|||||||
try testing.expect($(array_info_matches([2]mut i64, i64, 2)))
|
try testing.expect($(array_info_matches([2]mut i64, i64, 2)))
|
||||||
try testing.expect($(array_info_matches([2;0]u8, u8, 2)))
|
try testing.expect($(array_info_matches([2;0]u8, u8, 2)))
|
||||||
}
|
}
|
||||||
|
distinct_reflection_exposes_immediate_backing test {
|
||||||
|
try testing.expect($(distinct_info_matches(TestInner, u16)))
|
||||||
|
try testing.expect($(distinct_info_matches(TestOuter, TestInner)))
|
||||||
|
try testing.expect($(distinct_info_matches(TestOuterAlias, TestInner)))
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
enum_field_struct_defaults test {
|
enum_field_struct_defaults test {
|
||||||
names TestNames = {
|
names TestNames = {
|
||||||
|
|||||||
+2
-2
@@ -1,11 +1,11 @@
|
|||||||
import "io"
|
import "io"
|
||||||
import "enums"
|
import "enums/enummap"
|
||||||
import "hashmap"
|
import "hashmap"
|
||||||
import "arraylist"
|
import "arraylist"
|
||||||
import "strmap"
|
import "strmap"
|
||||||
|
|
||||||
Io :: alias io.Io
|
Io :: alias io.Io
|
||||||
EnumMap :: alias enums.EnumMap
|
EnumMap :: alias enummap.EnumMap
|
||||||
ArrayList :: alias arraylist.ArrayList
|
ArrayList :: alias arraylist.ArrayList
|
||||||
StringHashMap :: alias hashmap.StringHashMap
|
StringHashMap :: alias hashmap.StringHashMap
|
||||||
StringMap :: alias strmap.StringMap
|
StringMap :: alias strmap.StringMap
|
||||||
|
|||||||
@@ -0,0 +1 @@
|
|||||||
|
# todo
|
||||||
|
|||||||
Reference in New Issue
Block a user