82 lines
1.7 KiB
Plaintext
82 lines
1.7 KiB
Plaintext
import "@std/debug"
|
|
import "@source/strpool"
|
|
|
|
TokenId :: distinct u32
|
|
|
|
NO_TOKEN :: maxval!(TokenId)
|
|
|
|
Token :: struct {
|
|
start uint
|
|
kind TokenKind
|
|
str_id strpool.StringId = strpool.NO_STR
|
|
}
|
|
|
|
TokenKind :: enum {
|
|
# keywords
|
|
if, else
|
|
for, while
|
|
proc, return
|
|
|
|
# literals
|
|
ident, string
|
|
int, float
|
|
|
|
# comparison
|
|
equal_equal, not_equal
|
|
less, less_equal
|
|
greater, greater_equal
|
|
|
|
# assignment
|
|
double_colon, colon_equal
|
|
equal
|
|
|
|
# arithmetic
|
|
plus, minus
|
|
star, slash
|
|
|
|
# assignment & arithmetic
|
|
plus_equal, minus_equal
|
|
star_equal, slash_equal
|
|
|
|
# delimiters
|
|
open_paren, close_paren
|
|
open_bracket, close_bracket
|
|
open_curly, close_curly
|
|
|
|
colon
|
|
newline
|
|
|
|
# special
|
|
eof
|
|
invalid
|
|
}
|
|
|
|
render_token proc(tok Token, program []u8) void {
|
|
match tok.kind {
|
|
.ident: {
|
|
str :: strpool.get_str(&strpool.STRINGS, tok.str_id) orelse "null"
|
|
debug.print("{}({})\n", {tok.kind, str})
|
|
}
|
|
.string: {
|
|
res :: scan_string(tok.start, program) catch |_| {
|
|
debug.print("{}({})\n", {tok.kind, "null"})
|
|
return
|
|
}
|
|
debug.print("{}({})\n", {tok.kind, program[tok.start..res.end]})
|
|
}
|
|
.int, .float: {
|
|
res :: scan_number(tok.start, program) catch |_| {
|
|
debug.print("{}({})\n", {tok.kind, "null"})
|
|
return
|
|
}
|
|
debug.print("{}({})\n", {tok.kind, program[tok.start..res.end]})
|
|
}
|
|
else: debug.print("{}\n", {tok.kind})
|
|
}
|
|
}
|
|
|
|
token_id proc(idx uint) TokenId {
|
|
debug.assert(u64(idx) < u64(NO_TOKEN))
|
|
return TokenId(idx)
|
|
}
|