79 lines
1.5 KiB
Plaintext
79 lines
1.5 KiB
Plaintext
import "@std/debug"
|
|
import "@source/diagnostics"
|
|
import "@source/strpool"
|
|
|
|
TokenId :: distinct u32
|
|
NO_TOKEN :: maxval!(TokenId)
|
|
|
|
TokenData :: union {
|
|
str_id strpool.StringId
|
|
diagnostic diagnostics.DiagnosticId
|
|
}
|
|
|
|
Token :: struct {
|
|
span diagnostics.Span
|
|
kind TokenKind
|
|
data TokenData = TokenData{ str_id = strpool.NO_STR }
|
|
}
|
|
|
|
TokenKind :: enum {
|
|
# keywords
|
|
if, else
|
|
for, while
|
|
proc, return
|
|
|
|
# literals
|
|
ident, string
|
|
int, float
|
|
|
|
# comparison
|
|
equal_equal, not_equal
|
|
less, less_equal
|
|
greater, greater_equal
|
|
|
|
# assignment
|
|
double_colon, colon_equal
|
|
equal
|
|
|
|
# arithmetic
|
|
plus, minus
|
|
star, slash
|
|
|
|
# assignment & arithmetic
|
|
plus_equal, minus_equal
|
|
star_equal, slash_equal
|
|
|
|
# delimiters
|
|
open_paren, close_paren
|
|
open_bracket, close_bracket
|
|
open_curly, close_curly
|
|
|
|
comma
|
|
colon
|
|
newline
|
|
|
|
# special
|
|
eof
|
|
invalid
|
|
}
|
|
|
|
render_token proc(tok Token, program []u8) void {
|
|
match tok.kind {
|
|
.ident: {
|
|
if (strpool.get_str(&strpool.STRINGS, tok.data.str_id)) |str| {
|
|
debug.print("{}({})\n", {tok.kind, str})
|
|
} else {
|
|
debug.print("{}({})\n", {tok.kind, program[tok.span.start..tok.span.end]})
|
|
}
|
|
}
|
|
.string, .int, .float:
|
|
debug.print("{}({})\n", {tok.kind, program[tok.span.start..tok.span.end]})
|
|
else: debug.print("{}\n", {tok.kind})
|
|
}
|
|
}
|
|
|
|
token_id proc(idx uint) TokenId {
|
|
debug.assert(u64(idx) < u64(NO_TOKEN))
|
|
return TokenId(idx)
|
|
}
|