105 lines
1.8 KiB
Plaintext
105 lines
1.8 KiB
Plaintext
import "@std/debug"
|
|
import "@source/diagnostics"
|
|
import "@source/strpool"
|
|
|
|
TokenId :: distinct u32
|
|
no_token :: maxval!(TokenId)
|
|
|
|
TokenData :: union {
|
|
str_id strpool.StringId
|
|
diagnostic diagnostics.DiagnosticId
|
|
}
|
|
|
|
Token :: struct {
|
|
span diagnostics.Span
|
|
kind TokenKind
|
|
data TokenData = TokenData{ str_id = strpool.no_str }
|
|
}
|
|
|
|
TokenKind :: enum {
|
|
# keywords
|
|
if, else
|
|
for, while
|
|
match
|
|
inline
|
|
proc, return
|
|
yield
|
|
break, continue
|
|
try, catch
|
|
defer, errdefer
|
|
true, false
|
|
and, or
|
|
xor
|
|
struct, c_struct
|
|
opaque
|
|
enum, union
|
|
distinct, alias
|
|
import, test
|
|
mut
|
|
c_func
|
|
null
|
|
undefined
|
|
|
|
# literals
|
|
ident, string
|
|
int, float
|
|
|
|
# comparison
|
|
equal_equal, not_equal
|
|
less, less_equal
|
|
greater, greater_equal
|
|
|
|
# assignment
|
|
double_colon, colon_equal
|
|
equal
|
|
|
|
# arithmetic
|
|
plus, minus
|
|
star, slash
|
|
|
|
# bitwise
|
|
tilde
|
|
shift_left, shift_right, shift_left_saturating
|
|
|
|
range, range_inclusive, ellipsis
|
|
at, ampersand, caret, dollar
|
|
question, semicolon
|
|
pipe
|
|
|
|
# logical
|
|
bang
|
|
|
|
# assignment & arithmetic
|
|
plus_equal, minus_equal
|
|
star_equal, slash_equal
|
|
ampersand_equal, pipe_equal, xor_equal
|
|
shift_left_equal, shift_right_equal, shift_left_saturating_equal
|
|
|
|
# delimiters
|
|
open_paren, close_paren
|
|
open_bracket, close_bracket
|
|
open_curly, close_curly
|
|
|
|
comma, dot
|
|
colon
|
|
newline
|
|
|
|
# special
|
|
eof
|
|
invalid
|
|
}
|
|
|
|
render_token proc(token Token, program []u8) void {
|
|
match token.kind {
|
|
.ident, .string, .int, .float:
|
|
debug.print("{}({})\n", {token.kind, program[token.span.start..token.span.end]})
|
|
else:
|
|
debug.print("{}\n", {token.kind})
|
|
}
|
|
}
|
|
|
|
token_id proc(index uint) TokenId {
|
|
debug.assert(u64(index) < u64(no_token))
|
|
return TokenId(index)
|
|
}
|