59 lines
903 B
Plaintext
59 lines
903 B
Plaintext
import "@std/debug"
|
|
import "@source/strpool"
|
|
|
|
TokenId :: distinct u32
|
|
|
|
NO_ID_TOKEN :: maxval!(TokenId)
|
|
|
|
Token :: struct {
|
|
start uint
|
|
kind TokenKind
|
|
str_id strpool.StringId = strpool.NO_ID_STR
|
|
}
|
|
|
|
TokenKind :: enum {
|
|
# keywords
|
|
if, else
|
|
for, while
|
|
proc, return
|
|
|
|
# literals
|
|
ident, string
|
|
int, float
|
|
|
|
# comparison
|
|
equal_equal, not_equal
|
|
less, less_equal
|
|
greater, greater_equal
|
|
|
|
# assignment
|
|
double_colon, colon_equal
|
|
equal
|
|
|
|
# arithmetic
|
|
plus, minus
|
|
star, slash
|
|
|
|
# assignment & arithmetic
|
|
plus_equal, minus_equal
|
|
star_equal, slash_equal
|
|
|
|
# delimiters
|
|
open_paren, close_paren
|
|
open_bracket, close_bracket
|
|
open_curly, close_curly
|
|
|
|
colon
|
|
newline
|
|
|
|
# special
|
|
eof
|
|
invalid
|
|
}
|
|
|
|
@hide
|
|
token_id proc(idx uint) TokenId {
|
|
debug.assert(u64(idx) < u64(NO_ID_TOKEN))
|
|
return TokenId(idx)
|
|
}
|