honey/source/lexer/token.hon

import "@std/debug"
import "@source/diagnostics"
import "@source/strpool"

TokenId :: distinct u32
no_token :: maxval!(TokenId)

TokenData :: union {
    str_id strpool.StringId
    diagnostic diagnostics.DiagnosticId
}

Token :: struct {
    span diagnostics.Span
    kind TokenKind
    data TokenData = TokenData{ str_id = strpool.no_str }
}

TokenKind :: enum {
    # keywords
    if, else
    for, while
    proc, return
    true, false

    # literals
    ident, string
    int, float

    # comparison
    equal_equal, not_equal
    less, less_equal
    greater, greater_equal

    # assignment
    double_colon, colon_equal
    equal

    # arithmetic
    plus, minus
    star, slash

    # assignment & arithmetic
    plus_equal, minus_equal
    star_equal, slash_equal

    # delimiters
    open_paren, close_paren
    open_bracket, close_bracket
    open_curly, close_curly

    comma
    colon
    newline

    # special
    eof
    invalid
}

render_token proc(token Token, program []u8) void {
    match token.kind {
        .ident: {
            if (strpool.get_str(&strpool.strings, token.data.str_id)) |str| {
                debug.print("{}({})\n", {token.kind, str})
            } else {
                debug.print("{}({})\n", {token.kind, program[token.span.start..token.span.end]})
            }
        }
        .string, .int, .float:
            debug.print("{}({})\n", {token.kind, program[token.span.start..token.span.end]})
        else: debug.print("{}\n", {token.kind})
    }
}

token_id proc(index uint) TokenId {
    debug.assert(u64(index) < u64(no_token))
    return TokenId(index)
}