refactor: rename lexer files to scan
This commit is contained in:
@@ -0,0 +1,323 @@
|
||||
import "@std/mem"
|
||||
import "@std/strmap"
|
||||
import "@std/arraylist"
|
||||
|
||||
import "@source/diagnostics"
|
||||
import "@source/strpool"
|
||||
|
||||
ScanError :: struct {
|
||||
code ErrorCode
|
||||
end usize
|
||||
}
|
||||
|
||||
@hide:file
|
||||
ErrorCode :: enum {
|
||||
float_must_end_with_digit,
|
||||
unterminated_string,
|
||||
}
|
||||
SourceError :: enum { source_too_large }
|
||||
|
||||
|
||||
keywords strmap.StaticStringMap(TokenKind) :: strmap.init([
|
||||
{ "proc", .proc },
|
||||
{ "return", .return },
|
||||
{ "if", .if },
|
||||
{ "for", .for },
|
||||
{ "else", .else },
|
||||
{ "while", .while },
|
||||
])
|
||||
|
||||
State :: struct {
|
||||
tokens arraylist.ArrayList(Token)
|
||||
}
|
||||
|
||||
init proc(allocator mem.Allocator) State {
|
||||
return State{ tokens = arraylist.init(allocator) }
|
||||
}
|
||||
|
||||
deinit proc(state @mut State) void {
|
||||
arraylist.deinit(&state.tokens)
|
||||
}
|
||||
|
||||
scan proc(
|
||||
state @mut State,
|
||||
program []u8,
|
||||
diagnostic_store @mut diagnostics.Store,
|
||||
) void ! (mem.AllocError | strpool.InternError | diagnostics.CapacityError | SourceError) {
|
||||
if (program.len > usize(maxval!(u32))) return .source_too_large
|
||||
cursor := 0
|
||||
while cursor < program.len {
|
||||
char :: program[cursor]
|
||||
|
||||
# whitespace
|
||||
if char == '\n' {
|
||||
try add_token(state, Token{ span = make_span(cursor, cursor + 1), kind = .newline })
|
||||
cursor += 1
|
||||
continue
|
||||
} else if is_whitespace(char) {
|
||||
cursor += 1
|
||||
continue
|
||||
}
|
||||
|
||||
# comments
|
||||
if char == '#' {
|
||||
while (cursor < program.len and program[cursor] != '\n') cursor += 1
|
||||
if (cursor < program.len) cursor += 1
|
||||
continue
|
||||
}
|
||||
|
||||
# identifiers and keywords
|
||||
if is_alpha(char) or char == '_' {
|
||||
start :: cursor
|
||||
result :: scan_ident(start, program)
|
||||
try add_token(state, Token{
|
||||
span = make_span(start, result.end),
|
||||
kind = result.kind,
|
||||
data = TokenData{ str_id = result.str_id },
|
||||
})
|
||||
cursor = result.end
|
||||
continue
|
||||
}
|
||||
|
||||
# numeric literals
|
||||
if is_digit(char) {
|
||||
start :: cursor
|
||||
result :: scan_number(start, program) catch |err| {
|
||||
id :: try diagnostics.add(
|
||||
diagnostic_store,
|
||||
diagnostic_code(err.code),
|
||||
make_span(start, err.end),
|
||||
)
|
||||
try add_token(state, Token{
|
||||
span = make_span(start, err.end),
|
||||
kind = .invalid,
|
||||
data = TokenData{ diagnostic = id },
|
||||
})
|
||||
cursor = err.end
|
||||
continue
|
||||
}
|
||||
kind :: if (result.has_decimal) .float else .int
|
||||
try add_token(state, Token{ span = make_span(start, result.end), kind = kind })
|
||||
cursor = result.end
|
||||
continue
|
||||
}
|
||||
|
||||
# string literals
|
||||
if char == '"' {
|
||||
start :: cursor
|
||||
result :: scan_string(start, program) catch |err| {
|
||||
id :: try diagnostics.add(
|
||||
diagnostic_store,
|
||||
diagnostic_code(err.code),
|
||||
make_span(start, err.end),
|
||||
)
|
||||
try add_token(state, Token{
|
||||
span = make_span(start, err.end),
|
||||
kind = .invalid,
|
||||
data = TokenData{ diagnostic = id },
|
||||
})
|
||||
cursor = err.end
|
||||
continue
|
||||
}
|
||||
try add_token(state, Token{ span = make_span(start, result.end), kind = .string })
|
||||
cursor = result.end
|
||||
continue
|
||||
}
|
||||
|
||||
# immutable decl, mutable decl, or single colon
|
||||
if char == ':' {
|
||||
if cursor + 1 < program.len and program[cursor + 1] == ':' {
|
||||
try add_token(state, Token{ span = make_span(cursor, cursor + 2), kind = .double_colon })
|
||||
cursor += 2
|
||||
continue
|
||||
}
|
||||
if cursor + 1 < program.len and program[cursor + 1] == '=' {
|
||||
try add_token(state, Token{ span = make_span(cursor, cursor + 2), kind = .colon_equal })
|
||||
cursor += 2
|
||||
continue
|
||||
}
|
||||
try add_token(state, Token{ span = make_span(cursor, cursor + 1), kind = .colon })
|
||||
cursor += 1
|
||||
continue
|
||||
}
|
||||
|
||||
# compound arithmetic assignment
|
||||
if char == '+' {
|
||||
if cursor + 1 < program.len and program[cursor + 1] == '=' {
|
||||
try add_token(state, Token{ span = make_span(cursor, cursor + 2), kind = .plus_equal })
|
||||
cursor += 2
|
||||
continue
|
||||
}
|
||||
try add_token(state, Token{ span = make_span(cursor, cursor + 1), kind = .plus })
|
||||
cursor += 1
|
||||
continue
|
||||
}
|
||||
if char == '-' {
|
||||
if cursor + 1 < program.len and program[cursor + 1] == '=' {
|
||||
try add_token(state, Token{ span = make_span(cursor, cursor + 2), kind = .minus_equal })
|
||||
cursor += 2
|
||||
continue
|
||||
}
|
||||
try add_token(state, Token{ span = make_span(cursor, cursor + 1), kind = .minus })
|
||||
cursor += 1
|
||||
continue
|
||||
}
|
||||
if char == '*' {
|
||||
if cursor + 1 < program.len and program[cursor + 1] == '=' {
|
||||
try add_token(state, Token{ span = make_span(cursor, cursor + 2), kind = .star_equal })
|
||||
cursor += 2
|
||||
continue
|
||||
}
|
||||
try add_token(state, Token{ span = make_span(cursor, cursor + 1), kind = .star })
|
||||
cursor += 1
|
||||
continue
|
||||
}
|
||||
if char == '/' {
|
||||
if cursor + 1 < program.len and program[cursor + 1] == '=' {
|
||||
try add_token(state, Token{ span = make_span(cursor, cursor + 2), kind = .slash_equal })
|
||||
cursor += 2
|
||||
continue
|
||||
}
|
||||
try add_token(state, Token{ span = make_span(cursor, cursor + 1), kind = .slash })
|
||||
cursor += 1
|
||||
continue
|
||||
}
|
||||
|
||||
# single character tokens
|
||||
kind ?TokenKind :: match char {
|
||||
'=': .equal
|
||||
',': .comma
|
||||
'(': .open_paren
|
||||
')': .close_paren
|
||||
'[': .open_bracket
|
||||
']': .close_bracket
|
||||
'{': .open_curly
|
||||
'}': .close_curly
|
||||
else: null
|
||||
}
|
||||
if (kind) |value| {
|
||||
try add_token(state, Token{ span = make_span(cursor, cursor + 1), kind = value })
|
||||
cursor += 1
|
||||
continue
|
||||
}
|
||||
|
||||
# invalid character
|
||||
id :: try diagnostics.add(
|
||||
diagnostic_store,
|
||||
.invalid_character,
|
||||
make_span(cursor, cursor + 1),
|
||||
)
|
||||
try add_token(state, Token{
|
||||
span = make_span(cursor, cursor + 1),
|
||||
kind = .invalid,
|
||||
data = TokenData{ diagnostic = id },
|
||||
})
|
||||
cursor += 1
|
||||
}
|
||||
|
||||
try add_token(state, Token{ span = make_span(cursor, cursor), kind = .eof })
|
||||
}
|
||||
|
||||
ScanIdentResult :: struct {
|
||||
end usize
|
||||
kind TokenKind
|
||||
str_id strpool.StringId
|
||||
}
|
||||
|
||||
scan_ident proc(start usize, program []u8) ScanIdentResult {
|
||||
cursor := start + 1
|
||||
# scan whole identifier
|
||||
while (cursor < program.len and (
|
||||
is_alpha(program[cursor]) or
|
||||
is_digit(program[cursor]) or
|
||||
program[cursor] == '_'
|
||||
)) cursor += 1
|
||||
|
||||
# don't intern keywords (already O(1) lookup via token kind)
|
||||
kind :: strmap.get(&keywords, program[start..cursor]) orelse .ident
|
||||
str_id :: if (kind == .ident)
|
||||
strpool.intern(&strpool.STRINGS, program[start..cursor]) catch strpool.NO_STR
|
||||
else
|
||||
strpool.NO_STR
|
||||
return ScanIdentResult{ end = cursor, kind = kind, str_id = str_id }
|
||||
}
|
||||
|
||||
ScanNumResult :: struct {
|
||||
end usize
|
||||
has_decimal bool
|
||||
}
|
||||
|
||||
scan_number proc(start usize, program []u8) ScanNumResult ! ScanError {
|
||||
cursor := start
|
||||
has_decimal := false
|
||||
# scan integer part
|
||||
while (cursor < program.len and is_digit(program[cursor])) cursor += 1
|
||||
# check for decimal
|
||||
if cursor < program.len and program[cursor] == '.' {
|
||||
has_decimal = true
|
||||
cursor += 1
|
||||
}
|
||||
# assert non-terminating decimal
|
||||
if has_decimal and (cursor >= program.len or !is_digit(program[cursor])) {
|
||||
return ScanError{ code = .float_must_end_with_digit, end = cursor }
|
||||
}
|
||||
# scan fractional part
|
||||
while (cursor < program.len and is_digit(program[cursor])) cursor += 1
|
||||
return ScanNumResult{ end = cursor, has_decimal = has_decimal }
|
||||
}
|
||||
|
||||
ScanStrResult :: struct { end usize }
|
||||
|
||||
scan_string proc(start usize, program []u8) ScanStrResult ! ScanError {
|
||||
cursor := start + 1 # skip first `"`
|
||||
# scan entire string
|
||||
while cursor < program.len and program[cursor] != '"' and program[cursor] != '\n' : cursor += 1 {
|
||||
# ignore escaped characters
|
||||
if (program[cursor] == '\\' and cursor + 1 < program.len) cursor += 1
|
||||
}
|
||||
# assert string terminal
|
||||
if (cursor >= program.len or program[cursor] != '"') {
|
||||
return ScanError{ code = .unterminated_string, end = cursor }
|
||||
}
|
||||
return ScanStrResult{ end = cursor + 1 } # skip last `"`
|
||||
}
|
||||
|
||||
@hide
|
||||
is_whitespace proc(char u8) bool {
|
||||
return char == ' ' or char == '\t' or char == '\n' or char == '\r'
|
||||
}
|
||||
|
||||
@hide
|
||||
is_alpha proc(char u8) bool {
|
||||
return match char {
|
||||
'a'..='z', 'A'..='Z': true
|
||||
else: false
|
||||
}
|
||||
}
|
||||
|
||||
@hide
|
||||
is_digit proc(char u8) bool {
|
||||
return match char {
|
||||
'0'..='9': true
|
||||
else: false
|
||||
}
|
||||
}
|
||||
|
||||
@hide:file
|
||||
diagnostic_code proc(code ErrorCode) diagnostics.Code {
|
||||
return match code {
|
||||
.float_must_end_with_digit: .float_must_end_with_digit
|
||||
.unterminated_string: .unterminated_string
|
||||
}
|
||||
}
|
||||
|
||||
@hide:file
|
||||
make_span proc(start, end usize) diagnostics.Span {
|
||||
return diagnostics.Span{ start = u32(start), end = u32(end) }
|
||||
}
|
||||
|
||||
@hide
|
||||
add_token proc(state @mut State, token Token) void ! mem.AllocError {
|
||||
_ = token_id(state.tokens.items.len)
|
||||
try arraylist.append(&state.tokens, token)
|
||||
}
|
||||
Reference in New Issue
Block a user