152 lines
3.6 KiB
Plaintext
152 lines
3.6 KiB
Plaintext
import "@std"
|
|
import "@std/debug"
|
|
import "@std/mem"
|
|
import "@std/arraylist"
|
|
|
|
TokenKind :: enum {
|
|
ident
|
|
int
|
|
float
|
|
string
|
|
|
|
equal
|
|
double_colon
|
|
|
|
left_paren
|
|
right_paren
|
|
left_curly
|
|
right_curly
|
|
|
|
newline
|
|
|
|
invalid
|
|
eof
|
|
}
|
|
|
|
Token :: struct {
|
|
kind TokenKind
|
|
start int
|
|
}
|
|
|
|
hide is_whitespace func(char u8) bool {
|
|
return char == ' ' or char == '\t' or char == '\n' or char == '\r'
|
|
}
|
|
|
|
hide is_alpha func(char u8) bool {
|
|
return match char {
|
|
'a'..'z', 'A'..'Z': true
|
|
else: false
|
|
}
|
|
}
|
|
|
|
hide is_digit func(char u8) bool {
|
|
return match char {
|
|
'0'..'9': true
|
|
else: false
|
|
}
|
|
}
|
|
|
|
scan func(tokens @mut std.ArrayList(Token), input []u8) void ! mem.AllocError {
|
|
cursor usize = 0
|
|
while cursor < input.len {
|
|
char :: input[cursor]
|
|
|
|
# whitespace
|
|
if char == '\n' {
|
|
try arraylist.append(tokens, Token{ kind = .newline, start = cursor })
|
|
cursor += 1
|
|
continue
|
|
} else if is_whitespace(char) {
|
|
cursor += 1
|
|
continue
|
|
}
|
|
|
|
# comments
|
|
if char == '#' {
|
|
while cursor < input.len and input[cursor] != '\n' : cursor += 1 {}
|
|
cursor += 1
|
|
continue
|
|
}
|
|
|
|
# identifiers and keywords
|
|
if is_alpha(char) or char == '_' {
|
|
start :: cursor
|
|
cursor += 1
|
|
while cursor < input.len and (is_alpha(input[cursor]) or is_digit(input[cursor]) or input[cursor] == '_') {
|
|
cursor += 1
|
|
}
|
|
try arraylist.append(tokens, Token{ kind = .ident, start = start })
|
|
continue
|
|
}
|
|
|
|
# integers literals
|
|
if is_digit(char) {
|
|
start :: cursor
|
|
cursor += 1
|
|
while cursor < input.len and is_digit(input[cursor]) {
|
|
cursor += 1
|
|
}
|
|
try arraylist.append(tokens, Token{ kind = .int, start = start })
|
|
continue
|
|
}
|
|
|
|
# string literals
|
|
if char == '"' {
|
|
start :: cursor
|
|
cursor += 1
|
|
|
|
while cursor < input.len and input[cursor] != '"' and input[cursor] != '\n' : cursor += 1 {
|
|
# ignore escaped characters
|
|
if (input[cursor] == '\\' and cursor + 1 < input.len) cursor += 1
|
|
}
|
|
|
|
if cursor < input.len and input[cursor] == '"' {
|
|
cursor += 1
|
|
try arraylist.append(tokens, Token{ kind = .string, start = start })
|
|
} else {
|
|
try arraylist.append(tokens, Token{ kind = .invalid, start = start })
|
|
}
|
|
|
|
continue
|
|
}
|
|
|
|
# mutable assignment
|
|
if char == '=' {
|
|
try arraylist.append(tokens, Token{ kind = .equal, start = cursor })
|
|
cursor += 1
|
|
continue
|
|
}
|
|
|
|
# immutable assignment
|
|
cursor += 1
|
|
if cursor < input.len and char == ':' and input[cursor] == ':' {
|
|
try arraylist.append(tokens, Token{ kind = .double_colon, start = cursor })
|
|
cursor += 1
|
|
continue
|
|
}
|
|
|
|
# invalid character
|
|
try arraylist.append(tokens, Token{ kind = .invalid, start = cursor })
|
|
}
|
|
try arraylist.append(tokens, Token{ kind = .eof, start = cursor })
|
|
}
|
|
|
|
program ::
|
|
`# these are immutable
|
|
`x :: 32
|
|
`y :: 3.2
|
|
|
|
main func() void {
|
|
tokens std.ArrayList(Token) = arraylist.init(mem.c_allocator)
|
|
defer arraylist.deinit(&tokens)
|
|
|
|
scan(&tokens, program) catch |_| {
|
|
debug.print("failed to scan: out of memory\n", {})
|
|
return
|
|
}
|
|
|
|
for tokens.items |token| {
|
|
debug.print("{}\n", { token.kind })
|
|
}
|
|
}
|