import "@std" import "@std/mem" import "@std/arraylist" #import "@std/static_string_map" #keywords std.StaticStringMap(TokenKind) :: static_string_map.init([ # { "func", .func }, # { "return", .return }, # { "if", .if }, # { "for", .for }, # { "else", .else }, #]) TokenIndex :: alias usize ScanDiagnostic :: struct { token TokenIndex message []u8 } State :: struct { tokens std.ArrayList(Token) diagnostics std.ArrayList(ScanDiagnostic) } init func(allocator mem.Allocator) State { return State { tokens = arraylist.init(allocator), diagnostics = arraylist.init(allocator), } } deinit func(state @mut State) void { arraylist.deinit(&state.tokens) arraylist.deinit(&state.diagnostics) } scan func(state @mut State, input []u8) void ! mem.AllocError { tokens :: &state.tokens diagnostics :: &state.diagnostics cursor usize = 0 while cursor < input.len { char :: input[cursor] # whitespace if char == '\n' { try arraylist.append(tokens, Token{ kind = .newline, start = cursor }) cursor += 1 continue } else if is_whitespace(char) { cursor += 1 continue } # comments if char == '#' { while cursor < input.len and input[cursor] != '\n' : cursor += 1 {} cursor += 1 continue } # identifiers and keywords if is_alpha(char) or char == '_' { start :: cursor cursor += 1 while cursor < input.len and (is_alpha(input[cursor]) or is_digit(input[cursor]) or input[cursor] == '_') { cursor += 1 } kind :: ident_keyword_map(input[start..cursor]) try arraylist.append(tokens, Token{ kind = kind, start = start }) continue } # numeric literals if is_digit(char) { start :: cursor has_decimal bool = false # scan integer part while cursor < input.len and is_digit(input[cursor]) { cursor += 1 } # check for decimal point if cursor < input.len and input[cursor] == '.' { has_decimal = true cursor += 1 } # assert that decimals follow the decimal point if has_decimal and cursor < input.len and !is_digit(input[cursor]) { token :: tokens.items.len try arraylist.append(tokens, Token{ kind = .invalid, start = start }) try arraylist.append(diagnostics, ScanDiagnostic{ token = token, message = "float must end with a digit" }) continue } # scan decimal part while cursor < input.len and is_digit(input[cursor]) : cursor += 1 {} if has_decimal { try arraylist.append(tokens, Token{ kind = .float, start = start }) } else { try arraylist.append(tokens, Token{ kind = .int, start = start }) } continue } # string literals if char == '"' { start :: cursor cursor += 1 while cursor < input.len and input[cursor] != '"' and input[cursor] != '\n' : cursor += 1 { # ignore escaped characters if (input[cursor] == '\\' and cursor + 1 < input.len) cursor += 1 } if cursor < input.len and input[cursor] == '"' { cursor += 1 try arraylist.append(tokens, Token{ kind = .string, start = start }) } else { try arraylist.append(tokens, Token{ kind = .invalid, start = start }) } continue } # mutable assignment if char == '=' { try arraylist.append(tokens, Token{ kind = .equal, start = cursor }) cursor += 1 continue } # immutable assignment if cursor + 1 < input.len and char == ':' and input[cursor + 1] == ':' { try arraylist.append(tokens, Token{ kind = .double_colon, start = cursor }) cursor += 2 continue } # parentheses if char == '(' { try arraylist.append(tokens, Token{ kind = .open_paren, start = cursor }) cursor += 1 continue } else if char == ')' { try arraylist.append(tokens, Token{ kind = .close_paren, start = cursor }) cursor += 1 continue } # curly braces if char == '{' { try arraylist.append(tokens, Token{ kind = .open_curly, start = cursor }) cursor += 1 continue } else if char == '}' { try arraylist.append(tokens, Token{ kind = .close_curly, start = cursor }) cursor += 1 continue } # invalid character token :: tokens.items.len try arraylist.append(tokens, Token{ kind = .invalid, start = cursor }) try arraylist.append(diagnostics, ScanDiagnostic{ token = token, message = "invalid character" }) } try arraylist.append(tokens, Token{ kind = .eof, start = cursor }) } hide is_whitespace func(char u8) bool { return char == ' ' or char == '\t' or char == '\n' or char == '\r' } hide is_alpha func(char u8) bool { return match char { 'a'..='z', 'A'..='Z': true else: false } } hide is_digit func(char u8) bool { return match char { '0'..='9': true else: false } } # fixme: replace with a static string map once implemented hide ident_keyword_map func(ident []u8) TokenKind { if mem.eql(u8, ident, "if") return .if if mem.eql(u8, ident, "else") return .else if mem.eql(u8, ident, "for") return .for if mem.eql(u8, ident, "while") return .while if mem.eql(u8, ident, "func") return .func if mem.eql(u8, ident, "return") return .return return .ident } # fixme(brolang): this function fails to infer the enum type from the return values due to the optional #hide ident_keyword_map func(ident []u8) ?TokenKind { # if mem.eql(u8, ident, "if") return .if # if mem.eql(u8, ident, "else") return .else # if mem.eql(u8, ident, "for") return .for # if mem.eql(u8, ident, "while") return .while # if mem.eql(u8, ident, "func") return .func # if mem.eql(u8, ident, "return") return .return # return null #}