diff --git a/source/lexer.hon b/source/lexer.hon new file mode 100644 index 0000000..0cd3d87 --- /dev/null +++ b/source/lexer.hon @@ -0,0 +1,106 @@ +import "@std" +import "@std/mem" +import "@std/arraylist" + +scan func(tokens @mut std.ArrayList(Token), input []u8) void ! mem.AllocError { + cursor usize = 0 + while cursor < input.len { + char :: input[cursor] + + # whitespace + if char == '\n' { + try arraylist.append(tokens, Token{ kind = .newline, start = cursor }) + cursor += 1 + continue + } else if is_whitespace(char) { + cursor += 1 + continue + } + + # comments + if char == '#' { + while cursor < input.len and input[cursor] != '\n' : cursor += 1 {} + cursor += 1 + continue + } + + # identifiers and keywords + if is_alpha(char) or char == '_' { + start :: cursor + cursor += 1 + while cursor < input.len and (is_alpha(input[cursor]) or is_digit(input[cursor]) or input[cursor] == '_') { + cursor += 1 + } + try arraylist.append(tokens, Token{ kind = .ident, start = start }) + continue + } + + # integers literals + if is_digit(char) { + start :: cursor + cursor += 1 + while cursor < input.len and is_digit(input[cursor]) { + cursor += 1 + } + try arraylist.append(tokens, Token{ kind = .int, start = start }) + continue + } + + # string literals + if char == '"' { + start :: cursor + cursor += 1 + + while cursor < input.len and input[cursor] != '"' and input[cursor] != '\n' : cursor += 1 { + # ignore escaped characters + if (input[cursor] == '\\' and cursor + 1 < input.len) cursor += 1 + } + + if cursor < input.len and input[cursor] == '"' { + cursor += 1 + try arraylist.append(tokens, Token{ kind = .string, start = start }) + } else { + try arraylist.append(tokens, Token{ kind = .invalid, start = start }) + } + + continue + } + + # mutable assignment + if char == '=' { + try arraylist.append(tokens, Token{ kind = .equal, start = cursor }) + cursor += 1 + continue + } + + # immutable assignment + cursor += 1 + if cursor < input.len and char == ':' and input[cursor] == ':' { + try arraylist.append(tokens, Token{ kind = .double_colon, start = cursor }) + cursor += 1 + continue + } + + # invalid character + try arraylist.append(tokens, Token{ kind = .invalid, start = cursor }) + } + try arraylist.append(tokens, Token{ kind = .eof, start = cursor }) +} + +hide is_whitespace func(char u8) bool { + return char == ' ' or char == '\t' or char == '\n' or char == '\r' +} + +hide is_alpha func(char u8) bool { + return match char { + 'a'..'z', 'A'..'Z': true + else: false + } +} + +hide is_digit func(char u8) bool { + return match char { + '0'..'9': true + else: false + } +} diff --git a/source/main.hon b/source/main.hon index 21bc5b1..655b735 100644 --- a/source/main.hon +++ b/source/main.hon @@ -3,134 +3,6 @@ import "@std/debug" import "@std/mem" import "@std/arraylist" -TokenKind :: enum { - ident - int - float - string - - equal - double_colon - - left_paren - right_paren - left_curly - right_curly - - newline - - invalid - eof -} - -Token :: struct { - kind TokenKind - start int -} - -hide is_whitespace func(char u8) bool { - return char == ' ' or char == '\t' or char == '\n' or char == '\r' -} - -hide is_alpha func(char u8) bool { - return match char { - 'a'..'z', 'A'..'Z': true - else: false - } -} - -hide is_digit func(char u8) bool { - return match char { - '0'..'9': true - else: false - } -} - -scan func(tokens @mut std.ArrayList(Token), input []u8) void ! mem.AllocError { - cursor usize = 0 - while cursor < input.len { - char :: input[cursor] - - # whitespace - if char == '\n' { - try arraylist.append(tokens, Token{ kind = .newline, start = cursor }) - cursor += 1 - continue - } else if is_whitespace(char) { - cursor += 1 - continue - } - - # comments - if char == '#' { - while cursor < input.len and input[cursor] != '\n' : cursor += 1 {} - cursor += 1 - continue - } - - # identifiers and keywords - if is_alpha(char) or char == '_' { - start :: cursor - cursor += 1 - while cursor < input.len and (is_alpha(input[cursor]) or is_digit(input[cursor]) or input[cursor] == '_') { - cursor += 1 - } - try arraylist.append(tokens, Token{ kind = .ident, start = start }) - continue - } - - # integers literals - if is_digit(char) { - start :: cursor - cursor += 1 - while cursor < input.len and is_digit(input[cursor]) { - cursor += 1 - } - try arraylist.append(tokens, Token{ kind = .int, start = start }) - continue - } - - # string literals - if char == '"' { - start :: cursor - cursor += 1 - - while cursor < input.len and input[cursor] != '"' and input[cursor] != '\n' : cursor += 1 { - # ignore escaped characters - if (input[cursor] == '\\' and cursor + 1 < input.len) cursor += 1 - } - - if cursor < input.len and input[cursor] == '"' { - cursor += 1 - try arraylist.append(tokens, Token{ kind = .string, start = start }) - } else { - try arraylist.append(tokens, Token{ kind = .invalid, start = start }) - } - - continue - } - - # mutable assignment - if char == '=' { - try arraylist.append(tokens, Token{ kind = .equal, start = cursor }) - cursor += 1 - continue - } - - # immutable assignment - cursor += 1 - if cursor < input.len and char == ':' and input[cursor] == ':' { - try arraylist.append(tokens, Token{ kind = .double_colon, start = cursor }) - cursor += 1 - continue - } - - # invalid character - try arraylist.append(tokens, Token{ kind = .invalid, start = cursor }) - } - try arraylist.append(tokens, Token{ kind = .eof, start = cursor }) -} - program :: `# these are immutable `x :: 32 diff --git a/source/token.hon b/source/token.hon new file mode 100644 index 0000000..28833dc --- /dev/null +++ b/source/token.hon @@ -0,0 +1,24 @@ +TokenKind :: enum { + ident + int + float + string + + equal + double_colon + + left_paren + right_paren + left_curly + right_curly + + newline + + invalid + eof +} + +Token :: struct { + kind TokenKind + start int +}