checkpoint

source/lexer/lexer.hon

@@ -0,0 +1,213 @@
+import "@std"
+import "@std/mem"
+import "@std/arraylist"
+#import "@std/static_string_map"
+
+#keywords std.StaticStringMap(TokenKind) :: static_string_map.init([
+#	{ "func", .func },
+#	{ "return", .return },
+#	{ "if", .if },
+#	{ "for", .for },
+#	{ "else", .else },
+#])
+
+TokenIndex :: alias usize
+
+ScanDiagnostic :: struct {
+    token TokenIndex
+    message []u8
+}
+
+State :: struct {
+    tokens std.ArrayList(Token)
+    diagnostics std.ArrayList(ScanDiagnostic)
+}
+
+init func(allocator mem.Allocator) State {
+    return State {
+        tokens = arraylist.init(allocator),
+        diagnostics = arraylist.init(allocator),
+    }
+}
+
+deinit func(state @mut State) void {
+    arraylist.deinit(&state.tokens)
+    arraylist.deinit(&state.diagnostics)
+}
+
+scan func(state @mut State, input []u8) void ! mem.AllocError {
+	tokens :: &state.tokens
+	diagnostics :: &state.diagnostics
+
+    cursor usize = 0
+    while cursor < input.len {
+        char :: input[cursor]
+
+        # whitespace
+        if char == '\n' {
+	        try arraylist.append(tokens, Token{ kind = .newline, start = cursor })
+	        cursor += 1
+			continue
+        } else if is_whitespace(char) {
+            cursor += 1
+            continue
+        }
+
+        # comments
+        if char == '#' {
+            while cursor < input.len and input[cursor] != '\n' : cursor += 1 {}
+            cursor += 1
+            continue
+        }
+
+        # identifiers and keywords
+        if is_alpha(char) or char == '_' {
+            start :: cursor
+            cursor += 1
+            while cursor < input.len and (is_alpha(input[cursor]) or is_digit(input[cursor]) or input[cursor] == '_') {
+                cursor += 1
+            }
+            kind :: ident_keyword_map(input[start..cursor])
+            try arraylist.append(tokens, Token{ kind = kind, start = start })
+            continue
+        }
+
+        # numeric literals
+        if is_digit(char) {
+            start :: cursor
+            has_decimal bool = false
+
+            # scan integer part
+            while cursor < input.len and is_digit(input[cursor]) {
+                cursor += 1
+            }
+
+            # check for decimal point
+            if cursor < input.len and input[cursor] == '.' {
+                has_decimal = true
+                cursor += 1
+            }
+
+            # assert that decimals follow the decimal point
+            if has_decimal and cursor < input.len and !is_digit(input[cursor]) {
+                token :: tokens.items.len
+	            try arraylist.append(tokens, Token{ kind = .invalid, start = start })
+	            try arraylist.append(diagnostics, ScanDiagnostic{ token = token, message = "float must end with a digit" })
+	            continue
+            }
+
+            # scan decimal part
+            while cursor < input.len and is_digit(input[cursor]) : cursor += 1 {}
+
+            if has_decimal {
+                try arraylist.append(tokens, Token{ kind = .float, start = start })
+            } else {
+                try arraylist.append(tokens, Token{ kind = .int, start = start })
+            }
+            continue
+        }
+
+        # string literals
+        if char == '"' {
+            start :: cursor
+            cursor += 1
+
+            while cursor < input.len and input[cursor] != '"' and input[cursor] != '\n' : cursor += 1 {
+	            # ignore escaped characters
+	            if (input[cursor] == '\\' and cursor + 1 < input.len) cursor += 1
+            }
+
+            if cursor < input.len and input[cursor] == '"' {
+                cursor += 1
+	            try arraylist.append(tokens, Token{ kind = .string, start = start })
+            } else {
+                try arraylist.append(tokens, Token{ kind = .invalid, start = start })
+            }
+
+            continue
+        }
+
+        # mutable assignment
+        if char == '=' {
+            try arraylist.append(tokens, Token{ kind = .equal, start = cursor })
+            cursor += 1
+            continue
+        }
+
+        # immutable assignment
+        if cursor + 1 < input.len and char == ':' and input[cursor + 1] == ':' {
+            try arraylist.append(tokens, Token{ kind = .double_colon, start = cursor })
+            cursor += 2
+            continue
+        }
+
+        # parentheses
+        if char == '(' {
+            try arraylist.append(tokens, Token{ kind = .open_paren, start = cursor })
+            cursor += 1
+            continue
+        } else if char == ')' {
+            try arraylist.append(tokens, Token{ kind = .close_paren, start = cursor })
+            cursor += 1
+            continue
+        }
+
+        # curly braces
+        if char == '{' {
+            try arraylist.append(tokens, Token{ kind = .open_curly, start = cursor })
+            cursor += 1
+            continue
+        } else if char == '}' {
+            try arraylist.append(tokens, Token{ kind = .close_curly, start = cursor })
+            cursor += 1
+            continue
+        }
+
+        # invalid character
+        token :: tokens.items.len
+        try arraylist.append(tokens, Token{ kind = .invalid, start = cursor })
+        try arraylist.append(diagnostics, ScanDiagnostic{ token = token, message = "invalid character" })
+    }
+
+    try arraylist.append(tokens, Token{ kind = .eof, start = cursor })
+}
+
+hide is_whitespace func(char u8) bool {
+    return char == ' ' or char == '\t' or char == '\n' or char == '\r'
+}
+
+hide is_alpha func(char u8) bool {
+    return match char {
+        'a'..='z', 'A'..='Z': true
+        else: false
+    }
+}
+
+hide is_digit func(char u8) bool {
+    return match char {
+        '0'..='9': true
+        else: false
+    }
+}
+
+# fixme: replace with a static string map once implemented
+hide ident_keyword_map func(ident []u8) TokenKind {
+    if mem.eql(u8, ident, "if") return .if
+    if mem.eql(u8, ident, "else") return .else
+    if mem.eql(u8, ident, "for") return .for
+    if mem.eql(u8, ident, "while") return .while
+    if mem.eql(u8, ident, "func") return .func
+    if mem.eql(u8, ident, "return") return .return
+    return .ident
+}
+
+# fixme(brolang): this function fails to infer the enum type from the return values due to the optional
+#hide ident_keyword_map func(ident []u8) ?TokenKind {
+#    if mem.eql(u8, ident, "if") return .if
+#    if mem.eql(u8, ident, "else") return .else
+#    if mem.eql(u8, ident, "for") return .for
+#    if mem.eql(u8, ident, "while") return .while
+#    if mem.eql(u8, ident, "func") return .func
+#    if mem.eql(u8, ident, "return") return .return
+#    return null
+#}

source/lexer/token.hon

@@ -0,0 +1,31 @@
+TokenKind :: enum {
+    if
+    else
+    for
+    while
+    func
+    return
+
+    ident
+    int
+    float
+    string
+
+    equal
+    double_colon
+
+    open_paren
+    close_paren
+    open_curly
+    close_curly
+
+    newline
+
+    invalid
+    eof
+}
+
+Token :: struct {
+    kind TokenKind
+    start int
+}