refactor a bit

source/lexer.hon

@@ -0,0 +1,106 @@
+import "@std"
+import "@std/mem"
+import "@std/arraylist"
+
+scan func(tokens @mut std.ArrayList(Token), input []u8) void ! mem.AllocError {
+    cursor usize = 0
+    while cursor < input.len {
+        char :: input[cursor]
+
+        # whitespace
+        if char == '\n' {
+	        try arraylist.append(tokens, Token{ kind = .newline, start = cursor })
+	        cursor += 1
+			continue
+        } else if is_whitespace(char) {
+            cursor += 1
+            continue
+        }
+
+        # comments
+        if char == '#' {
+            while cursor < input.len and input[cursor] != '\n' : cursor += 1 {}
+            cursor += 1
+            continue
+        }
+
+        # identifiers and keywords
+        if is_alpha(char) or char == '_' {
+            start :: cursor
+            cursor += 1
+            while cursor < input.len and (is_alpha(input[cursor]) or is_digit(input[cursor]) or input[cursor] == '_') {
+                cursor += 1
+            }
+            try arraylist.append(tokens, Token{ kind = .ident, start = start })
+            continue
+        }
+
+        # integers literals
+        if is_digit(char) {
+            start :: cursor
+            cursor += 1
+            while cursor < input.len and is_digit(input[cursor]) {
+                cursor += 1
+            }
+            try arraylist.append(tokens, Token{ kind = .int, start = start })
+            continue
+        }
+
+        # string literals
+        if char == '"' {
+            start :: cursor
+            cursor += 1
+
+            while cursor < input.len and input[cursor] != '"' and input[cursor] != '\n' : cursor += 1 {
+	            # ignore escaped characters
+	            if (input[cursor] == '\\' and cursor + 1 < input.len) cursor += 1
+            }
+
+            if cursor < input.len and input[cursor] == '"' {
+                cursor += 1
+	            try arraylist.append(tokens, Token{ kind = .string, start = start })
+            } else {
+                try arraylist.append(tokens, Token{ kind = .invalid, start = start })
+            }
+
+            continue
+        }
+
+        # mutable assignment
+        if char == '=' {
+            try arraylist.append(tokens, Token{ kind = .equal, start = cursor })
+            cursor += 1
+            continue
+        }
+
+        # immutable assignment
+        cursor += 1
+        if cursor < input.len and char == ':' and input[cursor] == ':' {
+            try arraylist.append(tokens, Token{ kind = .double_colon, start = cursor })
+            cursor += 1
+            continue
+        }
+
+        # invalid character
+        try arraylist.append(tokens, Token{ kind = .invalid, start = cursor })
+    }
+    try arraylist.append(tokens, Token{ kind = .eof, start = cursor })
+}
+
+hide is_whitespace func(char u8) bool {
+    return char == ' ' or char == '\t' or char == '\n' or char == '\r'
+}
+
+hide is_alpha func(char u8) bool {
+    return match char {
+        'a'..'z', 'A'..'Z': true
+        else: false
+    }
+}
+
+hide is_digit func(char u8) bool {
+    return match char {
+        '0'..'9': true
+        else: false
+    }
+}

source/main.hon

@@ -3,134 +3,6 @@ import "@std/debug"
 import "@std/mem"
 import "@std/arraylist"
 
-TokenKind :: enum {
-    ident
-    int
-    float
-    string
-
-    equal
-    double_colon
-
-    left_paren
-    right_paren
-    left_curly
-    right_curly
-
-    newline
-
-    invalid
-    eof
-}
-
-Token :: struct {
-    kind TokenKind
-    start int
-}
-
-hide is_whitespace func(char u8) bool {
-    return char == ' ' or char == '\t' or char == '\n' or char == '\r'
-}
-
-hide is_alpha func(char u8) bool {
-    return match char {
-        'a'..'z', 'A'..'Z': true
-        else: false
-    }
-}
-
-hide is_digit func(char u8) bool {
-    return match char {
-        '0'..'9': true
-        else: false
-    }
-}
-
-scan func(tokens @mut std.ArrayList(Token), input []u8) void ! mem.AllocError {
-    cursor usize = 0
-    while cursor < input.len {
-        char :: input[cursor]
-
-        # whitespace
-        if char == '\n' {
-	        try arraylist.append(tokens, Token{ kind = .newline, start = cursor })
-	        cursor += 1
-			continue
-        } else if is_whitespace(char) {
-            cursor += 1
-            continue
-        }
-
-        # comments
-        if char == '#' {
-            while cursor < input.len and input[cursor] != '\n' : cursor += 1 {}
-            cursor += 1
-            continue
-        }
-
-        # identifiers and keywords
-        if is_alpha(char) or char == '_' {
-            start :: cursor
-            cursor += 1
-            while cursor < input.len and (is_alpha(input[cursor]) or is_digit(input[cursor]) or input[cursor] == '_') {
-                cursor += 1
-            }
-            try arraylist.append(tokens, Token{ kind = .ident, start = start })
-            continue
-        }
-
-        # integers literals
-        if is_digit(char) {
-            start :: cursor
-            cursor += 1
-            while cursor < input.len and is_digit(input[cursor]) {
-                cursor += 1
-            }
-            try arraylist.append(tokens, Token{ kind = .int, start = start })
-            continue
-        }
-
-        # string literals
-        if char == '"' {
-            start :: cursor
-            cursor += 1
-
-            while cursor < input.len and input[cursor] != '"' and input[cursor] != '\n' : cursor += 1 {
-	            # ignore escaped characters
-	            if (input[cursor] == '\\' and cursor + 1 < input.len) cursor += 1
-            }
-
-            if cursor < input.len and input[cursor] == '"' {
-                cursor += 1
-	            try arraylist.append(tokens, Token{ kind = .string, start = start })
-            } else {
-                try arraylist.append(tokens, Token{ kind = .invalid, start = start })
-            }
-
-            continue
-        }
-
-        # mutable assignment
-        if char == '=' {
-            try arraylist.append(tokens, Token{ kind = .equal, start = cursor })
-            cursor += 1
-            continue
-        }
-
-        # immutable assignment
-        cursor += 1
-        if cursor < input.len and char == ':' and input[cursor] == ':' {
-            try arraylist.append(tokens, Token{ kind = .double_colon, start = cursor })
-            cursor += 1
-            continue
-        }
-
-        # invalid character
-        try arraylist.append(tokens, Token{ kind = .invalid, start = cursor })
-    }
-    try arraylist.append(tokens, Token{ kind = .eof, start = cursor })
-}
-
 program ::
 	`# these are immutable
 	`x :: 32

source/token.hon

@@ -0,0 +1,24 @@
+TokenKind :: enum {
+    ident
+    int
+    float
+    string
+
+    equal
+    double_colon
+
+    left_paren
+    right_paren
+    left_curly
+    right_curly
+
+    newline
+
+    invalid
+    eof
+}
+
+Token :: struct {
+    kind TokenKind
+    start int
+}