refactor a bit

2026-07-15 22:48:09 +02:00
parent 73da0d95c4
commit 3e8db2c15e
3 changed files with 130 additions and 128 deletions
@@ -0,0 +1,106 @@
+import "@std"
+import "@std/mem"
+import "@std/arraylist"
+
+scan func(tokens @mut std.ArrayList(Token), input []u8) void ! mem.AllocError {
+    cursor usize = 0
+    while cursor < input.len {
+        char :: input[cursor]
+
+        # whitespace
+        if char == '\n' {
+	        try arraylist.append(tokens, Token{ kind = .newline, start = cursor })
+	        cursor += 1
+			continue
+        } else if is_whitespace(char) {
+            cursor += 1
+            continue
+        }
+
+        # comments
+        if char == '#' {
+            while cursor < input.len and input[cursor] != '\n' : cursor += 1 {}
+            cursor += 1
+            continue
+        }
+
+        # identifiers and keywords
+        if is_alpha(char) or char == '_' {
+            start :: cursor
+            cursor += 1
+            while cursor < input.len and (is_alpha(input[cursor]) or is_digit(input[cursor]) or input[cursor] == '_') {
+                cursor += 1
+            }
+            try arraylist.append(tokens, Token{ kind = .ident, start = start })
+            continue
+        }
+
+        # integers literals
+        if is_digit(char) {
+            start :: cursor
+            cursor += 1
+            while cursor < input.len and is_digit(input[cursor]) {
+                cursor += 1
+            }
+            try arraylist.append(tokens, Token{ kind = .int, start = start })
+            continue
+        }
+
+        # string literals
+        if char == '"' {
+            start :: cursor
+            cursor += 1
+
+            while cursor < input.len and input[cursor] != '"' and input[cursor] != '\n' : cursor += 1 {
+	            # ignore escaped characters
+	            if (input[cursor] == '\\' and cursor + 1 < input.len) cursor += 1
+            }
+
+            if cursor < input.len and input[cursor] == '"' {
+                cursor += 1
+	            try arraylist.append(tokens, Token{ kind = .string, start = start })
+            } else {
+                try arraylist.append(tokens, Token{ kind = .invalid, start = start })
+            }
+
+            continue
+        }
+
+        # mutable assignment
+        if char == '=' {
+            try arraylist.append(tokens, Token{ kind = .equal, start = cursor })
+            cursor += 1
+            continue
+        }
+
+        # immutable assignment
+        cursor += 1
+        if cursor < input.len and char == ':' and input[cursor] == ':' {
+            try arraylist.append(tokens, Token{ kind = .double_colon, start = cursor })
+            cursor += 1
+            continue
+        }
+
+        # invalid character
+        try arraylist.append(tokens, Token{ kind = .invalid, start = cursor })
+    }
+    try arraylist.append(tokens, Token{ kind = .eof, start = cursor })
+}
+
+hide is_whitespace func(char u8) bool {
+    return char == ' ' or char == '\t' or char == '\n' or char == '\r'
+}
+
+hide is_alpha func(char u8) bool {
+    return match char {
+        'a'..'z', 'A'..'Z': true
+        else: false
+    }
+}
+
+hide is_digit func(char u8) bool {
+    return match char {
+        '0'..'9': true
+        else: false
+    }
+}