checkpoint

source/lexer.hon

@@ -1,106 +0,0 @@
-import "@std"
-import "@std/mem"
-import "@std/arraylist"
-
-scan func(tokens @mut std.ArrayList(Token), input []u8) void ! mem.AllocError {
-    cursor usize = 0
-    while cursor < input.len {
-        char :: input[cursor]
-
-        # whitespace
-        if char == '\n' {
-	        try arraylist.append(tokens, Token{ kind = .newline, start = cursor })
-	        cursor += 1
-			continue
-        } else if is_whitespace(char) {
-            cursor += 1
-            continue
-        }
-
-        # comments
-        if char == '#' {
-            while cursor < input.len and input[cursor] != '\n' : cursor += 1 {}
-            cursor += 1
-            continue
-        }
-
-        # identifiers and keywords
-        if is_alpha(char) or char == '_' {
-            start :: cursor
-            cursor += 1
-            while cursor < input.len and (is_alpha(input[cursor]) or is_digit(input[cursor]) or input[cursor] == '_') {
-                cursor += 1
-            }
-            try arraylist.append(tokens, Token{ kind = .ident, start = start })
-            continue
-        }
-
-        # integers literals
-        if is_digit(char) {
-            start :: cursor
-            cursor += 1
-            while cursor < input.len and is_digit(input[cursor]) {
-                cursor += 1
-            }
-            try arraylist.append(tokens, Token{ kind = .int, start = start })
-            continue
-        }
-
-        # string literals
-        if char == '"' {
-            start :: cursor
-            cursor += 1
-
-            while cursor < input.len and input[cursor] != '"' and input[cursor] != '\n' : cursor += 1 {
-	            # ignore escaped characters
-	            if (input[cursor] == '\\' and cursor + 1 < input.len) cursor += 1
-            }
-
-            if cursor < input.len and input[cursor] == '"' {
-                cursor += 1
-	            try arraylist.append(tokens, Token{ kind = .string, start = start })
-            } else {
-                try arraylist.append(tokens, Token{ kind = .invalid, start = start })
-            }
-
-            continue
-        }
-
-        # mutable assignment
-        if char == '=' {
-            try arraylist.append(tokens, Token{ kind = .equal, start = cursor })
-            cursor += 1
-            continue
-        }
-
-        # immutable assignment
-        cursor += 1
-        if cursor < input.len and char == ':' and input[cursor] == ':' {
-            try arraylist.append(tokens, Token{ kind = .double_colon, start = cursor })
-            cursor += 1
-            continue
-        }
-
-        # invalid character
-        try arraylist.append(tokens, Token{ kind = .invalid, start = cursor })
-    }
-    try arraylist.append(tokens, Token{ kind = .eof, start = cursor })
-}
-
-hide is_whitespace func(char u8) bool {
-    return char == ' ' or char == '\t' or char == '\n' or char == '\r'
-}
-
-hide is_alpha func(char u8) bool {
-    return match char {
-        'a'..'z', 'A'..'Z': true
-        else: false
-    }
-}
-
-hide is_digit func(char u8) bool {
-    return match char {
-        '0'..'9': true
-        else: false
-    }
-}

source/lexer/lexer.hon

@@ -0,0 +1,213 @@
+import "@std"
+import "@std/mem"
+import "@std/arraylist"
+#import "@std/static_string_map"
+
+#keywords std.StaticStringMap(TokenKind) :: static_string_map.init([
+#	{ "func", .func },
+#	{ "return", .return },
+#	{ "if", .if },
+#	{ "for", .for },
+#	{ "else", .else },
+#])
+
+TokenIndex :: alias usize
+
+ScanDiagnostic :: struct {
+    token TokenIndex
+    message []u8
+}
+
+State :: struct {
+    tokens std.ArrayList(Token)
+    diagnostics std.ArrayList(ScanDiagnostic)
+}
+
+init func(allocator mem.Allocator) State {
+    return State {
+        tokens = arraylist.init(allocator),
+        diagnostics = arraylist.init(allocator),
+    }
+}
+
+deinit func(state @mut State) void {
+    arraylist.deinit(&state.tokens)
+    arraylist.deinit(&state.diagnostics)
+}
+
+scan func(state @mut State, input []u8) void ! mem.AllocError {
+	tokens :: &state.tokens
+	diagnostics :: &state.diagnostics
+
+    cursor usize = 0
+    while cursor < input.len {
+        char :: input[cursor]
+
+        # whitespace
+        if char == '\n' {
+	        try arraylist.append(tokens, Token{ kind = .newline, start = cursor })
+	        cursor += 1
+			continue
+        } else if is_whitespace(char) {
+            cursor += 1
+            continue
+        }
+
+        # comments
+        if char == '#' {
+            while cursor < input.len and input[cursor] != '\n' : cursor += 1 {}
+            cursor += 1
+            continue
+        }
+
+        # identifiers and keywords
+        if is_alpha(char) or char == '_' {
+            start :: cursor
+            cursor += 1
+            while cursor < input.len and (is_alpha(input[cursor]) or is_digit(input[cursor]) or input[cursor] == '_') {
+                cursor += 1
+            }
+            kind :: ident_keyword_map(input[start..cursor])
+            try arraylist.append(tokens, Token{ kind = kind, start = start })
+            continue
+        }
+
+        # numeric literals
+        if is_digit(char) {
+            start :: cursor
+            has_decimal bool = false
+
+            # scan integer part
+            while cursor < input.len and is_digit(input[cursor]) {
+                cursor += 1
+            }
+
+            # check for decimal point
+            if cursor < input.len and input[cursor] == '.' {
+                has_decimal = true
+                cursor += 1
+            }
+
+            # assert that decimals follow the decimal point
+            if has_decimal and cursor < input.len and !is_digit(input[cursor]) {
+                token :: tokens.items.len
+	            try arraylist.append(tokens, Token{ kind = .invalid, start = start })
+	            try arraylist.append(diagnostics, ScanDiagnostic{ token = token, message = "float must end with a digit" })
+	            continue
+            }
+
+            # scan decimal part
+            while cursor < input.len and is_digit(input[cursor]) : cursor += 1 {}
+
+            if has_decimal {
+                try arraylist.append(tokens, Token{ kind = .float, start = start })
+            } else {
+                try arraylist.append(tokens, Token{ kind = .int, start = start })
+            }
+            continue
+        }
+
+        # string literals
+        if char == '"' {
+            start :: cursor
+            cursor += 1
+
+            while cursor < input.len and input[cursor] != '"' and input[cursor] != '\n' : cursor += 1 {
+	            # ignore escaped characters
+	            if (input[cursor] == '\\' and cursor + 1 < input.len) cursor += 1
+            }
+
+            if cursor < input.len and input[cursor] == '"' {
+                cursor += 1
+	            try arraylist.append(tokens, Token{ kind = .string, start = start })
+            } else {
+                try arraylist.append(tokens, Token{ kind = .invalid, start = start })
+            }
+
+            continue
+        }
+
+        # mutable assignment
+        if char == '=' {
+            try arraylist.append(tokens, Token{ kind = .equal, start = cursor })
+            cursor += 1
+            continue
+        }
+
+        # immutable assignment
+        if cursor + 1 < input.len and char == ':' and input[cursor + 1] == ':' {
+            try arraylist.append(tokens, Token{ kind = .double_colon, start = cursor })
+            cursor += 2
+            continue
+        }
+
+        # parentheses
+        if char == '(' {
+            try arraylist.append(tokens, Token{ kind = .open_paren, start = cursor })
+            cursor += 1
+            continue
+        } else if char == ')' {
+            try arraylist.append(tokens, Token{ kind = .close_paren, start = cursor })
+            cursor += 1
+            continue
+        }
+
+        # curly braces
+        if char == '{' {
+            try arraylist.append(tokens, Token{ kind = .open_curly, start = cursor })
+            cursor += 1
+            continue
+        } else if char == '}' {
+            try arraylist.append(tokens, Token{ kind = .close_curly, start = cursor })
+            cursor += 1
+            continue
+        }
+
+        # invalid character
+        token :: tokens.items.len
+        try arraylist.append(tokens, Token{ kind = .invalid, start = cursor })
+        try arraylist.append(diagnostics, ScanDiagnostic{ token = token, message = "invalid character" })
+    }
+
+    try arraylist.append(tokens, Token{ kind = .eof, start = cursor })
+}
+
+hide is_whitespace func(char u8) bool {
+    return char == ' ' or char == '\t' or char == '\n' or char == '\r'
+}
+
+hide is_alpha func(char u8) bool {
+    return match char {
+        'a'..='z', 'A'..='Z': true
+        else: false
+    }
+}
+
+hide is_digit func(char u8) bool {
+    return match char {
+        '0'..='9': true
+        else: false
+    }
+}
+
+# fixme: replace with a static string map once implemented
+hide ident_keyword_map func(ident []u8) TokenKind {
+    if mem.eql(u8, ident, "if") return .if
+    if mem.eql(u8, ident, "else") return .else
+    if mem.eql(u8, ident, "for") return .for
+    if mem.eql(u8, ident, "while") return .while
+    if mem.eql(u8, ident, "func") return .func
+    if mem.eql(u8, ident, "return") return .return
+    return .ident
+}
+
+# fixme(brolang): this function fails to infer the enum type from the return values due to the optional
+#hide ident_keyword_map func(ident []u8) ?TokenKind {
+#    if mem.eql(u8, ident, "if") return .if
+#    if mem.eql(u8, ident, "else") return .else
+#    if mem.eql(u8, ident, "for") return .for
+#    if mem.eql(u8, ident, "while") return .while
+#    if mem.eql(u8, ident, "func") return .func
+#    if mem.eql(u8, ident, "return") return .return
+#    return null
+#}

source/lexer/token.hon

@@ -1,4 +1,11 @@
 TokenKind :: enum {
+    if
+    else
+    for
+    while
+    func
+    return
+
     ident
     int
     float
@@ -7,10 +14,10 @@ TokenKind :: enum {
     equal
     double_colon
 
-    left_paren
+    open_paren
-    right_paren
+    close_paren
-    left_curly
+    open_curly
-    right_curly
+    close_curly
 
     newline
 

source/main.hon

@@ -1,11 +1,15 @@
-import "@std"
 import "@std/debug"
 import "@std/mem"
-import "@std/arraylist"
 
 test import "@std/enums"
 test import "@std/arraylist"
 test import "@std/hashmap"
+test import "@std/static_string_map"
+
+import "@source/strpool"
+import "@source/lexer"
+
+test import "@source/strpool"
 
 program ::
 	`# these are immutable
@@ -14,21 +18,37 @@ program ::
 	`
 	`# these are mutable
 	`z u32 = 54
+	`
+	`# keywords
+	`if
+	`else
+	`for
+	`while
+	`func
+	`return
+	`
+	`main func() void {}
+
+# cross-cutting concern, hence global singleton
+strings StringPool = undefined
 
 main func() void {
+    strings = strpool.init(mem.c_allocator)
+    defer strpool.deinit(&strings)
+
     debug.print("PROGRAM::[[\n{}\n]]\n\n", {program})
 
-    tokens std.ArrayList(Token) = arraylist.init(mem.c_allocator)
+    scan_state lexer.State = lexer.init(mem.c_allocator)
-    defer arraylist.deinit(&tokens)
+    defer lexer.deinit(&scan_state)
 
-    scan(&tokens, program) catch |_| {
+    lexer.scan(&scan_state, program) catch |err| {
-        debug.print("failed to scan: out of memory\n", {})
+        debug.print("failed to scan: {}\n", {err})
         return
     }
 
     debug.print("TOKENS::[[\n", {})
-    for tokens.items |token| {
+    for scan_state.tokens.items |token| {
-        debug.print("{}\n", { token.kind })
+        debug.print("{}\n", {token.kind})
     }
     debug.print("]]\n", {})
 }

std/hashmap/hashmap.hon

@@ -40,8 +40,8 @@ init func(
 	}
 }
 
-# free the entries in the hash map.
+#! free the entries in the hash map.
-# note: this operation invalidates the map.
+#! note: this operation invalidates the map.
 deinit func(
 	$K, $V type,
 	$hash_key func(key K) usize,

std/meta/meta.hon

@@ -1,6 +1,8 @@
-Layout :: enum {
+Layout :: enum { auto, c }
-	auto
+
-	c
+ArrayInfo :: struct {
+	child type
+	len usize
 }
 
 FieldInfo :: struct {
@@ -28,7 +30,7 @@ TypeInfo :: union(enum) {
 	bool void
 	integer void
 	float void
-	array void
+	array ArrayInfo
 	pointer void
 	slice void
 	range void
@@ -44,9 +46,9 @@ TypeInfo :: union(enum) {
 EnumFieldStruct func($E, $Field type, $default ?Field) type {
 	match typeinfo!(E) {
 		.enum |info|: {
-		names [info.fields.len]mut []u8 = undefined
+    		names [info.fields.len]mut []u8 = undefined
-		field_types [info.fields.len]mut type = undefined
+    		field_types [info.fields.len]mut type = undefined
-		defaults [info.fields.len]mut ?Field = undefined
+    		defaults [info.fields.len]mut ?Field = undefined
 			expand for info.fields |field, index| {
 				names[index] = field.name
 				field_types[index] = Field

std/meta/meta.test.hon

@@ -7,6 +7,22 @@ TestTokenKind :: enum(u8) {
 }
 
 TestNames :: alias EnumFieldStruct(TestTokenKind, ?[]u8, some!(null))
+TestArrayAlias :: alias [3]u16
+
+hide array_info_matches func($Array, $Child type, $len usize) bool {
+	match typeinfo!(Array) {
+		.array |info|: return info.child == Child and info.len == len
+		else: return false
+	}
+}
+
+array_reflection_exposes_child_and_logical_length test {
+	try testing.expect($(array_info_matches([4]i32, i32, 4)))
+	try testing.expect($(array_info_matches([0]bool, bool, 0)))
+	try testing.expect($(array_info_matches(TestArrayAlias, u16, 3)))
+	try testing.expect($(array_info_matches([2]mut i64, i64, 2)))
+	try testing.expect($(array_info_matches([2;0]u8, u8, 2)))
+}
 
 enum_field_struct_defaults test {
 	names TestNames = {

std/static_string_map/static_string_map.hon

@@ -0,0 +1,93 @@
+import "@std/mem"
+
+StaticStringMap func($V type) type {
+	return struct {
+		keys [][]u8
+		values []V
+		len_indexes []u32
+		min_len u32
+		max_len u32
+	}
+}
+
+hide Pair func($V type) type {
+	return struct { []u8, V }
+}
+
+init func($V type, $N usize, $entries [N]Pair(V)) StaticStringMap(V) {
+	if N > usize(maxval!(u32)) {
+		compile_error!("static string map has too many entries")
+	}
+
+	keys [N]mut []u8 = undefined
+	values [N]mut V = undefined
+
+	# assert no duplicate keys
+	for entries |entry, i| {
+	    if entry.0.len > usize(maxval!(u32)) {
+			compile_error!("static string map key is too long")
+	    }
+
+		for (0..i) |prior| if mem.eql(u8, entry.0, entries[prior].0) {
+			compile_error!("duplicate static string map key")
+		}
+
+		keys[i] = entry.0
+		values[i] = entry.1
+	}
+
+	if N == 0 {
+		len_indexes [0]u32 = undefined
+		return StaticStringMap(V){
+			keys = keys[..],
+			values = values[..],
+			len_indexes = len_indexes[..],
+			min_len = 0,
+			max_len = 0,
+		}
+	}
+
+	# fixme: insertion sort is compile-time O(N^2); replace if large maps affect builds
+	for 1..N |i| {
+		key :: keys[i]
+		value :: values[i]
+		j usize = i
+		while j > 0 and keys[j - 1].len > key.len : j -= 1 {
+			keys[j] = keys[j - 1]
+			values[j] = values[j - 1]
+		}
+		keys[j] = key
+		values[j] = value
+	}
+
+	min_len u32 :: u32(keys[0].len)
+	max_len u32 :: u32(keys[N - 1].len)
+	len_indexes [usize(max_len) + 1]mut u32 = undefined
+	entry_index usize = 0
+	for 0..=(usize(max_len)) |length| { # fixme: for casts and function calls, we should be able to omit the surrounding parentheses in the range
+		while entry_index < N and keys[entry_index].len < length : entry_index += 1 {}
+		len_indexes[length] = u32(entry_index)
+	}
+
+	return StaticStringMap(V) {
+		keys = keys[..],
+		values = values[..],
+		len_indexes = len_indexes[..],
+		min_len = min_len,
+		max_len = max_len,
+	}
+}
+
+get func($V type, map @StaticStringMap(V), key []u8) ?V {
+	if (map.keys.len == 0 or key.len > maxval!(u32)) return null
+
+	length u32 = u32(key.len)
+	if (length < map.min_len or length > map.max_len) return null
+
+	idx usize = usize(map.len_indexes[usize(length)])
+	while idx < map.keys.len : idx += 1 {
+		candidate :: map.keys[idx]
+		if (candidate.len != key.len) return null
+		if mem.eql(u8, candidate, key) return map.values[idx]
+	}
+}

std/std.hon

@@ -1,7 +1,11 @@
 import "io"
 import "enums"
+import "hashmap"
 import "arraylist"
+import "static_string_map"
 
 Io :: alias io.Io
-ArrayList :: alias arraylist.ArrayList
 EnumMap :: alias enums.EnumMap
+ArrayList :: alias arraylist.ArrayList
+StringHashMap :: alias hashmap.StringHashMap
+StaticStringMap :: alias static_string_map.StaticStringMap

std/testing/testing.hon

@@ -6,14 +6,18 @@ Error :: enum {
 }
 
 SourceLocation :: struct {
-	file   []u8
+	file []u8
-	line   usize
+	line usize
 	column usize
 }
 
 expect func(condition bool, location SourceLocation) void ! Error {
 	if !condition {
-		debug.print("{s}:{d}:{d}: expectation failed\n", {location.file, location.line, location.column})
+		debug.print("{s}:{d}:{d}: expectation failed\n", {
+			location.file,
+			location.line,
+			location.column,
+		})
 		return .expectation_failed
 	}
 }
@@ -26,20 +30,38 @@ expect_equal func($T type, expected, actual T, location SourceLocation) void ! E
 					try expect_equal(expected_value, actual_value, location)
 					return
 				}
-				debug.print("{s}:{d}:{d}: expected an optional value, found null\n", {location.file, location.line, location.column})
+				debug.print("{s}:{d}:{d}: expected an optional value, found null\n", {
+				    location.file,
+				    location.line,
+				    location.column,
+				})
 				return .expectation_failed
 			}
 			if actual |_| {
-				debug.print("{s}:{d}:{d}: expected null, found an optional value\n", {location.file, location.line, location.column})
+				debug.print("{s}:{d}:{d}: expected null, found an optional value\n", {
+				    location.file,
+				    location.line,
+				    location.column,
+				})
 				return .expectation_failed
 			}
 		}
 		.slice: if !mem.eql(expected, actual) {
-			debug.print("{s}:{d}:{d}: expected and actual slices differ\n", {location.file, location.line, location.column})
+			debug.print("{s}:{d}:{d}: expected and actual slices differ\n", {
+				location.file,
+				location.line,
+				location.column,
+			})
 			return .expectation_failed
 		}
 		else: if expected != actual {
-			debug.print("{s}:{d}:{d}: expected {}, found {}\n", {location.file, location.line, location.column, expected, actual})
+			debug.print("{s}:{d}:{d}: expected {}, found {}\n", {
+				location.file,
+				location.line,
+				location.column,
+				expected,
+				actual,
+			})
 			return .expectation_failed
 		}
 	}