diff --git a/source/lexer/lexer.hon b/source/lexer/lexer.hon index 6bfd666..48ab535 100644 --- a/source/lexer/lexer.hon +++ b/source/lexer/lexer.hon @@ -5,7 +5,7 @@ import "@std/strmap" import "@source/strpool" -keywords std.StaticStringMap(TokenKind) :: strmap.init([ +keywords std.StringMap(TokenKind) :: strmap.init([ { "proc", .proc }, { "return", .return }, { "if", .if }, @@ -58,6 +58,9 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE # comments if char == '#' { + # fixme(brolang): while currently requires curly braces but this should be legal + #while (cursor < input.len and input[cursor] != '\n') cursor += 1 + while cursor < input.len and input[cursor] != '\n' : cursor += 1 {} continue } @@ -96,9 +99,7 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE has_decimal bool = false # scan integer part - while cursor < input.len and is_digit(input[cursor]) { - cursor += 1 - } + while cursor < input.len and is_digit(input[cursor]) : cursor += 1 {} # check for decimal point if cursor < input.len and input[cursor] == '.' { @@ -110,18 +111,21 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE if has_decimal and (cursor >= input.len or !is_digit(input[cursor])) { token :: tokens.items.len try arraylist.append(tokens, Token{ kind = .invalid, start = start }) - try arraylist.append(diagnostics, ScanDiagnostic{ token = token, message = "float must end with a digit" }) + try arraylist.append(diagnostics, ScanDiagnostic{ + token = token, + message = "float must end with a digit", + }) continue } # scan decimal part while cursor < input.len and is_digit(input[cursor]) : cursor += 1 {} - if has_decimal { + if (has_decimal) try arraylist.append(tokens, Token{ kind = .float, start = start }) - } else { + else try arraylist.append(tokens, Token{ kind = .int, start = start }) - } + continue } @@ -184,7 +188,10 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE # invalid character token :: tokens.items.len try arraylist.append(tokens, Token{ kind = .invalid, start = cursor }) - try arraylist.append(diagnostics, ScanDiagnostic{ token = token, message = "invalid character" }) + try arraylist.append(diagnostics, ScanDiagnostic{ + token = token, + message = "invalid character", + }) cursor += 1 } diff --git a/source/strpool/strpool.hon b/source/strpool/strpool.hon index a21ba6c..5c55de1 100644 --- a/source/strpool/strpool.hon +++ b/source/strpool/strpool.hon @@ -1,7 +1,6 @@ import "@std/mem" import "@std/arraylist" import "@std/hashmap" -import "@std/testing" # cross-cutting concern, hence global singleton (owned by main.hon) strings StringPool = undefined @@ -64,25 +63,3 @@ get_str proc(pool @StringPool, id StringId) ?[]u8 { get_id proc(pool @StringPool, str []u8) ?StringId { return hashmap.get(&pool.ids, str) } - -handles_intern test { - pool StringPool = init(mem.c_allocator) - defer deinit(&pool) - - input [5]mut u8 = ['h', 'e', 'l', 'l', 'o'] - id :: try intern(&pool, input[..]) - duplicate :: try intern(&pool, "hello") - input[0] = 'j' - - same_id :: get_id(&pool, "hello") - mutated_id :: get_id(&pool, input[..]) - hello_str :: get_str(&pool, id) - invalid_str :: get_str(&pool, id + 1) - - try testing.expect_equal(0, id) - try testing.expect_equal(id, duplicate) - try testing.expect_equal(id, same_id?) - try testing.expect_equal(null, mutated_id) - try testing.expect_equal(null, invalid_str) - try testing.expect_equal("hello", hello_str?) -} diff --git a/source/strpool/strpool.test.hon b/source/strpool/strpool.test.hon new file mode 100644 index 0000000..1d5d48c --- /dev/null +++ b/source/strpool/strpool.test.hon @@ -0,0 +1,24 @@ +import "@std/mem" +import "@std/testing" + +handles_intern test { + pool StringPool = init(mem.c_allocator) + defer deinit(&pool) + + input [5]mut u8 = ['h', 'e', 'l', 'l', 'o'] + id :: try intern(&pool, input[..]) + duplicate :: try intern(&pool, "hello") + input[0] = 'j' + + same_id :: get_id(&pool, "hello") + mutated_id :: get_id(&pool, input[..]) + hello_str :: get_str(&pool, id) + invalid_str :: get_str(&pool, id + 1) + + try testing.expect_equal(0, id) + try testing.expect_equal(id, duplicate) + try testing.expect_equal(id, same_id?) + try testing.expect_equal(null, mutated_id) + try testing.expect_equal(null, invalid_str) + try testing.expect_equal("hello", hello_str?) +} diff --git a/std/std.hon b/std/std.hon index 0c05558..813b250 100644 --- a/std/std.hon +++ b/std/std.hon @@ -8,4 +8,4 @@ Io :: alias io.Io EnumMap :: alias enums.EnumMap ArrayList :: alias arraylist.ArrayList StringHashMap :: alias hashmap.StringHashMap -StaticStringMap :: alias strmap.StaticStringMap +StringMap :: alias strmap.StringMap diff --git a/std/strmap/strmap.hon b/std/strmap/strmap.hon index e8022cd..de45d93 100644 --- a/std/strmap/strmap.hon +++ b/std/strmap/strmap.hon @@ -1,6 +1,6 @@ import "@std/mem" -StaticStringMap proc($V type) type { +StringMap proc($V type) type { return struct { keys [][]u8 values []V @@ -14,7 +14,8 @@ hide Pair proc($V type) type { return struct { []u8, V } } -init proc($V type, $N usize, $entries [N]Pair(V)) StaticStringMap(V) { +#! initializes a static string map from a list of key-value pairs (constructed at compile-time). +init proc($V type, $N usize, $entries [N]Pair(V)) StringMap(V) { if N > usize(maxval!(u32)) { compile_error!("static string map has too many entries") } @@ -38,7 +39,7 @@ init proc($V type, $N usize, $entries [N]Pair(V)) StaticStringMap(V) { if N == 0 { len_indexes [0]u32 = undefined - return StaticStringMap(V){ + return StringMap(V){ keys = keys[..], values = values[..], len_indexes = len_indexes[..], @@ -47,7 +48,7 @@ init proc($V type, $N usize, $entries [N]Pair(V)) StaticStringMap(V) { } } - # fixme: insertion sort is compile-time O(N^2); replace if large maps affect builds + # fixme: insertion sort is compile-time O(N²); replace if large maps affect builds for 1..N |i| { key :: keys[i] value :: values[i] @@ -69,7 +70,7 @@ init proc($V type, $N usize, $entries [N]Pair(V)) StaticStringMap(V) { len_indexes[length] = u32(entry_index) } - return StaticStringMap(V) { + return StringMap(V) { keys = keys[..], values = values[..], len_indexes = len_indexes[..], @@ -78,7 +79,7 @@ init proc($V type, $N usize, $entries [N]Pair(V)) StaticStringMap(V) { } } -get proc($V type, map @StaticStringMap(V), key []u8) ?V { +get proc($V type, map @StringMap(V), key []u8) ?V { if (map.keys.len == 0 or key.len > maxval!(u32)) return null length u32 = u32(key.len) @@ -87,7 +88,7 @@ get proc($V type, map @StaticStringMap(V), key []u8) ?V { idx usize = usize(map.len_indexes[usize(length)]) while idx < map.keys.len : idx += 1 { candidate :: map.keys[idx] - if (candidate.len != key.len) return null + if (candidate.len != key.len) return null # key not found if mem.eql(u8, candidate, key) return map.values[idx] } return null