hl-valdemar/honey
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Wiki Activity

refactor and func -> proc rename

Browse Source
This commit is contained in:
Valdemar Høgsvig Lorenzen
2026-07-23 11:07:30 +02:00
parent 0338e35e75
commit ad54a00893
18 changed files with 317 additions and 311 deletions
Download Patch File Download Diff File Expand all files Collapse all files
source/lexer/lexer.hon
+40 -43
View File
@@ -1,15 +1,18 @@
import "@std"
import "@std/mem"
import "@std/arraylist"
#import "@std/static_string_map"
import "@std/strmap"
#keywords std.StaticStringMap(TokenKind) :: static_string_map.init([
# { "func", .func },
# { "return", .return },
# { "if", .if },
# { "for", .for },
# { "else", .else },
#])
import "@source/strpool"
keywords std.StaticStringMap(TokenKind) :: strmap.init([
{ "proc", .proc },
{ "return", .return },
{ "if", .if },
{ "for", .for },
{ "else", .else },
{ "while", .while },
])
TokenIndex :: alias usize
@@ -23,19 +26,19 @@ State :: struct {
diagnostics std.ArrayList(ScanDiagnostic)
}
init func(allocator mem.Allocator) State {
init proc(allocator mem.Allocator) State {
return State {
tokens = arraylist.init(allocator),
diagnostics = arraylist.init(allocator),
}
}
deinit func(state @mut State) void {
deinit proc(state @mut State) void {
arraylist.deinit(&state.tokens)
arraylist.deinit(&state.diagnostics)
}
scan func(state @mut State, input []u8) void ! mem.AllocError {
scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternError) {
tokens :: &state.tokens
diagnostics :: &state.diagnostics
@@ -56,7 +59,6 @@ scan func(state @mut State, input []u8) void ! mem.AllocError {
# comments
if char == '#' {
while cursor < input.len and input[cursor] != '\n' : cursor += 1 {}
cursor += 1
continue
}
@@ -64,11 +66,27 @@ scan func(state @mut State, input []u8) void ! mem.AllocError {
if is_alpha(char) or char == '_' {
start :: cursor
cursor += 1
while cursor < input.len and (is_alpha(input[cursor]) or is_digit(input[cursor]) or input[cursor] == '_') {
cursor += 1
}
kind :: ident_keyword_map(input[start..cursor])
try arraylist.append(tokens, Token{ kind = kind, start = start })
# scan whole identifier
while (cursor < input.len and (
is_alpha(input[cursor]) or
is_digit(input[cursor]) or
input[cursor] == '_'
)) : cursor += 1 {}
kind :: strmap.get(&keywords, input[start..cursor]) orelse .ident
# don't intern keywords (already O(1) lookup via token kind)
str_id :: if (kind == .ident)
try strpool.intern(&strpool.strings, input[start..cursor])
else
strpool.NoId
try arraylist.append(tokens, Token{
kind = kind,
start = start,
str_id = str_id
})
continue
}
@@ -89,7 +107,7 @@ scan func(state @mut State, input []u8) void ! mem.AllocError {
}
# assert that decimals follow the decimal point
if has_decimal and cursor < input.len and !is_digit(input[cursor]) {
if has_decimal and (cursor >= input.len or !is_digit(input[cursor])) {
token :: tokens.items.len
try arraylist.append(tokens, Token{ kind = .invalid, start = start })
try arraylist.append(diagnostics, ScanDiagnostic{ token = token, message = "float must end with a digit" })
@@ -167,47 +185,26 @@ scan func(state @mut State, input []u8) void ! mem.AllocError {
token :: tokens.items.len
try arraylist.append(tokens, Token{ kind = .invalid, start = cursor })
try arraylist.append(diagnostics, ScanDiagnostic{ token = token, message = "invalid character" })
cursor += 1
}
try arraylist.append(tokens, Token{ kind = .eof, start = cursor })
}
hide is_whitespace func(char u8) bool {
hide is_whitespace proc(char u8) bool {
return char == ' ' or char == '\t' or char == '\n' or char == '\r'
}
hide is_alpha func(char u8) bool {
hide is_alpha proc(char u8) bool {
return match char {
'a'..='z', 'A'..='Z': true
else: false
}
}
hide is_digit func(char u8) bool {
hide is_digit proc(char u8) bool {
return match char {
'0'..='9': true
else: false
}
}
# fixme: replace with a static string map once implemented
hide ident_keyword_map func(ident []u8) TokenKind {
if mem.eql(u8, ident, "if") return .if
if mem.eql(u8, ident, "else") return .else
if mem.eql(u8, ident, "for") return .for
if mem.eql(u8, ident, "while") return .while
if mem.eql(u8, ident, "func") return .func
if mem.eql(u8, ident, "return") return .return
return .ident
}
# fixme(brolang): this function fails to infer the enum type from the return values due to the optional
#hide ident_keyword_map func(ident []u8) ?TokenKind {
# if mem.eql(u8, ident, "if") return .if
# if mem.eql(u8, ident, "else") return .else
# if mem.eql(u8, ident, "for") return .for
# if mem.eql(u8, ident, "while") return .while
# if mem.eql(u8, ident, "func") return .func
# if mem.eql(u8, ident, "return") return .return
# return null
#}
source/lexer/lexer.test.hon
+22
View File
@@ -0,0 +1,22 @@
import "@source/strpool"
import "@std/mem"
import "@std/testing"
handles_keywords_identifiers_and_error_progress test {
strpool.strings = strpool.init(mem.c_allocator)
defer strpool.deinit(&strpool.strings)
state State = init(mem.c_allocator)
defer deinit(&state)
try scan(&state, "if name # comment\n@1.")
try testing.expect_equal(6, state.tokens.items.len)
try testing.expect_equal(TokenKind.if, state.tokens.items[0].kind)
try testing.expect_equal(TokenKind.ident, state.tokens.items[1].kind)
try testing.expect_equal(TokenKind.newline, state.tokens.items[2].kind)
try testing.expect_equal(TokenKind.invalid, state.tokens.items[3].kind)
try testing.expect_equal(TokenKind.invalid, state.tokens.items[4].kind)
try testing.expect_equal(TokenKind.eof, state.tokens.items[5].kind)
try testing.expect_equal(2, state.diagnostics.items.len)
try testing.expect_equal("name", strpool.get_str(&strpool.strings, state.tokens.items[1].str_id)?)
}
source/lexer/token.hon
+4 -1
View File
@@ -1,9 +1,11 @@
import "@source/strpool"
TokenKind :: enum {
if
else
for
while
func
proc
return
ident
@@ -28,4 +30,5 @@ TokenKind :: enum {
Token :: struct {
kind TokenKind
start int
str_id strpool.StringId = strpool.NoId
}
source/main.hon
+7 -9
View File
@@ -4,12 +4,13 @@ import "@std/mem"
test import "@std/enums"
test import "@std/arraylist"
test import "@std/hashmap"
test import "@std/static_string_map"
test import "@std/strmap"
import "@source/strpool"
import "@source/lexer"
test import "@source/strpool"
test import "@source/lexer"
program ::
`# these are immutable
@@ -24,17 +25,14 @@ program ::
`else
`for
`while
`func
`proc
`return
`
`main func() void {}
`main proc() void {}
# cross-cutting concern, hence global singleton
strings StringPool = undefined
main func() void {
strings = strpool.init(mem.c_allocator)
defer strpool.deinit(&strings)
main proc() void {
strpool.strings = strpool.init(mem.c_allocator)
defer strpool.deinit(&strpool.strings)
debug.print("PROGRAM::[[\n{}\n]]\n\n", {program})
source/strpool/strpool.hon
+17 -9
View File
@@ -3,7 +3,13 @@ import "@std/arraylist"
import "@std/hashmap"
import "@std/testing"
StringId :: alias usize
# cross-cutting concern, hence global singleton (owned by main.hon)
strings StringPool = undefined
InternError :: enum { out_of_space }
StringId :: alias u32
NoId :: maxval!(StringId)
StringPool :: struct {
ids hashmap.StringHashMap(StringId) # string → id
@@ -11,7 +17,7 @@ StringPool :: struct {
allocator mem.Allocator
}
init func(allocator mem.Allocator) StringPool {
init proc(allocator mem.Allocator) StringPool {
return StringPool{
ids = hashmap.init(allocator),
strings = arraylist.init(allocator),
@@ -19,7 +25,7 @@ init func(allocator mem.Allocator) StringPool {
}
}
deinit func(pool @mut StringPool) void {
deinit proc(pool @mut StringPool) void {
hashmap.deinit(&pool.ids)
for pool.strings.items |str| {
mem.free(pool.allocator, str)
@@ -27,10 +33,12 @@ deinit func(pool @mut StringPool) void {
arraylist.deinit(&pool.strings)
}
intern func(pool @mut StringPool, str []u8) StringId ! mem.AllocError {
intern proc(pool @mut StringPool, str []u8) StringId ! (mem.AllocError | InternError) {
if hashmap.get(&pool.ids, str) |id| return id
id :: pool.strings.items.len
if (pool.strings.items.len >= usize(NoId)) return .out_of_space
id StringId :: StringId(pool.strings.items.len)
owned_str []mut u8 :: try mem.alloc(u8, pool.allocator, str.len)
errdefer mem.free(pool.allocator, owned_str)
memcopy!(owned_str, str)
@@ -48,12 +56,12 @@ intern func(pool @mut StringPool, str []u8) StringId ! mem.AllocError {
return id
}
get_str func(pool @StringPool, id StringId) ?[]u8 {
if (id >= pool.strings.items.len) return null
return pool.strings.items[id]
get_str proc(pool @StringPool, id StringId) ?[]u8 {
if (usize(id) >= pool.strings.items.len) return null
return pool.strings.items[usize(id)]
}
get_id func(pool @StringPool, str []u8) ?StringId {
get_id proc(pool @StringPool, str []u8) ?StringId {
return hashmap.get(&pool.ids, str)
}