hl-valdemar/honey
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Wiki Activity

parse const decls

Browse Source
This commit is contained in:
Valdemar Høgsvig Lorenzen
2026-07-25 00:26:47 +02:00
parent c7f527e3c3
commit 90c7195d4b
28 changed files with 1452 additions and 1134 deletions
Download Patch File Download Diff File Expand all files Collapse all files
source/ast/ast.hon
+1
View File
@@ -0,0 +1 @@
# todo: move ast stuff from parser module to here
source/lexer/lexer.hon
+127 -57
View File
@@ -1,29 +1,52 @@
import "@std"
import "@std/mem"
import "@std/arraylist"
import "@std/strmap"
import "@std/enums/enummap"
import "@std/arraylist"
import "@std/debug"
import "@source/strpool"
ErrorCode :: enum {
invalid_character,
float_must_end_with_digit,
}
ErrorDetails :: struct {
name []u8
message []u8
}
error_msg_map std.EnumMap(ErrorCode, ErrorDetails) :: enummap.init({
invalid_character = ErrorDetails {
name = "L0",
message = "invalid character",
},
float_must_end_with_digit = ErrorDetails {
name = "L1",
message = "float must end with a digit",
},
})
keywords std.StringMap(TokenKind) :: strmap.init([
{ "proc", .proc },
{ "return", .return },
{ "if", .if },
{ "for", .for },
{ "else", .else },
{ "while", .while },
{ "proc", .proc },
{ "return", .return },
{ "if", .if },
{ "for", .for },
{ "else", .else },
{ "while", .while },
])
TokenIndex :: alias usize
TokenId :: distinct u32
ScanDiagnostic :: struct {
token TokenIndex
message []u8
Diagnostic :: struct {
code ErrorCode
token TokenId
}
State :: struct {
tokens std.ArrayList(Token)
diagnostics std.ArrayList(ScanDiagnostic)
diagnostics std.ArrayList(Diagnostic)
}
init proc(allocator mem.Allocator) State {
@@ -39,8 +62,8 @@ deinit proc(state @mut State) void {
}
scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternError) {
tokens :: &state.tokens
diagnostics :: &state.diagnostics
tokens :: &state.tokens
diagnostics :: &state.diagnostics
cursor usize = 0
while cursor < input.len {
@@ -48,9 +71,9 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE
# whitespace
if char == '\n' {
try arraylist.append(tokens, Token{ kind = .newline, start = cursor })
cursor += 1
continue
try add_token(tokens, Token{ kind = .newline, start = cursor })
cursor += 1
continue
} else if is_whitespace(char) {
cursor += 1
continue
@@ -58,10 +81,8 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE
# comments
if char == '#' {
# fixme(brolang): while currently requires curly braces but this should be legal
#while (cursor < input.len and input[cursor] != '\n') cursor += 1
while cursor < input.len and input[cursor] != '\n' : cursor += 1 {}
while (cursor < input.len and input[cursor] != '\n') cursor += 1
cursor += 1 # also skip newline
continue
}
@@ -75,7 +96,7 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE
is_alpha(input[cursor]) or
is_digit(input[cursor]) or
input[cursor] == '_'
)) : cursor += 1 {}
)) cursor += 1
kind :: strmap.get(&keywords, input[start..cursor]) orelse .ident
@@ -83,9 +104,9 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE
str_id :: if (kind == .ident)
try strpool.intern(&strpool.strings, input[start..cursor])
else
strpool.NoId
strpool.NO_ID
try arraylist.append(tokens, Token{
try add_token(tokens, Token{
kind = kind,
start = start,
str_id = str_id
@@ -99,7 +120,7 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE
has_decimal bool = false
# scan integer part
while cursor < input.len and is_digit(input[cursor]) : cursor += 1 {}
while (cursor < input.len and is_digit(input[cursor])) cursor += 1
# check for decimal point
if cursor < input.len and input[cursor] == '.' {
@@ -109,22 +130,17 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE
# assert that decimals follow the decimal point
if has_decimal and (cursor >= input.len or !is_digit(input[cursor])) {
token :: tokens.items.len
try arraylist.append(tokens, Token{ kind = .invalid, start = start })
try arraylist.append(diagnostics, ScanDiagnostic{
token = token,
message = "float must end with a digit",
})
continue
token :: token_id(tokens.items.len)
try arraylist.append(diagnostics, Diagnostic{ token = token, code = .float_must_end_with_digit })
try add_token(tokens, Token{ kind = .invalid, start = start })
continue
}
# scan decimal part
while cursor < input.len and is_digit(input[cursor]) : cursor += 1 {}
while (cursor < input.len and is_digit(input[cursor])) cursor += 1
if (has_decimal)
try arraylist.append(tokens, Token{ kind = .float, start = start })
else
try arraylist.append(tokens, Token{ kind = .int, start = start })
kind :: if (has_decimal) .float else .int
try add_token(tokens, Token{ kind = kind, start = start })
continue
}
@@ -135,15 +151,15 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE
cursor += 1
while cursor < input.len and input[cursor] != '"' and input[cursor] != '\n' : cursor += 1 {
# ignore escaped characters
if (input[cursor] == '\\' and cursor + 1 < input.len) cursor += 1
# ignore escaped characters
if (input[cursor] == '\\' and cursor + 1 < input.len) cursor += 1
}
if cursor < input.len and input[cursor] == '"' {
cursor += 1
try arraylist.append(tokens, Token{ kind = .string, start = start })
try add_token(tokens, Token{ kind = .string, start = start })
} else {
try arraylist.append(tokens, Token{ kind = .invalid, start = start })
try add_token(tokens, Token{ kind = .invalid, start = start })
}
continue
@@ -151,51 +167,95 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE
# mutable assignment
if char == '=' {
try arraylist.append(tokens, Token{ kind = .equal, start = cursor })
try add_token(tokens, Token{ kind = .equal, start = cursor })
cursor += 1
continue
}
# immutable assignment
if cursor + 1 < input.len and char == ':' and input[cursor + 1] == ':' {
try arraylist.append(tokens, Token{ kind = .double_colon, start = cursor })
cursor += 2
# immutable assignment or single colon
if char == ':' {
if cursor + 1 < input.len and input[cursor + 1] == ':' {
try add_token(tokens, Token{ kind = .double_colon, start = cursor })
cursor += 2
continue
}
try add_token(tokens, Token{ kind = .colon, start = cursor })
cursor += 1
continue
}
# parentheses
if char == '(' {
try arraylist.append(tokens, Token{ kind = .open_paren, start = cursor })
try add_token(tokens, Token{ kind = .open_paren, start = cursor })
cursor += 1
continue
} else if char == ')' {
try arraylist.append(tokens, Token{ kind = .close_paren, start = cursor })
try add_token(tokens, Token{ kind = .close_paren, start = cursor })
cursor += 1
continue
}
# curly braces
if char == '{' {
try arraylist.append(tokens, Token{ kind = .open_curly, start = cursor })
try add_token(tokens, Token{ kind = .open_curly, start = cursor })
cursor += 1
continue
} else if char == '}' {
try arraylist.append(tokens, Token{ kind = .close_curly, start = cursor })
try add_token(tokens, Token{ kind = .close_curly, start = cursor })
cursor += 1
continue
}
# invalid character
token :: tokens.items.len
try arraylist.append(tokens, Token{ kind = .invalid, start = cursor })
try arraylist.append(diagnostics, ScanDiagnostic{
token = token,
message = "invalid character",
})
token :: token_id(tokens.items.len)
try arraylist.append(diagnostics, Diagnostic{ token = token, code = .invalid_character })
try add_token(tokens, Token{ kind = .invalid, start = cursor })
cursor += 1
}
try arraylist.append(tokens, Token{ kind = .eof, start = cursor })
try add_token(tokens, Token{ kind = .eof, start = cursor })
}
#! returns the cursor position after scanning a number.
scan_number proc(start usize, input []u8) usize {
debug.assert(is_digit(input[start]))
has_decimal bool = false
cursor usize = start
# scan integer part
while cursor < input.len and is_digit(input[cursor]) : cursor += 1 {}
# check for decimal point
if cursor < input.len and input[cursor] == '.' {
has_decimal = true
cursor += 1
}
# assert that decimals follow the decimal point
debug.assert(!(has_decimal and (cursor >= input.len or !is_digit(input[cursor]))))
# scan decimal part
while cursor < input.len and is_digit(input[cursor]) : cursor += 1 {}
return cursor
}
#! returns the cursor position after scanning a string.
scan_string proc(start usize, input []u8) usize {
debug.assert(input[start] == '"')
cursor usize = start + 1
while cursor < input.len and input[cursor] != '"' and input[cursor] != '\n' : cursor += 1 {
# ignore escaped characters
if (input[cursor] == '\\' and cursor + 1 < input.len) cursor += 1
}
debug.assert(cursor < input.len and input[cursor] == '"')
cursor += 1
return cursor
}
hide is_whitespace proc(char u8) bool {
@@ -215,3 +275,13 @@ hide is_digit proc(char u8) bool {
else: false
}
}
hide add_token proc(tokens @mut std.ArrayList(Token), token Token) void ! mem.AllocError {
_ = token_id(tokens.items.len)
try arraylist.append(tokens, token)
}
hide token_id proc(idx uint) TokenId {
debug.assert(u64(idx) < u64(maxval!(TokenId)))
return TokenId(idx)
}
source/lexer/lexer.test.hon
+13 -14
View File
@@ -3,20 +3,19 @@ import "@std/mem"
import "@std/testing"
handles_keywords_identifiers_and_error_progress test {
strpool.strings = strpool.init(mem.c_allocator)
defer strpool.deinit(&strpool.strings)
strpool.strings = strpool.init(mem.c_allocator)
defer strpool.deinit(&strpool.strings)
state State = init(mem.c_allocator)
defer deinit(&state)
try scan(&state, "if name # comment\n@1.")
state State = init(mem.c_allocator)
defer deinit(&state)
try scan(&state, "if name # comment\n@1.")
try testing.expect_equal(6, state.tokens.items.len)
try testing.expect_equal(TokenKind.if, state.tokens.items[0].kind)
try testing.expect_equal(TokenKind.ident, state.tokens.items[1].kind)
try testing.expect_equal(TokenKind.newline, state.tokens.items[2].kind)
try testing.expect_equal(TokenKind.invalid, state.tokens.items[3].kind)
try testing.expect_equal(TokenKind.invalid, state.tokens.items[4].kind)
try testing.expect_equal(TokenKind.eof, state.tokens.items[5].kind)
try testing.expect_equal(2, state.diagnostics.items.len)
try testing.expect_equal("name", strpool.get_str(&strpool.strings, state.tokens.items[1].str_id)?)
try testing.expect_equal(5, state.tokens.items.len)
try testing.expect_equal(TokenKind.if, state.tokens.items[0].kind)
try testing.expect_equal(TokenKind.ident, state.tokens.items[1].kind)
try testing.expect_equal(TokenKind.invalid, state.tokens.items[2].kind)
try testing.expect_equal(TokenKind.invalid, state.tokens.items[3].kind)
try testing.expect_equal(TokenKind.eof, state.tokens.items[4].kind)
try testing.expect_equal(2, state.diagnostics.items.len)
try testing.expect_equal("name", strpool.get_str(&strpool.strings, state.tokens.items[1].str_id)?)
}
source/lexer/token.hon
+32 -28
View File
@@ -1,34 +1,38 @@
import "@source/strpool"
TokenKind :: enum {
if
else
for
while
proc
return
ident
int
float
string
equal
double_colon
open_paren
close_paren
open_curly
close_curly
newline
invalid
eof
}
Token :: struct {
kind TokenKind
start int
str_id strpool.StringId = strpool.NoId
str_id strpool.StringId = strpool.NO_ID
}
TokenKind :: enum {
# keywords
if, else
for, while
proc, return
# literals
ident, string
int, float
# comparison
equal_equal, not_equal
less, less_equal
greater, greater_equal
# assignment
equal, double_colon
# delimiters
open_paren, close_paren
open_bracket, close_bracket
open_curly, close_curly
colon
newline
# special
eof
invalid
}
source/main.hon
+42 -17
View File
@@ -1,34 +1,24 @@
import "@std/debug"
import "@std/mem"
import "@std/enums/enummap"
test import "@std/enums"
test import "@std/enums/enummap"
test import "@std/arraylist"
test import "@std/hashmap"
test import "@std/strmap"
import "@source/strpool"
import "@source/lexer"
import "@source/parser"
test import "@source/strpool"
test import "@source/lexer"
program ::
`# these are immutable
`x :: 32
`y :: 3.2
`
`# these are mutable
`z u32 = 54
`
`# keywords
`if
`else
`for
`while
`proc
`return
`
`main proc() void {}
`# literals
`x :: 123
# todo: consider making current aliased types (IDs) distinct instead
main proc() void {
strpool.strings = strpool.init(mem.c_allocator)
@@ -48,5 +38,40 @@ main proc() void {
for scan_state.tokens.items |token| {
debug.print("{}\n", {token.kind})
}
debug.print("]]\n\n", {})
debug.print("TOKENS::DIAGNOSTICS::[[\n", {})
for scan_state.diagnostics.items |diagnostic| {
details :: enummap.get(&lexer.error_msg_map, diagnostic.code)
debug.print("{}: {}\n", {details?.name, details?.message})
}
debug.print("]]\n\n", {})
parse_state parser.State = parser.init(mem.c_allocator)
defer parser.deinit(&parse_state)
parser.parse(&parse_state, scan_state.tokens.items) catch |err| {
debug.print("failed to parse: {}\n", {err})
return
}
debug.print("AST::[[\n", {})
for parse_state.nodes.items |node, id| {
token :: scan_state.tokens.items[node.main_token]
end :: if (token.kind == .int or token.kind == .float)
lexer.scan_number(token.start, program)
else if (token.kind == .string)
lexer.scan_string(token.start, program)
else
token.start
debug.print("{} (id = {}): {}\n", {
node.kind,
id,
program[token.start..end],
#node.data0,
#node.data1,
})
}
debug.print("]]\n", {})
}
source/parser/parser.hon
+156
View File
@@ -0,0 +1,156 @@
import "@std"
import "@std/mem"
import "@std/arraylist"
import "@std/debug"
import "@std/enums/enummap"
import "@source/lexer"
NodeId :: distinct u32
ExtraId :: distinct NodeId
NO_ID :: maxval!(NodeId)
Diagnostic :: struct {
code ErrorCode
node NodeId
}
ParseError :: alias ErrorCode | mem.AllocError
ErrorCode :: enum {
unexpected_token,
}
ErrorDetails :: struct {
name []u8
message []u8
}
error_msg_map std.EnumMap(ErrorCode, ErrorDetails) :: enummap.init({
unexpected_token = ErrorDetails {
name = "P0",
message = "unexpected token",
},
})
hide Token :: alias lexer.Token
hide TokenId :: alias lexer.TokenId
hide TokenKind :: alias lexer.TokenKind
NodeData :: union {
node_id NodeId
extra_id ExtraId
}
Node :: struct {
kind NodeKind
main_token TokenId
data0 NodeData = NodeData{ node_id = NO_ID }
data1 NodeData = NodeData{ node_id = NO_ID }
}
NodeKind :: enum {
literal_int
literal_float
literal_string
expr_unary
expr_binary
decl
invalid
}
State :: struct {
nodes std.ArrayList(Node)
#! array of indexes into the nodes array.
extra std.ArrayList(NodeId)
next_token TokenId = TokenId(0)
}
init proc(allocator mem.Allocator) State {
return State{
nodes = arraylist.init(allocator),
extra = arraylist.init(allocator),
}
}
deinit proc(state @mut State) void {
arraylist.deinit(&state.nodes)
arraylist.deinit(&state.extra)
}
parse proc(state @mut State, tokens []Token) void ! ParseError {
_ = try parse_decl(state, tokens)
}
parse_decl proc(state @mut State, tokens []Token) NodeId ! ParseError {
# expect identifier
try expect(state, tokens, .ident)
ident_token :: state.next_token
state.next_token += 1
# expect `::` (immutable assignment)
try expect(state, tokens, .double_colon)
state.next_token += 1
# expect expression
expr :: try parse_primary(state, tokens)
# expect statement terminator (newline)
try expect_either(state, tokens, &[.newline, .eof])
state.next_token += 1
decl :: try add_node(&state.nodes, Node{
kind = .decl,
main_token = ident_token,
data0 = NodeData{ node_id = expr },
})
return decl
}
parse_primary proc(state @mut State, tokens []Token) NodeId ! mem.AllocError {
start_token :: state.next_token
state.next_token += 1
return match tokens[start_token].kind {
.int: try add_node(&state.nodes, Node{
kind = .literal_int,
main_token = start_token,
})
.float: try add_node(&state.nodes, Node{
kind = .literal_float,
main_token = start_token,
})
.string: try add_node(&state.nodes, Node{
kind = .literal_string,
main_token = start_token,
})
else: try add_node(&state.nodes, Node{
kind = .invalid,
main_token = start_token,
})
}
}
hide expect proc(state @mut State, tokens []Token, token_kind TokenKind) void ! ErrorCode {
if (tokens[state.next_token].kind != token_kind) return .unexpected_token
}
hide expect_either proc(state @mut State, tokens []Token, token_kinds []TokenKind) void ! ErrorCode {
for (token_kinds) |kind| if (tokens[state.next_token].kind == kind) return
return .unexpected_token
}
hide add_node proc(nodes @mut std.ArrayList(Node), node Node) NodeId ! mem.AllocError {
id :: node_id(nodes.items.len)
try arraylist.append(nodes, node)
return id
}
hide node_id proc(idx uint) NodeId {
debug.assert(u64(idx) < u64(NO_ID))
return NodeId(idx)
}
source/strpool/strpool.hon
+33 -32
View File
@@ -8,58 +8,59 @@ strings StringPool = undefined
InternError :: enum { out_of_space }
StringId :: alias u32
NoId :: maxval!(StringId)
NO_ID :: maxval!(StringId)
StringPool :: struct {
ids hashmap.StringHashMap(StringId) # string → id
strings arraylist.ArrayList([]u8) # id → owned string
allocator mem.Allocator
ids hashmap.StringHashMap(StringId) # string → id
strings arraylist.ArrayList([]u8) # id → owned string
allocator mem.Allocator
}
init proc(allocator mem.Allocator) StringPool {
return StringPool{
ids = hashmap.init(allocator),
strings = arraylist.init(allocator),
allocator = allocator,
}
return StringPool{
ids = hashmap.init(allocator),
strings = arraylist.init(allocator),
allocator = allocator,
}
}
deinit proc(pool @mut StringPool) void {
hashmap.deinit(&pool.ids)
for pool.strings.items |str| {
mem.free(pool.allocator, str)
}
arraylist.deinit(&pool.strings)
hashmap.deinit(&pool.ids)
for pool.strings.items |str| {
mem.free(pool.allocator, str)
}
arraylist.deinit(&pool.strings)
}
intern proc(pool @mut StringPool, str []u8) StringId ! (mem.AllocError | InternError) {
if hashmap.get(&pool.ids, str) |id| return id
if hashmap.get(&pool.ids, str) |id| return id
if (pool.strings.items.len >= usize(NoId)) return .out_of_space
id StringId :: StringId(pool.strings.items.len)
if (pool.strings.items.len >= usize(NO_ID)) return .out_of_space
id StringId :: StringId(pool.strings.items.len)
owned_str []mut u8 :: try mem.alloc(u8, pool.allocator, str.len)
errdefer mem.free(pool.allocator, owned_str)
memcopy!(owned_str, str)
owned_str []mut u8 :: try mem.alloc(u8, pool.allocator, str.len)
errdefer mem.free(pool.allocator, owned_str)
memcopy!(owned_str, str)
try arraylist.append(&pool.strings, owned_str)
errdefer _ = arraylist.pop(&pool.strings)
try arraylist.append(&pool.strings, owned_str)
errdefer _ = arraylist.pop(&pool.strings)
hashmap.put(&pool.ids, owned_str, id) catch |err| {
match err {
.key_exists: unreachable
else: return err
}
}
hashmap.put(&pool.ids, owned_str, id) catch |err| {
match err {
.key_exists: unreachable
else: return err
}
}
return id
return id
}
get_str proc(pool @StringPool, id StringId) ?[]u8 {
if (usize(id) >= pool.strings.items.len) return null
return pool.strings.items[usize(id)]
if (usize(id) >= pool.strings.items.len) return null
return pool.strings.items[usize(id)]
}
get_id proc(pool @StringPool, str []u8) ?StringId {
return hashmap.get(&pool.ids, str)
return hashmap.get(&pool.ids, str)
}
source/strpool/strpool.test.hon
+16 -16
View File
@@ -2,23 +2,23 @@ import "@std/mem"
import "@std/testing"
handles_intern test {
pool StringPool = init(mem.c_allocator)
defer deinit(&pool)
pool StringPool = init(mem.c_allocator)
defer deinit(&pool)
input [5]mut u8 = ['h', 'e', 'l', 'l', 'o']
id :: try intern(&pool, input[..])
duplicate :: try intern(&pool, "hello")
input[0] = 'j'
input [5]mut u8 = ['h', 'e', 'l', 'l', 'o']
id :: try intern(&pool, input[..])
duplicate :: try intern(&pool, "hello")
input[0] = 'j'
same_id :: get_id(&pool, "hello")
mutated_id :: get_id(&pool, input[..])
hello_str :: get_str(&pool, id)
invalid_str :: get_str(&pool, id + 1)
same_id :: get_id(&pool, "hello")
mutated_id :: get_id(&pool, input[..])
hello_str :: get_str(&pool, id)
invalid_str :: get_str(&pool, id + 1)
try testing.expect_equal(0, id)
try testing.expect_equal(id, duplicate)
try testing.expect_equal(id, same_id?)
try testing.expect_equal(null, mutated_id)
try testing.expect_equal(null, invalid_str)
try testing.expect_equal("hello", hello_str?)
try testing.expect_equal(0, id)
try testing.expect_equal(id, duplicate)
try testing.expect_equal(id, same_id?)
try testing.expect_equal(null, mutated_id)
try testing.expect_equal(null, invalid_str)
try testing.expect_equal("hello", hello_str?)
}