hl-valdemar/honey
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Wiki Activity

lexer touchups

Browse Source
This commit is contained in:
Valdemar Høgsvig Lorenzen
2026-08-05 08:53:51 +02:00
parent a2ccf60eb1
commit 1689c4db3c
4 changed files with 113 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
source/ast/ast.hon
+19
View File
@@ -48,3 +48,22 @@ NodeKind :: enum {
invalid
}
# todo: render ast nodes as source code
render proc(node @Node, tokens []Token) void { }
render_literal proc(node @Node, tokens []Token, program []u8) []u8 ! lexer.ScanError {
tok :: tokens[node.main_token]
match node.kind {
.literal_int, .literal_float: {
result :: try lexer.scan_number(tok.start, program)
return program[tok.start..result.end]
}
.literal_string: {
result :: try lexer.scan_string(tok.start, program)
return program[tok.start..result.end]
}
else: unreachable
}
}
source/lexer/lexer.hon
+66 -71
View File
@@ -7,9 +7,10 @@ import "@std/debug"
import "@source/strpool"
ErrorCode :: enum {
ScanError :: enum {
invalid_character,
float_must_end_with_digit,
unterminated_string,
}
ErrorDetails :: struct {
@@ -17,7 +18,7 @@ ErrorDetails :: struct {
message []u8
}
error_msg_map std.EnumMap(ErrorCode, ErrorDetails) :: enummap.init({
error_msg_map std.EnumMap(ScanError, ErrorDetails) :: enummap.init({
invalid_character = ErrorDetails {
name = "L0",
message = "invalid character",
@@ -40,8 +41,8 @@ keywords std.StringMap(TokenKind) :: strmap.init([
TokenId :: distinct u32
Diagnostic :: struct {
code ErrorCode
token TokenId
code ScanError
}
State :: struct {
@@ -61,13 +62,13 @@ deinit proc(state @mut State) void {
arraylist.deinit(&state.diagnostics)
}
scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternError) {
scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.InternError) {
tokens :: &state.tokens
diagnostics :: &state.diagnostics
cursor usize = 0
while cursor < input.len {
char :: input[cursor]
while cursor < program.len {
char :: program[cursor]
# whitespace
if char == '\n' {
@@ -81,7 +82,7 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE
# comments
if char == '#' {
while (cursor < input.len and input[cursor] != '\n') cursor += 1
while (cursor < program.len and program[cursor] != '\n') cursor += 1
cursor += 1 # also skip newline
continue
}
@@ -92,17 +93,17 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE
cursor += 1
# scan whole identifier
while (cursor < input.len and (
is_alpha(input[cursor]) or
is_digit(input[cursor]) or
input[cursor] == '_'
while (cursor < program.len and (
is_alpha(program[cursor]) or
is_digit(program[cursor]) or
program[cursor] == '_'
)) cursor += 1
kind :: strmap.get(&keywords, input[start..cursor]) orelse .ident
kind :: strmap.get(&keywords, program[start..cursor]) orelse .ident
# don't intern keywords (already O(1) lookup via token kind)
str_id :: if (kind == .ident)
try strpool.intern(&strpool.strings, input[start..cursor])
try strpool.intern(&strpool.strings, program[start..cursor])
else
strpool.NO_ID
@@ -117,51 +118,35 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE
# numeric literals
if is_digit(char) {
start :: cursor
has_decimal bool = false
# scan integer part
while (cursor < input.len and is_digit(input[cursor])) cursor += 1
# check for decimal point
if cursor < input.len and input[cursor] == '.' {
has_decimal = true
cursor += 1
}
# assert that decimals follow the decimal point
if has_decimal and (cursor >= input.len or !is_digit(input[cursor])) {
result :: scan_number(start, program) catch |err| {
token :: token_id(tokens.items.len)
try arraylist.append(diagnostics, Diagnostic{ token = token, code = .float_must_end_with_digit })
try add_token(tokens, Token{ kind = .invalid, start = start })
try arraylist.append(diagnostics, Diagnostic{ token = token, code = err })
cursor += 1
while (cursor < program.len and (is_digit(program[cursor]) or program[cursor] == '.')) cursor += 1
continue
}
# scan decimal part
while (cursor < input.len and is_digit(input[cursor])) cursor += 1
kind :: if (has_decimal) .float else .int
kind :: if (result.has_decimal) .float else .int
try add_token(tokens, Token{ kind = kind, start = start })
cursor = result.end
continue
}
# string literals
if char == '"' {
start :: cursor
cursor += 1
while cursor < input.len and input[cursor] != '"' and input[cursor] != '\n' : cursor += 1 {
# ignore escaped characters
if (input[cursor] == '\\' and cursor + 1 < input.len) cursor += 1
}
if cursor < input.len and input[cursor] == '"' {
cursor += 1
try add_token(tokens, Token{ kind = .string, start = start })
} else {
result :: scan_string(start, program) catch |err| {
token :: token_id(tokens.items.len)
try add_token(tokens, Token{ kind = .invalid, start = start })
try arraylist.append(diagnostics, Diagnostic{ token = token, code = err })
cursor += 1
while cursor < program.len and program[cursor] != '\n' : cursor += 1 {}
continue
}
try add_token(tokens, Token{ kind = .string, start = start })
cursor = result.end
continue
}
@@ -174,7 +159,7 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE
# immutable assignment or single colon
if char == ':' {
if cursor + 1 < input.len and input[cursor + 1] == ':' {
if cursor + 1 < program.len and program[cursor + 1] == ':' {
try add_token(tokens, Token{ kind = .double_colon, start = cursor })
cursor += 2
continue
@@ -217,45 +202,55 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE
try add_token(tokens, Token{ kind = .eof, start = cursor })
}
#! returns the cursor position after scanning a number.
scan_number proc(start usize, input []u8) usize {
debug.assert(is_digit(input[start]))
ScanNumResult :: struct {
end usize
has_decimal bool
}
has_decimal bool = false
scan_number proc(start usize, program []u8) ScanNumResult ! ScanError {
cursor usize = start
has_decimal bool = false
# scan integer part
while cursor < input.len and is_digit(input[cursor]) : cursor += 1 {}
while (cursor < program.len and is_digit(program[cursor])) cursor += 1
# check for decimal point
if cursor < input.len and input[cursor] == '.' {
# check for decimal
if cursor < program.len and program[cursor] == '.' {
has_decimal = true
cursor += 1
}
# assert that decimals follow the decimal point
debug.assert(!(has_decimal and (cursor >= input.len or !is_digit(input[cursor]))))
# scan decimal part
while cursor < input.len and is_digit(input[cursor]) : cursor += 1 {}
return cursor
}
#! returns the cursor position after scanning a string.
scan_string proc(start usize, input []u8) usize {
debug.assert(input[start] == '"')
cursor usize = start + 1
while cursor < input.len and input[cursor] != '"' and input[cursor] != '\n' : cursor += 1 {
# ignore escaped characters
if (input[cursor] == '\\' and cursor + 1 < input.len) cursor += 1
# assert non-terminating decimal
if has_decimal and (cursor >= program.len or !is_digit(program[cursor])) {
return .float_must_end_with_digit
}
debug.assert(cursor < input.len and input[cursor] == '"')
cursor += 1
# scan fractional part
while (cursor < program.len and is_digit(program[cursor])) cursor += 1
return cursor
return ScanNumResult{
end = cursor,
has_decimal = has_decimal,
}
}
ScanStrResult :: struct { end usize }
scan_string proc(start usize, program []u8) ScanStrResult ! ScanError {
cursor usize = start + 1 # skip first `"`
# scan entire string
while cursor < program.len and program[cursor] != '"' and program[cursor] != '\n' : cursor += 1 {
# ignore escaped characters
if (program[cursor] == '\\' and cursor + 1 < program.len) cursor += 1
}
# assert string terminal
if (cursor >= program.len or program[cursor] != '"') return .unterminated_string
return ScanStrResult{
end = cursor + 1, # skip last `"`
}
}
hide is_whitespace proc(char u8) bool {
source/lexer/lexer.test.hon
+6 -3
View File
@@ -8,14 +8,17 @@ handles_keywords_identifiers_and_error_progress test {
state State = init(mem.c_allocator)
defer deinit(&state)
try scan(&state, "if name # comment\n@1.")
try scan(&state, "if name # comment\n@1. 2 3.5 \"hi\"")
try testing.expect_equal(5, state.tokens.items.len)
try testing.expect_equal(8, state.tokens.items.len)
try testing.expect_equal(TokenKind.if, state.tokens.items[0].kind)
try testing.expect_equal(TokenKind.ident, state.tokens.items[1].kind)
try testing.expect_equal(TokenKind.invalid, state.tokens.items[2].kind)
try testing.expect_equal(TokenKind.invalid, state.tokens.items[3].kind)
try testing.expect_equal(TokenKind.eof, state.tokens.items[4].kind)
try testing.expect_equal(TokenKind.int, state.tokens.items[4].kind)
try testing.expect_equal(TokenKind.float, state.tokens.items[5].kind)
try testing.expect_equal(TokenKind.string, state.tokens.items[6].kind)
try testing.expect_equal(TokenKind.eof, state.tokens.items[7].kind)
try testing.expect_equal(2, state.diagnostics.items.len)
try testing.expect_equal("name", strpool.get_str(&strpool.strings, state.tokens.items[1].str_id)?)
}
source/main.hon
+22 -7
View File
@@ -16,7 +16,8 @@ test import "@source/lexer"
program ::
`# literals
`x int :: 123
`x int :: 123.9
`y :: "hello"
main proc() void {
strpool.strings = strpool.init(mem.c_allocator)
@@ -56,12 +57,17 @@ main proc() void {
for parse_state.nodes.items |node, id| {
token :: scan_state.tokens.items[node.main_token]
end :: if (token.kind == .int or token.kind == .float)
lexer.scan_number(token.start, program)
else if (token.kind == .string)
lexer.scan_string(token.start, program)
else
token.start
end :: if (token.kind == .int or token.kind == .float) lbl: {
res :: lexer.scan_number(token.start, program) catch {
yield :lbl token.start
}
yield :lbl res.end
} else if (token.kind == .string) lbl: {
res :: lexer.scan_string(token.start, program) catch {
yield :lbl token.start
}
yield :lbl res.end
} else token.start
debug.print("{} (id = {}): {}\n", {
node.kind,
@@ -71,5 +77,14 @@ main proc() void {
#node.data1,
})
}
debug.print("]]\n\n", {})
debug.print("AST Render::[[\n", {})
literal :: ast.render_literal(
&parse_state.nodes.items[1],
scan_state.tokens.items,
program,
) catch "<invalid>"
debug.print("generated literal: {}\n", { literal })
debug.print("]]\n", {})
}