hl-valdemar/brolang
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Wiki Activity

better escape code support

Browse Source
This commit is contained in:
Valdemar Høgsvig Lorenzen
2026-08-11 21:16:25 +02:00
parent c2343b54bb
commit 378eb637c6
5 changed files with 63 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
LANGUAGE.md
+1 -1
View File
@@ -37,7 +37,7 @@ roadmap and milestone history.
- pointer-to-array `.len`, indexing, slicing, `.ptr` on slices and pointers-to-arrays, implicit address-taking for array-variable slices, and pointer/slice sentinel weakening
- `ptrcast!(T, ptr)` as a first-pass pointer-child retype that preserves optionality, pointer kind, mutability, and sentinel shape
- unsafe `constcast!(value)` for restoring mutability to pointers, optional pointers, and slices without changing their child type or shape
- UTF-8 string literals as immutable pointers to static zero-terminated byte arrays, plus raw backtick multiline strings
- UTF-8 string literals as immutable pointers to static zero-terminated byte arrays, supporting `\\`, `\"`, `\n`, `\r`, `\t`, `\0`, and `\xNN` escapes, plus raw backtick multiline strings
- narrow immutable zero-terminated byte pointer/slice conversion to `*c_char` / `?*c_char` without general `u8`/`c_char` interchange
- optionals with `null`, `orelse`, postfix `?`, conditional `if`/`while` unwraps, guarded unwraps, and left-to-right short-circuiting multi-unwraps
- nominal distinct types with explicit scalar backing conversion during construction and explicit scalar backing extraction, native enums with optional explicit integer backing and explicit backing-to-scalar casts, contextual enum literals, and imported C enums as target-backed integer aliases
compiler/lexer/lexer.odin
+17 -5
View File
@@ -8,6 +8,10 @@ is_identifier_start :: proc(value: byte) -> bool {
return value == '_' || value >= 'a' && value <= 'z' || value >= 'A' && value <= 'Z'
}
is_hex_digit :: proc(value: byte) -> bool {
return value >= '0' && value <= '9' || value >= 'a' && value <= 'f' || value >= 'A' && value <= 'F'
}
is_identifier_continue :: proc(value: byte) -> bool {
return is_identifier_start(value) || value >= '0' && value <= '9'
}
@@ -319,13 +323,21 @@ lex :: proc(
for cursor < len(bytes) && bytes[cursor] != '"' && bytes[cursor] != '\n' {
if bytes[cursor] == '\\' {
cursor += 1
if cursor >= len(bytes) ||
(bytes[cursor] != '\\' && bytes[cursor] != '"' && bytes[cursor] != 'n' &&
bytes[cursor] != 'r' && bytes[cursor] != 't' && bytes[cursor] != '0') {
valid_escape := cursor < len(bytes) &&
(bytes[cursor] == '\\' || bytes[cursor] == '"' || bytes[cursor] == 'n' ||
bytes[cursor] == 'r' || bytes[cursor] == 't' || bytes[cursor] == '0')
if cursor < len(bytes) && bytes[cursor] == 'x' {
valid_escape = cursor+2 < len(bytes) &&
is_hex_digit(bytes[cursor+1]) && is_hex_digit(bytes[cursor+2])
if valid_escape {
cursor += 2
}
}
if !valid_escape {
source.add(
diagnostics,
source.Span{file=source_file.id, start=source.Offset(max(cursor-1, start)), end=source.Offset(min(cursor+1, len(bytes)))},
"strings only support '\\\\', '\\\"', '\\n', '\\r', '\\t', and '\\0' escapes",
source.Span{file=source_file.id, start=source.Offset(max(cursor-1, start)), end=source.Offset(min(cursor+3, len(bytes)))},
"strings only support '\\\\', '\\\"', '\\n', '\\r', '\\t', '\\0', and '\\xNN' escapes",
)
valid = false
}
compiler/parser/parser.odin
+13
View File
@@ -219,6 +219,16 @@ decode_character :: proc(parser: ^Parser, tok: token.Token) -> (u64, bool) {
return u64(value), width == len(contents)
}
decode_hex_digit :: proc(value: byte) -> byte {
if value >= '0' && value <= '9' {
return value - '0'
}
if value >= 'a' && value <= 'f' {
return value - 'a' + 10
}
return value - 'A' + 10
}
parse_type_constant :: proc(parser: ^Parser) -> (u64, bool) {
negative := false
if _, ok := allow(parser, .Minus); ok {
@@ -3243,6 +3253,9 @@ decode_import_path :: proc(parser: ^Parser, tok: token.Token) -> string {
case 'r': value = '\r'
case 't': value = '\t'
case '0': value = 0
case 'x':
value = decode_hex_digit(text[index+1])*16 + decode_hex_digit(text[index+2])
index += 2
case:
}
}
compiler_tests.odin
+30 -7
View File
@@ -1257,6 +1257,29 @@ main func() void {}
testing.expect_value(t, module.imports[2].path, "dir\"name\\tail")
}
@(test)
honey_hex_byte_string_escapes_decode_to_bytes :: proc(t: ^testing.T) {
text := `value :: "\x1b[31m\x00\xFf"
main func() void {}
`
source_file := source.Source{path="test.hon", text=text}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
module := parser.parse(&stream, &source_file, &diagnostics)
defer ast.destroy_module(&module)
testing.expect_value(t, len(diagnostics.items), 0)
testing.expect_value(t, len(module.strings), 1)
testing.expect_value(t, len(module.strings[0]), 7)
testing.expect_value(t, module.strings[0][0], byte(0x1b))
testing.expect_value(t, module.strings[0][5], byte(0))
testing.expect_value(t, module.strings[0][6], byte(0xff))
}
@(test)
parser_accepts_chained_field_access :: proc(t: ^testing.T) {
text := `main func() void {
@@ -1278,7 +1301,7 @@ parser_accepts_chained_field_access :: proc(t: ^testing.T) {
@(test)
lexer_diagnoses_invalid_import_strings :: proc(t: ^testing.T) {
text := "import \"bad\\q\"\nimport \"unterminated\n"
text := "import \"bad\\q\"\nimport \"bad\\xg0\"\nimport \"bad\\x1\"\nimport \"unterminated\n"
source_file := source.Source{path="test.bro", text=text}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
@@ -1287,7 +1310,7 @@ lexer_diagnoses_invalid_import_strings :: proc(t: ^testing.T) {
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
testing.expect_value(t, len(diagnostics.items), 2)
testing.expect_value(t, len(diagnostics.items), 4)
}
@(test)
@@ -15568,12 +15591,12 @@ dependency_passes test {
defer delete(stderr)
output := string(stderr)
testing.expect_value(t, state.exit_code, 1)
testing.expect(t, strings.contains(output, "root.root_passes...[ok]"))
testing.expect(t, strings.contains(output, "root.root_fails...[failed]"))
testing.expect(t, strings.contains(output, "root.root_passes...[\x1b[92mok\x1b[0m]"))
testing.expect(t, strings.contains(output, "root.root_fails...[\x1b[91mfailure\x1b[0m]"))
testing.expect(t, strings.contains(output, "expected 42, found 41"))
testing.expect(t, strings.contains(output, "root.root_continues...[ok]"))
testing.expect(t, strings.contains(output, "root.root_errors...[failed]"))
testing.expect(t, strings.contains(output, "dependency.dependency_passes...[ok]"))
testing.expect(t, strings.contains(output, "root.root_continues...[\x1b[92mok\x1b[0m]"))
testing.expect(t, strings.contains(output, "root.root_errors...[\x1b[91mfailure\x1b[0m]"))
testing.expect(t, strings.contains(output, "dependency.dependency_passes...[\x1b[92mok\x1b[0m]"))
testing.expect(t, strings.contains(output, root_path))
testing.expect(t, strings.contains(output, "3 passed, 2 failed"))
}
std/testing/testing.hon
+2 -2
View File
@@ -73,10 +73,10 @@ expect_type func($Expected, $Actual type, _ Actual, location SourceLocation) voi
run func(name []u8, callback *func() void ! Error) bool {
callback() catch |_| {
debug.print("{s}...[failed]\n", {name,})
debug.print("{s}...[\x1b[91mfailure\x1b[0m]\n", {name,})
return false
}
debug.print("{s}...[ok]\n", {name,})
debug.print("{s}...[\x1b[92mok\x1b[0m]\n", {name,})
return true
}