From 378eb637c6bd6e05a939b026cd8435f656676616 Mon Sep 17 00:00:00 2001 From: hl-valdemar Date: Tue, 11 Aug 2026 21:16:25 +0200 Subject: [PATCH] better escape code support --- LANGUAGE.md | 2 +- compiler/lexer/lexer.odin | 22 +++++++++++++++++----- compiler/parser/parser.odin | 13 +++++++++++++ compiler_tests.odin | 37 ++++++++++++++++++++++++++++++------- std/testing/testing.hon | 4 ++-- 5 files changed, 63 insertions(+), 15 deletions(-) diff --git a/LANGUAGE.md b/LANGUAGE.md index 991bdfc..b1697c0 100644 --- a/LANGUAGE.md +++ b/LANGUAGE.md @@ -37,7 +37,7 @@ roadmap and milestone history. - pointer-to-array `.len`, indexing, slicing, `.ptr` on slices and pointers-to-arrays, implicit address-taking for array-variable slices, and pointer/slice sentinel weakening - `ptrcast!(T, ptr)` as a first-pass pointer-child retype that preserves optionality, pointer kind, mutability, and sentinel shape - unsafe `constcast!(value)` for restoring mutability to pointers, optional pointers, and slices without changing their child type or shape -- UTF-8 string literals as immutable pointers to static zero-terminated byte arrays, plus raw backtick multiline strings +- UTF-8 string literals as immutable pointers to static zero-terminated byte arrays, supporting `\\`, `\"`, `\n`, `\r`, `\t`, `\0`, and `\xNN` escapes, plus raw backtick multiline strings - narrow immutable zero-terminated byte pointer/slice conversion to `*c_char` / `?*c_char` without general `u8`/`c_char` interchange - optionals with `null`, `orelse`, postfix `?`, conditional `if`/`while` unwraps, guarded unwraps, and left-to-right short-circuiting multi-unwraps - nominal distinct types with explicit scalar backing conversion during construction and explicit scalar backing extraction, native enums with optional explicit integer backing and explicit backing-to-scalar casts, contextual enum literals, and imported C enums as target-backed integer aliases diff --git a/compiler/lexer/lexer.odin b/compiler/lexer/lexer.odin index 135a8ce..463c4c0 100644 --- a/compiler/lexer/lexer.odin +++ b/compiler/lexer/lexer.odin @@ -8,6 +8,10 @@ is_identifier_start :: proc(value: byte) -> bool { return value == '_' || value >= 'a' && value <= 'z' || value >= 'A' && value <= 'Z' } +is_hex_digit :: proc(value: byte) -> bool { + return value >= '0' && value <= '9' || value >= 'a' && value <= 'f' || value >= 'A' && value <= 'F' +} + is_identifier_continue :: proc(value: byte) -> bool { return is_identifier_start(value) || value >= '0' && value <= '9' } @@ -319,13 +323,21 @@ lex :: proc( for cursor < len(bytes) && bytes[cursor] != '"' && bytes[cursor] != '\n' { if bytes[cursor] == '\\' { cursor += 1 - if cursor >= len(bytes) || - (bytes[cursor] != '\\' && bytes[cursor] != '"' && bytes[cursor] != 'n' && - bytes[cursor] != 'r' && bytes[cursor] != 't' && bytes[cursor] != '0') { + valid_escape := cursor < len(bytes) && + (bytes[cursor] == '\\' || bytes[cursor] == '"' || bytes[cursor] == 'n' || + bytes[cursor] == 'r' || bytes[cursor] == 't' || bytes[cursor] == '0') + if cursor < len(bytes) && bytes[cursor] == 'x' { + valid_escape = cursor+2 < len(bytes) && + is_hex_digit(bytes[cursor+1]) && is_hex_digit(bytes[cursor+2]) + if valid_escape { + cursor += 2 + } + } + if !valid_escape { source.add( diagnostics, - source.Span{file=source_file.id, start=source.Offset(max(cursor-1, start)), end=source.Offset(min(cursor+1, len(bytes)))}, - "strings only support '\\\\', '\\\"', '\\n', '\\r', '\\t', and '\\0' escapes", + source.Span{file=source_file.id, start=source.Offset(max(cursor-1, start)), end=source.Offset(min(cursor+3, len(bytes)))}, + "strings only support '\\\\', '\\\"', '\\n', '\\r', '\\t', '\\0', and '\\xNN' escapes", ) valid = false } diff --git a/compiler/parser/parser.odin b/compiler/parser/parser.odin index 2b2c4b6..d9279e7 100644 --- a/compiler/parser/parser.odin +++ b/compiler/parser/parser.odin @@ -219,6 +219,16 @@ decode_character :: proc(parser: ^Parser, tok: token.Token) -> (u64, bool) { return u64(value), width == len(contents) } +decode_hex_digit :: proc(value: byte) -> byte { + if value >= '0' && value <= '9' { + return value - '0' + } + if value >= 'a' && value <= 'f' { + return value - 'a' + 10 + } + return value - 'A' + 10 +} + parse_type_constant :: proc(parser: ^Parser) -> (u64, bool) { negative := false if _, ok := allow(parser, .Minus); ok { @@ -3243,6 +3253,9 @@ decode_import_path :: proc(parser: ^Parser, tok: token.Token) -> string { case 'r': value = '\r' case 't': value = '\t' case '0': value = 0 + case 'x': + value = decode_hex_digit(text[index+1])*16 + decode_hex_digit(text[index+2]) + index += 2 case: } } diff --git a/compiler_tests.odin b/compiler_tests.odin index bc27921..08d7ba3 100644 --- a/compiler_tests.odin +++ b/compiler_tests.odin @@ -1257,6 +1257,29 @@ main func() void {} testing.expect_value(t, module.imports[2].path, "dir\"name\\tail") } +@(test) +honey_hex_byte_string_escapes_decode_to_bytes :: proc(t: ^testing.T) { + text := `value :: "\x1b[31m\x00\xFf" +main func() void {} +` + source_file := source.Source{path="test.hon", text=text} + diagnostics := source.init_diagnostics(&source_file) + defer source.destroy_diagnostics(&diagnostics) + symbols := symbol.init_table() + defer symbol.destroy_table(&symbols) + stream := lexer.lex(&source_file, &diagnostics, &symbols) + defer delete(stream.items) + module := parser.parse(&stream, &source_file, &diagnostics) + defer ast.destroy_module(&module) + + testing.expect_value(t, len(diagnostics.items), 0) + testing.expect_value(t, len(module.strings), 1) + testing.expect_value(t, len(module.strings[0]), 7) + testing.expect_value(t, module.strings[0][0], byte(0x1b)) + testing.expect_value(t, module.strings[0][5], byte(0)) + testing.expect_value(t, module.strings[0][6], byte(0xff)) +} + @(test) parser_accepts_chained_field_access :: proc(t: ^testing.T) { text := `main func() void { @@ -1278,7 +1301,7 @@ parser_accepts_chained_field_access :: proc(t: ^testing.T) { @(test) lexer_diagnoses_invalid_import_strings :: proc(t: ^testing.T) { - text := "import \"bad\\q\"\nimport \"unterminated\n" + text := "import \"bad\\q\"\nimport \"bad\\xg0\"\nimport \"bad\\x1\"\nimport \"unterminated\n" source_file := source.Source{path="test.bro", text=text} diagnostics := source.init_diagnostics(&source_file) defer source.destroy_diagnostics(&diagnostics) @@ -1287,7 +1310,7 @@ lexer_diagnoses_invalid_import_strings :: proc(t: ^testing.T) { stream := lexer.lex(&source_file, &diagnostics, &symbols) defer delete(stream.items) - testing.expect_value(t, len(diagnostics.items), 2) + testing.expect_value(t, len(diagnostics.items), 4) } @(test) @@ -15568,12 +15591,12 @@ dependency_passes test { defer delete(stderr) output := string(stderr) testing.expect_value(t, state.exit_code, 1) - testing.expect(t, strings.contains(output, "root.root_passes...[ok]")) - testing.expect(t, strings.contains(output, "root.root_fails...[failed]")) + testing.expect(t, strings.contains(output, "root.root_passes...[\x1b[92mok\x1b[0m]")) + testing.expect(t, strings.contains(output, "root.root_fails...[\x1b[91mfailure\x1b[0m]")) testing.expect(t, strings.contains(output, "expected 42, found 41")) - testing.expect(t, strings.contains(output, "root.root_continues...[ok]")) - testing.expect(t, strings.contains(output, "root.root_errors...[failed]")) - testing.expect(t, strings.contains(output, "dependency.dependency_passes...[ok]")) + testing.expect(t, strings.contains(output, "root.root_continues...[\x1b[92mok\x1b[0m]")) + testing.expect(t, strings.contains(output, "root.root_errors...[\x1b[91mfailure\x1b[0m]")) + testing.expect(t, strings.contains(output, "dependency.dependency_passes...[\x1b[92mok\x1b[0m]")) testing.expect(t, strings.contains(output, root_path)) testing.expect(t, strings.contains(output, "3 passed, 2 failed")) } diff --git a/std/testing/testing.hon b/std/testing/testing.hon index 76188ca..bb01a12 100644 --- a/std/testing/testing.hon +++ b/std/testing/testing.hon @@ -73,10 +73,10 @@ expect_type func($Expected, $Actual type, _ Actual, location SourceLocation) voi run func(name []u8, callback *func() void ! Error) bool { callback() catch |_| { - debug.print("{s}...[failed]\n", {name,}) + debug.print("{s}...[\x1b[91mfailure\x1b[0m]\n", {name,}) return false } - debug.print("{s}...[ok]\n", {name,}) + debug.print("{s}...[\x1b[92mok\x1b[0m]\n", {name,}) return true }