feat: support existing io.print formatting

source/lexer/scan.hon

@@ -16,6 +16,7 @@ ScanError :: struct {
 ErrorCode :: enum {
     float_must_end_with_digit,
     unterminated_string,
+    unterminated_character,
 }
 SourceError :: enum { source_too_large }
 
@@ -153,9 +154,9 @@ scan proc(
             continue
         }
 
-        if char == '"' {
+        if (char == '"' or char == '\'') {
             start :: cursor
-            result :: scan_string(start, program) catch |err| {
+            result :: scan_quoted(start, program, char) catch |err| {
                 id :: try diagnostics.add(
                     diagnostic_store,
                     diagnostic_code(err.code),
@@ -169,9 +170,10 @@ scan proc(
                 cursor = err.end
                 continue
             }
+            kind :: if (char == '\'') TokenKind.character else .string
             try add_token(state, Token{
                 span = make_span(source, start, result.end),
-                kind = .string,
+                kind = kind,
             })
             cursor = result.end
             continue
@@ -440,19 +442,20 @@ scan_number proc(start usize, program []u8, allow_decimal bool) ScanNumResult !
 ScanStrResult :: struct { end usize }
 
 @hide:file
-scan_string proc(start usize, program []u8) ScanStrResult ! ScanError {
-    cursor := start + 1 # skip first `"`
-    while cursor < program.len and program[cursor] != '"' and program[cursor] != '\n' : cursor += 1 {
-        # escaped quotes do not terminate the string.
-        if (program[cursor] == '\\' and cursor + 1 < program.len) cursor += 1
+scan_quoted proc(start usize, program []u8, quote u8) ScanStrResult ! ScanError {
+    cursor := start + 1
+    while (cursor < program.len and program[cursor] != quote and program[cursor] != '\n') : cursor += 1 {
+        # an escaped delimiter is content, but a newline still ends an invalid literal.
+        if (program[cursor] == '\\' and cursor + 1 < program.len and program[cursor + 1] != '\n') cursor += 1
     }
-    if (cursor >= program.len or program[cursor] != '"') {
+    if (cursor >= program.len or program[cursor] != quote) {
+        code :: if (quote == '\'') ErrorCode.unterminated_character else .unterminated_string
         return ScanError{
-            code = .unterminated_string,
+            code = code,
             end = cursor,
         }
     }
-    return ScanStrResult{ end = cursor + 1 } # skip last `"`
+    return ScanStrResult{ end = cursor + 1 }
 }
 
 @hide:file
@@ -480,6 +483,7 @@ is_digit proc(char u8) bool {
 diagnostic_codes :: $enummap.init(ErrorCode, diagnostics.Code, {
     float_must_end_with_digit = .float_must_end_with_digit,
     unterminated_string = .unterminated_string,
+    unterminated_character = .unterminated_character,
 })
 
 @hide:file

source/lexer/scan.test.hon

@@ -3,6 +3,32 @@ import "@std/testing"
 import "@source/diagnostics"
 import "@source/strpool"
 
+scans_character_literals_and_preserves_unterminated_line_provenance test {
+    pool := strpool.init(mem.c_allocator)
+    defer strpool.deinit(&pool)
+    diagnostic_store := diagnostics.init(mem.c_allocator)
+    defer diagnostics.deinit(&diagnostic_store)
+    state := init(mem.c_allocator)
+    defer deinit(&state)
+    source :: "'a' '\\n' '\\'' '\\\\' 'é' '\\\nnext :: 1"
+    state.source_id = diagnostics.SourceId(7)
+    try scan(&state, source, &pool, &diagnostic_store)
+    expected [11]TokenKind :: [
+        .character, .character, .character, .character, .character,
+        .invalid, .newline, .ident, .double_colon, .int, .eof,
+    ]
+    try testing.expect_equal(expected.len, state.tokens.items.len)
+    for (expected) |kind, index| try testing.expect_equal(kind, state.tokens.items[index].kind)
+    try testing.expect_equal(1, diagnostics.count(&diagnostic_store))
+    invalid :: state.tokens.items[5]
+    diagnostic :: diagnostics.get(&diagnostic_store, invalid.data.diagnostic)?
+    try testing.expect_equal(diagnostics.Code.unterminated_character, diagnostic.code)
+    try testing.expect_equal(diagnostics.SourceId(7), diagnostic.span.source)
+    try testing.expect_equal(invalid.span.start, diagnostic.span.start)
+    try testing.expect_equal(invalid.span.end, diagnostic.span.end)
+    try testing.expect_equal(invalid.span.end, state.tokens.items[6].span.start)
+}
+
 reports_invalid_input_and_keeps_scanning test {
     pool := strpool.init(mem.c_allocator)
     defer strpool.deinit(&pool)

source/lexer/token.hon

@@ -42,7 +42,7 @@ TokenKind :: enum {
     undefined
 
     # literals
-    ident, string
+    ident, string, character
     int, float
 
     # comparison
@@ -92,7 +92,7 @@ TokenKind :: enum {
 
 render_token proc(token Token, program []u8) void {
     match token.kind {
-        .ident, .string, .int, .float:
+        .ident, .string, .character, .int, .float:
             debug.print("{}({})\n", {token.kind, program[token.span.start..token.span.end]})
         else:
             debug.print("{}\n", {token.kind})