diff --git a/source/ast/ast.hon b/source/ast/ast.hon index 0ed083e..3ef3338 100644 --- a/source/ast/ast.hon +++ b/source/ast/ast.hon @@ -48,3 +48,22 @@ NodeKind :: enum { invalid } + +# todo: render ast nodes as source code + +render proc(node @Node, tokens []Token) void { } + +render_literal proc(node @Node, tokens []Token, program []u8) []u8 ! lexer.ScanError { + tok :: tokens[node.main_token] + match node.kind { + .literal_int, .literal_float: { + result :: try lexer.scan_number(tok.start, program) + return program[tok.start..result.end] + } + .literal_string: { + result :: try lexer.scan_string(tok.start, program) + return program[tok.start..result.end] + } + else: unreachable + } +} diff --git a/source/lexer/lexer.hon b/source/lexer/lexer.hon index 1728f55..3766925 100644 --- a/source/lexer/lexer.hon +++ b/source/lexer/lexer.hon @@ -7,9 +7,10 @@ import "@std/debug" import "@source/strpool" -ErrorCode :: enum { +ScanError :: enum { invalid_character, float_must_end_with_digit, + unterminated_string, } ErrorDetails :: struct { @@ -17,7 +18,7 @@ ErrorDetails :: struct { message []u8 } -error_msg_map std.EnumMap(ErrorCode, ErrorDetails) :: enummap.init({ +error_msg_map std.EnumMap(ScanError, ErrorDetails) :: enummap.init({ invalid_character = ErrorDetails { name = "L0", message = "invalid character", @@ -40,8 +41,8 @@ keywords std.StringMap(TokenKind) :: strmap.init([ TokenId :: distinct u32 Diagnostic :: struct { - code ErrorCode token TokenId + code ScanError } State :: struct { @@ -61,13 +62,13 @@ deinit proc(state @mut State) void { arraylist.deinit(&state.diagnostics) } -scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternError) { +scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.InternError) { tokens :: &state.tokens diagnostics :: &state.diagnostics cursor usize = 0 - while cursor < input.len { - char :: input[cursor] + while cursor < program.len { + char :: program[cursor] # whitespace if char == '\n' { @@ -81,7 +82,7 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE # comments if char == '#' { - while (cursor < input.len and input[cursor] != '\n') cursor += 1 + while (cursor < program.len and program[cursor] != '\n') cursor += 1 cursor += 1 # also skip newline continue } @@ -92,17 +93,17 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE cursor += 1 # scan whole identifier - while (cursor < input.len and ( - is_alpha(input[cursor]) or - is_digit(input[cursor]) or - input[cursor] == '_' + while (cursor < program.len and ( + is_alpha(program[cursor]) or + is_digit(program[cursor]) or + program[cursor] == '_' )) cursor += 1 - kind :: strmap.get(&keywords, input[start..cursor]) orelse .ident + kind :: strmap.get(&keywords, program[start..cursor]) orelse .ident # don't intern keywords (already O(1) lookup via token kind) str_id :: if (kind == .ident) - try strpool.intern(&strpool.strings, input[start..cursor]) + try strpool.intern(&strpool.strings, program[start..cursor]) else strpool.NO_ID @@ -117,51 +118,35 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE # numeric literals if is_digit(char) { start :: cursor - has_decimal bool = false - - # scan integer part - while (cursor < input.len and is_digit(input[cursor])) cursor += 1 - - # check for decimal point - if cursor < input.len and input[cursor] == '.' { - has_decimal = true - cursor += 1 - } - - # assert that decimals follow the decimal point - if has_decimal and (cursor >= input.len or !is_digit(input[cursor])) { + result :: scan_number(start, program) catch |err| { token :: token_id(tokens.items.len) - try arraylist.append(diagnostics, Diagnostic{ token = token, code = .float_must_end_with_digit }) try add_token(tokens, Token{ kind = .invalid, start = start }) + try arraylist.append(diagnostics, Diagnostic{ token = token, code = err }) + cursor += 1 + while (cursor < program.len and (is_digit(program[cursor]) or program[cursor] == '.')) cursor += 1 continue } - # scan decimal part - while (cursor < input.len and is_digit(input[cursor])) cursor += 1 - - kind :: if (has_decimal) .float else .int + kind :: if (result.has_decimal) .float else .int try add_token(tokens, Token{ kind = kind, start = start }) - + cursor = result.end continue } # string literals if char == '"' { start :: cursor - cursor += 1 - - while cursor < input.len and input[cursor] != '"' and input[cursor] != '\n' : cursor += 1 { - # ignore escaped characters - if (input[cursor] == '\\' and cursor + 1 < input.len) cursor += 1 - } - - if cursor < input.len and input[cursor] == '"' { - cursor += 1 - try add_token(tokens, Token{ kind = .string, start = start }) - } else { + result :: scan_string(start, program) catch |err| { + token :: token_id(tokens.items.len) try add_token(tokens, Token{ kind = .invalid, start = start }) + try arraylist.append(diagnostics, Diagnostic{ token = token, code = err }) + cursor += 1 + while cursor < program.len and program[cursor] != '\n' : cursor += 1 {} + continue } + try add_token(tokens, Token{ kind = .string, start = start }) + cursor = result.end continue } @@ -174,7 +159,7 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE # immutable assignment or single colon if char == ':' { - if cursor + 1 < input.len and input[cursor + 1] == ':' { + if cursor + 1 < program.len and program[cursor + 1] == ':' { try add_token(tokens, Token{ kind = .double_colon, start = cursor }) cursor += 2 continue @@ -217,45 +202,55 @@ scan proc(state @mut State, input []u8) void ! (mem.AllocError | strpool.InternE try add_token(tokens, Token{ kind = .eof, start = cursor }) } -#! returns the cursor position after scanning a number. -scan_number proc(start usize, input []u8) usize { - debug.assert(is_digit(input[start])) +ScanNumResult :: struct { + end usize + has_decimal bool +} - has_decimal bool = false +scan_number proc(start usize, program []u8) ScanNumResult ! ScanError { cursor usize = start + has_decimal bool = false # scan integer part - while cursor < input.len and is_digit(input[cursor]) : cursor += 1 {} + while (cursor < program.len and is_digit(program[cursor])) cursor += 1 - # check for decimal point - if cursor < input.len and input[cursor] == '.' { + # check for decimal + if cursor < program.len and program[cursor] == '.' { has_decimal = true cursor += 1 } - # assert that decimals follow the decimal point - debug.assert(!(has_decimal and (cursor >= input.len or !is_digit(input[cursor])))) - - # scan decimal part - while cursor < input.len and is_digit(input[cursor]) : cursor += 1 {} - - return cursor -} - -#! returns the cursor position after scanning a string. -scan_string proc(start usize, input []u8) usize { - debug.assert(input[start] == '"') - - cursor usize = start + 1 - while cursor < input.len and input[cursor] != '"' and input[cursor] != '\n' : cursor += 1 { - # ignore escaped characters - if (input[cursor] == '\\' and cursor + 1 < input.len) cursor += 1 + # assert non-terminating decimal + if has_decimal and (cursor >= program.len or !is_digit(program[cursor])) { + return .float_must_end_with_digit } - debug.assert(cursor < input.len and input[cursor] == '"') - cursor += 1 + # scan fractional part + while (cursor < program.len and is_digit(program[cursor])) cursor += 1 - return cursor + return ScanNumResult{ + end = cursor, + has_decimal = has_decimal, + } +} + +ScanStrResult :: struct { end usize } + +scan_string proc(start usize, program []u8) ScanStrResult ! ScanError { + cursor usize = start + 1 # skip first `"` + + # scan entire string + while cursor < program.len and program[cursor] != '"' and program[cursor] != '\n' : cursor += 1 { + # ignore escaped characters + if (program[cursor] == '\\' and cursor + 1 < program.len) cursor += 1 + } + + # assert string terminal + if (cursor >= program.len or program[cursor] != '"') return .unterminated_string + + return ScanStrResult{ + end = cursor + 1, # skip last `"` + } } hide is_whitespace proc(char u8) bool { diff --git a/source/lexer/lexer.test.hon b/source/lexer/lexer.test.hon index 4a0181b..a3fc9f9 100644 --- a/source/lexer/lexer.test.hon +++ b/source/lexer/lexer.test.hon @@ -8,14 +8,17 @@ handles_keywords_identifiers_and_error_progress test { state State = init(mem.c_allocator) defer deinit(&state) - try scan(&state, "if name # comment\n@1.") + try scan(&state, "if name # comment\n@1. 2 3.5 \"hi\"") - try testing.expect_equal(5, state.tokens.items.len) + try testing.expect_equal(8, state.tokens.items.len) try testing.expect_equal(TokenKind.if, state.tokens.items[0].kind) try testing.expect_equal(TokenKind.ident, state.tokens.items[1].kind) try testing.expect_equal(TokenKind.invalid, state.tokens.items[2].kind) try testing.expect_equal(TokenKind.invalid, state.tokens.items[3].kind) - try testing.expect_equal(TokenKind.eof, state.tokens.items[4].kind) + try testing.expect_equal(TokenKind.int, state.tokens.items[4].kind) + try testing.expect_equal(TokenKind.float, state.tokens.items[5].kind) + try testing.expect_equal(TokenKind.string, state.tokens.items[6].kind) + try testing.expect_equal(TokenKind.eof, state.tokens.items[7].kind) try testing.expect_equal(2, state.diagnostics.items.len) try testing.expect_equal("name", strpool.get_str(&strpool.strings, state.tokens.items[1].str_id)?) } diff --git a/source/main.hon b/source/main.hon index 32c1c35..63bafb8 100644 --- a/source/main.hon +++ b/source/main.hon @@ -16,7 +16,8 @@ test import "@source/lexer" program :: `# literals - `x int :: 123 + `x int :: 123.9 + `y :: "hello" main proc() void { strpool.strings = strpool.init(mem.c_allocator) @@ -56,12 +57,17 @@ main proc() void { for parse_state.nodes.items |node, id| { token :: scan_state.tokens.items[node.main_token] - end :: if (token.kind == .int or token.kind == .float) - lexer.scan_number(token.start, program) - else if (token.kind == .string) - lexer.scan_string(token.start, program) - else - token.start + end :: if (token.kind == .int or token.kind == .float) lbl: { + res :: lexer.scan_number(token.start, program) catch { + yield :lbl token.start + } + yield :lbl res.end + } else if (token.kind == .string) lbl: { + res :: lexer.scan_string(token.start, program) catch { + yield :lbl token.start + } + yield :lbl res.end + } else token.start debug.print("{} (id = {}): {}\n", { node.kind, @@ -71,5 +77,14 @@ main proc() void { #node.data1, }) } + debug.print("]]\n\n", {}) + + debug.print("AST Render::[[\n", {}) + literal :: ast.render_literal( + &parse_state.nodes.items[1], + scan_state.tokens.items, + program, + ) catch "" + debug.print("generated literal: {}\n", { literal }) debug.print("]]\n", {}) }