import "@std/mem" import "@std/testing" import "@source/diagnostics" import "@source/strpool" reports_invalid_input_and_keeps_scanning test { strpool.strings = strpool.init(mem.c_allocator) defer strpool.deinit(&strpool.strings) diagnostic_store := diagnostics.init(mem.c_allocator) defer diagnostics.deinit(&diagnostic_store) state := init(mem.c_allocator) defer deinit(&state) try scan(&state, "if name # comment\n$1. 2 3.5 \"hi\"", &diagnostic_store) try testing.expect_equal(8, state.tokens.items.len) try testing.expect_equal(TokenKind.if, state.tokens.items[0].kind) try testing.expect_equal(TokenKind.ident, state.tokens.items[1].kind) try testing.expect_equal(TokenKind.invalid, state.tokens.items[2].kind) try testing.expect_equal(TokenKind.invalid, state.tokens.items[3].kind) try testing.expect_equal(TokenKind.int, state.tokens.items[4].kind) try testing.expect_equal(TokenKind.float, state.tokens.items[5].kind) try testing.expect_equal(TokenKind.string, state.tokens.items[6].kind) try testing.expect_equal(TokenKind.eof, state.tokens.items[7].kind) try testing.expect_equal(2, diagnostics.count(&diagnostic_store)) try testing.expect_equal( "name", strpool.get_str(&strpool.strings, state.tokens.items[1].data.str_id)?, ) try testing.expect_equal(u32(3), state.tokens.items[1].span.start) try testing.expect_equal(u32(7), state.tokens.items[1].span.end) } reserves_boolean_keywords_and_scans_comparisons test { strpool.strings = strpool.init(mem.c_allocator) defer strpool.deinit(&strpool.strings) diagnostic_store := diagnostics.init(mem.c_allocator) defer diagnostics.deinit(&diagnostic_store) state := init(mem.c_allocator) defer deinit(&state) source :: "true false bool trueish == != < <= > >=" try scan(&state, source, &diagnostic_store) expected [11]TokenKind :: [ TokenKind.true, .false, .ident, .ident, .equal_equal, .not_equal, .less, .less_equal, .greater, .greater_equal, .eof, ] try testing.expect_equal(expected.len, state.tokens.items.len) for (expected) |kind, index| try testing.expect_equal(kind, state.tokens.items[index].kind) try testing.expect_equal(u32(0), state.tokens.items[0].span.start) try testing.expect_equal(u32(4), state.tokens.items[0].span.end) try testing.expect_equal(u32(5), state.tokens.items[1].span.start) try testing.expect_equal(u32(10), state.tokens.items[1].span.end) try testing.expect_equal("bool", strpool.get_str(&strpool.strings, state.tokens.items[2].data.str_id)?) try testing.expect_equal(0, diagnostics.count(&diagnostic_store)) } scans_logical_operators_without_splitting_identifiers test { strpool.strings = strpool.init(mem.c_allocator) defer strpool.deinit(&strpool.strings) diagnostic_store := diagnostics.init(mem.c_allocator) defer diagnostics.deinit(&diagnostic_store) state := init(mem.c_allocator) defer deinit(&state) source :: "and or android orbit and_1 or2 ! != !! !true" try scan(&state, source, &diagnostic_store) expected [13]TokenKind :: [ TokenKind.and, .or, .ident, .ident, .ident, .ident, .bang, .not_equal, .bang, .bang, .bang, .true, .eof, ] try testing.expect_equal(expected.len, state.tokens.items.len) for (expected) |kind, index| try testing.expect_equal(kind, state.tokens.items[index].kind) try testing.expect_equal("android", strpool.get_str(&strpool.strings, state.tokens.items[2].data.str_id)?) try testing.expect_equal("orbit", strpool.get_str(&strpool.strings, state.tokens.items[3].data.str_id)?) try testing.expect_equal("and_1", strpool.get_str(&strpool.strings, state.tokens.items[4].data.str_id)?) try testing.expect_equal("or2", strpool.get_str(&strpool.strings, state.tokens.items[5].data.str_id)?) try testing.expect_equal(u32(31), state.tokens.items[6].span.start) try testing.expect_equal(u32(32), state.tokens.items[6].span.end) try testing.expect_equal(u32(33), state.tokens.items[7].span.start) try testing.expect_equal(u32(35), state.tokens.items[7].span.end) try testing.expect_equal(0, diagnostics.count(&diagnostic_store)) } scans_loop_controls_with_exact_keyword_boundaries test { strpool.strings = strpool.init(mem.c_allocator) defer strpool.deinit(&strpool.strings) diagnostic_store := diagnostics.init(mem.c_allocator) defer diagnostics.deinit(&diagnostic_store) state := init(mem.c_allocator) defer deinit(&state) source :: "break continue breakfast continued break_1 continue2" try scan(&state, source, &diagnostic_store) expected [7]TokenKind :: [.break, .continue, .ident, .ident, .ident, .ident, .eof] try testing.expect_equal(expected.len, state.tokens.items.len) for (expected) |kind, index| try testing.expect_equal(kind, state.tokens.items[index].kind) try testing.expect_equal(u32(0), state.tokens.items[0].span.start) try testing.expect_equal(u32(5), state.tokens.items[0].span.end) try testing.expect_equal(u32(6), state.tokens.items[1].span.start) try testing.expect_equal(u32(14), state.tokens.items[1].span.end) try testing.expect_equal("breakfast", strpool.get_str(&strpool.strings, state.tokens.items[2].data.str_id)?) try testing.expect_equal("continued", strpool.get_str(&strpool.strings, state.tokens.items[3].data.str_id)?) try testing.expect_equal(0, diagnostics.count(&diagnostic_store)) } scans_ranges_without_consuming_float_dots test { strpool.strings = strpool.init(mem.c_allocator) defer strpool.deinit(&strpool.strings) diagnostic_store := diagnostics.init(mem.c_allocator) defer diagnostics.deinit(&diagnostic_store) state := init(mem.c_allocator) defer deinit(&state) try scan(&state, "0..10 0..=10 1.5..2.5 |@item, _| ... 1.", &diagnostic_store) expected [19]TokenKind :: [ .int, .range, .int, .int, .range_inclusive, .int, .float, .range, .float, .pipe, .at, .ident, .comma, .ident, .pipe, .range, .invalid, .invalid, .eof, ] try testing.expect_equal(expected.len, state.tokens.items.len) for (expected) |kind, index| try testing.expect_equal(kind, state.tokens.items[index].kind) try testing.expect_equal(u32(1), state.tokens.items[1].span.start) try testing.expect_equal(u32(3), state.tokens.items[1].span.end) try testing.expect_equal(u32(7), state.tokens.items[4].span.start) try testing.expect_equal(u32(10), state.tokens.items[4].span.end) try testing.expect_equal(2, diagnostics.count(&diagnostic_store)) }