feat: support compound updates and bitwise operators

source/lexer/scan.hon

@@ -41,6 +41,7 @@ keywords strmap.StaticStringMap(TokenKind) :: strmap.init([
     { "false", .false },
     { "and", .and },
     { "or", .or },
+    { "xor", .xor },
     { "mut", .mut },
     { "struct", .struct },
     { "c_struct", .c_struct },
@@ -107,12 +108,16 @@ scan proc(
         if is_alpha(char) or char == '_' {
             start :: cursor
             result :: try scan_ident(start, program, pool)
+            compound :: result.kind == .xor and result.end < program.len and program[result.end] == '='
+            extra :: if (compound) usize(1) else 0
+            end :: result.end + extra
+            kind :: if (compound) TokenKind.xor_equal else result.kind
             try add_token(state, Token{
-                span = make_span(source, start, result.end),
-                kind = result.kind,
+                span = make_span(source, start, end),
+                kind = kind,
                 data = TokenData{ str_id = result.str_id },
             })
-            cursor = result.end
+            cursor = end
             continue
         }
 
@@ -281,12 +286,37 @@ scan proc(
             continue
         }
 
+        if (
+            (char == '<' or char == '>') and cursor + 1 < program.len
+            and program[cursor + 1] == char
+        ) {
+            saturating :: char == '<' and cursor + 2 < program.len and program[cursor + 2] == '|'
+            width :: if (saturating) usize(3) else 2
+            compound :: cursor + width < program.len and program[cursor + width] == '='
+            kind TokenKind := .shift_right
+            if (saturating) {
+                kind = if (compound) .shift_left_saturating_equal else .shift_left_saturating
+            } else if (char == '<') {
+                kind = if (compound) .shift_left_equal else .shift_left
+            } else if (compound) kind = .shift_right_equal
+            extra :: if (compound) usize(1) else 0
+            end :: cursor + width + extra
+            try add_token(state, Token{
+                span = make_span(source, cursor, end),
+                kind = kind,
+            })
+            cursor = end
+            continue
+        }
+
         if (cursor + 1 < program.len and program[cursor + 1] == '=') {
             comparison ?TokenKind :: match char {
                 '=': .equal_equal
                 '!': .not_equal
                 '<': .less_equal
                 '>': .greater_equal
+                '&': .ampersand_equal
+                '|': .pipe_equal
                 else: null
             }
             if (comparison) |value| {
@@ -302,6 +332,7 @@ scan proc(
         kind ?TokenKind :: match char {
             '=': .equal
             '!': .bang
+            '~': .tilde
             '<': .less
             '>': .greater
             ',': .comma

source/lexer/scan.test.hon

@@ -11,7 +11,7 @@ reports_invalid_input_and_keeps_scanning test {
     state := init(mem.c_allocator)
     defer deinit(&state)
 
-    try scan(&state, "if name # comment\n~1. 2 3.5 \"hi\"", &pool, &diagnostic_store)
+    try scan(&state, "if name # comment\n%1. 2 3.5 \"hi\"", &pool, &diagnostic_store)
 
     try testing.expect_equal(9, state.tokens.items.len)
     try testing.expect_equal(TokenKind.if, state.tokens.items[0].kind)
@@ -266,7 +266,7 @@ scans_import_keywords_and_preserves_source_in_tokens_and_diagnostics test {
     state := init(mem.c_allocator)
     defer deinit(&state)
     state.source_id = diagnostics.SourceId(7)
-    try scan(&state, "import test ~\n1.\n\"unfinished", &pool, &diagnostic_store)
+    try scan(&state, "import test %\n1.\n\"unfinished", &pool, &diagnostic_store)
     try testing.expect_equal(TokenKind.import, state.tokens.items[0].kind)
     try testing.expect_equal(TokenKind.test, state.tokens.items[1].kind)
     for (state.tokens.items) |token| try testing.expect_equal(state.source_id, token.span.source)
@@ -290,3 +290,29 @@ reserves_yield_without_splitting_identifiers test {
     try testing.expect_equal("yield", strpool.get_str(&pool, state.tokens.items[0].data.str_id)?)
     try testing.expect_equal(0, diagnostics.count(&diagnostic_store))
 }
+
+scans_bitwise_operators_and_contiguous_compounds test {
+    pool := strpool.init(mem.c_allocator)
+    defer strpool.deinit(&pool)
+    diagnostic_store := diagnostics.init(mem.c_allocator)
+    defer diagnostics.deinit(&diagnostic_store)
+    state := init(mem.c_allocator)
+    defer deinit(&state)
+    source :: "~ & | xor << >> <<| += -= *= /= &= |= xor= <<= >>= <<|= xor_flag xor = << |= <<| ="
+    try scan(&state, source, &pool, &diagnostic_store)
+    expected [25]TokenKind :: [
+        .tilde, .ampersand, .pipe, .xor, .shift_left, .shift_right, .shift_left_saturating,
+        .plus_equal, .minus_equal, .star_equal, .slash_equal,
+        .ampersand_equal, .pipe_equal, .xor_equal,
+        .shift_left_equal, .shift_right_equal, .shift_left_saturating_equal,
+        .ident, .xor, .equal, .shift_left, .pipe_equal, .shift_left_saturating, .equal, .eof,
+    ]
+    try testing.expect_equal(expected.len, state.tokens.items.len)
+    for (expected) |kind, index| try testing.expect_equal(kind, state.tokens.items[index].kind)
+    xor_span :: state.tokens.items[13].span
+    try testing.expect_equal("xor=", source[usize(xor_span.start)..usize(xor_span.end)])
+    saturating_span :: state.tokens.items[16].span
+    try testing.expect_equal("<<|=", source[usize(saturating_span.start)..usize(saturating_span.end)])
+    try testing.expect_equal("xor_flag", strpool.get_str(&pool, state.tokens.items[17].data.str_id)?)
+    try testing.expect_equal(0, diagnostics.count(&diagnostic_store))
+}

source/lexer/token.hon

@@ -29,6 +29,7 @@ TokenKind :: enum {
     defer, errdefer
     true, false
     and, or
+    xor
     struct, c_struct
     opaque
     enum, union
@@ -56,6 +57,10 @@ TokenKind :: enum {
     plus, minus
     star, slash
 
+    # bitwise
+    tilde
+    shift_left, shift_right, shift_left_saturating
+
     range, range_inclusive, ellipsis
     at, ampersand, caret, dollar
     question, semicolon
@@ -67,6 +72,8 @@ TokenKind :: enum {
     # assignment & arithmetic
     plus_equal, minus_equal
     star_equal, slash_equal
+    ampersand_equal, pipe_equal, xor_equal
+    shift_left_equal, shift_right_equal, shift_left_saturating_equal
 
     # delimiters
     open_paren, close_paren