import "@std/mem" import "@std/arraylist" import "@std/enummap" import "@source/diagnostics" import "@source/lexer" import "@source/ast" import "@source/strpool" @hide:file SyntaxError :: enum { unexpected_token } @hide:file ParseError :: alias SyntaxError | mem.AllocError | diagnostics.CapacityError @hide:file BindingPowerScalar :: distinct u32 @hide:file BindingPower :: struct { left BindingPowerScalar right BindingPowerScalar } @hide:file State :: struct { builder @mut ast.Builder diagnostics @mut diagnostics.Store tokens []lexer.Token allocator mem.Allocator next_token usize paren_depth usize = 0 allow_record_literals bool = true allow_bit_or bool = true match_arm_body bool = false } @hide:file ParameterName :: struct { token lexer.Token start u32 comptime bool } #! borrows `tokens` and `source` while parsing; the returned Tree borrows `source` until ast.deinit. #! `allocator` supplies returned node storage and temporary lists released before return. #! the caller owns the Tree and must release it with ast.deinit. parse proc( allocator mem.Allocator, tokens []lexer.Token, source []u8, diagnostic_store @mut diagnostics.Store, ) ast.Tree ! (mem.AllocError | diagnostics.CapacityError) { builder := ast.init_builder(allocator, source) defer ast.deinit_builder(&builder) root :: try parse_source(&builder, tokens, diagnostic_store) return ast.finish(&builder, root) } #! appends one source to shared node storage; token spans must identify the builder's current source. #! infrastructure failure rolls back all nodes, extra data, spans, and diagnostics added by this parse. parse_source proc( builder @mut ast.Builder, tokens []lexer.Token, diagnostic_store @mut diagnostics.Store, ) ast.NodeId ! (mem.AllocError | diagnostics.CapacityError) { node_count :: builder.storage.nodes.items.len extra_count :: builder.storage.extra.items.len diagnostic_count :: diagnostic_store.list.items.len errdefer { builder.storage.nodes.items = builder.storage.nodes.items.ptr[..node_count] builder.storage.spans.items = builder.storage.spans.items.ptr[..node_count] builder.storage.extra.items = builder.storage.extra.items.ptr[..extra_count] diagnostic_store.list.items = diagnostic_store.list.items.ptr[..diagnostic_count] } state := State{ builder = builder, diagnostics = diagnostic_store, tokens = tokens, allocator = builder.storage.nodes.allocator, next_token = 0, } return try parse_package(&state) } @hide:file parse_package proc(state @mut State) ast.NodeId ! (mem.AllocError | diagnostics.CapacityError) { declarations arraylist.ArrayList(ast.NodeId) := arraylist.init(state.allocator) defer arraylist.deinit(&declarations) skip_newlines(state) while !next_is(state, .eof) { start :: state.next_token declaration :: parse_decl(state) catch |err| { match err { .unexpected_token: { recovered :: try recover_decl(state, start) try arraylist.append(&declarations, recovered) skip_newlines(state) continue } else: return err } } try arraylist.append(&declarations, declaration) skip_newlines(state) } return try ast.add_package(state.builder, ast.Span{ start = 0, end = u32(state.builder.source.len), source = state.builder.source_id, }, declarations.items) } @hide:file parse_decl proc(state @mut State) ast.NodeId ! ParseError { if (next_is(state, .at)) { start :: try consume(state, .at) name :: try consume(state, .ident) _ = try consume(state, .colon) qualifier :: try consume(state, .ident) skip_newlines(state) declaration :: try parse_decl(state) if ( !mem.eql(state.builder.source[usize(name.span.start)..usize(name.span.end)], "hide") or !mem.eql(state.builder.source[usize(qualifier.span.start)..usize(qualifier.span.end)], "file") ) { span :: ast.Span{ start = start.span.start, end = qualifier.span.end, source = start.span.source } diagnostic :: try diagnostics.add(state.diagnostics, .unexpected_token, span) return try ast.add_invalid_decl(state.builder, span, diagnostic) } ast.set_file_local(state.builder, declaration) return declaration } if ( next_is(state, .import) or next_is(state, .test) or (next_is(state, .ident) and next_n_is(state, 1, .double_colon) and next_n_is(state, 2, .import)) ) return try parse_import_decl(state) if (next_is(state, .ident) and (next_n_is(state, 1, .proc) or next_n_is(state, 1, .c_func))) return try parse_func_decl(state) return try parse_statement_decl(state) } @hide:file parse_import_decl proc(state @mut State) ast.NodeId ! ParseError { start :: current(state).span test_only :: next_is(state, .test) if (test_only) state.next_token += 1 namespace ?strpool.StringId := null if next_is(state, .ident) { name :: try consume(state, .ident) namespace = name.data.str_id _ = try consume(state, .double_colon) } _ = try consume(state, .import) token :: try consume(state, .string) path :: ast.Span{ start = token.span.start + 1, end = token.span.end - 1, source = token.span.source } # import paths are literal filesystem paths, not escaped string values. valid := path.start != path.end for (state.builder.source[usize(path.start)..usize(path.end)]) |byte| { if (byte < 32 or byte == 127 or byte == '\\') valid = false } span :: ast.Span{ start = start.start, end = token.span.end, source = start.source } if (!valid) { diagnostic :: try diagnostics.add(state.diagnostics, .invalid_import_path, path) return try ast.add_invalid_decl(state.builder, span, diagnostic) } return try ast.add_import_decl(state.builder, span, path, namespace, test_only) } @hide:file parse_func_decl proc(state @mut State) ast.NodeId ! ParseError { name :: try consume(state, .ident) external :: next_is(state, .c_func) _ = try consume_either(state, &[.proc, .c_func]) _ = try consume(state, .open_paren) params arraylist.ArrayList(ast.NodeId) := arraylist.init(state.allocator) defer arraylist.deinit(¶ms) variadic :: try parse_parameters(state, ¶ms) _ = try consume(state, .close_paren) return_type :: try parse_type(state) body :: if (external) try ast.add_block(state.builder, ast.builder_span(state.builder, return_type), &[]) else try parse_block(state) body_span :: ast.builder_span(state.builder, body) id :: try ast.add_function_decl( state.builder, ast.Span{ start = name.span.start, end = body_span.end, source = name.span.source, }, name.span, name.data.str_id, return_type, params.items, body, ) if (external) ast.set_function_external(state.builder, id) if (variadic) ast.set_callable_variadic(state.builder, id) return id } @hide:file parse_parameters proc( state @mut State, params @mut arraylist.ArrayList(ast.NodeId), ) bool ! ParseError { skip_newlines(state) names arraylist.ArrayList(ParameterName) := arraylist.init(state.allocator) defer arraylist.deinit(&names) while !next_is(state, .close_paren) { if (next_is(state, .ellipsis)) { state.next_token += 1 skip_newlines(state) if (!next_is(state, .close_paren)) return .unexpected_token return true } arraylist.clear(&names) while true { start :: current(state).span.start comptime :: next_is(state, .dollar) if (comptime) state.next_token += 1 token :: try consume(state, .ident) try arraylist.append(&names, ParameterName{ token = token, start = start, comptime = comptime, }) if (starts_type(current(state).kind)) break _ = try consume(state, .comma) } type_id :: try parse_type(state) type_end :: state.tokens[state.next_token - 1].span.end for (names.items) |parameter_name| { param_id :: try ast.add_param( state.builder, ast.Span{ start = parameter_name.start, end = type_end, source = parameter_name.token.span.source }, parameter_name.token.span, parameter_name.token.data.str_id, type_id, ) if (parameter_name.comptime) ast.set_param_comptime(state.builder, param_id) try arraylist.append(params, param_id) } skip_newlines(state) if next_is(state, .comma) { state.next_token += 1 skip_newlines(state) } } return false } @hide:file parse_block proc(state @mut State) ast.NodeId ! ParseError { open :: try consume(state, .open_curly) previous_depth :: state.paren_depth previous_record_literals :: state.allow_record_literals previous_bit_or :: state.allow_bit_or previous_match_arm_body :: state.match_arm_body state.paren_depth = 0 state.allow_record_literals = true state.allow_bit_or = true state.match_arm_body = false defer { state.paren_depth = previous_depth state.allow_record_literals = previous_record_literals state.allow_bit_or = previous_bit_or state.match_arm_body = previous_match_arm_body } statements arraylist.ArrayList(ast.NodeId) := arraylist.init(state.allocator) defer arraylist.deinit(&statements) skip_newlines(state) while !next_is(state, .close_curly) and !next_is(state, .eof) { start :: state.next_token statement :: parse_statement(state, false) catch |err| { match err { .unexpected_token: { recovered :: try recover_statement(state, start) try arraylist.append(&statements, recovered) skip_newlines(state) continue } else: return err } } try arraylist.append(&statements, statement) skip_newlines(state) } close :: try consume(state, .close_curly) return try ast.add_block( state.builder, ast.Span{ start = open.span.start, end = close.span.end, source = open.span.source, }, statements.items, ) } @hide:file parse_statement proc(state @mut State, allow_else bool) ast.NodeId ! ParseError { if ( next_is(state, .ident) and (next_n_is(state, 1, .ident) or next_n_is(state, 1, .struct) or next_n_is(state, 1, .c_struct) or next_n_is(state, 1, .opaque) or next_n_is(state, 1, .enum) or next_n_is(state, 1, .union) or next_n_is(state, 1, .distinct) or next_n_is(state, 1, .match) or next_n_is(state, 1, .at) or next_n_is(state, 1, .question) or (next_n_is(state, 1, .open_paren) and pointer_binding_starts(state)) or (next_n_is(state, 1, .star) and pointer_binding_starts(state)) or (next_n_is(state, 1, .open_bracket) and array_starts_type(state)) or next_n_is(state, 1, .double_colon) or next_n_is(state, 1, .colon_equal)) ) return try parse_statement_decl(state) match current(state).kind { .if: return try parse_if(state) .while: return try parse_while(state, allow_else) .for, .inline: return try parse_for(state, allow_else) .return: return try parse_return(state, allow_else) .yield: return try parse_yield(state, allow_else) .break, .continue: return try parse_loop_control(state, allow_else) .defer, .errdefer: return try parse_cleanup(state, allow_else) .open_curly: return try ast.add_block_stmt(state.builder, try parse_block(state)) .ident, .int, .float, .string, .true, .false, .minus, .bang, .tilde, .open_paren, .open_bracket, .struct, .c_struct, .opaque, .c_func, .enum, .union, .distinct, .alias, .dot, .match, .ampersand, .at, .star, .question, .null, .undefined, .try, .dollar: { expression :: try parse_expr(state) if (is_assignment_operator(current(state).kind)) return try parse_assignment(state, expression, allow_else) return try ast.add_expression_stmt(state.builder, expression) } else: return .unexpected_token } } @hide:file parse_error_capture proc(state @mut State) ?ast.NodeId ! ParseError { capture ?ast.NodeId := null if (!next_is(state, .pipe)) return capture state.next_token += 1 capture = try parse_capture_name(state) _ = try consume(state, .pipe) return capture } @hide:file parse_cleanup proc(state @mut State, allow_else bool) ast.NodeId ! ParseError { keyword :: current(state) state.next_token += 1 error_only :: keyword.kind == .errdefer capture ?ast.NodeId :: if (error_only) try parse_error_capture(state) else null body :: try parse_arm(state, allow_else) return try ast.add_cleanup_stmt(state.builder, ast.Span{ start = keyword.span.start, end = ast.builder_span(state.builder, body).end, source = keyword.span.source, }, body, capture, error_only) } @hide:file parse_return proc(state @mut State, allow_else bool) ast.NodeId ! ParseError { keyword :: try consume(state, .return) if ( next_is(state, .newline) or next_is(state, .close_curly) or next_is(state, .eof) or (state.match_arm_body and next_is(state, .comma)) or (allow_else and next_is(state, .else)) ) return try ast.add_return_stmt(state.builder, keyword.span, null) value :: try parse_expr(state) if ( !next_is(state, .newline) and !next_is(state, .close_curly) and !next_is(state, .eof) and !(allow_else and next_is(state, .else)) and !(state.match_arm_body and next_is(state, .comma)) ) return .unexpected_token # include consumed grouping punctuation, which has no separate expression node. end :: state.tokens[state.next_token - 1].span.end return try ast.add_return_stmt( state.builder, ast.Span{ start = keyword.span.start, end = end, source = keyword.span.source, }, value, ) } @hide:file parse_yield proc(state @mut State, allow_else bool) ast.NodeId ! ParseError { keyword :: try consume(state, .yield) value :: if ( next_is(state, .newline) or next_is(state, .close_curly) or next_is(state, .eof) or (state.match_arm_body and next_is(state, .comma)) or (allow_else and next_is(state, .else)) ) { diagnostic :: try diagnostics.add(state.diagnostics, .unexpected_token, keyword.span) yield try ast.add_invalid_expr(state.builder, keyword.span, diagnostic) } else { yield try parse_expr(state) } if ( !next_is(state, .newline) and !next_is(state, .close_curly) and !next_is(state, .eof) and !(allow_else and next_is(state, .else)) and !(state.match_arm_body and next_is(state, .comma)) ) return .unexpected_token return try ast.add_yield_stmt(state.builder, ast.Span{ start = keyword.span.start, end = state.tokens[state.next_token - 1].span.end, source = keyword.span.source, }, value) } @hide:file parse_loop_control proc(state @mut State, allow_else bool) ast.NodeId ! ParseError { keyword :: current(state) state.next_token += 1 if ( !next_is(state, .newline) and !next_is(state, .close_curly) and !next_is(state, .eof) and !(allow_else and next_is(state, .else)) and !(state.match_arm_body and next_is(state, .comma)) ) return .unexpected_token if (keyword.kind == .break) return try ast.add_break_stmt(state.builder, keyword.span) return try ast.add_continue_stmt(state.builder, keyword.span) } @hide:file parse_assignment proc(state @mut State, target ast.NodeId, allow_else bool) ast.NodeId ! ParseError { operator :: enummap.get(&assignment_operators, current(state).kind) state.next_token += 1 value := try parse_expr(state) if ( !next_is(state, .newline) and !next_is(state, .close_curly) and !next_is(state, .eof) and !(allow_else and next_is(state, .else)) and !(state.match_arm_body and next_is(state, .comma)) ) return .unexpected_token span :: ast.Span{ start = ast.builder_span(state.builder, target).start, end = state.tokens[state.next_token - 1].span.end, source = state.builder.source_id, } if (operator) |binary| value = try ast.add_binary(state.builder, span, binary, target, value) id :: try ast.add_assignment_stmt(state.builder, span, target, value) if (operator != null) ast.set_assignment_compound(state.builder, id) return id } @hide:file parse_if proc(state @mut State) ast.NodeId ! ParseError { keyword :: try consume(state, .if) condition :: try parse_guard(state) capture :: try parse_optional_capture(state) then_block :: try parse_arm(state, true) after_then :: state.next_token skip_newlines(state) alternative ?ast.NodeId := null if (next_is(state, .else) and !next_n_is(state, 1, .colon)) { state.next_token += 1 alternative = try parse_arm(state, true) } else { state.next_token = after_then } end :: if (alternative) |block| ast.builder_span(state.builder, block).end else ast.builder_span(state.builder, then_block).end id :: try ast.add_if_stmt( state.builder, ast.Span{ start = keyword.span.start, end = end, source = keyword.span.source, }, condition, then_block, alternative, ) if (capture) |node| ast.set_capture(state.builder, id, node) return id } @hide:file parse_if_expr proc(state @mut State) ast.NodeId ! ParseError { keyword :: try consume(state, .if) condition :: try parse_guard(state) capture :: try parse_optional_capture(state) then_value :: try parse_expr(state) skip_newlines(state) if (!next_is(state, .else) or next_n_is(state, 1, .colon)) return .unexpected_token state.next_token += 1 skip_newlines(state) else_value :: try parse_expr(state) span :: ast.Span{ start = keyword.span.start, end = ast.builder_span(state.builder, else_value).end, source = keyword.span.source, } # canonical matches share inference, specialization and branch execution. patterns [1]ast.NodeId :: [try ast.add_boolean(state.builder, keyword.span, true)] then_arm :: try ast.add_match_arm( state.builder, ast.builder_span(state.builder, then_value), patterns[..], capture, then_value, true, false, ) else_arm :: try ast.add_match_arm( state.builder, ast.builder_span(state.builder, else_value), &[], null, else_value, true, true, ) arms [2]ast.NodeId :: [then_arm, else_arm] result :: try ast.add_match(state.builder, span, condition, arms[..]) if (capture != null) ast.set_match_optional(state.builder, result) return result } @hide:file parse_while proc(state @mut State, allow_else bool) ast.NodeId ! ParseError { keyword :: try consume(state, .while) condition :: try parse_guard(state) capture :: try parse_optional_capture(state) body :: try parse_arm(state, allow_else) id :: try ast.add_while_stmt( state.builder, ast.Span{ start = keyword.span.start, end = ast.builder_span(state.builder, body).end, source = keyword.span.source, }, condition, body, ) if (capture) |node| ast.set_capture(state.builder, id, node) return id } @hide:file parse_optional_capture proc(state @mut State) ?ast.NodeId ! ParseError { missing ?ast.NodeId :: null if (!next_is(state, .pipe)) return missing state.next_token += 1 name :: try consume(state, .ident) _ = try consume(state, .pipe) return try ast.add_param(state.builder, name.span, name.span, name.data.str_id, maxval!(ast.NodeId)) } @hide:file CaptureList :: struct { item ?ast.NodeId = null index ?ast.NodeId = null pointer bool = false indexed bool = false } @hide:file parse_capture_name proc(state @mut State) ?ast.NodeId ! ParseError { name :: try consume(state, .ident) capture ?ast.NodeId := null if ( name.span.end != name.span.start + 1 or state.builder.source[usize(name.span.start)] != '_' ) capture = try ast.add_identifier( state.builder, name.span, name.data.str_id, ) return capture } @hide:file parse_captures proc(state @mut State) CaptureList ! ParseError { _ = try consume(state, .pipe) captures := CaptureList{} if next_is(state, .at) { captures.pointer = true state.next_token += 1 } captures.item = try parse_capture_name(state) if next_is(state, .comma) { captures.indexed = true state.next_token += 1 captures.index = try parse_capture_name(state) } _ = try consume(state, .pipe) return captures } @hide:file parse_for proc(state @mut State, allow_else bool) ast.NodeId ! ParseError { keyword :: current(state) inlined :: next_is(state, .inline) if (inlined) state.next_token += 1 _ = try consume(state, .for) skip_newlines(state) iterable_start :: state.next_token iterable := try parse_guard_expr(state) grouped :: parenthesized_guard(state, iterable_start) skip_newlines(state) captures :: parse_captures(state) catch |err| { match err { .unexpected_token: {} else: return err } diagnostic :: try diagnostics.add( state.diagnostics, .unexpected_token, current(state).span, ) iterable = try ast.add_invalid_expr( state.builder, ast.builder_span(state.builder, iterable), diagnostic, ) while ( !next_is(state, .open_curly) and !next_is(state, .close_curly) and !next_is(state, .newline) and !next_is(state, .eof) and !next_is(state, .else) ) state.next_token += 1 yield CaptureList{} } after_captures :: state.next_token skip_newlines(state) if ( !next_is(state, .open_curly) and !grouped and ast.builder_expr_kind(state.builder, iterable) != .call and ast.builder_expr_kind(state.builder, iterable) != .invalid ) { diagnostic :: try diagnostics.add(state.diagnostics, .unexpected_token, current(state).span) iterable = try ast.add_invalid_expr( state.builder, ast.builder_span(state.builder, iterable), diagnostic, ) } # a malformed capture list must not swallow the next line's sibling. if ( ast.builder_expr_kind(state.builder, iterable) == .invalid and !next_is(state, .open_curly) ) state.next_token = after_captures body :: try parse_arm(state, allow_else) id :: try ast.add_for_stmt( state.builder, ast.Span{ start = keyword.span.start, end = ast.builder_span(state.builder, body).end, source = keyword.span.source, }, iterable, captures.item, captures.index, captures.pointer, captures.indexed, body, ) if (inlined) ast.set_for_inline(state.builder, id) return id } @hide:file parse_guard_expr proc(state @mut State) ast.NodeId ! ParseError { previous :: state.allow_record_literals previous_bit_or :: state.allow_bit_or state.allow_record_literals = false # top-level header pipes delimit captures; nested expressions may use bitwise OR. state.allow_bit_or = false defer { state.allow_record_literals = previous state.allow_bit_or = previous_bit_or } return try parse_expr(state) } @hide:file parse_guard proc(state @mut State) ast.NodeId ! ParseError { condition_start :: state.next_token condition := parse_guard_expr(state) catch |err| { match err { .unexpected_token: {} else: return err } diagnostic :: try diagnostics.add(state.diagnostics, .unexpected_token, current(state).span) yield try ast.add_invalid_expr(state.builder, current(state).span, diagnostic) } if (ast.builder_expr_kind(state.builder, condition) == .invalid) { # discard a malformed guard's tail, but leave its body and enclosing block intact. while ( !next_is(state, .open_curly) and !next_is(state, .close_curly) and !next_is(state, .newline) and !next_is(state, .eof) and !next_is(state, .else) and !next_is(state, .return) and !next_is(state, .if) and !next_is(state, .while) and !next_is(state, .for) and !next_is(state, .inline) and !next_is(state, .break) and !next_is(state, .continue) ) { if (next_is(state, .ident) and is_assignment_operator(state.tokens[state.next_token + 1].kind)) break state.next_token += 1 } } if ( !next_is(state, .open_curly) and !next_is(state, .pipe) and !parenthesized_guard(state, condition_start) and ast.builder_expr_kind(state.builder, condition) != .call and ast.builder_expr_kind(state.builder, condition) != .comptime_expr and ast.builder_expr_kind(state.builder, condition) != .invalid ) { diagnostic :: try diagnostics.add(state.diagnostics, .unexpected_token, current(state).span) condition = try ast.add_invalid_expr( state.builder, ast.builder_span(state.builder, condition), diagnostic, ) } return condition } @hide:file parenthesized_guard proc(state @State, start usize) bool { if (state.tokens[start].kind != .open_paren) return false depth usize := 0 for start..state.next_token |index| { kind :: state.tokens[index].kind if (kind == .open_paren) depth += 1 if (kind == .close_paren) { depth -= 1 if (depth == 0) return index + 1 == state.next_token } } return false } @hide:file parse_arm proc(state @mut State, allow_else bool) ast.NodeId ! ParseError { if next_is(state, .open_curly) return try parse_block(state) start :: state.next_token statement :: parse_statement(state, allow_else) catch |err| { match err { .unexpected_token: {} else: return err } diagnostic :: try unexpected_diagnostic(state) span :: synchronize(state, start, true, true, false) yield try ast.add_invalid_stmt(state.builder, span, diagnostic) } statements [1]ast.NodeId :: [statement] return try ast.add_block( state.builder, ast.builder_span(state.builder, statement), statements[..], ) } @hide:file parse_statement_decl proc(state @mut State) ast.NodeId ! ParseError { name :: try consume(state, .ident) type_id ?ast.NodeId :: if (starts_type(current(state).kind)) try parse_type(state) else null assignment :: try consume_either(state, &[.double_colon, .colon_equal]) initializer :: try parse_expr(state) end :: ast.builder_span(state.builder, initializer).end mutability :: if (assignment.kind == .double_colon) .constant else .variable return try ast.add_statement_decl( state.builder, ast.Span{ start = name.span.start, end = end, source = name.span.source, }, name.span, name.data.str_id, type_id, mutability, initializer, ) } @hide:file parse_type proc(state @mut State) ast.NodeId ! ParseError { skip_operand_newlines(state) start :: current(state).span left := try parse_type_atom(state) while true { skip_continuation_newlines(state) if (!next_is(state, .pipe)) break state.next_token += 1 right :: try parse_type_atom(state) left = try ast.add_binary_type(state.builder, ast.Span{ start = start.start, end = state.tokens[state.next_token - 1].span.end, source = start.source, }, left, right, false) } if (next_is(state, .bang)) { state.next_token += 1 right :: try parse_type(state) left = try ast.add_binary_type(state.builder, ast.Span{ start = start.start, end = state.tokens[state.next_token - 1].span.end, source = start.source, }, left, right, true) } return left } @hide:file parse_type_atom proc(state @mut State) ast.NodeId ! ParseError { skip_operand_newlines(state) if (next_is(state, .open_paren)) { state.next_token += 1 state.paren_depth += 1 defer state.paren_depth -= 1 child :: try parse_type(state) _ = try consume(state, .close_paren) return child } if (next_is(state, .question)) { token :: current(state) state.next_token += 1 child :: try parse_type_atom(state) return try ast.add_optional_type(state.builder, ast.Span{ start = token.span.start, end = state.tokens[state.next_token - 1].span.end, source = token.span.source, }, child) } if (next_is(state, .at) or next_is(state, .star)) { token :: current(state) state.next_token += 1 mutable :: next_is(state, .mut) if (mutable) state.next_token += 1 if (next_is(state, .proc) or next_is(state, .c_func)) return try parse_function_type(state, token.span, mutable or (token.kind == .star and !next_is(state, .c_func))) element :: try parse_type_atom(state) return try ast.add_pointer_type(state.builder, ast.Span{ start = token.span.start, end = state.tokens[state.next_token - 1].span.end, source = token.span.source, }, element, mutable, token.kind == .star) } if next_is(state, .open_bracket) { open :: try consume(state, .open_bracket) if (next_is(state, .close_bracket) or next_is(state, .semicolon)) { sentinel ?ast.NodeId := null if (next_is(state, .semicolon)) { state.next_token += 1 sentinel = try parse_expr(state) } _ = try consume(state, .close_bracket) mutable :: next_is(state, .mut) if (mutable) state.next_token += 1 element :: try parse_type_atom(state) return try ast.add_slice_type(state.builder, ast.Span{ start = open.span.start, end = state.tokens[state.next_token - 1].span.end, source = open.span.source, }, element, mutable, sentinel) } count ?ast.NodeId := null if ( next_is(state, .ident) and current(state).span.end == current(state).span.start + 1 and state.builder.source[usize(current(state).span.start)] == '_' ) { state.next_token += 1 } else { count = try parse_expr(state) } _ = try consume(state, .close_bracket) mutable :: next_is(state, .mut) if (mutable) state.next_token += 1 element :: try parse_type_atom(state) return try ast.add_array_type( state.builder, ast.Span{ start = open.span.start, end = state.tokens[state.next_token - 1].span.end, source = open.span.source, }, count, element, mutable, ) } if (next_is(state, .proc) or next_is(state, .c_func)) return try parse_function_type(state, current(state).span, false) if (next_is(state, .struct) or next_is(state, .c_struct)) return try parse_record_type(state) if (next_is(state, .opaque)) { token :: try consume(state, .opaque) id :: try ast.add_record_type(state.builder, token.span, &[]) ast.set_record_opaque(state.builder, id) return id } if next_is(state, .enum) return try parse_enum_type(state) if next_is(state, .union) return try parse_union_type(state) if next_is(state, .match) return try parse_match(state) if next_is(state, .if) return try parse_if_expr(state) if (next_is(state, .distinct) or next_is(state, .alias)) return try parse_type_wrapper(state) token :: try consume(state, .ident) type := try ast.add_identifier(state.builder, token.span, token.data.str_id) while true { if (next_is(state, .open_paren)) { type = try parse_call(state, type, token.span.start, false) continue } if (next_is(state, .open_bracket)) { type = try parse_index(state, type, token.span.start) continue } if (!next_is(state, .dot)) break state.next_token += 1 member :: try consume_member(state, false) member_id :: try ast.add_identifier(state.builder, member.span, member.data.str_id) type = try ast.add_selection(state.builder, ast.Span{ start = token.span.start, end = member.span.end, source = token.span.source, }, type, member_id) } return type } @hide:file parse_function_type proc(state @mut State, start ast.Span, invalid bool) ast.NodeId ! ParseError { external :: next_is(state, .c_func) _ = try consume_either(state, &[.proc, .c_func]) _ = try consume(state, .open_paren) params := arraylist.init(ast.NodeId, state.allocator) defer arraylist.deinit(¶ms) variadic :: try parse_parameters(state, ¶ms) _ = try consume(state, .close_paren) result :: try parse_type(state) span :: ast.Span{ start = start.start, end = state.tokens[state.next_token - 1].span.end, source = start.source, } if (invalid) { diagnostic :: try diagnostics.add(state.diagnostics, .invalid_pointer_type, span) return try ast.add_invalid_expr(state.builder, span, diagnostic) } id :: try ast.add_function_type(state.builder, span, result, params.items) if (external) ast.set_callable_foreign(state.builder, id) if (variadic) ast.set_callable_variadic(state.builder, id) return id } @hide:file type_starters :: $enummap.init_default(lexer.TokenKind, bool, false, { ident = true, open_bracket = true, struct = true, c_struct = true, opaque = true, enum = true, union = true, distinct = true, alias = true, match = true, if = true, at = true, star = true, question = true, open_paren = true, proc = true, c_func = true, }) @hide:file starts_type proc(kind lexer.TokenKind) bool { return enummap.get(&type_starters, kind) != null } @hide:file member_names :: $enummap.init_default(lexer.TokenKind, bool, false, { ident = true, if = true, else = true, for = true, inline = true, while = true, proc = true, return = true, yield = true, break = true, continue = true, true = true, false = true, and = true, or = true, xor = true, mut = true, struct = true, c_struct = true, opaque = true, enum = true, union = true, distinct = true, alias = true, match = true, import = true, test = true, c_func = true, null = true, undefined = true, try = true, catch = true, defer = true, errdefer = true, }) @hide:file member_name proc(kind lexer.TokenKind) bool { return enummap.get(&member_names, kind) != null } @hide:file consume_member proc(state @mut State, numeric bool) lexer.Token ! SyntaxError { token :: current(state) if (!member_name(token.kind) and !(numeric and token.kind == .int)) return .unexpected_token state.next_token += 1 return token } @hide:file parse_type_wrapper proc(state @mut State) ast.NodeId ! ParseError { keyword :: current(state) state.next_token += 1 operand :: try parse_type(state) return try ast.add_type_operand(state.builder, ast.Span{ start = keyword.span.start, end = ast.builder_span(state.builder, operand).end, source = keyword.span.source, }, operand, keyword.kind == .alias) } @hide:file parse_enum_type proc(state @mut State) ast.NodeId ! ParseError { keyword :: try consume(state, .enum) backing ?ast.NodeId := null if next_is(state, .open_paren) { state.next_token += 1 backing = try parse_type(state) _ = try consume(state, .close_paren) } skip_newlines(state) _ = try consume(state, .open_curly) members arraylist.ArrayList(ast.NodeId) := arraylist.init(state.allocator) defer arraylist.deinit(&members) skip_newlines(state) while (!next_is(state, .close_curly) and !next_is(state, .eof)) { if ( next_is(state, .ident) and (next_n_is(state, 1, .double_colon) or next_n_is(state, 1, .colon_equal) or next_n_is(state, 1, .proc)) ) return try invalid_record(state, keyword.span.start, false) start :: state.next_token member :: parse_enum_member(state) catch |err| { match err { .unexpected_token: {} else: return err } diagnostic :: try unexpected_diagnostic(state) span :: recover_record_field(state, start) invalid :: try ast.add_invalid_expr(state.builder, span, diagnostic) yield try ast.add_field(state.builder, span, null, null, invalid) } try arraylist.append(&members, member) if next_is(state, .close_curly) break if next_is(state, .comma) state.next_token += 1 else if (!next_is(state, .newline)) return try invalid_record(state, keyword.span.start, true) skip_newlines(state) } if (!next_is(state, .close_curly)) return try invalid_record(state, keyword.span.start, false) close :: try consume(state, .close_curly) return try ast.add_enum_type(state.builder, ast.Span{ start = keyword.span.start, end = close.span.end, source = keyword.span.source, }, backing, members.items) } @hide:file parse_enum_member proc(state @mut State) ast.NodeId ! ParseError { token :: try consume_member(state, false) name :: try ast.add_identifier(state.builder, token.span, token.data.str_id) value ?ast.NodeId := null if next_is(state, .equal) { state.next_token += 1 value = try parse_expr(state) } end :: if (value) |id| ast.builder_span(state.builder, id).end else token.span.end return try ast.add_field(state.builder, ast.Span{ start = token.span.start, end = end, source = token.span.source, }, name, null, value) } @hide:file parse_union_type proc(state @mut State) ast.NodeId ! ParseError { keyword :: try consume(state, .union) tag ?ast.NodeId := null auto_tag := false if next_is(state, .open_paren) { state.next_token += 1 if next_is(state, .enum) { state.next_token += 1 auto_tag = true } else { tag = try parse_type(state) } _ = try consume(state, .close_paren) } skip_newlines(state) _ = try consume(state, .open_curly) fields arraylist.ArrayList(ast.NodeId) := arraylist.init(state.allocator) defer arraylist.deinit(&fields) skip_newlines(state) while (!next_is(state, .close_curly) and !next_is(state, .eof)) { if ( next_is(state, .ident) and (next_n_is(state, 1, .double_colon) or next_n_is(state, 1, .colon_equal) or next_n_is(state, 1, .proc)) ) return try invalid_record(state, keyword.span.start, false) start :: state.next_token field :: parse_union_field(state, auto_tag or tag != null) catch |err| { match err { .unexpected_token: {} else: return err } diagnostic :: try unexpected_diagnostic(state) span :: recover_record_field(state, start) invalid :: try ast.add_invalid_expr(state.builder, span, diagnostic) yield try ast.add_field(state.builder, span, null, invalid, null) } try arraylist.append(&fields, field) if next_is(state, .close_curly) break if next_is(state, .comma) state.next_token += 1 else if (!next_is(state, .newline)) return try invalid_record(state, keyword.span.start, true) skip_newlines(state) } if (!next_is(state, .close_curly)) return try invalid_record(state, keyword.span.start, false) close :: try consume(state, .close_curly) return try ast.add_union_type(state.builder, ast.Span{ start = keyword.span.start, end = close.span.end, source = keyword.span.source, }, tag, auto_tag, fields.items) } @hide:file parse_union_field proc(state @mut State, tagged bool) ast.NodeId ! ParseError { token :: if (tagged) try consume_member(state, false) else try consume(state, .ident) name :: try ast.add_identifier(state.builder, token.span, token.data.str_id) type :: try parse_type(state) return try ast.add_field(state.builder, ast.Span{ start = token.span.start, end = ast.builder_span(state.builder, type).end, source = token.span.source, }, name, type, null) } @hide:file parse_match proc(state @mut State) ast.NodeId ! ParseError { keyword :: try consume(state, .match) skip_newlines(state) subject :: try parse_guard_expr(state) skip_newlines(state) _ = try consume(state, .open_curly) previous_depth :: state.paren_depth state.paren_depth = 0 defer state.paren_depth = previous_depth arms arraylist.ArrayList(ast.NodeId) := arraylist.init(state.allocator) defer arraylist.deinit(&arms) skip_newlines(state) while (!next_is(state, .close_curly) and !next_is(state, .eof)) { if ( next_is(state, .ident) and (next_n_is(state, 1, .double_colon) or next_n_is(state, 1, .colon_equal) or next_n_is(state, 1, .proc)) ) return try invalid_record(state, keyword.span.start, false) start :: state.next_token arm :: parse_match_arm(state) catch |err| { match err { .unexpected_token: {} else: return err } diagnostic :: try unexpected_diagnostic(state) span :: recover_record_field(state, start) invalid :: try ast.add_invalid_expr(state.builder, span, diagnostic) yield try ast.add_match_arm(state.builder, span, &[], null, invalid, true, false) } try arraylist.append(&arms, arm) if next_is(state, .close_curly) break if next_is(state, .comma) state.next_token += 1 else if (!next_is(state, .newline)) return try invalid_record(state, keyword.span.start, true) skip_newlines(state) } if (!next_is(state, .close_curly)) return try invalid_record(state, keyword.span.start, false) close :: try consume(state, .close_curly) return try ast.add_match(state.builder, ast.Span{ start = keyword.span.start, end = close.span.end, source = keyword.span.source, }, subject, arms.items) } @hide:file parse_match_arm proc(state @mut State) ast.NodeId ! ParseError { start :: current(state).span patterns arraylist.ArrayList(ast.NodeId) := arraylist.init(state.allocator) defer arraylist.deinit(&patterns) is_inline :: next_is(state, .inline) if (is_inline) state.next_token += 1 catch_all :: !is_inline and next_is(state, .else) capture ?ast.NodeId := null tag_capture ?ast.NodeId := null tag_capture_present := false if (catch_all) state.next_token += 1 else if (!is_inline) { while true { try arraylist.append(&patterns, try parse_guard_expr(state)) if (!next_is(state, .comma)) break state.next_token += 1 skip_newlines(state) } } if (is_inline or (!catch_all and next_is(state, .pipe))) { _ = try consume(state, .pipe) # captures are by value; pointer capture tokens are not identifiers. token :: try consume(state, .ident) if ( token.span.end != token.span.start + 1 or state.builder.source[usize(token.span.start)] != '_' ) capture = try ast.add_identifier(state.builder, token.span, token.data.str_id) if (is_inline and next_is(state, .comma)) { state.next_token += 1 tag_capture_present = true tag :: try consume(state, .ident) if ( tag.span.end != tag.span.start + 1 or state.builder.source[usize(tag.span.start)] != '_' ) tag_capture = try ast.add_identifier(state.builder, tag.span, tag.data.str_id) } _ = try consume(state, .pipe) } _ = try consume(state, .colon) skip_newlines(state) previous :: state.match_arm_body state.match_arm_body = true defer state.match_arm_body = previous statement_body :: match current(state).kind { .open_curly, .return, .yield, .break, .continue, .if, .while, .for, .inline, .defer, .errdefer: true else: false } body ast.NodeId := undefined expression := false if (statement_body or named_binding_starts(state)) { body = try parse_arm(state, false) } else { value :: try parse_expr(state) if (is_assignment_operator(current(state).kind)) { statement :: try parse_assignment(state, value, false) statements [1]ast.NodeId :: [statement] body = try ast.add_block(state.builder, ast.builder_span(state.builder, statement), statements[..]) } else { body = value expression = true } tree :: ast.view(state.builder, value) if (expression and ast.expr_kind(&tree, value) == .match_expr and ast.match_arm_count(&tree, value) > 0) { nested :: ast.match_arm_view(&tree, ast.match_arm(&tree, value, 0)) if (!nested.expression) { statement :: try ast.add_expression_stmt(state.builder, value) statements [1]ast.NodeId :: [statement] body = try ast.add_block(state.builder, ast.builder_span(state.builder, value), statements[..]) expression = false } } } arm :: try ast.add_match_arm(state.builder, ast.Span{ start = start.start, end = ast.builder_span(state.builder, body).end, source = start.source, }, patterns.items, capture, body, expression, catch_all) if (is_inline) ast.set_match_arm_inline(state.builder, arm, tag_capture, tag_capture_present) return arm } @hide:file parse_record_type proc(state @mut State) ast.NodeId ! ParseError { keyword :: try consume_either(state, &[.struct, .c_struct]) id :: try parse_record(state, null, keyword.span.start, true) if (keyword.kind == .c_struct and ast.builder_expr_kind(state.builder, id) == .record_type) ast.set_record_c_layout(state.builder, id) return id } @hide:file parse_record_literal proc( state @mut State, type ?ast.NodeId, start u32, ) ast.NodeId ! ParseError { return try parse_record(state, type, start, false) } @hide:file parse_record proc( state @mut State, type ?ast.NodeId, start u32, declaration bool, ) ast.NodeId ! ParseError { open :: try consume(state, .open_curly) previous :: state.allow_record_literals state.allow_record_literals = true defer state.allow_record_literals = previous fields arraylist.ArrayList(ast.NodeId) := arraylist.init(state.allocator) defer arraylist.deinit(&fields) skip_newlines(state) while (!next_is(state, .close_curly) and !next_is(state, .eof)) { # leave the next declaration intact when a record's closing brace is missing. if ( next_is(state, .ident) and (next_n_is(state, 1, .double_colon) or next_n_is(state, 1, .colon_equal) or next_n_is(state, 1, .proc)) ) return try invalid_record(state, start, false) field_start :: state.next_token field :: parse_record_field(state, declaration) catch |err| { match err { .unexpected_token: {} else: return err } diagnostic :: try unexpected_diagnostic(state) span :: recover_record_field(state, field_start) invalid :: try ast.add_invalid_expr(state.builder, span, diagnostic) recovered_type ?ast.NodeId :: if (declaration) invalid else null recovered_value ?ast.NodeId :: if (declaration) null else invalid yield try ast.add_field(state.builder, span, null, recovered_type, recovered_value) } try arraylist.append(&fields, field) if next_is(state, .close_curly) break if next_is(state, .comma) { state.next_token += 1 skip_newlines(state) continue } if (declaration and next_is(state, .newline)) { skip_newlines(state) continue } after_field :: state.next_token skip_newlines(state) if next_is(state, .close_curly) break state.next_token = after_field return try invalid_record(state, start, true) } if (!next_is(state, .close_curly)) return try invalid_record(state, start, false) close :: try consume(state, .close_curly) span :: ast.Span{ start = start, end = close.span.end, source = open.span.source } if (declaration) return try ast.add_record_type(state.builder, span, fields.items) return try ast.add_record_literal(state.builder, span, type, fields.items) } @hide:file parse_record_field proc(state @mut State, declaration bool) ast.NodeId ! ParseError { start :: current(state).span start_token :: state.next_token name ?ast.NodeId := null type ?ast.NodeId := null value ?ast.NodeId := null named :: ( member_name(current(state).kind) and ((!declaration and next_n_is(state, 1, .equal)) or (declaration and starts_type(state.tokens[state.next_token + 1].kind))) ) if (named) { token :: if (declaration) try consume(state, .ident) else try consume_member(state, false) name = try ast.add_identifier(state.builder, token.span, token.data.str_id) } if (declaration) { type = try parse_type(state) if next_is(state, .equal) { state.next_token += 1 value = try parse_expr(state) } } else { if (named) _ = try consume(state, .equal) value = try parse_expr(state) } end :: if (state.next_token > start_token) state.tokens[state.next_token - 1].span.end else start.end return try ast.add_field(state.builder, ast.Span{ start = start.start, end = end, source = start.source, }, name, type, value) } @hide:file recover_record_field proc(state @mut State, start usize) ast.Span { depth usize := 0 for (start..state.next_token) |index| { kind :: state.tokens[index].kind if (kind == .open_curly or kind == .open_paren or kind == .open_bracket) depth += 1 if ((kind == .close_curly or kind == .close_paren or kind == .close_bracket) and depth > 0) depth -= 1 } while (!next_is(state, .eof)) { kind :: current(state).kind if (depth == 0 and (kind == .comma or kind == .newline or kind == .close_curly)) break if (kind == .open_curly or kind == .open_paren or kind == .open_bracket) depth += 1 if (kind == .close_curly or kind == .close_paren or kind == .close_bracket) { if (depth == 0) break depth -= 1 } state.next_token += 1 } first :: state.tokens[start].span end :: if (state.next_token > start) state.tokens[state.next_token - 1].span.end else first.end return ast.Span{ start = first.start, end = end, source = first.source, } } @hide:file invalid_record proc(state @mut State, start u32, recover_tail bool) ast.NodeId ! ParseError { diagnostic :: try unexpected_diagnostic(state) if (recover_tail) { while (!next_is(state, .newline) and !next_is(state, .close_curly) and !next_is(state, .eof)) { before :: state.next_token _ = recover_record_field(state, before) if (next_is(state, .comma)) state.next_token += 1 if (state.next_token == before) break } if (next_is(state, .close_curly)) state.next_token += 1 } return try ast.add_invalid_expr(state.builder, ast.Span{ start = start, end = state.tokens[state.next_token - 1].span.end, source = state.builder.source_id, }, diagnostic) } @hide:file parse_expr proc(state @mut State) ast.NodeId ! ParseError { return try parse_expr_bp(state, BindingPowerScalar(0)) } @hide:file parse_expr_bp proc(state @mut State, min_bp BindingPowerScalar) ast.NodeId ! ParseError { skip_operand_newlines(state) token := current(state) expression_start :: state.next_token left := if (enummap.get(&prefix_binding_powers, token.kind)) |right_bp| { state.next_token += 1 operand :: try parse_expr_bp(state, right_bp) operand_span :: ast.builder_span(state.builder, operand) span :: ast.Span{ start = token.span.start, end = operand_span.end, source = token.span.source, } operator ast.UnaryOperator :: match token.kind { .bang: .logical_not .tilde: .bit_not .ampersand: .address_of else: .negate } prefixed :: if (token.kind == .try) try ast.add_try_expr(state.builder, span, operand) else try ast.add_unary(state.builder, span, operator, operand) yield prefixed } else { yield try parse_primary(state) } left_grouped := parenthesized_guard(state, expression_start) while true { skip_continuation_newlines(state) token = current(state) if (token.kind == .caret or token.kind == .question) { state.next_token += 1 operator ast.UnaryOperator :: if (token.kind == .caret) .dereference else .unwrap left = try ast.add_unary(state.builder, ast.Span{ start = state.tokens[expression_start].span.start, end = token.span.end, source = token.span.source, }, operator, left) left_grouped = false continue } # adjacent bang calls stay intrinsics; spaced 'T ! (E)' is a fallible type. if ( token.kind == .bang and next_n_is(state, 1, .open_paren) and state.tokens[state.next_token - 1].span.end == token.span.start ) { state.next_token += 1 left = try parse_call(state, left, state.tokens[expression_start].span.start, true) left_grouped = false continue } if (token.kind == .open_paren) { left = try parse_call(state, left, state.tokens[expression_start].span.start, false) left_grouped = false continue } if (token.kind == .open_bracket) { left = try parse_index(state, left, state.tokens[expression_start].span.start) left_grouped = false continue } if (token.kind == .open_curly and state.allow_record_literals) { left = try parse_record_literal(state, left, state.tokens[expression_start].span.start) left_grouped = false continue } if (token.kind == .dot) { state.next_token += 1 member :: try consume_member(state, true) member_id :: if (member.kind == .int) try ast.add_literal(state.builder, .int, member.span) else try ast.add_identifier(state.builder, member.span, member.data.str_id) left = try ast.add_selection( state.builder, ast.Span{ start = state.tokens[expression_start].span.start, end = member.span.end, source = member.span.source, }, left, member_id, ) left_grouped = false continue } if (token.kind == .catch) { if (min_bp > BindingPowerScalar(0)) break state.next_token += 1 capture :: try parse_error_capture(state) skip_operand_newlines(state) handler_is_block :: next_is(state, .open_curly) handler :: if (handler_is_block) try parse_block(state) else try parse_expr_bp(state, BindingPowerScalar(0)) left = try ast.add_catch_expr(state.builder, ast.Span{ start = state.tokens[expression_start].span.start, end = state.tokens[state.next_token - 1].span.end, source = token.span.source, }, left, capture, handler, handler_is_block) left_grouped = false continue } if (token.kind == .bang) { if (BindingPowerScalar(1) < min_bp) break state.next_token += 1 right :: try parse_type(state) left = try ast.add_binary_type(state.builder, ast.Span{ start = state.tokens[expression_start].span.start, end = state.tokens[state.next_token - 1].span.end, source = token.span.source, }, left, right, token.kind == .bang) left_grouped = false continue } if (token.kind == .range or token.kind == .range_inclusive) { if (min_bp > BindingPowerScalar(0)) break left = try validate_range_bound(state, left, left_grouped) state.next_token += 1 right_start :: state.next_token right := try parse_expr_bp(state, BindingPowerScalar(1)) right = try validate_range_bound(state, right, parenthesized_guard(state, right_start)) left = try ast.add_range( state.builder, ast.Span{ start = state.tokens[expression_start].span.start, end = state.tokens[state.next_token - 1].span.end, source = state.builder.source_id, }, left, right, token.kind == .range_inclusive, ) left_grouped = false continue } if (token.kind == .pipe and !state.allow_bit_or) break if (enummap.get(&infix_binding_powers, token.kind)) |power| { if (power.left < min_bp) break state.next_token += 1 right :: try parse_expr_bp(state, power.right) left_span :: ast.builder_span(state.builder, left) right_span :: ast.builder_span(state.builder, right) left = try ast.add_binary( state.builder, ast.Span{ start = left_span.start, end = right_span.end, source = left_span.source, }, enummap.get(&binary_operators, token.kind)?, left, right, ) left_grouped = false continue } break } return left } @hide:file validate_range_bound proc( state @mut State, bound ast.NodeId, grouped bool, ) ast.NodeId ! (mem.AllocError | diagnostics.CapacityError) { if (grouped) return bound match ast.builder_expr_kind(state.builder, bound) { .unary, .binary, .range: { span :: ast.builder_span(state.builder, bound) diagnostic :: try diagnostics.add(state.diagnostics, .unexpected_token, span) return try ast.add_invalid_expr(state.builder, span, diagnostic) } else: return bound } } @hide:file parse_call proc(state @mut State, callee ast.NodeId, start u32, builtin bool) ast.NodeId ! ParseError { _ = try consume(state, .open_paren) previous :: state.allow_record_literals previous_bit_or :: state.allow_bit_or state.allow_bit_or = true defer state.allow_bit_or = previous_bit_or state.allow_record_literals = true defer state.allow_record_literals = previous state.paren_depth += 1 defer state.paren_depth -= 1 skip_argument_newlines(state) arguments arraylist.ArrayList(ast.NodeId) := arraylist.init(state.allocator) defer arraylist.deinit(&arguments) argument_start :: state.next_token node_count :: state.builder.storage.nodes.items.len extra_count :: state.builder.storage.extra.items.len diagnostic_count :: state.diagnostics.list.items.len while (!is_call_boundary(current(state).kind)) { argument :: try parse_expr(state) try arraylist.append(&arguments, argument) if (!next_is(state, .comma)) break state.next_token += 1 skip_argument_newlines(state) } if (!previous and !next_is(state, .close_paren)) { # a guard's body may look like a record argument until its missing ')' is known. state.next_token = argument_start state.builder.storage.nodes.items = state.builder.storage.nodes.items.ptr[..node_count] state.builder.storage.spans.items = state.builder.storage.spans.items.ptr[..node_count] state.builder.storage.extra.items = state.builder.storage.extra.items.ptr[..extra_count] state.diagnostics.list.items = state.diagnostics.list.items.ptr[..diagnostic_count] return .unexpected_token } close :: try consume(state, .close_paren) span :: ast.Span{ start = start, end = close.span.end, source = state.builder.source_id } if (builtin) return try ast.add_builtin_call(state.builder, span, callee, arguments.items) return try ast.add_call(state.builder, span, callee, arguments.items) } @hide:file parse_index proc(state @mut State, base ast.NodeId, start u32) ast.NodeId ! ParseError { _ = try consume(state, .open_bracket) previous :: state.allow_record_literals previous_bit_or :: state.allow_bit_or state.allow_bit_or = previous_bit_or or index_has_close(state) defer state.allow_bit_or = previous_bit_or state.allow_record_literals = true defer state.allow_record_literals = previous first ?ast.NodeId := null if (!next_is(state, .range)) first = try parse_expr_bp(state, BindingPowerScalar(1)) if (next_is(state, .range)) { state.next_token += 1 end ?ast.NodeId := null sentinel ?ast.NodeId := null if (!next_is(state, .close_bracket) and !next_is(state, .semicolon)) end = try parse_expr(state) if (next_is(state, .semicolon)) { state.next_token += 1 sentinel = try parse_expr(state) } close :: try consume(state, .close_bracket) return try ast.add_slice(state.builder, ast.Span{ start = start, end = close.span.end, source = state.builder.source_id, }, base, first, end, sentinel) } if !next_is(state, .close_bracket) { diagnostic :: try diagnostics.add(state.diagnostics, .unexpected_token, current(state).span) return try ast.add_invalid_expr(state.builder, ast.Span{ start = start, end = ast.builder_span(state.builder, first?).end, source = state.builder.source_id, }, diagnostic) } close :: try consume(state, .close_bracket) return try ast.add_index(state.builder, ast.Span{ start = start, end = close.span.end, source = state.builder.source_id, }, base, first?) } @hide:file index_has_close proc(state @State) bool { # keep malformed guard indices from consuming a capture list and its body. depth := usize(1) for (state.next_token)..(state.tokens.len) |index| { kind :: state.tokens[index].kind if (kind == .eof or (kind == .newline and depth == 1)) return false if (kind == .open_paren or kind == .open_bracket or kind == .open_curly) depth += 1 if (kind == .close_paren or kind == .close_bracket or kind == .close_curly) { depth -= 1 if (depth == 0) return kind == .close_bracket } } return false } @hide:file array_starts_type proc(state @State) bool { depth usize := 0 curly_depth := usize(0) for (state.next_token)..(state.tokens.len) |i| { kind :: state.tokens[i].kind if (kind == .open_bracket) depth += 1 if (kind == .close_bracket) { depth -= 1 if (depth == 0) { if (i + 1 >= state.tokens.len) return false next :: state.tokens[i + 1].kind return starts_type(next) or next == .mut } } if (kind == .open_curly) curly_depth += 1 if (kind == .close_curly) { if (curly_depth == 0) return false curly_depth -= 1 } if (kind == .eof) return false } return false } @hide:file parse_array_literal proc(state @mut State) ast.NodeId ! ParseError { open :: try consume(state, .open_bracket) previous :: state.allow_record_literals previous_bit_or :: state.allow_bit_or state.allow_bit_or = true defer state.allow_bit_or = previous_bit_or state.allow_record_literals = true defer state.allow_record_literals = previous elements arraylist.ArrayList(ast.NodeId) := arraylist.init(state.allocator) defer arraylist.deinit(&elements) after_open :: state.next_token skip_newlines(state) if named_binding_starts(state) { state.next_token = after_open return try invalid_array_literal(state, open.span.start, open.span.end) } while !next_is(state, .close_bracket) { element :: try parse_expr(state) try arraylist.append(&elements, element) after_element :: state.next_token skip_newlines(state) if next_is(state, .close_bracket) break if !next_is(state, .comma) { state.next_token = after_element return try invalid_array_literal(state, open.span.start, ast.builder_span(state.builder, element).end) } state.next_token += 1 after_comma :: state.next_token skip_newlines(state) if named_binding_starts(state) { state.next_token = after_comma return try invalid_array_literal(state, open.span.start, ast.builder_span(state.builder, element).end) } } close :: try consume(state, .close_bracket) return try ast.add_array_literal(state.builder, ast.Span{ start = open.span.start, end = close.span.end, source = open.span.source, }, elements.items) } @hide:file invalid_array_literal proc(state @mut State, start, end u32) ast.NodeId ! ParseError { diagnostic :: try diagnostics.add(state.diagnostics, .unexpected_token, current(state).span) return try ast.add_invalid_expr(state.builder, ast.Span{ start = start, end = end, source = state.builder.source_id, }, diagnostic) } @hide:file pointer_binding_starts proc(state @State) bool { depth := usize(0) for (state.next_token + 1)..(state.tokens.len) |i| { kind :: state.tokens[i].kind if (kind == .open_curly or kind == .open_bracket or kind == .open_paren) depth += 1 if (kind == .close_curly or kind == .close_bracket or kind == .close_paren) { if (depth == 0) return false depth -= 1 } if (depth != 0) continue if (kind == .double_colon or kind == .colon_equal) return true if (kind == .newline or kind == .eof or is_assignment_operator(kind)) return false } return false } @hide:file named_binding_starts proc(state @State) bool { return ( next_is(state, .ident) and (next_n_is(state, 1, .double_colon) or next_n_is(state, 1, .colon_equal) or next_n_is(state, 1, .ident) or next_n_is(state, 1, .proc) or next_n_is(state, 1, .match) or next_n_is(state, 1, .if) or next_n_is(state, 1, .at) or next_n_is(state, 1, .question) or (next_n_is(state, 1, .open_paren) and pointer_binding_starts(state)) or (next_n_is(state, 1, .star) and pointer_binding_starts(state))) ) } @hide:file brace_starts_block proc(state @State) bool { first := state.next_token + 1 while (first < state.tokens.len and state.tokens[first].kind == .newline) first += 1 # empty records and braces without statement markers retain record syntax. if (first >= state.tokens.len or state.tokens[first].kind == .close_curly) return false depth := usize(0) for (state.next_token)..(state.tokens.len) |index| { kind :: state.tokens[index].kind if (kind == .eof) return false if (depth > 0 and (state.tokens[first].kind == .if or state.tokens[first].kind == .open_curly) and (kind == .yield or kind == .return or kind == .defer or kind == .errdefer) and state.tokens[index - 1].kind != .dot and !(index + 1 < state.tokens.len and state.tokens[index + 1].kind == .equal)) return true if (depth == 1) { if (kind == .double_colon or kind == .colon_equal) return true if (kind != .equal and is_assignment_operator(kind)) return true previous :: if (index > first) state.tokens[index - 1].kind else lexer.TokenKind.open_curly if (previous != .dot and !(index + 1 < state.tokens.len and state.tokens[index + 1].kind == .equal)) { match kind { .yield, .return, .break, .continue, .while, .for, .inline, .defer, .errdefer: return true else: {} } } } if (kind == .open_curly or kind == .open_paren or kind == .open_bracket) depth += 1 if (kind == .close_curly or kind == .close_paren or kind == .close_bracket) { if (depth == 0) return false depth -= 1 if (depth == 0) return false } } return false } @hide:file parse_primary proc(state @mut State) ast.NodeId ! ParseError { token :: current(state) match token.kind { .at, .star, .question, .proc, .c_func, .opaque: return try parse_type(state) .struct, .c_struct: return try parse_record_type(state) .enum: return try parse_enum_type(state) .union: return try parse_union_type(state) .distinct, .alias: return try parse_type_wrapper(state) .match: return try parse_match(state) .if: return try parse_if_expr(state) .dollar: { state.next_token += 1 _ = try consume(state, .open_paren) previous :: state.allow_record_literals previous_bit_or :: state.allow_bit_or state.allow_bit_or = true defer state.allow_bit_or = previous_bit_or state.allow_record_literals = true defer state.allow_record_literals = previous state.paren_depth += 1 defer state.paren_depth -= 1 expression :: try parse_expr(state) skip_operand_newlines(state) close :: try consume(state, .close_paren) return try ast.add_comptime_expr(state.builder, ast.Span{ start = token.span.start, end = close.span.end, source = token.span.source, }, expression) } .dot: { state.next_token += 1 member :: try consume_member(state, false) member_id :: try ast.add_identifier(state.builder, member.span, member.data.str_id) return try ast.add_enum_literal(state.builder, ast.Span{ start = token.span.start, end = member.span.end, source = token.span.source, }, member_id) } .open_curly: { if (state.allow_record_literals and brace_starts_block(state)) { body :: try parse_block(state) return try ast.add_block_expr(state.builder, ast.builder_span(state.builder, body), body) } if (state.allow_record_literals) return try parse_record_literal(state, null, token.span.start) diagnostic :: try diagnostics.add(state.diagnostics, .unexpected_token, token.span) return try ast.add_invalid_expr(state.builder, token.span, diagnostic) } .open_bracket: { if array_starts_type(state) return try parse_type(state) return try parse_array_literal(state) } .open_paren: { state.next_token += 1 previous :: state.allow_record_literals previous_bit_or :: state.allow_bit_or state.allow_bit_or = true defer state.allow_bit_or = previous_bit_or state.allow_record_literals = true defer state.allow_record_literals = previous state.paren_depth += 1 defer state.paren_depth -= 1 expression :: try parse_expr(state) if next_is(state, .close_paren) { state.next_token += 1 return expression } diagnostic :: try diagnostics.add( state.diagnostics, .unexpected_token, current(state).span, ) return try ast.add_invalid_expr(state.builder, ast.Span{ start = token.span.start, end = ast.builder_span(state.builder, expression).end, source = token.span.source, }, diagnostic) } .int: { state.next_token += 1 return try ast.add_literal(state.builder, .int, token.span) } .float: { state.next_token += 1 return try ast.add_literal(state.builder, .float, token.span) } .string: { state.next_token += 1 return try ast.add_literal(state.builder, .string, token.span) } .true, .false: { state.next_token += 1 return try ast.add_literal(state.builder, .bool, token.span) } .null: { state.next_token += 1 return try ast.add_literal(state.builder, .null, token.span) } .undefined: { state.next_token += 1 return try ast.add_literal(state.builder, .undefined, token.span) } .ident: { state.next_token += 1 return try ast.add_identifier(state.builder, token.span, token.data.str_id) } else: { diagnostic :: try unexpected_diagnostic(state) if (!is_expression_recovery_boundary(token.kind)) state.next_token += 1 return try ast.add_invalid_expr(state.builder, token.span, diagnostic) } } } @hide:file is_expression_boundary proc(kind lexer.TokenKind) bool { return match kind { .close_paren, .newline, .close_curly, .open_curly, .else, .return, .yield, .if, .while, .for, .inline, .defer, .errdefer, .eof: true else: false } } @hide:file is_call_boundary proc(kind lexer.TokenKind) bool { return ( kind == .break or kind == .continue or (kind != .open_curly and is_expression_boundary(kind)) ) } @hide:file is_expression_recovery_boundary proc(kind lexer.TokenKind) bool { return ( kind == .comma or kind == .close_bracket or kind == .and or kind == .or or kind == .pipe or kind == .catch or is_expression_boundary(kind) ) } @hide:file skip_operand_newlines proc(state @mut State) void { if (state.paren_depth == 0 or !next_is(state, .newline)) return start :: state.next_token skip_newlines(state) kind :: current(state).kind if (kind != .close_paren and is_expression_boundary(kind)) state.next_token = start # a missing operand must not consume the declaration or assignment on the next line. if ( named_binding_starts(state) or (next_is(state, .ident) and is_assignment_operator(state.tokens[state.next_token + 1].kind)) ) state.next_token = start } @hide:file skip_argument_newlines proc(state @mut State) void { skip_operand_newlines(state) if (!next_is(state, .newline)) return start :: state.next_token skip_newlines(state) if (!next_is(state, .open_curly)) state.next_token = start } @hide:file skip_continuation_newlines proc(state @mut State) void { if (state.paren_depth == 0 or !next_is(state, .newline)) return start :: state.next_token skip_newlines(state) kind :: current(state).kind if ( kind != .close_paren and kind != .comma and kind != .open_paren and kind != .close_bracket and kind != .open_bracket and kind != .range and kind != .range_inclusive and kind != .pipe and kind != .bang and kind != .catch and enummap.get(&infix_binding_powers, kind) == null ) state.next_token = start } @hide:file recover_decl proc( state @mut State, start usize, ) ast.NodeId ! (mem.AllocError | diagnostics.CapacityError) { diagnostic :: try unexpected_diagnostic(state) span :: synchronize(state, start, false, false, true) return try ast.add_invalid_decl(state.builder, span, diagnostic) } @hide:file recover_statement proc( state @mut State, start usize, ) ast.NodeId ! (mem.AllocError | diagnostics.CapacityError) { diagnostic :: try unexpected_diagnostic(state) span :: synchronize(state, start, true, false, true) return try ast.add_invalid_stmt(state.builder, span, diagnostic) } @hide:file synchronize proc( state @mut State, start usize, stop_at_curly, allow_else, consume_newline bool, ) ast.Span { first :: state.tokens[start].span end := first.end curly usize := 0 paren usize := 0 square usize := 0 for (start..state.next_token) |i| { kind :: state.tokens[i].kind if (kind == .open_paren) paren += 1 if (kind == .close_paren and paren > 0) paren -= 1 if (kind == .open_bracket) square += 1 if (kind == .close_bracket and square > 0) square -= 1 } while !next_is(state, .eof) { if (curly == 0) { if (stop_at_curly and next_is(state, .close_curly)) break if (allow_else and next_is(state, .else)) break if ( next_is(state, .newline) and !recovery_closes_delimiters(state, paren, square) ) break } kind :: current(state).kind if (kind == .open_curly) curly += 1 if (kind == .close_curly and curly > 0) curly -= 1 if (kind == .open_paren) paren += 1 if (kind == .close_paren and paren > 0) paren -= 1 if (kind == .open_bracket) square += 1 if (kind == .close_bracket and square > 0) square -= 1 end = current(state).span.end state.next_token += 1 } if (consume_newline and next_is(state, .newline)) { end = current(state).span.end state.next_token += 1 } return ast.Span{ start = first.start, end = end, source = first.source, } } @hide:file recovery_closes_delimiters proc(state @State, paren_depth, square_depth usize) bool { if (paren_depth == 0 and square_depth == 0) return false paren := paren_depth square := square_depth for (state.next_token)..(state.tokens.len) |i| { kind :: state.tokens[i].kind match kind { .open_curly, .close_curly, .eof, .if, .else, .while, .for, .inline, .return, .break, .continue, .defer, .errdefer: return false .ident: { if (i + 1 < state.tokens.len) { next :: state.tokens[i + 1].kind if (next == .double_colon or next == .colon_equal or next == .ident or next == .proc) return false } } else: {} } if (kind == .open_paren) paren += 1 if (kind == .close_paren and paren > 0) paren -= 1 if (kind == .open_bracket) square += 1 if (kind == .close_bracket and square > 0) square -= 1 if (paren == 0 and square == 0) return true } return false } @hide:file unexpected_diagnostic proc(state @mut State) diagnostics.DiagnosticId ! (mem.AllocError | diagnostics.CapacityError) { token :: current(state) if (token.kind == .invalid and token.data.diagnostic != diagnostics.no_diagnostic) return token.data.diagnostic return try diagnostics.add(state.diagnostics, .unexpected_token, token.span) } @hide:file skip_newlines proc(state @mut State) void { while next_is(state, .newline) state.next_token += 1 } @hide:file current proc(state @State) lexer.Token { if (state.next_token >= state.tokens.len) return state.tokens[state.tokens.len - 1] return state.tokens[state.next_token] } @hide:file next_is proc(state @State, kind lexer.TokenKind) bool { return current(state).kind == kind } @hide:file next_n_is proc(state @State, count usize, kind lexer.TokenKind) bool { index :: state.next_token + count return index < state.tokens.len and state.tokens[index].kind == kind } @hide:file consume proc(state @mut State, kind lexer.TokenKind) lexer.Token ! SyntaxError { token :: current(state) if (token.kind != kind) return .unexpected_token state.next_token += 1 return token } @hide:file consume_either proc(state @mut State, kinds []lexer.TokenKind) lexer.Token ! SyntaxError { token :: current(state) for (kinds) |kind| { if token.kind == kind { state.next_token += 1 return token } } return .unexpected_token } @hide:file prefix_binding_powers :: $enummap.init_default( lexer.TokenKind, BindingPowerScalar, BindingPowerScalar(0), { minus = BindingPowerScalar(50), bang = BindingPowerScalar(50), tilde = BindingPowerScalar(50), ampersand = BindingPowerScalar(50), try = BindingPowerScalar(50), }) @hide:file infix_binding_powers :: $enummap.init_default(lexer.TokenKind, BindingPower, BindingPower{ left = BindingPowerScalar(0), right = BindingPowerScalar(0) }, { or = BindingPower{ left = BindingPowerScalar(2), right = BindingPowerScalar(3) }, and = BindingPower{ left = BindingPowerScalar(4), right = BindingPowerScalar(5) }, equal_equal = BindingPower{ left = BindingPowerScalar(10), right = BindingPowerScalar(11) }, not_equal = BindingPower{ left = BindingPowerScalar(10), right = BindingPowerScalar(11) }, less = BindingPower{ left = BindingPowerScalar(20), right = BindingPowerScalar(21) }, less_equal = BindingPower{ left = BindingPowerScalar(20), right = BindingPowerScalar(21) }, greater = BindingPower{ left = BindingPowerScalar(20), right = BindingPowerScalar(21) }, greater_equal = BindingPower{ left = BindingPowerScalar(20), right = BindingPowerScalar(21) }, ampersand = BindingPower{ left = BindingPowerScalar(22), right = BindingPowerScalar(23) }, xor = BindingPower{ left = BindingPowerScalar(22), right = BindingPowerScalar(23) }, pipe = BindingPower{ left = BindingPowerScalar(22), right = BindingPowerScalar(23) }, shift_left = BindingPower{ left = BindingPowerScalar(25), right = BindingPowerScalar(26) }, shift_right = BindingPower{ left = BindingPowerScalar(25), right = BindingPowerScalar(26) }, shift_left_saturating = BindingPower{ left = BindingPowerScalar(25), right = BindingPowerScalar(26) }, plus = BindingPower{ left = BindingPowerScalar(30), right = BindingPowerScalar(31) }, minus = BindingPower{ left = BindingPowerScalar(30), right = BindingPowerScalar(31) }, star = BindingPower{ left = BindingPowerScalar(40), right = BindingPowerScalar(41) }, slash = BindingPower{ left = BindingPowerScalar(40), right = BindingPowerScalar(41) }, }, ) @hide:file binary_operators :: $enummap.init_default( lexer.TokenKind, ast.BinaryOperator, ast.BinaryOperator.add, { and = .logical_and, or = .logical_or, plus = .add, minus = .subtract, star = .multiply, slash = .divide, ampersand = .bit_and, pipe = .bit_or, xor = .bit_xor, shift_left = .shift_left, shift_right = .shift_right, shift_left_saturating = .shift_left_saturating, equal_equal = .equal, not_equal = .not_equal, less = .less, less_equal = .less_equal, greater = .greater, greater_equal = .greater_equal, }, ) @hide:file is_assignment_operator proc(kind lexer.TokenKind) bool { return kind == .equal or enummap.get(&assignment_operators, kind) != null } @hide:file assignment_operators :: $enummap.init_default(lexer.TokenKind, ast.BinaryOperator, ast.BinaryOperator.add, { plus_equal = .add, minus_equal = .subtract, star_equal = .multiply, slash_equal = .divide, ampersand_equal = .bit_and, pipe_equal = .bit_or, xor_equal = .bit_xor, shift_left_equal = .shift_left, shift_right_equal = .shift_right, shift_left_saturating_equal = .shift_left_saturating, })