brolang/compiler/parser/parser.odin

package parser

import "../ast"
import "../source"
import "../symbol"
import "../token"
import "../types"
import "base:intrinsics"
import "core:fmt"
import "core:strconv"
import "core:strings"
import "core:unicode/utf8"

Parser :: struct {
	tokens:      ^token.Stream,
	source_file: ^source.Source,
	diagnostics: ^source.Diagnostics,
	module:      ast.Module,
	pkg:         ast.Package_Id,
	file:        ast.File_Id,
	cursor:      int,
	delimiter_depth: int,
}

MAX_EXPRESSION_NESTING :: 256

token_text :: proc(parser: ^Parser, tok: token.Token) -> string {
	if tok.span.end < tok.span.start || int(tok.span.end) > len(parser.source_file.text) {
		return ""
	}
	return parser.source_file.text[int(tok.span.start):int(tok.span.end)]
}

span_from :: proc(first, last: source.Span) -> source.Span {
	return source.Span{file=first.file, start=first.start, end=last.end}
}

current :: proc(parser: ^Parser) -> token.Token {
	return parser.tokens.items[min(parser.cursor, len(parser.tokens.items)-1)]
}

previous :: proc(parser: ^Parser) -> token.Token {
	return parser.tokens.items[max(parser.cursor-1, 0)]
}

advance :: proc(parser: ^Parser) -> token.Token {
	result := current(parser)
	if result.kind != .Eof {
		parser.cursor += 1
	}
	return result
}

allow :: proc(parser: ^Parser, kind: token.Kind) -> (token.Token, bool) {
	if current(parser).kind == kind {
		return advance(parser), true
	}
	return current(parser), false
}

skip_newlines :: proc(parser: ^Parser) {
	for current(parser).kind == .Newline {
		advance(parser)
	}
}

add_expr :: proc(parser: ^Parser, expr: ast.Expr) -> ast.Expr_Id {
	id := ast.expr_id(len(parser.module.exprs))
	append(&parser.module.exprs, expr)
	return id
}

invalid_expr :: proc(parser: ^Parser, span: source.Span, message: string) -> ast.Expr_Id {
	id := source.add(parser.diagnostics, span, message)
	return add_expr(parser, ast.Expr{
		kind=.Invalid,
		span=span,
		left=ast.INVALID_EXPR,
		right=ast.INVALID_EXPR,
		diagnostic=id,
	})
}

is_type_token :: proc(kind: token.Kind) -> bool {
	#partial switch kind {
	case .Keyword_Int, .Keyword_I8, .Keyword_I16, .Keyword_I32, .Keyword_I64,
	     .Keyword_U8, .Keyword_U16, .Keyword_U32, .Keyword_U64,
	     .Keyword_Isize, .Keyword_Usize, .Keyword_F32, .Keyword_F64,
	     .Keyword_C_Char, .Keyword_C_Schar, .Keyword_C_Uchar,
	     .Keyword_C_Short, .Keyword_C_Ushort, .Keyword_C_Int, .Keyword_C_Uint,
	     .Keyword_C_Long, .Keyword_C_Ulong, .Keyword_C_Longlong, .Keyword_C_Ulonglong,
	     .Keyword_C_Float, .Keyword_C_Double, .Keyword_C_Longdouble,
	     .Keyword_Void, .Identifier, .Question, .At, .Star, .Left_Bracket:
		return true
	}
	return false
}

decode_character :: proc(parser: ^Parser, tok: token.Token) -> (u64, bool) {
	text := token_text(parser, tok)
	if len(text) < 3 {
		return 0, false
	}
	contents := text[1:len(text)-1]
	if len(contents) == 2 && contents[0] == '\\' {
		switch contents[1] {
		case '0':  return 0, true
		case 'n':  return '\n', true
		case 'r':  return '\r', true
		case 't':  return '\t', true
		case '\\': return '\\', true
		case '\'': return '\'', true
		}
		return 0, false
	}
	value, width := utf8.decode_rune_in_string(contents)
	return u64(value), width == len(contents)
}

parse_type_constant :: proc(parser: ^Parser) -> (u64, bool) {
	negative := false
	if _, ok := allow(parser, .Minus); ok {
		negative = true
	}
	tok := current(parser)
	if tok.kind == .Integer {
		advance(parser)
		value, ok := parse_integer_magnitude(token_text(parser, tok))
		if !ok {
			return 0, false
		}
		if negative {
			return transmute(u64)-i64(value), true
		}
		return value, true
	}
	if !negative && tok.kind == .Character {
		advance(parser)
		return decode_character(parser, tok)
	}
	source.add(parser.diagnostics, tok.span, "expected an integer or character constant")
	return 0, false
}

parse_type :: proc(parser: ^Parser) -> ast.Type_Syntax {
	tok := current(parser)
	if tok.kind == .Question {
		advance(parser)
		child := parse_type(parser)
		return types.intern(&parser.module.type_store, types.Node{kind=.Optional, child=child})
	}
	if tok.kind == .At || tok.kind == .Star {
		many := tok.kind == .Star
		advance(parser)
		_, mutable := allow(parser, .Keyword_Mut)
		child := parse_type(parser)
		return types.intern(&parser.module.type_store, types.Node{
			kind=.Pointer,
			child=child,
			mutable=mutable,
			many=many,
		})
	}
	if tok.kind == .Left_Bracket {
		advance(parser)
		node := types.Node{}
		if _, ok := allow(parser, .Right_Bracket); ok {
			node.kind = .Slice
		} else if _, ok := allow(parser, .Semicolon); ok {
			node.kind = .Slice
			node.has_sentinel = true
			node.sentinel, _ = parse_type_constant(parser)
			if _, ok = allow(parser, .Right_Bracket); !ok {
				source.add(parser.diagnostics, current(parser).span, "expected ']' after sentinel slice type")
			}
		} else {
			node.kind = .Array
			if _, ok := allow(parser, .Underscore); ok {
				node.inferred_count = true
			} else {
				count, ok := parse_type_constant(parser)
				if ok {
					node.count = count
				}
			}
			if _, ok := allow(parser, .Semicolon); ok {
				node.has_sentinel = true
				node.sentinel, _ = parse_type_constant(parser)
			}
			if _, ok := allow(parser, .Right_Bracket); !ok {
				source.add(parser.diagnostics, current(parser).span, "expected ']' after array type")
			}
		}
		_, node.mutable = allow(parser, .Keyword_Mut)
		node.child = parse_type(parser)
		return types.intern(&parser.module.type_store, node)
	}
	#partial switch tok.kind {
	case .Keyword_Int:
		advance(parser)
		return types.INT
	case .Keyword_I8:
		advance(parser)
		return types.I8
	case .Keyword_I16:
		advance(parser)
		return types.I16
	case .Keyword_I32:
		advance(parser)
		return types.I32
	case .Keyword_I64:
		advance(parser)
		return types.I64
	case .Keyword_U8:
		advance(parser)
		return types.U8
	case .Keyword_U16:
		advance(parser)
		return types.U16
	case .Keyword_U32:
		advance(parser)
		return types.U32
	case .Keyword_U64:
		advance(parser)
		return types.U64
	case .Keyword_Isize:
		advance(parser)
		return types.ISIZE
	case .Keyword_Usize:
		advance(parser)
		return types.USIZE
	case .Keyword_F32:
		advance(parser)
		return types.F32
	case .Keyword_F64:
		advance(parser)
		return types.F64
	case .Keyword_C_Char:
		advance(parser)
		return types.C_CHAR
	case .Keyword_C_Schar:
		advance(parser)
		return types.C_SCHAR
	case .Keyword_C_Uchar:
		advance(parser)
		return types.C_UCHAR
	case .Keyword_C_Short:
		advance(parser)
		return types.C_SHORT
	case .Keyword_C_Ushort:
		advance(parser)
		return types.C_USHORT
	case .Keyword_C_Int:
		advance(parser)
		return types.C_INT
	case .Keyword_C_Uint:
		advance(parser)
		return types.C_UINT
	case .Keyword_C_Long:
		advance(parser)
		return types.C_LONG
	case .Keyword_C_Ulong:
		advance(parser)
		return types.C_ULONG
	case .Keyword_C_Longlong:
		advance(parser)
		return types.C_LONGLONG
	case .Keyword_C_Ulonglong:
		advance(parser)
		return types.C_ULONGLONG
	case .Keyword_C_Float:
		advance(parser)
		return types.C_FLOAT
	case .Keyword_C_Double:
		advance(parser)
		return types.C_DOUBLE
	case .Keyword_C_Longdouble:
		advance(parser)
		return types.C_LONGDOUBLE
	case .Keyword_Void:
		advance(parser)
		return types.VOID
	case .Identifier:
		first := advance(parser)
		name := first
		qualifier := symbol.INVALID
		if _, ok := allow(parser, .Dot); ok {
			qualifier = first.symbol
			if current(parser).kind != .Identifier {
				source.add(parser.diagnostics, current(parser).span, "expected a type name after '.'")
				return types.INVALID
			}
			name = advance(parser)
		}
		return types.named(
			&parser.module.type_store,
			u32(parser.pkg),
			u32(name.symbol),
			u32(qualifier),
			u32(parser.file),
		)
	}
	source.add(parser.diagnostics, tok.span, "expected a type")
	return types.INVALID
}

skip_parenthesized :: proc(parser: ^Parser) -> source.Span {
	start := current(parser)
	depth := 0
	end := start
	for current(parser).kind != .Eof {
		tok := advance(parser)
		end = tok
		if tok.kind == .Left_Paren {
			depth += 1
		} else if tok.kind == .Right_Paren {
			depth -= 1
			if depth == 0 {
				break
			}
		}
	}
	return span_from(start.span, end.span)
}

parse_call :: proc(parser: ^Parser, qualifier: symbol.Id, first, name: token.Token, nesting: int) -> ast.Expr_Id {
	if nesting >= MAX_EXPRESSION_NESTING {
		span := skip_parenthesized(parser)
		return invalid_expr(parser, span, "expression nesting exceeds 256 levels")
	}
	left_paren := advance(parser)
	parser.delimiter_depth += 1
	defer parser.delimiter_depth -= 1
	args: [dynamic]ast.Expr_Id
	args.allocator = parser.module.allocator
	skip_newlines(parser)
	for current(parser).kind != .Right_Paren && current(parser).kind != .Eof {
		append(&args, parse_expression_bp(parser, 0, nesting+1))
		skip_newlines(parser)
		if _, ok := allow(parser, .Comma); ok {
			skip_newlines(parser)
			continue
		}
		break
	}
	right_paren, ok := allow(parser, .Right_Paren)
	if !ok {
		source.add(parser.diagnostics, current(parser).span, "expected ')' after call arguments")
		right_paren = left_paren
	}
	return add_expr(parser, ast.Expr{
		kind=.Call,
		span=source.Span{file=name.span.file, start=first.span.start, end=right_paren.span.end},
		qualifier=qualifier,
		name=name.symbol,
		args=args[:],
		left=ast.INVALID_EXPR,
		right=ast.INVALID_EXPR,
		diagnostic=source.INVALID_DIAGNOSTIC,
	})
}

parse_array_literal :: proc(parser: ^Parser, nesting: int) -> ast.Expr_Id {
	start := advance(parser)
	parser.delimiter_depth += 1
	defer parser.delimiter_depth -= 1
	args: [dynamic]ast.Expr_Id
	args.allocator = parser.module.allocator
	skip_newlines(parser)
	for current(parser).kind != .Right_Bracket && current(parser).kind != .Eof {
		append(&args, parse_expression_bp(parser, 0, nesting+1))
		skip_newlines(parser)
		if _, ok := allow(parser, .Comma); ok {
			skip_newlines(parser)
			continue
		}
		break
	}
	end, ok := allow(parser, .Right_Bracket)
	if !ok {
		source.add(parser.diagnostics, current(parser).span, "expected ']' after array literal")
		end = start
	}
	return add_expr(parser, ast.Expr{
		kind=.Array,
		span=span_from(start.span, end.span),
		args=args[:],
		left=ast.INVALID_EXPR,
		right=ast.INVALID_EXPR,
		diagnostic=source.INVALID_DIAGNOSTIC,
	})
}

parse_struct_literal :: proc(
	parser: ^Parser,
	qualifier: symbol.Id,
	first, name: token.Token,
	nesting: int,
) -> ast.Expr_Id {
	left_brace := advance(parser)
	parser.delimiter_depth += 1
	defer parser.delimiter_depth -= 1
	args: [dynamic]ast.Expr_Id
	args.allocator = parser.module.allocator
	skip_newlines(parser)
	for current(parser).kind != .Right_Brace && current(parser).kind != .Eof {
		field := current(parser)
		if field.kind != .Identifier {
			source.add(parser.diagnostics, field.span, "expected a keyed struct field initializer")
			break
		}
		advance(parser)
		if _, ok := allow(parser, .Equal); !ok {
			source.add(parser.diagnostics, current(parser).span, "expected '=' after struct field name")
		}
		skip_newlines(parser)
		value := parse_expression_bp(parser, 0, nesting+1)
		append(&args, add_expr(parser, ast.Expr{
			kind=.Keyed,
			span=span_from(field.span, parser.module.exprs[value].span),
			name=field.symbol,
			left=value,
			right=ast.INVALID_EXPR,
			diagnostic=source.INVALID_DIAGNOSTIC,
		}))
		skip_newlines(parser)
		if _, ok := allow(parser, .Comma); ok {
			skip_newlines(parser)
			continue
		}
		break
	}
	right_brace, ok := allow(parser, .Right_Brace)
	if !ok {
		source.add(parser.diagnostics, current(parser).span, "expected '}' after struct literal")
		right_brace = left_brace
	}
	return add_expr(parser, ast.Expr{
		kind=.Struct_Literal,
		span=source.Span{file=name.span.file, start=first.span.start, end=right_brace.span.end},
		qualifier=qualifier,
		name=name.symbol,
		args=args[:],
		left=ast.INVALID_EXPR,
		right=ast.INVALID_EXPR,
		diagnostic=source.INVALID_DIAGNOSTIC,
	})
}

parse_integer_magnitude :: proc(text: string) -> (u64, bool) {
	value: u64
	for byte in transmute([]byte)text {
		if byte < '0' || byte > '9' {
			return 0, false
		}
		next, overflow := intrinsics.overflow_mul(value, u64(10))
		if overflow {
			return 0, false
		}
		value, overflow = intrinsics.overflow_add(next, u64(byte-'0'))
		if overflow {
			return 0, false
		}
	}
	return value, len(text) > 0
}

parse_primary :: proc(parser: ^Parser, nesting: int) -> ast.Expr_Id {
	tok := current(parser)
	#partial switch tok.kind {
	case .Integer:
		advance(parser)
		value, ok := parse_integer_magnitude(token_text(parser, tok))
		if !ok {
			return invalid_expr(parser, tok.span, "integer literal magnitude does not fit in u64")
		}
		return add_expr(parser, ast.Expr{
			kind=.Integer,
			span=tok.span,
			integer=value,
			left=ast.INVALID_EXPR,
			right=ast.INVALID_EXPR,
			diagnostic=source.INVALID_DIAGNOSTIC,
		})
	case .Character:
		advance(parser)
		value, ok := decode_character(parser, tok)
		if !ok {
			return invalid_expr(parser, tok.span, "character literal must contain one Unicode code point")
		}
		return add_expr(parser, ast.Expr{
			kind=.Integer,
			span=tok.span,
			integer=value,
			left=ast.INVALID_EXPR,
			right=ast.INVALID_EXPR,
			diagnostic=source.INVALID_DIAGNOSTIC,
		})
	case .Float:
		advance(parser)
		value, ok := strconv.parse_f64(token_text(parser, tok))
		if !ok {
			return invalid_expr(parser, tok.span, "invalid floating-point literal")
		}
		return add_expr(parser, ast.Expr{
			kind=.Float,
			span=tok.span,
			integer=transmute(u64)value,
			left=ast.INVALID_EXPR,
			right=ast.INVALID_EXPR,
			diagnostic=source.INVALID_DIAGNOSTIC,
		})
	case .String:
		advance(parser)
		value := decode_import_path(parser, tok)
		id := u64(len(parser.module.strings))
		append(&parser.module.strings, value)
		return add_expr(parser, ast.Expr{
			kind=.String,
			span=tok.span,
			integer=id,
			left=ast.INVALID_EXPR,
			right=ast.INVALID_EXPR,
			diagnostic=source.INVALID_DIAGNOSTIC,
		})
	case .Keyword_None:
		advance(parser)
		return add_expr(parser, ast.Expr{
			kind=.None,
			span=tok.span,
			left=ast.INVALID_EXPR,
			right=ast.INVALID_EXPR,
			diagnostic=source.INVALID_DIAGNOSTIC,
		})
	case .Left_Bracket:
		return parse_array_literal(parser, nesting)
	case .Identifier:
		first := advance(parser)
		name := first
		qualifier := symbol.INVALID
		if _, ok := allow(parser, .Dot); ok {
			if current(parser).kind != .Identifier {
				return invalid_expr(parser, current(parser).span, "expected a package member after '.'")
			}
			qualifier = first.symbol
			name = advance(parser)
		}
		if current(parser).kind == .Left_Paren {
			return parse_call(parser, qualifier, first, name, nesting)
		}
		if current(parser).kind == .Left_Brace {
			return parse_struct_literal(parser, qualifier, first, name, nesting)
		}
		return add_expr(parser, ast.Expr{
			kind=.Name,
			span=span_from(first.span, name.span),
			qualifier=qualifier,
			name=name.symbol,
			left=ast.INVALID_EXPR,
			right=ast.INVALID_EXPR,
			diagnostic=source.INVALID_DIAGNOSTIC,
		})
	case .Underscore:
		advance(parser)
		return invalid_expr(parser, tok.span, "'_' is a write-only sink and cannot be read")
	case .Left_Paren:
		if nesting >= MAX_EXPRESSION_NESTING {
			span := skip_parenthesized(parser)
			return invalid_expr(parser, span, "expression nesting exceeds 256 levels")
		}
		advance(parser)
		parser.delimiter_depth += 1
		defer parser.delimiter_depth -= 1
		skip_newlines(parser)
		expr := parse_expression_bp(parser, 0, nesting+1)
		skip_newlines(parser)
		if _, ok := allow(parser, .Right_Paren); !ok {
			source.add(parser.diagnostics, current(parser).span, "expected ')'")
		}
		return expr
	case .Invalid:
		advance(parser)
		return add_expr(parser, ast.Expr{
			kind=.Invalid,
			span=tok.span,
			left=ast.INVALID_EXPR,
			right=ast.INVALID_EXPR,
			diagnostic=tok.diagnostic,
		})
	}
	if tok.kind != .Newline && tok.kind != .Right_Brace && tok.kind != .Eof {
		advance(parser)
	}
	return invalid_expr(parser, tok.span, "expected an expression")
}

infix_binding_power :: proc(kind: token.Kind) -> (left, right: int, ok: bool) {
	#partial switch kind {
	case .Keyword_Orelse:
		return 2, 3, true
	case .Plus:
		return 10, 11, true
	}
	return 0, 0, false
}

prefix_binding_power :: proc(kind: token.Kind) -> (right: int, ok: bool) {
	#partial switch kind {
	case .Minus, .Ampersand:
		return 20, true
	}
	return 0, false
}

parse_expression_bp :: proc(parser: ^Parser, minimum_binding_power, nesting: int) -> ast.Expr_Id {
	if nesting > MAX_EXPRESSION_NESTING {
		tok := current(parser)
		if tok.kind != .Newline && tok.kind != .Right_Brace && tok.kind != .Eof {
			advance(parser)
		}
		return invalid_expr(parser, tok.span, "expression nesting exceeds 256 levels")
	}
	left := ast.INVALID_EXPR
	if right_power, ok := prefix_binding_power(current(parser).kind); ok {
		operator := advance(parser)
		if parser.delimiter_depth > 0 {
			skip_newlines(parser)
		}
		operand := parse_expression_bp(parser, right_power, nesting+1)
		operand_expr := parser.module.exprs[operand]
		left = add_expr(parser, ast.Expr{
			kind=.Address if operator.kind == .Ampersand else .Negate,
			span=span_from(operator.span, operand_expr.span),
			left=operand,
			right=ast.INVALID_EXPR,
			diagnostic=source.INVALID_DIAGNOSTIC,
		})
	} else {
		left = parse_primary(parser, nesting)
	}
	if parser.delimiter_depth > 0 {
		skip_newlines(parser)
	}
	for {
		if current(parser).kind == .Caret || current(parser).kind == .Question {
			operator := advance(parser)
			left_expr := parser.module.exprs[left]
			left = add_expr(parser, ast.Expr{
				kind=.Deref if operator.kind == .Caret else .Unwrap,
				span=span_from(left_expr.span, operator.span),
				left=left,
				right=ast.INVALID_EXPR,
				diagnostic=source.INVALID_DIAGNOSTIC,
			})
			continue
		}
		if current(parser).kind == .Dot {
			advance(parser)
			field := current(parser)
			if field.kind != .Identifier {
				left = invalid_expr(parser, field.span, "expected a field name after '.'")
				continue
			}
			advance(parser)
			left_expr := parser.module.exprs[left]
			left = add_expr(parser, ast.Expr{
				kind=.Field,
				span=span_from(left_expr.span, field.span),
				name=field.symbol,
				left=left,
				right=ast.INVALID_EXPR,
				diagnostic=source.INVALID_DIAGNOSTIC,
			})
			continue
		}
		if current(parser).kind == .Left_Bracket {
			start_token := advance(parser)
			parser.delimiter_depth += 1
			skip_newlines(parser)
			start_expr := ast.INVALID_EXPR
			end_expr := ast.INVALID_EXPR
			slicing := false
			if _, ok := allow(parser, .Range); ok {
				slicing = true
			} else {
				start_expr = parse_expression_bp(parser, 0, nesting+1)
				skip_newlines(parser)
				if _, ok := allow(parser, .Range); ok {
					slicing = true
				}
			}
			skip_newlines(parser)
			if slicing && current(parser).kind != .Right_Bracket {
				end_expr = parse_expression_bp(parser, 0, nesting+1)
				skip_newlines(parser)
			}
			end_token, ok := allow(parser, .Right_Bracket)
			if !ok {
				source.add(parser.diagnostics, current(parser).span, "expected ']' after index or slice")
				end_token = start_token
			}
			parser.delimiter_depth -= 1
			left_expr := parser.module.exprs[left]
			if slicing {
				args := make([]ast.Expr_Id, 2, parser.module.allocator)
				args[0] = start_expr
				args[1] = end_expr
				left = add_expr(parser, ast.Expr{
					kind=.Slice,
					span=span_from(left_expr.span, end_token.span),
					args=args,
					left=left,
					right=ast.INVALID_EXPR,
					diagnostic=source.INVALID_DIAGNOSTIC,
				})
			} else {
				left = add_expr(parser, ast.Expr{
					kind=.Index,
					span=span_from(left_expr.span, end_token.span),
					left=left,
					right=start_expr,
					diagnostic=source.INVALID_DIAGNOSTIC,
				})
			}
			continue
		}
		left_power, right_power, ok := infix_binding_power(current(parser).kind)
		if !ok || left_power < minimum_binding_power {
			break
		}
		operator := advance(parser)
		skip_newlines(parser)
		right := parse_expression_bp(parser, right_power, nesting+1)
		left_expr := parser.module.exprs[left]
		right_expr := parser.module.exprs[right]
		left = add_expr(parser, ast.Expr{
			kind=.Orelse if operator.kind == .Keyword_Orelse else .Add,
			span=span_from(left_expr.span, right_expr.span),
			left=left,
			right=right,
			diagnostic=source.INVALID_DIAGNOSTIC,
		})
		if parser.delimiter_depth > 0 {
			skip_newlines(parser)
		}
	}
	return left
}

parse_expression :: proc(parser: ^Parser) -> ast.Expr_Id {
	return parse_expression_bp(parser, 0, 0)
}

finish_statement :: proc(parser: ^Parser, allow_closing_brace := false) -> source.Diagnostic_Id {
	if current(parser).kind == .Newline {
		skip_newlines(parser)
		return source.INVALID_DIAGNOSTIC
	}
	if current(parser).kind == .Eof || allow_closing_brace && current(parser).kind == .Right_Brace {
		return source.INVALID_DIAGNOSTIC
	}
	diagnostic := source.add(
		parser.diagnostics,
		current(parser).span,
		"completed statements must be followed by a newline",
	)
	for current(parser).kind != .Newline &&
	    current(parser).kind != .Right_Brace &&
	    current(parser).kind != .Eof {
		advance(parser)
	}
	skip_newlines(parser)
	return diagnostic
}

parse_return :: proc(parser: ^Parser) -> ast.Stmt_Id {
	start := advance(parser)
	skip_newlines(parser)
	if current(parser).kind == .Underscore {
		end := advance(parser)
		id := ast.stmt_id(len(parser.module.statements))
		append(&parser.module.statements, ast.Stmt{
			kind=.Return,
			span=span_from(start.span, end.span),
			name=end.symbol,
			expr=ast.INVALID_EXPR,
			diagnostic=source.INVALID_DIAGNOSTIC,
		})
		return id
	}
	expr := parse_expression(parser)
	id := ast.stmt_id(len(parser.module.statements))
	append(&parser.module.statements, ast.Stmt{
		kind=.Return,
		span=span_from(start.span, parser.module.exprs[expr].span),
		expr=expr,
		diagnostic=source.INVALID_DIAGNOSTIC,
	})
	return id
}

starts_declared_type :: proc(parser: ^Parser) -> bool {
	if current(parser).kind != .Left_Bracket {
		return is_type_token(current(parser).kind)
	}
	depth := 0
	cursor := parser.cursor
	for cursor < len(parser.tokens.items) {
		kind := parser.tokens.items[cursor].kind
		if kind == .Left_Bracket {
			depth += 1
		} else if kind == .Right_Bracket {
			depth -= 1
			if depth == 0 {
				cursor += 1
				break
			}
		}
		cursor += 1
	}
	if cursor < len(parser.tokens.items) && parser.tokens.items[cursor].kind == .Keyword_Mut {
		cursor += 1
	}
	return cursor < len(parser.tokens.items) && is_type_token(parser.tokens.items[cursor].kind)
}

parse_statement :: proc(parser: ^Parser) -> ast.Stmt_Id {
	if current(parser).kind == .Keyword_Return {
		return parse_return(parser)
	}

	if current(parser).kind == .Identifier || current(parser).kind == .Underscore {
		start_cursor := parser.cursor
		name := advance(parser)
		type_syntax := types.INVALID
		had_type := false
		if starts_declared_type(parser) {
			type_syntax = parse_type(parser)
			had_type = true
		}
		operator := current(parser)
		if operator.kind == .Colon_Colon || operator.kind == .Equal {
			advance(parser)
			skip_newlines(parser)
			expr := parse_expression(parser)
			kind := ast.Stmt_Kind.Assignment
			immutable := false
			if operator.kind == .Colon_Colon || had_type {
				kind = .Declaration
				immutable = operator.kind == .Colon_Colon
			}
			id := ast.stmt_id(len(parser.module.statements))
			append(&parser.module.statements, ast.Stmt{
				kind=kind,
				span=span_from(name.span, parser.module.exprs[expr].span),
				name=name.symbol,
				type=type_syntax,
				immutable=immutable,
				target=ast.INVALID_EXPR,
				expr=expr,
				diagnostic=source.INVALID_DIAGNOSTIC,
			})
			return id
		}
		parser.cursor = start_cursor
	}

	expr := parse_expression(parser)
	if _, ok := allow(parser, .Equal); ok {
		skip_newlines(parser)
		value := parse_expression(parser)
		id := ast.stmt_id(len(parser.module.statements))
		append(&parser.module.statements, ast.Stmt{
			kind=.Assignment,
			span=span_from(parser.module.exprs[expr].span, parser.module.exprs[value].span),
			target=expr,
			expr=value,
			diagnostic=source.INVALID_DIAGNOSTIC,
		})
		return id
	}
	id := ast.stmt_id(len(parser.module.statements))
	append(&parser.module.statements, ast.Stmt{
		kind=.Expression,
		span=parser.module.exprs[expr].span,
		expr=expr,
		diagnostic=source.INVALID_DIAGNOSTIC,
	})
	return id
}

parse_params :: proc(parser: ^Parser) -> ([]ast.Param, bool) {
	params: [dynamic]ast.Param
	params.allocator = parser.module.allocator
	variadic := false
	skip_newlines(parser)
	for current(parser).kind != .Right_Paren && current(parser).kind != .Eof {
		if current(parser).kind == .Ellipsis {
			marker := advance(parser)
			if variadic {
				source.add(parser.diagnostics, marker.span, "duplicate variadic marker")
			}
			variadic = true
			skip_newlines(parser)
			if _, ok := allow(parser, .Comma); ok {
				skip_newlines(parser)
			}
			if current(parser).kind != .Right_Paren {
				source.add(parser.diagnostics, current(parser).span, "variadic marker must be the final parameter")
			}
			continue
		}
		names: [dynamic]token.Token
		names.allocator = parser.module.allocator
		for {
			if current(parser).kind != .Identifier {
				source.add(parser.diagnostics, current(parser).span, "expected parameter name")
				break
			}
			append(&names, advance(parser))
			if is_type_token(current(parser).kind) {
				break
			}
			if _, ok := allow(parser, .Comma); !ok {
				source.add(parser.diagnostics, current(parser).span, "expected ',' or parameter type")
				break
			}
			skip_newlines(parser)
		}
		type_syntax := parse_type(parser)
		for name in names {
			append(&params, ast.Param{name=name.symbol, span=name.span, type=type_syntax})
		}
		delete(names)
		skip_newlines(parser)
		if _, ok := allow(parser, .Comma); ok {
			skip_newlines(parser)
			continue
		}
		break
	}
	return params[:], variadic
}

parse_function :: proc(parser: ^Parser, name: token.Token, c_abi: bool) {
	func_token := advance(parser)
	if _, ok := allow(parser, .Left_Paren); !ok {
		source.add(parser.diagnostics, current(parser).span, "expected '(' after 'func'")
	}
	params, variadic := parse_params(parser)
	if _, ok := allow(parser, .Right_Paren); !ok {
		source.add(parser.diagnostics, current(parser).span, "expected ')' after parameters")
	}
	skip_newlines(parser)
	result := parse_type(parser)
	end := previous(parser)
	ended_by_newline := current(parser).kind == .Newline
	if current(parser).kind == .Newline {
		skip_newlines(parser)
	}
	if current(parser).kind != .Left_Brace {
		if !ended_by_newline && current(parser).kind != .Eof {
			_ = finish_statement(parser)
		}
		_ = ast.function_id(len(parser.module.functions))
		append(&parser.module.functions, ast.Function{
			span=span_from(name.span, end.span),
			name=name.symbol,
			pkg=parser.pkg,
			file=parser.file,
			c_abi=c_abi,
			has_body=false,
			variadic=variadic,
			params=params,
			result=result,
			diagnostic=source.INVALID_DIAGNOSTIC,
		})
		return
	}
	advance(parser)

	body: [dynamic]ast.Stmt_Id
	body.allocator = parser.module.allocator
	skip_newlines(parser)
	for current(parser).kind != .Right_Brace && current(parser).kind != .Eof {
		append(&body, parse_statement(parser))
		if diagnostic := finish_statement(parser, true); diagnostic != source.INVALID_DIAGNOSTIC {
			statement_id := ast.stmt_id(len(parser.module.statements))
			append(&parser.module.statements, ast.Stmt{
				kind=.Invalid,
				span=current(parser).span,
				expr=ast.INVALID_EXPR,
				diagnostic=diagnostic,
			})
			append(&body, statement_id)
		}
	}
	end = current(parser)
	if _, ok := allow(parser, .Right_Brace); !ok {
		source.add(parser.diagnostics, current(parser).span, "expected '}' after function body")
		end = func_token
	}
	_ = ast.function_id(len(parser.module.functions))
	append(&parser.module.functions, ast.Function{
		span=span_from(name.span, end.span),
		name=name.symbol,
		pkg=parser.pkg,
		file=parser.file,
		c_abi=c_abi,
		has_body=true,
		variadic=variadic,
		params=params,
		result=result,
		body=body[:],
		diagnostic=source.INVALID_DIAGNOSTIC,
	})
}

parse_struct :: proc(parser: ^Parser, name: token.Token, c_layout: bool) {
	start := advance(parser)
	id := types.named(&parser.module.type_store, u32(parser.pkg), u32(name.symbol))
	ended_by_newline := current(parser).kind == .Newline
	skip_newlines(parser)
	if current(parser).kind != .Left_Brace {
		if !c_layout {
			source.add(parser.diagnostics, start.span, "native struct declarations require a body")
		}
		if !types.define_struct(&parser.module.type_store, id, nil, c_layout, true) {
			source.addf(parser.diagnostics, name.span, "duplicate type declaration '%s'", token_text(parser, name))
		}
		if !ended_by_newline {
			_ = finish_statement(parser)
		}
		return
	}
	advance(parser)
	fields: [dynamic]types.Field
	fields.allocator = parser.module.allocator
	defer delete(fields)
	skip_newlines(parser)
	for current(parser).kind != .Right_Brace && current(parser).kind != .Eof {
		if current(parser).kind != .Identifier {
			source.add(parser.diagnostics, current(parser).span, "expected a struct field name")
			for current(parser).kind != .Newline &&
			    current(parser).kind != .Right_Brace &&
			    current(parser).kind != .Eof {
				advance(parser)
			}
			skip_newlines(parser)
			continue
		}
		field_name := advance(parser)
		field_type := parse_type(parser)
		append(&fields, types.Field{name=u32(field_name.symbol), type=field_type})
		if _, ok := allow(parser, .Comma); ok {
			skip_newlines(parser)
			continue
		}
		_ = finish_statement(parser, true)
	}
	if _, ok := allow(parser, .Right_Brace); !ok {
		source.add(parser.diagnostics, current(parser).span, "expected '}' after struct fields")
	}
	if !types.define_struct(&parser.module.type_store, id, fields[:], c_layout, false) {
		source.addf(parser.diagnostics, name.span, "duplicate type declaration '%s'", token_text(parser, name))
	}
	_ = finish_statement(parser)
}

decode_import_path :: proc(parser: ^Parser, tok: token.Token) -> string {
	text := token_text(parser, tok)
	if len(text) < 2 {
		return fmt.aprintf("", allocator=parser.module.allocator)
	}
	builder := strings.builder_make(parser.module.allocator)
	defer strings.builder_destroy(&builder)
	for index := 1; index < len(text)-1; index += 1 {
		value := text[index]
		if value == '\\' && index+1 < len(text)-1 {
			index += 1
			value = text[index]
			switch value {
			case 'n': value = '\n'
			case 'r': value = '\r'
			case 't': value = '\t'
			case '0': value = 0
			case:
			}
		}
		strings.write_byte(&builder, value)
	}
	return fmt.aprintf("%s", strings.to_string(builder), allocator=parser.module.allocator)
}

parse_import :: proc(parser: ^Parser, alias: token.Token, start: token.Token) {
	skip_newlines(parser)
	path_token := current(parser)
	if path_token.kind != .String {
		source.add(parser.diagnostics, path_token.span, "expected an import path string")
		if path_token.kind != .Newline && path_token.kind != .Eof {
			advance(parser)
		}
		_ = ast.import_id(len(parser.module.imports))
		append(&parser.module.imports, ast.Import{
			span=start.span,
			alias=alias.symbol,
			pkg=parser.pkg,
			file=parser.file,
			target=ast.INVALID_PACKAGE,
			valid=false,
			diagnostic=source.INVALID_DIAGNOSTIC,
		})
		_ = finish_statement(parser)
		return
	}
	advance(parser)
	_ = ast.import_id(len(parser.module.imports))
	append(&parser.module.imports, ast.Import{
		span=span_from(start.span, path_token.span),
		alias=alias.symbol,
		path=decode_import_path(parser, path_token),
		pkg=parser.pkg,
		file=parser.file,
		target=ast.INVALID_PACKAGE,
		valid=true,
		diagnostic=source.INVALID_DIAGNOSTIC,
	})
	_ = finish_statement(parser)
}

parse_top_level :: proc(parser: ^Parser) {
	if current(parser).kind == .Keyword_Import {
		start := advance(parser)
		parse_import(parser, token.Token{}, start)
		return
	}
	if current(parser).kind != .Identifier {
		source.add(parser.diagnostics, current(parser).span, "expected a top-level declaration")
		for current(parser).kind != .Newline && current(parser).kind != .Eof {
			advance(parser)
		}
		_ = finish_statement(parser)
		return
	}
	name := advance(parser)
	if current(parser).kind == .Colon_Colon {
		saved := parser.cursor
		advance(parser)
		skip_newlines(parser)
		if current(parser).kind == .Keyword_Import {
			start := advance(parser)
			parse_import(parser, name, start)
			return
		}
		parser.cursor = saved
	}
	type_syntax := types.INVALID
	if is_type_token(current(parser).kind) {
		type_syntax = parse_type(parser)
	}
	operator := current(parser)
	if operator.kind != .Colon_Colon && operator.kind != .Equal {
		source.add(parser.diagnostics, operator.span, "expected '::' or '=' after top-level name")
		_ = finish_statement(parser)
		return
	}
	advance(parser)
	skip_newlines(parser)

	if operator.kind == .Colon_Colon &&
	   (current(parser).kind == .Keyword_Func || current(parser).kind == .Keyword_C_Func) {
		c_abi := current(parser).kind == .Keyword_C_Func
		parse_function(parser, name, c_abi)
		return
	}
	if operator.kind == .Colon_Colon &&
	   (current(parser).kind == .Keyword_Struct || current(parser).kind == .Keyword_C_Struct) {
		parse_struct(parser, name, current(parser).kind == .Keyword_C_Struct)
		return
	}

	expr := parse_expression(parser)
	_ = ast.global_id(len(parser.module.globals))
	append(&parser.module.globals, ast.Global{
		span=span_from(name.span, parser.module.exprs[expr].span),
		name=name.symbol,
		pkg=parser.pkg,
		file=parser.file,
		type=type_syntax,
		immutable=operator.kind == .Colon_Colon,
		expr=expr,
		diagnostic=source.INVALID_DIAGNOSTIC,
	})
	_ = finish_statement(parser)
}

parse :: proc(
	stream: ^token.Stream,
	source_file: ^source.Source,
	diagnostics: ^source.Diagnostics,
	allocator := context.allocator,
) -> ast.Module {
	parser := Parser{
		tokens=stream,
		source_file=source_file,
		diagnostics=diagnostics,
		module=ast.init_module(allocator),
	}
	skip_newlines(&parser)
	for current(&parser).kind != .Eof {
		parse_top_level(&parser)
		skip_newlines(&parser)
	}
	return parser.module
}

parse_into :: proc(
	stream: ^token.Stream,
	source_file: ^source.Source,
	diagnostics: ^source.Diagnostics,
	module: ^ast.Module,
	pkg: ast.Package_Id,
	file: ast.File_Id,
) {
	parser := Parser{
		tokens=stream,
		source_file=source_file,
		diagnostics=diagnostics,
		module=module^,
		pkg=pkg,
		file=file,
	}
	skip_newlines(&parser)
	for current(&parser).kind != .Eof {
		parse_top_level(&parser)
		skip_newlines(&parser)
	}
	module^ = parser.module
}