brolang/compiler/loader/loader.odin

package loader

import "../ast"
import "../cimport"
import "../lexer"
import "../parser"
import "../source"
import "../symbol"
import "../target"
import "../types"
import "core:fmt"
import "core:math"
import "core:mem"
import "core:os"
import "core:path/filepath"
import "core:slice"
import "core:strings"

State :: struct {
	module:          ^ast.Module,
	sources:         ^source.Store,
	diagnostics:     ^source.Diagnostics,
	symbols:         ^symbol.Table,
	token_allocator: mem.Allocator,
	allocator:       mem.Allocator,
	c_options:       cimport.Options,
	selected:        target.Target,
	record_identities: [dynamic]string,
	record_types:      [dynamic]types.Type,
	root_failed:     bool,
}

is_identifier :: proc(value: string) -> bool {
	if len(value) == 0 {
		return false
	}
	is_start := proc(value: byte) -> bool {
		return value == '_' || value >= 'a' && value <= 'z' || value >= 'A' && value <= 'Z'
	}
	if !is_start(value[0]) {
		return false
	}
	for byte_value in transmute([]byte)value[1:] {
		if !is_start(byte_value) && !(byte_value >= '0' && byte_value <= '9') {
			return false
		}
	}
	return true
}

find_package :: proc(state: ^State, path: string) -> ast.Package_Id {
	for pkg, id in state.module.packages {
		if pkg.path == path {
			return ast.package_id(id)
		}
	}
	return ast.INVALID_PACKAGE
}

add_placeholder :: proc(state: ^State, path: string) -> ast.Package_Id {
	if existing := find_package(state, path); existing != ast.INVALID_PACKAGE {
		return existing
	}
	id := ast.package_id(len(state.module.packages))
	append(&state.module.packages, ast.Package{
		path=strings.clone(path, state.allocator),
		name=symbol.intern(state.symbols, filepath.base(path)),
		available=false,
	})
	return id
}

read_package_files :: proc(state: ^State, path: string) -> ([]os.File_Info, bool) {
	handle, open_error := os.open(path, os.O_RDONLY)
	if open_error != nil {
		return nil, false
	}
	defer os.close(handle)
	entries, read_error := os.read_dir(handle, -1, state.allocator)
	if read_error != nil {
		return nil, false
	}
	slice.sort_by(entries, proc(a, b: os.File_Info) -> bool {
		return a.name < b.name
	})
	files: [dynamic]os.File_Info
	files.allocator = state.allocator
	for entry in entries {
		if !entry.is_dir && filepath.ext(entry.name) == ".bro" {
			append(&files, entry)
		} else {
			os.file_info_delete(entry, state.allocator)
		}
	}
	delete(entries, state.allocator)
	return files[:], true
}

resolve_import_path :: proc(state: ^State, importing_path, import_path: string) -> (string, bool) {
	if filepath.is_abs(import_path) {
		return "", false
	}
	joined, join_error := filepath.join({importing_path, import_path}, state.allocator)
	if join_error != nil {
		return "", false
	}
	canonical, ok := filepath.abs(joined, state.allocator)
	if ok {
		delete(joined, state.allocator)
		return canonical, true
	}
	return joined, false
}

header_package_name :: proc(path: string, symbols: ^symbol.Table) -> symbol.Id {
	base := filepath.base(path)
	extension := filepath.ext(base)
	if len(extension) > 0 {
		base = base[:len(base)-len(extension)]
	}
	return symbol.intern(symbols, base)
}

find_record_identity :: proc(state: ^State, identity: string) -> types.Type {
	for existing, index in state.record_identities {
		if existing == identity {
			return state.record_types[index]
		}
	}
	return types.INVALID
}

translate_c_type :: proc(
	state: ^State,
	result: ^cimport.Result,
	value: cimport.Type_Id,
	pkg: ast.Package_Id,
	record_mapping: []types.Type,
	type_mapping: []types.Type,
) -> types.Type {
	if value == cimport.INVALID_TYPE || int(value) < 0 || int(value) >= len(result.types) {
		return types.INVALID
	}
	if types.is_valid(type_mapping[value]) {
		return type_mapping[value]
	}
	item := result.types[value]
	translated := types.INVALID
	switch item.kind {
	case .Invalid:      translated = types.INVALID
	case .Void:         translated = types.VOID
	case .C_Char:       translated = types.C_CHAR
	case .C_Schar:      translated = types.C_SCHAR
	case .C_Uchar:      translated = types.C_UCHAR
	case .C_Short:      translated = types.C_SHORT
	case .C_Ushort:     translated = types.C_USHORT
	case .C_Int:        translated = types.C_INT
	case .C_Uint:       translated = types.C_UINT
	case .C_Long:       translated = types.C_LONG
	case .C_Ulong:      translated = types.C_ULONG
	case .C_Longlong:   translated = types.C_LONGLONG
	case .C_Ulonglong:  translated = types.C_ULONGLONG
	case .C_Float:      translated = types.C_FLOAT
	case .C_Double:     translated = types.C_DOUBLE
	case .C_Longdouble: translated = types.C_LONGDOUBLE
	case .Pointer:
		child := translate_c_type(state, result, item.child, pkg, record_mapping, type_mapping)
		if types.is_valid(child) {
			pointer := types.pointer(&state.module.type_store, child, item.mutable, true)
			translated = types.optional(&state.module.type_store, pointer)
		}
	case .Array:
		child := translate_c_type(state, result, item.child, pkg, record_mapping, type_mapping)
		if types.is_valid(child) {
			translated = types.array(&state.module.type_store, child, item.count, item.mutable)
		}
	case .Function:
		params := make([]types.Type, len(item.params), state.allocator)
		for param, index in item.params {
			params[index] = translate_c_type(state, result, param, pkg, record_mapping, type_mapping)
			if !types.is_valid(params[index]) {
				delete(params, state.allocator)
				type_mapping[value] = types.INVALID
				return types.INVALID
			}
		}
		result_type := translate_c_type(state, result, item.child, pkg, record_mapping, type_mapping)
		if types.is_valid(result_type) {
			translated = types.function(&state.module.type_store, params, result_type, true, item.variadic)
		}
		delete(params, state.allocator)
	case .Record:
		if int(item.record) < len(record_mapping) {
			translated = record_mapping[item.record]
		}
	}
	type_mapping[value] = translated
	return translated
}

function_signatures_equal :: proc(left: ast.Function, params: []ast.Param, result: types.Type, variadic: bool) -> bool {
	if left.result != result || left.variadic != variadic || len(left.params) != len(params) {
		return false
	}
	for param, index in params {
		if left.params[index].type != param.type {
			return false
		}
	}
	return true
}

RECORD_DEPENDENCY_PENDING :: "C record contains an incomplete or unsupported record field"

record_layout_reason :: proc(
	store: ^types.Store,
	record: cimport.Record,
	fields: []types.Field,
	selected: target.Target,
) -> string {
	if len(fields) == 0 {
		if record.size > 0 {
			return "anonymous C record fields are not supported"
		}
		return "empty C records are not supported"
	}
	if len(fields) != len(record.fields) || record.alignment == 0 {
		return "C record metadata is incomplete"
	}
	size: u64
	alignment: u64 = 1
	if record.kind == .Union {
		for field in fields {
			if !types.is_runtime_value(field.type, store) {
				return RECORD_DEPENDENCY_PENDING
			}
			if field.offset != 0 {
				return "non-natural C record layouts are not supported"
			}
			size = max(size, types.size(field.type, store, selected))
			alignment = max(alignment, u64(types.alignment_of(field.type, store, selected)))
		}
	} else {
		offset: u64
		for field in fields {
			if !types.is_runtime_value(field.type, store) {
				return RECORD_DEPENDENCY_PENDING
			}
			field_alignment := u64(types.alignment_of(field.type, store, selected))
			offset = (offset+field_alignment-1)/field_alignment*field_alignment
			if field.offset != offset {
				if field.offset < offset {
					return "packed C records are not supported"
				}
				return "non-natural C record layouts are not supported"
			}
			offset += types.size(field.type, store, selected)
			alignment = max(alignment, field_alignment)
		}
		size = offset
	}
	size = (size+alignment-1)/alignment*alignment
	if u64(record.alignment) > alignment {
		return "over-aligned C records are not supported"
	}
	if u64(record.alignment) < alignment || record.size < size {
		return "packed C records are not supported"
	}
	if record.size != size {
		return "non-natural C record layouts are not supported"
	}
	return ""
}

c_record_by_value_reason :: proc(result: ^cimport.Result, value: cimport.Type_Id, depth := 0) -> string {
	if depth > 64 || value == cimport.INVALID_TYPE || int(value) < 0 || int(value) >= len(result.types) {
		return ""
	}
	item := result.types[value]
	#partial switch item.kind {
	case .Pointer:
		return ""
	case .Array:
		return c_record_by_value_reason(result, item.child, depth+1)
	case .Record:
		if int(item.record) >= len(result.records) {
			return "C record metadata is incomplete"
		}
		record := result.records[item.record]
		if len(record.reason) > 0 {
			return record.reason
		}
		if !record.complete {
			return "incomplete C records are pointer-only"
		}
	}
	return ""
}

add_import_unsupported :: proc(state: ^State, pkg: ast.Package_Id, name: string, reason: string) {
	if len(name) == 0 || len(reason) == 0 {
		return
	}
	append(&state.module.unsupported, ast.Unsupported{
		pkg=pkg,
		name=symbol.intern(state.symbols, name),
		reason=strings.clone(reason, state.allocator),
	})
}

find_trampoline :: proc(result: ^cimport.Result, symbol: string) -> (cimport.Trampoline, bool) {
	for trampoline in result.trampolines {
		if trampoline.symbol == symbol {
			return trampoline, true
		}
	}
	return {}, false
}

add_c_trampoline :: proc(state: ^State, trampoline: cimport.Trampoline) {
	if len(trampoline.symbol) == 0 || len(trampoline.source) == 0 {
		return
	}
	for existing in state.module.c_trampolines {
		if existing.symbol == trampoline.symbol {
			return
		}
	}
	append(&state.module.c_trampolines, ast.Trampoline{
		symbol=strings.clone(trampoline.symbol, state.allocator),
		source=strings.clone(trampoline.source, state.allocator),
		header=strings.clone(trampoline.header, state.allocator),
	})
}

add_import_expr :: proc(state: ^State, expr: ast.Expr) -> ast.Expr_Id {
	id := ast.expr_id(len(state.module.exprs))
	append(&state.module.exprs, expr)
	return id
}

add_macro_value_expr :: proc(state: ^State, value: cimport.Macro_Value, span: source.Span) -> ast.Expr_Id {
	#partial switch value.kind {
	case .Integer:
		if value.negative {
			magnitude := value.integer
			operand := add_import_expr(state, ast.Expr{
				kind=.Integer, span=span, integer=magnitude,
				left=ast.INVALID_EXPR, right=ast.INVALID_EXPR,
				diagnostic=source.INVALID_DIAGNOSTIC,
			})
			return add_import_expr(state, ast.Expr{
				kind=.Negate, span=span, left=operand, right=ast.INVALID_EXPR,
				diagnostic=source.INVALID_DIAGNOSTIC,
			})
		}
		return add_import_expr(state, ast.Expr{
			kind=.Integer, span=span, integer=value.integer,
			left=ast.INVALID_EXPR, right=ast.INVALID_EXPR,
			diagnostic=source.INVALID_DIAGNOSTIC,
		})
	case .Float:
		return add_import_expr(state, ast.Expr{
			kind=.Float, span=span, integer=value.integer,
			left=ast.INVALID_EXPR, right=ast.INVALID_EXPR,
			diagnostic=source.INVALID_DIAGNOSTIC,
		})
	case:
	}
	return add_import_expr(state, ast.Expr{
		kind=.Invalid, span=span, left=ast.INVALID_EXPR, right=ast.INVALID_EXPR,
		diagnostic=source.add(state.diagnostics, span, "unsupported C macro value"),
	})
}

MAX_MACRO_ZERO_DEPTH :: 64
MAX_MACRO_ZERO_NODES :: 65_536

find_macro_record_name :: proc(
	state: ^State,
	pkg: ast.Package_Id,
	value: types.Type,
) -> (symbol.Id, bool) {
	resolved := types.resolve_alias(value, &state.module.type_store)
	for item, index in state.module.type_store.nodes {
		if item.pkg != u32(pkg) || item.name == 0 {
			continue
		}
		candidate := types.DYNAMIC_START+types.Type(index)
		if types.resolve_alias(candidate, &state.module.type_store) == resolved {
			return symbol.Id(item.name), true
		}
	}
	return symbol.INVALID, false
}

add_macro_zero_expr :: proc(
	state: ^State,
	pkg: ast.Package_Id,
	value_type: types.Type,
	span: source.Span,
	depth: int,
	remaining: ^int,
) -> (ast.Expr_Id, bool) {
	if depth > MAX_MACRO_ZERO_DEPTH || remaining^ <= 0 {
		return ast.INVALID_EXPR, false
	}
	remaining^ -= 1
	store := &state.module.type_store
	resolved := types.resolve_alias(value_type, store)
	#partial switch types.kind(resolved, store) {
	case .Scalar:
		kind := ast.Expr_Kind.Integer
		if types.is_float(resolved, state.selected) {
			kind = .Float
		}
		return add_import_expr(state, ast.Expr{
			kind=kind, span=span, integer=0,
			left=ast.INVALID_EXPR, right=ast.INVALID_EXPR,
			diagnostic=source.INVALID_DIAGNOSTIC,
		}), true
	case .Optional:
		if !types.is_optional_pointer(resolved, store) {
			return ast.INVALID_EXPR, false
		}
		return add_import_expr(state, ast.Expr{
			kind=.None, span=span,
			left=ast.INVALID_EXPR, right=ast.INVALID_EXPR,
			diagnostic=source.INVALID_DIAGNOSTIC,
		}), true
	case .Array:
		item, ok := types.node(store, resolved)
		if !ok || item.count > u64(max(int)) || item.count > u64(remaining^) {
			return ast.INVALID_EXPR, false
		}
		args := make([]ast.Expr_Id, int(item.count), state.allocator)
		for index in 0..<len(args) {
			value, value_ok := add_macro_zero_expr(
				state, pkg, item.child, span, depth+1, remaining,
			)
			if !value_ok {
				delete(args, state.allocator)
				return ast.INVALID_EXPR, false
			}
			args[index] = value
		}
		return add_import_expr(state, ast.Expr{
			kind=.Array, span=span, args=args,
			left=ast.INVALID_EXPR, right=ast.INVALID_EXPR,
			diagnostic=source.INVALID_DIAGNOSTIC,
		}), true
	case .Struct, .Union:
		item, ok := types.node(store, resolved)
		fields := types.fields_for(store, resolved)
		if !ok || len(fields) == 0 {
			return ast.INVALID_EXPR, false
		}
		name, found_name := find_macro_record_name(state, pkg, resolved)
		if !found_name {
			return ast.INVALID_EXPR, false
		}
		count := 1 if item.kind == .Union else len(fields)
		args := make([]ast.Expr_Id, count, state.allocator)
		for index in 0..<count {
			field := fields[index]
			value, value_ok := add_macro_zero_expr(
				state, pkg, field.type, span, depth+1, remaining,
			)
			if !value_ok {
				delete(args, state.allocator)
				return ast.INVALID_EXPR, false
			}
			args[index] = add_import_expr(state, ast.Expr{
				kind=.Keyed, span=span, name=symbol.Id(field.name), left=value,
				right=ast.INVALID_EXPR, diagnostic=source.INVALID_DIAGNOSTIC,
			})
		}
		return add_import_expr(state, ast.Expr{
			kind=.Struct_Literal, span=span, name=name, args=args,
			left=ast.INVALID_EXPR, right=ast.INVALID_EXPR,
			diagnostic=source.INVALID_DIAGNOSTIC,
		}), true
	case:
	}
	return ast.INVALID_EXPR, false
}

Converted_Macro_Value_Kind :: enum u8 {
	Invalid,
	Integer,
	Float,
	Null,
}

Converted_Macro_Value :: struct {
	kind:     Converted_Macro_Value_Kind,
	integer:  u64,
	negative: bool,
}

macro_integer_value :: proc(value: cimport.Macro_Value) -> i128 {
	magnitude := i128(value.integer)
	return -magnitude if value.negative else magnitude
}

convert_macro_integer :: proc(
	value: i128,
	destination: types.Type,
	selected: target.Target,
) -> (Converted_Macro_Value, bool) {
	bits := types.bits(destination, selected)
	if bits <= 0 || bits > 64 {
		return {}, false
	}
	if types.is_unsigned(destination, selected) {
		modulus := i128(1) << u32(bits)
		wrapped := value % modulus
		if wrapped < 0 {
			wrapped += modulus
		}
		return Converted_Macro_Value{
			kind=.Integer,
			integer=u64(wrapped),
		}, true
	}
	if !types.is_signed(destination, selected) {
		return {}, false
	}
	limit := i128(1) << u32(bits-1)
	if value < -limit || value >= limit {
		return {}, false
	}
	if value < 0 {
		return Converted_Macro_Value{
			kind=.Integer,
			integer=u64(-value),
			negative=true,
		}, true
	}
	return Converted_Macro_Value{kind=.Integer, integer=u64(value)}, true
}

macro_float_value :: proc(
	value: cimport.Macro_Value,
	source_type: types.Type,
	selected: target.Target,
) -> (f64, bool) {
	if value.kind != .Float || !types.is_float(source_type, selected) {
		return 0, false
	}
	number := transmute(f64)value.integer
	if types.bits(source_type, selected) == 32 {
		number = f64(f32(number))
	}
	if math.is_nan(number) || math.is_inf(number) {
		return 0, false
	}
	return number, true
}

convert_macro_float :: proc(
	number: f64,
	destination: types.Type,
	selected: target.Target,
) -> (Converted_Macro_Value, bool) {
	if !types.is_float(destination, selected) {
		return {}, false
	}
	converted := number
	if types.bits(destination, selected) == 32 {
		rounded := f64(f32(number))
		if math.is_inf(rounded) {
			return {}, false
		}
		converted = rounded
	}
	return Converted_Macro_Value{
		kind=.Float,
		integer=transmute(u64)converted,
	}, true
}

convert_macro_field_value :: proc(
	state: ^State,
	result: ^cimport.Result,
	value: cimport.Macro_Value,
	field_type: types.Type,
	pkg: ast.Package_Id,
	record_mapping: []types.Type,
	type_mapping: []types.Type,
) -> (Converted_Macro_Value, bool) {
	store := &state.module.type_store
	destination := types.resolve_alias(field_type, store)
	source_type := translate_c_type(
		state, result, value.type, pkg, record_mapping, type_mapping,
	)
	if !types.is_concrete_scalar(source_type) {
		return {}, false
	}
	if types.is_optional_pointer(destination, store) {
		if types.is_concrete_integer(source_type) &&
		   value.kind == .Integer && macro_integer_value(value) == 0 {
			return Converted_Macro_Value{kind=.Null}, true
		}
		return {}, false
	}
	if types.is_concrete_integer(source_type) {
		if value.kind != .Integer {
			return {}, false
		}
		integer := macro_integer_value(value)
		if types.is_concrete_integer(destination) {
			return convert_macro_integer(integer, destination, state.selected)
		}
		if types.is_float(destination, state.selected) {
			return convert_macro_float(f64(integer), destination, state.selected)
		}
		return {}, false
	}
	number, number_ok := macro_float_value(value, source_type, state.selected)
	if !number_ok {
		return {}, false
	}
	if types.is_float(destination, state.selected) {
		return convert_macro_float(number, destination, state.selected)
	}
	if types.is_concrete_integer(destination) {
		truncated := math.trunc(number)
		bits := types.bits(destination, state.selected)
		if bits <= 0 || bits > 64 {
			return {}, false
		}
		if types.is_signed(destination, state.selected) {
			limit := f64(i128(1) << u32(bits-1))
			if truncated < -limit || truncated >= limit {
				return {}, false
			}
		} else if types.is_unsigned(destination, state.selected) {
			limit := f64(i128(1) << u32(bits))
			if truncated < 0 || truncated >= limit {
				return {}, false
			}
		} else {
			return {}, false
		}
		return convert_macro_integer(i128(truncated), destination, state.selected)
	}
	return {}, false
}

add_converted_macro_value_expr :: proc(
	state: ^State,
	value: Converted_Macro_Value,
	span: source.Span,
) -> (ast.Expr_Id, bool) {
	#partial switch value.kind {
	case .Integer:
		return add_macro_value_expr(state, cimport.Macro_Value{
			kind=.Integer,
			integer=value.integer,
			negative=value.negative,
		}, span), true
	case .Float:
		return add_macro_value_expr(state, cimport.Macro_Value{
			kind=.Float,
			integer=value.integer,
		}, span), true
	case .Null:
		return add_import_expr(state, ast.Expr{
			kind=.None, span=span,
			left=ast.INVALID_EXPR, right=ast.INVALID_EXPR,
			diagnostic=source.INVALID_DIAGNOSTIC,
		}), true
	case:
	}
	return ast.INVALID_EXPR, false
}

add_macro_aggregate_expr :: proc(
	state: ^State,
	result: ^cimport.Result,
	macro: cimport.Macro_Constant,
	record_type: types.Type,
	pkg: ast.Package_Id,
	record_mapping: []types.Type,
	type_mapping: []types.Type,
	span: source.Span,
) -> (ast.Expr_Id, bool) {
	fields := types.fields_for(&state.module.type_store, record_type)
	union_record := types.is_union(record_type, &state.module.type_store)
	initializer_count := 1 if union_record else len(fields)
	if len(fields) == 0 || len(macro.values) > initializer_count {
		return ast.INVALID_EXPR, false
	}
	converted := make([]Converted_Macro_Value, len(macro.values), state.allocator)
	defer delete(converted, state.allocator)
	for value, index in macro.values {
		converted[index], _ = convert_macro_field_value(
			state,
			result,
			value,
			fields[index].type,
			pkg,
			record_mapping,
			type_mapping,
		)
		if converted[index].kind == .Invalid {
			return ast.INVALID_EXPR, false
		}
	}
	args := make([]ast.Expr_Id, initializer_count, state.allocator)
	remaining := MAX_MACRO_ZERO_NODES
	for index in 0..<initializer_count {
		field := fields[index]
		value_expr := ast.INVALID_EXPR
		value_ok := false
		if index < len(macro.values) {
			value_expr, value_ok = add_converted_macro_value_expr(
				state, converted[index], span,
			)
		} else {
			value_expr, value_ok = add_macro_zero_expr(
				state, pkg, field.type, span, 0, &remaining,
			)
		}
		if !value_ok {
			delete(args, state.allocator)
			return ast.INVALID_EXPR, false
		}
		args[index] = add_import_expr(state, ast.Expr{
			kind=.Keyed, span=span, name=symbol.Id(field.name), left=value_expr,
			right=ast.INVALID_EXPR, diagnostic=source.INVALID_DIAGNOSTIC,
		})
	}
	return add_import_expr(state, ast.Expr{
		kind=.Struct_Literal,
		span=span,
		name=symbol.intern(state.symbols, macro.type_name),
		args=args,
		left=ast.INVALID_EXPR,
		right=ast.INVALID_EXPR,
		diagnostic=source.INVALID_DIAGNOSTIC,
	}), true
}

find_global_in_package :: proc(module: ^ast.Module, pkg: ast.Package_Id, name: symbol.Id) -> (ast.Global_Id, bool) {
	for global, index in module.globals {
		if global.pkg == pkg && global.name == name {
			return ast.global_id(index), true
		}
	}
	return ast.INVALID_GLOBAL, false
}

find_function_in_package :: proc(module: ^ast.Module, pkg: ast.Package_Id, name: symbol.Id) -> bool {
	for function in module.functions {
		if function.pkg == pkg && function.name == name {
			return true
		}
	}
	return false
}

remove_value_declarations_in_package :: proc(
	state: ^State,
	pkg: ast.Package_Id,
	name: symbol.Id,
) {
	function_index := 0
	for function_index < len(state.module.functions) {
		function := state.module.functions[function_index]
		if function.pkg != pkg || function.name != name {
			function_index += 1
			continue
		}
		delete(function.params, state.allocator)
		delete(function.body, state.allocator)
		delete(function.link_name, state.allocator)
		delete(function.unsupported_reason, state.allocator)
		ordered_remove(&state.module.functions, function_index)
	}
	global_index := 0
	for global_index < len(state.module.globals) {
		global := state.module.globals[global_index]
		if global.pkg != pkg || global.name != name {
			global_index += 1
			continue
		}
		delete(global.link_name, state.allocator)
		ordered_remove(&state.module.globals, global_index)
	}
	unsupported_index := 0
	for unsupported_index < len(state.module.unsupported) {
		item := state.module.unsupported[unsupported_index]
		if item.pkg != pkg || item.name != name {
			unsupported_index += 1
			continue
		}
		delete(item.reason, state.allocator)
		ordered_remove(&state.module.unsupported, unsupported_index)
	}
}

add_external_variable_global :: proc(
	state: ^State,
	result: ^cimport.Result,
	pkg: ast.Package_Id,
	variable: cimport.Variable,
	record_mapping: []types.Type,
	type_mapping: []types.Type,
	span: source.Span,
) {
	name := symbol.intern(state.symbols, variable.name)
	variable_type := translate_c_type(state, result, variable.type, pkg, record_mapping, type_mapping)
	unsupported_reason := variable.reason
	if len(unsupported_reason) == 0 {
		unsupported_reason = c_record_by_value_reason(result, variable.type)
	}
	if len(unsupported_reason) == 0 && !types.is_runtime_value(variable_type, &state.module.type_store) {
		unsupported_reason = "C variable type is not supported"
	}
	if len(unsupported_reason) > 0 {
		add_import_unsupported(state, pkg, variable.name, unsupported_reason)
		return
	}
	if find_function_in_package(state.module, pkg, name) {
		add_import_unsupported(state, pkg, variable.name, "C variable conflicts with a function declaration")
		return
	}
	if existing, ok := find_global_in_package(state.module, pkg, name); ok {
		global := state.module.globals[existing]
		if !global.external || !types.equal(global.type, variable_type) || global.writable != variable.mutable {
			add_import_unsupported(state, pkg, variable.name, "conflicting C declarations for variable")
		}
		return
	}
	_ = ast.global_id(len(state.module.globals))
	append(&state.module.globals, ast.Global{
		span=span,
		name=name,
		link_name=strings.clone(variable.name, state.allocator),
		pkg=pkg,
		file=ast.INVALID_FILE,
		type=variable_type,
		immutable=true,
		external=true,
		writable=variable.mutable,
		expr=ast.INVALID_EXPR,
		diagnostic=source.INVALID_DIAGNOSTIC,
	})
}

add_macro_constant_global :: proc(
	state: ^State,
	result: ^cimport.Result,
	pkg: ast.Package_Id,
	macro: cimport.Macro_Constant,
	record_mapping: []types.Type,
	type_mapping: []types.Type,
	span: source.Span,
) {
	name := symbol.intern(state.symbols, macro.name)
	remove_value_declarations_in_package(state, pkg, name)
	if macro.aggregate {
		type_name := symbol.intern(state.symbols, macro.type_name)
		named := types.find_named(&state.module.type_store, u32(pkg), u32(type_name))
		record_type := types.resolve_alias(named, &state.module.type_store)
		if !types.is_record(record_type, &state.module.type_store) ||
		   types.is_opaque_struct(record_type, &state.module.type_store) {
			add_import_unsupported(state, pkg, macro.name, "C macro aggregate type is not supported")
			return
		}
		expr, ok := add_macro_aggregate_expr(
			state,
			result,
			macro,
			record_type,
			pkg,
			record_mapping,
			type_mapping,
			span,
		)
		if !ok {
			add_import_unsupported(state, pkg, macro.name, "C macro aggregate initializer is not representable")
			return
		}
		_ = ast.global_id(len(state.module.globals))
		append(&state.module.globals, ast.Global{
			span=span,
			name=name,
			pkg=pkg,
			file=ast.INVALID_FILE,
			type=record_type,
			immutable=true,
			expr=expr,
			diagnostic=source.INVALID_DIAGNOSTIC,
		})
		return
	}
	macro_type := translate_c_type(state, result, macro.type, pkg, record_mapping, type_mapping)
	if !types.is_runtime_value(macro_type, &state.module.type_store) {
		add_import_unsupported(state, pkg, macro.name, "C macro constant type is not supported")
		return
	}
	expr := add_macro_value_expr(state, macro.value, span)
	_ = ast.global_id(len(state.module.globals))
	append(&state.module.globals, ast.Global{
		span=span,
		name=name,
		pkg=pkg,
		file=ast.INVALID_FILE,
		type=macro_type,
		immutable=true,
		expr=expr,
		diagnostic=source.INVALID_DIAGNOSTIC,
	})
}

load_header :: proc(state: ^State, path: string, import_span: source.Span) -> ast.Package_Id {
	canonical, ok := filepath.abs(path, state.allocator)
	if !ok {
		id := add_placeholder(state, path)
		source.addf(state.diagnostics, import_span, "could not resolve C header '%s'", path)
		return id
	}
	if existing := find_package(state, canonical); existing != ast.INVALID_PACKAGE {
		delete(canonical, state.allocator)
		return existing
	}
	pkg_id := ast.package_id(len(state.module.packages))
	append(&state.module.packages, ast.Package{
		path=canonical,
		name=header_package_name(canonical, state.symbols),
		available=false,
		kind=.C_Header,
	})
	result := cimport.import_header(state.c_options, canonical, state.selected, state.allocator)
	defer cimport.destroy_result(&result)
	if !result.available {
		message := result.error_message if len(result.error_message) > 0 else "C header import failed"
		source.addf(state.diagnostics, import_span, "could not import C header '%s': %s", path, message)
		if result.infrastructure {
			state.root_failed = true
		}
		return pkg_id
	}
	state.module.packages[pkg_id].available = true

	record_mapping := make([]types.Type, len(result.records), state.allocator)
	defer delete(record_mapping, state.allocator)
	for record, index in result.records {
		record_type := find_record_identity(state, record.identity)
		if !types.is_valid(record_type) {
			name := record.name
			if len(name) == 0 {
				name = fmt.tprintf("__c_record_%d", len(state.record_types))
			}
			record_type = types.named(&state.module.type_store, u32(pkg_id), u32(symbol.intern(state.symbols, name)))
			_ = types.define_record(&state.module.type_store, record_type, nil, true, true, record.kind == .Union)
			append(&state.record_identities, strings.clone(record.identity, state.allocator))
			append(&state.record_types, record_type)
		}
		record_mapping[index] = record_type
	}
	type_mapping := make([]types.Type, len(result.types), state.allocator)
	defer delete(type_mapping, state.allocator)

	for _ in 0..<len(result.records) {
		changed := false
		for record, record_index in result.records {
			if !record.complete || len(record.reason) > 0 {
				continue
			}
			record_type := record_mapping[record_index]
			item, ok := types.node(&state.module.type_store, record_type)
			if !ok || (item.declared && !item.opaque) {
				continue
			}
			fields := make([]types.Field, len(record.fields), state.allocator)
			for field, field_index in record.fields {
				fields[field_index] = types.Field{
					name=u32(symbol.intern(state.symbols, field.name)),
					type=translate_c_type(state, &result, field.type, pkg_id, record_mapping, type_mapping),
					offset=field.offset,
				}
			}
			layout_reason := record_layout_reason(&state.module.type_store, record, fields, state.selected)
			if len(layout_reason) == 0 {
				changed = types.define_record(
					&state.module.type_store, record_type, fields, true, false,
					record.kind == .Union, record.size, record.alignment,
				) || changed
			} else if layout_reason != RECORD_DEPENDENCY_PENDING && len(record.reason) == 0 {
				delete(result.records[record_index].reason, result.allocator)
				result.records[record_index].reason = fmt.aprintf("%s", layout_reason, allocator=result.allocator)
			}
			delete(fields, state.allocator)
		}
		if !changed {
			break
		}
	}

	for alias in result.aliases {
		name := symbol.intern(state.symbols, alias.name)
		id := types.named(&state.module.type_store, u32(pkg_id), u32(name))
		child := translate_c_type(state, &result, alias.type, pkg_id, record_mapping, type_mapping)
		_ = types.define_alias(&state.module.type_store, id, child)
		if len(alias.reason) > 0 {
			append(&state.module.unsupported, ast.Unsupported{
				pkg=pkg_id,
				name=name,
				reason=strings.clone(alias.reason, state.allocator),
			})
		}
	}

	for function in result.functions {
		params := make([]ast.Param, len(function.params), state.allocator)
		for param_type, index in function.params {
			params[index] = ast.Param{
				name=symbol.intern(state.symbols, fmt.tprintf("arg%d", index)),
				span=import_span,
				type=translate_c_type(state, &result, param_type, pkg_id, record_mapping, type_mapping),
			}
		}
		function_result := translate_c_type(state, &result, function.result, pkg_id, record_mapping, type_mapping)
		unsupported_reason := function.reason
		if len(unsupported_reason) == 0 {
			for param_type in function.params {
				if reason := c_record_by_value_reason(&result, param_type); len(reason) > 0 {
					unsupported_reason = reason
					break
				}
			}
		}
		if len(unsupported_reason) == 0 {
			unsupported_reason = c_record_by_value_reason(&result, function.result)
		}
		if len(unsupported_reason) == 0 {
			for param in params {
				if types.contains_c_struct_by_value(param.type, &state.module.type_store) {
					unsupported_reason = "C record parameter is incomplete or has an unsupported layout"
					break
				}
			}
		}
		if len(unsupported_reason) == 0 &&
		   types.contains_c_struct_by_value(function_result, &state.module.type_store) {
			unsupported_reason = "C record result is incomplete or has an unsupported layout"
		}
		name := symbol.intern(state.symbols, function.name)
		duplicate := false
		for &existing in state.module.functions {
			if existing.pkg != pkg_id || existing.name != name {
				continue
			}
			duplicate = true
			if !function_signatures_equal(existing, params, function_result, function.variadic) && len(existing.unsupported_reason) == 0 {
				existing.unsupported_reason = fmt.aprintf(
					"conflicting C declarations for '%s'",
					function.name,
					allocator=state.allocator,
				)
			}
			break
		}
		if duplicate {
			delete(params, state.allocator)
			continue
		}
		append(&state.module.functions, ast.Function{
			span=import_span,
			name=name,
			pkg=pkg_id,
			file=ast.INVALID_FILE,
			c_abi=true,
			imported=true,
			has_body=false,
			variadic=function.variadic,
			params=params,
			result=function_result,
			link_name=strings.clone(function.link_name, state.allocator),
			unsupported_reason=strings.clone(unsupported_reason, state.allocator),
			diagnostic=source.INVALID_DIAGNOSTIC,
		})
		// Emit the wrapper only for a `static inline` that survives as supported.
		// cimport may translate its signature fine yet the by-value record layout
		// checks above can still reject it; a wrapper for an uncallable function
		// would just be dead external code.
		if len(unsupported_reason) == 0 && len(function.link_name) > 0 {
			if trampoline, ok := find_trampoline(&result, function.link_name); ok {
				add_c_trampoline(state, trampoline)
			}
		}
	}
	for variable in result.variables {
		add_external_variable_global(
			state, &result, pkg_id, variable, record_mapping, type_mapping, import_span,
		)
	}
	for macro in result.macros {
		add_macro_constant_global(
			state, &result, pkg_id, macro, record_mapping, type_mapping, import_span,
		)
	}
	for item in result.unsupported {
		name := symbol.intern(state.symbols, item.name)
		if item.final_macro {
			remove_value_declarations_in_package(state, pkg_id, name)
		}
		append(&state.module.unsupported, ast.Unsupported{
			pkg=pkg_id,
			name=name,
			reason=strings.clone(item.reason, state.allocator),
		})
	}
	return pkg_id
}

load_package :: proc(state: ^State, path: string, import_span: source.Span, is_root := false) -> ast.Package_Id {
	canonical, ok := filepath.abs(path, state.allocator)
	if !ok || !os.is_dir(path) {
		if is_root {
			state.root_failed = true
			if len(canonical) > 0 {
				delete(canonical, state.allocator)
			}
			return ast.INVALID_PACKAGE
		}
		placeholder := path
		if len(canonical) > 0 {
			placeholder = canonical
		}
		id := add_placeholder(state, placeholder)
		source.addf(state.diagnostics, import_span, "could not import package directory '%s'", path)
		if len(canonical) > 0 {
			delete(canonical, state.allocator)
		}
		return id
	}
	if existing := find_package(state, canonical); existing != ast.INVALID_PACKAGE {
		delete(canonical, state.allocator)
		return existing
	}

	pkg_id := ast.package_id(len(state.module.packages))
	append(&state.module.packages, ast.Package{
		path=canonical,
		name=symbol.intern(state.symbols, filepath.base(canonical)),
		available=true,
	})
	files, files_ok := read_package_files(state, canonical)
	if !files_ok {
		state.root_failed = true
		return pkg_id
	}
	if len(files) == 0 {
		if is_root {
			state.root_failed = true
		} else {
			source.addf(state.diagnostics, import_span, "package '%s' contains no readable .bro files", canonical)
			state.module.packages[pkg_id].available = false
		}
		os.file_info_slice_delete(files, state.allocator)
		return pkg_id
	}

	for file_info in files {
		if file_info.size < 0 || !source.fits_source_length(u64(file_info.size)) {
			source.addf(state.diagnostics, import_span, "source file '%s' exceeds the 4 GiB source limit", file_info.fullpath)
			state.root_failed = true
			continue
		}
		bytes, read_ok := os.read_entire_file(file_info.fullpath, state.sources.allocator)
		if !read_ok {
			state.root_failed = true
			continue
		}
		if !source.fits_source_length(u64(len(bytes))) {
			source.addf(state.diagnostics, import_span, "source file '%s' exceeds the 4 GiB source limit", file_info.fullpath)
			delete(bytes, state.sources.allocator)
			state.root_failed = true
			continue
		}
		source_id := source.add_source_owned(state.sources, file_info.fullpath, bytes)
		file_id := ast.file_id(len(state.module.files))
		append(&state.module.files, ast.File{source=source_id, pkg=pkg_id})
		stream := lexer.lex(&state.sources.items[source_id], state.diagnostics, state.symbols, state.token_allocator)
		parser.parse_into(&stream, &state.sources.items[source_id], state.diagnostics, state.module, pkg_id, file_id)
		delete(stream.items)
	}
	os.file_info_slice_delete(files, state.allocator)

	import_count := len(state.module.imports)
	for import_id in 0..<import_count {
		import_item := state.module.imports[import_id]
		if import_item.pkg != pkg_id || import_item.target != ast.INVALID_PACKAGE {
			continue
		}
		if filepath.is_abs(import_item.path) {
			state.module.imports[import_id].diagnostic = source.add(state.diagnostics, import_item.span, "absolute import paths are invalid")
			state.module.imports[import_id].valid = false
			state.module.imports[import_id].target = add_placeholder(state, import_item.path)
			continue
		}
		target_path, target_ok := resolve_import_path(state, canonical, import_item.path)
		target := load_header(state, target_path, import_item.span) if filepath.ext(import_item.path) == ".h" else load_package(state, target_path, import_item.span)
		state.module.imports[import_id].target = target
		if !target_ok || target == ast.INVALID_PACKAGE || !state.module.packages[target].available {
			state.module.imports[import_id].valid = false
		}
		delete(target_path, state.allocator)
	}
	return pkg_id
}

declaration_conflicts :: proc(module: ^ast.Module, pkg: ast.Package_Id, name: symbol.Id) -> bool {
	for function in module.functions {
		if function.pkg == pkg && function.name == name {
			return true
		}
	}
	for global in module.globals {
		if global.pkg == pkg && global.name == name {
			return true
		}
	}
	return false
}

validate_imports :: proc(state: ^State) {
	for import_item, import_id in state.module.imports {
		if !symbol.is_valid(import_item.alias) && import_item.target != ast.INVALID_PACKAGE {
			state.module.imports[import_id].alias = state.module.packages[import_item.target].name
		}
		alias := state.module.imports[import_id].alias
		alias_text := symbol.resolve(state.symbols, alias)
		if !is_identifier(alias_text) {
			state.module.imports[import_id].diagnostic = source.add(
				state.diagnostics,
				import_item.span,
				"import requires an explicit valid identifier alias",
			)
			state.module.imports[import_id].valid = false
		}
		if declaration_conflicts(state.module, import_item.pkg, alias) {
			state.module.imports[import_id].diagnostic = source.addf(
				state.diagnostics,
				import_item.span,
				"import alias '%s' conflicts with a package declaration",
				alias_text,
			)
			state.module.imports[import_id].valid = false
		}
		for previous in state.module.imports[:import_id] {
			if previous.file == import_item.file && previous.alias == alias {
				state.module.imports[import_id].diagnostic = source.addf(
					state.diagnostics,
					import_item.span,
					"duplicate import alias '%s' in the same file",
					alias_text,
				)
				state.module.imports[import_id].valid = false
				break
			}
		}
	}
}

find_type_import :: proc(module: ^ast.Module, file: ast.File_Id, alias: symbol.Id) -> ast.Import_Id {
	for import_item, index in module.imports {
		if import_item.file == file && import_item.alias == alias {
			return ast.import_id(index)
		}
	}
	return ast.INVALID_IMPORT
}

canonical_type :: proc(
	module: ^ast.Module,
	value: types.Type,
	mapping: []types.Type,
	visiting: []bool,
) -> types.Type {
	if value < types.DYNAMIC_START {
		return value
	}
	index := int(value-types.DYNAMIC_START)
	if index < 0 || index >= len(mapping) {
		return value
	}
	if types.is_valid(mapping[index]) {
		return mapping[index]
	}
	if visiting[index] {
		return value
	}
	visiting[index] = true
	defer visiting[index] = false
	item := module.type_store.nodes[index]
	if item.kind == .Named {
		if item.qualifier != 0 {
			import_id := find_type_import(module, ast.File_Id(item.file), symbol.Id(item.qualifier))
			if import_id != ast.INVALID_IMPORT {
				module.imports[import_id].used = true
				import_item := module.imports[import_id]
				resolved := types.find_named(&module.type_store, u32(import_item.target), item.name)
				if types.is_valid(resolved) {
					mapping[index] = canonical_type(module, resolved, mapping, visiting)
					return mapping[index]
				}
			}
		}
		mapping[index] = value
		return value
	}
	if item.kind == .Alias {
		resolved := canonical_type(module, item.child, mapping, visiting)
		mapping[index] = value if !types.is_valid(resolved) else resolved
		return mapping[index]
	}
	if item.kind == .Struct || item.kind == .Union {
		mapping[index] = value
		fields := types.fields_for(&module.type_store, value)
		for &field in fields {
			field.type = canonical_type(module, field.type, mapping, visiting)
		}
		return value
	}
	if item.kind == .Function {
		params := make([]types.Type, int(item.field_count), context.temp_allocator)
		for param, param_index in types.params_for(&module.type_store, value) {
			params[param_index] = canonical_type(module, param.type, mapping, visiting)
		}
		result := canonical_type(module, item.child, mapping, visiting)
		resolved := types.function(&module.type_store, params, result, item.c_abi, item.variadic)
		mapping[index] = resolved
		return resolved
	}
	if types.is_valid(item.child) {
		item.child = canonical_type(module, item.child, mapping, visiting)
	}
	resolved := types.intern(&module.type_store, item)
	mapping[index] = resolved
	return resolved
}

canonicalize_types :: proc(module: ^ast.Module, allocator: mem.Allocator) {
	original_count := len(module.type_store.nodes)
	mapping := make([]types.Type, original_count, allocator)
	visiting := make([]bool, original_count, allocator)
	defer delete(mapping, allocator)
	defer delete(visiting, allocator)
	for &function in module.functions {
		for &param in function.params {
			param.type = canonical_type(module, param.type, mapping, visiting)
		}
		function.result = canonical_type(module, function.result, mapping, visiting)
	}
	for &global in module.globals {
		global.type = canonical_type(module, global.type, mapping, visiting)
	}
	for &statement in module.statements {
		statement.type = canonical_type(module, statement.type, mapping, visiting)
	}
	for index := 0; index < original_count; index += 1 {
		_ = canonical_type(module, types.DYNAMIC_START+types.Type(index), mapping, visiting)
	}
}

load :: proc(
	root_path: string,
	sources: ^source.Store,
	diagnostics: ^source.Diagnostics,
	symbols: ^symbol.Table,
	token_allocator := context.allocator,
	allocator := context.allocator,
	c_options := cimport.Options{},
	selected := target.DEFAULT,
) -> (ast.Module, bool) {
	module := ast.init_module(allocator)
	state := State{
		module=&module,
		sources=sources,
		diagnostics=diagnostics,
		symbols=symbols,
		token_allocator=token_allocator,
		allocator=allocator,
		c_options=c_options,
		selected=selected,
	}
	state.record_identities.allocator = allocator
	state.record_types.allocator = allocator
	defer {
		for identity in state.record_identities {
			delete(identity, allocator)
		}
		delete(state.record_identities)
		delete(state.record_types)
	}
	root := load_package(&state, root_path, source.Span{}, true)
	if root != ast.Package_Id(0) && root != ast.INVALID_PACKAGE {
		state.root_failed = true
	}
	validate_imports(&state)
	canonicalize_types(&module, allocator)
	return module, !state.root_failed
}