feat: load canonical directory packages

source/ast/ast.hon

@@ -4,7 +4,7 @@
 #! * literal .. -> int | float | string | 'true' | 'false'
 #! * op ....... -> '+' | '-' | '*' | '/' | '==' | '!=' | '<' | '<=' | '>' | '>=' | 'and' | 'or'
 #! * expr ..... -> identifier | literal | type | '[' [expr {',' expr} [',']] ']' | '(' expr ')' | ('-' | '!') expr | expr op expr | expr '(' [expr {',' expr} [',']] ')' | expr '[' expr ']' | expr ('..' | '..=') expr
-#! * decl ..... -> identifier [type] binding expr
+#! * decl ..... -> identifier [type] binding expr | ['test'] [identifier '::'] 'import' string
 #! * block .... -> '{' {stmt} '}'
 #! * arm ...... -> block | stmt (single statements become canonical blocks)
 #! * stmt ..... -> decl | expr | expr '=' expr | 'return' [expr] | 'break' | 'continue' | 'if' expr arm ['else' arm] | 'while' expr arm | 'for' expr '|' ['@'] identifier [',' identifier] '|' arm
@@ -48,19 +48,22 @@ ExprKind :: enum {
 }
 
 DeclKind :: enum {
-    statement, function, invalid
+    statement
+    function
+    import
+    invalid
 }
 
 StmtKind :: enum {
-    declaration,
-    expression,
-    assignment_stmt,
-    return_stmt,
-    break_stmt,
-    continue_stmt,
-    if_stmt,
-    while_stmt,
-    for_stmt,
+    declaration
+    expression
+    assignment_stmt
+    return_stmt
+    break_stmt
+    continue_stmt
+    if_stmt
+    while_stmt
+    for_stmt
     invalid
 }
 
@@ -71,44 +74,45 @@ Mutability :: enum {
 UnaryOperator :: enum { negate, logical_not }
 
 BinaryOperator :: enum {
-    add, subtract,
+    add, subtract
     multiply, divide
-    equal, not_equal,
-    less, less_equal,
+    equal, not_equal
+    less, less_equal
     greater, greater_equal
     logical_and, logical_or
 }
 
 @hide:file
 NodeKind :: enum {
-    literal_int,
-    literal_float,
-    literal_string,
-    literal_bool,
-    expr_identifier,
-    expr_unary,
-    expr_binary,
-    expr_call,
-    expr_range,
-    expr_array_type,
-    expr_array_literal,
-    expr_index,
-    stmt_expr,
-    stmt_decl,
-    stmt_return,
-    stmt_break,
-    stmt_continue,
-    stmt_if,
-    stmt_assignment,
-    stmt_while,
-    stmt_for,
-    func_decl,
-    param_decl,
-    block,
-    package,
-    invalid_expr,
-    invalid_stmt,
-    invalid_decl,
+    literal_int
+    literal_float
+    literal_string
+    literal_bool
+    expr_identifier
+    expr_unary
+    expr_binary
+    expr_call
+    expr_range
+    expr_array_type
+    expr_array_literal
+    expr_index
+    stmt_expr
+    stmt_decl
+    stmt_return
+    stmt_break
+    stmt_continue
+    stmt_if
+    stmt_assignment
+    stmt_while
+    stmt_for
+    func_decl
+    param_decl
+    import_decl
+    block
+    package
+    invalid_expr
+    invalid_stmt
+    invalid_decl
 }
 
 @hide:file
@@ -135,21 +139,29 @@ Storage :: struct {
     nodes arraylist.ArrayList(Node)
     extra arraylist.ArrayList(u32)
     spans arraylist.ArrayList(Span)
+    sources arraylist.ArrayList(diagnostics.Source)
 }
 
-#! owns node storage, retains its allocator, and borrows source until finish or deinit_builder.
+#! owns node storage and source descriptors; borrows registered source paths and text.
 Builder :: struct {
     storage Storage
     source []u8
+    source_id diagnostics.SourceId = diagnostics.SourceId(0)
 }
 
-#! owns node storage and borrows source; source must remain valid until deinit.
+#! owns storage after finish, or borrows it after view; registered source paths and text remain borrowed.
 Tree :: struct {
     storage Storage
-    source []u8
     root NodeId
 }
 
+ImportView :: struct {
+    span Span
+    path Span
+    namespace ?strpool.StringId
+    test_only bool
+}
+
 StatementDeclView :: struct {
     span Span
     name Span
@@ -255,36 +267,78 @@ init_builder proc(allocator mem.Allocator, source []u8) Builder {
             nodes = arraylist.init(allocator),
             extra = arraylist.init(allocator),
             spans = arraylist.init(allocator),
+            sources = arraylist.init(allocator),
         },
         source = source,
     }
 }
 
-#! releases builder-owned node storage with its retained allocator, without freeing source.
+#! registers borrowed `path` and `text`; both must remain valid until deinit_builder or deinit.
+set_source proc(
+    builder @mut Builder, path, text []u8,
+) diagnostics.SourceId ! mem.AllocError {
+    id :: diagnostics.SourceId(builder.storage.sources.items.len)
+    try arraylist.append(&builder.storage.sources,
+        diagnostics.Source{
+            path = path,
+            text = text
+        })
+    builder.source = text
+    builder.source_id = id
+    return id
+}
+
+#! removes the latest source after its failed parse has rolled back all nodes.
+remove_source proc(builder @mut Builder) void {
+    debug.assert(builder.storage.sources.items.len > 0)
+    _ = arraylist.pop(&builder.storage.sources)
+    if (builder.storage.sources.items.len == 0) {
+        builder.source = &[]
+        builder.source_id = diagnostics.SourceId(0)
+        return
+    }
+    builder.source_id = diagnostics.SourceId(builder.storage.sources.items.len - 1)
+    builder.source = builder.storage.sources.items[usize(builder.source_id)].text
+}
+
+#! borrows builder storage; refresh after builder mutation and never pass this view to deinit.
+view proc(builder @Builder, root NodeId) Tree {
+    return Tree{ storage = builder.storage, root = root }
+}
+
+source_files proc(tree @Tree) []diagnostics.Source {
+    return tree.storage.sources.items
+}
+
+#! releases owned AST and source-descriptor storage, without freeing source paths or text.
 deinit_builder proc(builder @mut Builder) void {
     arraylist.deinit(&builder.storage.nodes)
     arraylist.deinit(&builder.storage.extra)
     arraylist.deinit(&builder.storage.spans)
+    arraylist.deinit(&builder.storage.sources)
 }
 
-#! transfers node storage and the source borrow to a caller-owned Tree; release with deinit.
+#! transfers storage and source borrows to a caller-owned Tree; release with deinit.
 finish proc(builder @mut Builder, root NodeId) Tree {
-    tree :: Tree{ storage = builder.storage, source = builder.source, root = root }
+    tree :: view(builder, root)
     allocator :: builder.storage.nodes.allocator
     builder.storage = Storage{
         nodes = arraylist.init(allocator),
         extra = arraylist.init(allocator),
         spans = arraylist.init(allocator),
+        sources = arraylist.init(allocator),
     }
     builder.source = &[]
+    builder.source_id = diagnostics.SourceId(0)
     return tree
 }
 
-#! releases owned node storage with its retained allocator, without freeing source.
+#! releases owned AST and source-descriptor storage, without freeing source paths or text.
 deinit proc(tree @mut Tree) void {
     arraylist.deinit(&tree.storage.nodes)
     arraylist.deinit(&tree.storage.extra)
     arraylist.deinit(&tree.storage.spans)
+    arraylist.deinit(&tree.storage.sources)
 }
 
 add_literal proc(builder @mut Builder, kind LiteralKind, span Span) NodeId ! mem.AllocError {
@@ -541,7 +595,29 @@ add_function_decl proc(
     }, span)
 }
 
-add_block proc(builder @mut Builder, span Span, statements []NodeId) NodeId ! mem.AllocError {
+add_import_decl proc(
+    builder @mut Builder,
+    span, path Span,
+    namespace ?strpool.StringId,
+    test_only bool,
+) NodeId ! mem.AllocError {
+    extra :: try add_extra(builder, path.start)
+    _ = try add_extra(builder, path.end)
+    alias_id :: if (namespace) |id| id else strpool.no_str
+    flag :: if (test_only) u32(1) else u32(0)
+    return try add_node(builder, Node{
+        kind = .import_decl,
+        main = NodeData{ extra = extra },
+        data0 = NodeData{ name_id = alias_id },
+        data1 = NodeData{ extra = flag },
+    }, span)
+}
+
+add_block proc(
+    builder @mut Builder,
+    span Span,
+    statements []NodeId,
+) NodeId ! mem.AllocError {
     start :: builder.storage.extra.items.len
     for (statements) |statement| _ = try add_extra(builder, u32(statement))
     return try add_node(builder, Node{
@@ -611,22 +687,41 @@ name_id proc(tree @Tree, id NodeId) strpool.StringId {
 }
 
 package_decl_count proc(tree @Tree) usize {
-    root :: get_node(&tree.storage, tree.root)
+    return package_decl_count_at(tree, tree.root)
+}
+
+package_decl proc(tree @Tree, index usize) NodeId {
+    return package_decl_at(tree, tree.root, index)
+}
+
+package_decl_count_at proc(tree @Tree, package NodeId) usize {
+    root :: get_node(&tree.storage, package)
     debug.assert(root.kind == .package)
     return usize(root.data1.extra - root.data0.extra)
 }
 
-package_decl proc(tree @Tree, index usize) NodeId {
-    root :: get_node(&tree.storage, tree.root)
+package_decl_at proc(tree @Tree, package NodeId, index usize) NodeId {
+    root :: get_node(&tree.storage, package)
     debug.assert(root.kind == .package)
     debug.assert(index < usize(root.data1.extra - root.data0.extra))
     return NodeId(tree.storage.extra.items[usize(root.data0.extra) + index])
 }
 
+builder_package_decl_count proc(builder @Builder, package NodeId) usize {
+    tree :: view(builder, package)
+    return package_decl_count(&tree)
+}
+
+builder_package_decl proc(builder @Builder, package NodeId, index usize) NodeId {
+    tree :: view(builder, package)
+    return package_decl(&tree, index)
+}
+
 decl_kind proc(tree @Tree, id NodeId) DeclKind {
     return match get_node(&tree.storage, id).kind {
         .stmt_decl: .statement
         .func_decl: .function
+        .import_decl: .import
         .invalid_decl: .invalid
         else: unreachable
     }
@@ -716,7 +811,7 @@ statement_decl proc(tree @Tree, id NodeId) StatementDeclView {
     raw_type :: tree.storage.extra.items[extra + 2]
     return StatementDeclView{
         span = get_span(&tree.storage, id),
-        name = span_from_extra(&tree.storage, extra),
+        name = span_from_extra(&tree.storage, extra, get_span(&tree.storage, id).source),
         type = maybe_node_id(raw_type),
         initializer = NodeId(tree.storage.extra.items[extra + 3]),
         mutability = node.data0.mutability,
@@ -729,12 +824,25 @@ function_decl proc(tree @Tree, id NodeId) FunctionDeclView {
     extra :: usize(node.main.extra)
     return FunctionDeclView{
         span = get_span(&tree.storage, id),
-        name = span_from_extra(&tree.storage, extra),
+        name = span_from_extra(&tree.storage, extra, get_span(&tree.storage, id).source),
         return_type = NodeId(tree.storage.extra.items[extra + 2]),
         body = node.data0.node_id,
     }
 }
 
+import_decl proc(tree @Tree, id NodeId) ImportView {
+    node :: get_node(&tree.storage, id)
+    debug.assert(node.kind == .import_decl)
+    span :: get_span(&tree.storage, id)
+    namespace ?strpool.StringId :: if (node.data0.name_id == strpool.no_str) null else node.data0.name_id
+    return ImportView{
+        span = span,
+        path = span_from_extra(&tree.storage, usize(node.main.extra), span.source),
+        namespace = namespace,
+        test_only = node.data1.extra != 0,
+    }
+}
+
 function_param_count proc(tree @Tree, id NodeId) usize {
     node :: get_node(&tree.storage, id)
     debug.assert(node.kind == .func_decl)
@@ -753,7 +861,7 @@ param proc(tree @Tree, id NodeId) ParamView {
     debug.assert(node.kind == .param_decl)
     return ParamView{
         span = get_span(&tree.storage, id),
-        name = span_from_extra(&tree.storage, usize(node.main.extra)),
+        name = span_from_extra(&tree.storage, usize(node.main.extra), get_span(&tree.storage, id).source),
         type = node.data0.node_id,
     }
 }
@@ -906,8 +1014,10 @@ invalid proc(tree @Tree, id NodeId) InvalidView {
 
 #! returns text borrowed from the source buffer; the caller must not free it.
 source_text proc(tree @Tree, span Span) []u8 {
-    debug.assert(span.start <= span.end and usize(span.end) <= tree.source.len)
-    return tree.source[usize(span.start)..usize(span.end)]
+    debug.assert(usize(span.source) < tree.storage.sources.items.len)
+    text :: tree.storage.sources.items[usize(span.source)].text
+    debug.assert(span.start <= span.end and usize(span.end) <= text.len)
+    return text[usize(span.start)..usize(span.end)]
 }
 
 builder_span proc(builder @Builder, id NodeId) Span {
@@ -915,16 +1025,19 @@ builder_span proc(builder @Builder, id NodeId) Span {
 }
 
 @hide:file
-span_from_extra proc(storage @Storage, start usize) Span {
+span_from_extra proc(storage @Storage, start usize, source diagnostics.SourceId) Span {
     return Span{
         start = storage.extra.items[start],
         end = storage.extra.items[start + 1],
+        source = source,
     }
 }
 
 @hide:file
 add_node proc(builder @mut Builder, node Node, span Span) NodeId ! mem.AllocError {
     debug.assert(builder.storage.nodes.items.len == builder.storage.spans.items.len)
+    if (builder.storage.sources.items.len == 0)
+        _ = try set_source(builder, "", builder.source)
     id :: node_id(builder.storage.nodes.items.len)
     try arraylist.reserve(&builder.storage.nodes, builder.storage.nodes.items.len + 1)
     try arraylist.reserve(&builder.storage.spans, builder.storage.spans.items.len + 1)

source/ast/ast.test.hon

@@ -1,6 +1,7 @@
 import "@std/mem"
 import "@std/testing"
 import "@source/strpool"
+import "@source/diagnostics"
 
 exposes_typed_declaration_views test {
     strings := strpool.init(mem.c_allocator)
@@ -74,3 +75,40 @@ exposes_array_type_literal_and_index_views test {
     try testing.expect_equal(literal, index_expr(&tree, indexed).base)
     try testing.expect_equal(count, index_expr(&tree, indexed).index)
 }
+
+registers_borrowed_sources_and_removes_failed_source test {
+    builder := init_builder(mem.c_allocator, "")
+    defer deinit_builder(&builder)
+    first :: try set_source(&builder, "first.hon", "first")
+    root :: try add_package(&builder, Span{ start = 0, end = 5, source = first }, &[])
+    failed :: try set_source(&builder, "failed.hon", "failed")
+    try testing.expect_equal(diagnostics.SourceId(1), failed)
+    remove_source(&builder)
+    try testing.expect_equal(first, builder.source_id)
+    try testing.expect_equal("first", builder.source)
+    next :: try set_source(&builder, "next.hon", "next")
+    try testing.expect_equal(failed, next)
+    # package roots may use a source other than the currently active source.
+    declarations [1]NodeId :: [root]
+    combined :: try add_package(&builder, Span{ start = 0, end = 5, source = first }, declarations[..])
+    tree :: view(&builder, combined)
+    try testing.expect_equal(first, node_span(&tree, combined).source)
+    try testing.expect_equal("first", source_text(&tree, node_span(&tree, combined)))
+    try testing.expect_equal(2, source_files(&tree).len)
+    try testing.expect_equal("first.hon", source_files(&tree)[usize(first)].path)
+    try testing.expect_equal("next.hon", source_files(&tree)[usize(next)].path)
+    try testing.expect_equal("next", source_files(&tree)[usize(next)].text)
+    try testing.expect_equal(1, builder_package_decl_count(&builder, combined))
+    try testing.expect_equal(root, builder_package_decl(&builder, combined, 0))
+}
+
+removes_only_registered_source_without_allocating_builder test {
+    builder := init_builder(mem.c_allocator, "")
+    defer deinit_builder(&builder)
+    _ = try set_source(&builder, "failed.hon", "failed")
+    remove_source(&builder)
+    try testing.expect_equal(diagnostics.SourceId(0), builder.source_id)
+    try testing.expect_equal("", builder.source)
+    replacement :: try set_source(&builder, "replacement.hon", "")
+    try testing.expect_equal(diagnostics.SourceId(0), replacement)
+}

source/ast/renderer/renderer.hon

@@ -26,6 +26,11 @@ render proc(renderer @mut Renderer, tree @ast.Tree) void ! mem.AllocError {
         match ast.decl_kind(tree, declaration) {
             .statement: try render_statement(renderer, tree, declaration, "  ")
             .function: try render_function(renderer, tree, declaration)
+            .import: {
+                try append_text(renderer, "  ")
+                try append_text(renderer, ast.source_text(tree, ast.import_decl(tree, declaration).span))
+                try append_text(renderer, "\n")
+            }
             .invalid: try append_text(renderer, "  invalid declaration\n")
         }
     }

source/ast/renderer/renderer.test.hon

@@ -31,3 +31,24 @@ renders_array_annotations_literals_and_indexed_assignments test {
         renderer.bytes.items,
     )
 }
+
+renders_import_declarations test {
+    strings := strpool.init(mem.c_allocator)
+    defer strpool.deinit(&strings)
+    diagnostic_store := diagnostics.init(mem.c_allocator)
+    defer diagnostics.deinit(&diagnostic_store)
+    source :: "import \"relative\"\nname :: import \"@root/pkg\"\ntest import \"test_only\"\n"
+    scan_state := lexer.init(mem.c_allocator)
+    defer lexer.deinit(&scan_state)
+    try lexer.scan(&scan_state, source, &strings, &diagnostic_store)
+    tree := try parser.parse(mem.c_allocator, scan_state.tokens.items, source, &diagnostic_store)
+    defer ast.deinit(&tree)
+    renderer := init(mem.c_allocator)
+    defer deinit(&renderer)
+    try render(&renderer, &tree)
+    try testing.expect_equal(
+        "package\n  import \"relative\"\n  name :: import \"@root/pkg\"\n  test import \"test_only\"\n",
+        renderer.bytes.items,
+    )
+    try testing.expect_equal(0, diagnostics.count(&diagnostic_store))
+}

source/compilation/compilation.hon

@@ -1,6 +1,8 @@
 import "@std/mem"
 import "@std/debug"
 import "@std/arraylist"
+import "@std/hashmap"
+import "@std/io"
 import "@source/strpool"
 import "@source/diagnostics"
 import "@source/lexer"
@@ -8,14 +10,15 @@ import "@source/parser"
 import "@source/ast"
 import "@source/sema"
 import "@source/lower"
+import "@source/loader"
 import "@source/ir"
 typecheck :: import "@source/sema/check"
 
 #! identities are valid only within their owning compilation.
-SourceId :: distinct usize
+SourceId :: alias diagnostics.SourceId
 PackageId :: distinct usize
 
-#! AST node IDs are source-local; retain both fields when referring across sources.
+#! node IDs belong to the compilation arena; source identifies their originating file.
 DeclarationId :: struct {
     source SourceId
     node ast.NodeId
@@ -25,58 +28,79 @@ Source :: struct {
     package PackageId
     path []u8
     bytes arraylist.ArrayList(u8)
-    tree ast.Tree
+    root ast.NodeId
 }
 
 Package :: struct {
     sources arraylist.ArrayList(SourceId)
+    path []u8 = &[]
+    root ?ast.NodeId = null
+    loading bool = false
 }
 
-#! owns sources, paths, ASTs, package membership and one identifier pool.
-#! borrows returned by source are invalidated by add_source or deinit; IDs remain stable.
+Import :: struct {
+    node ast.NodeId
+    target ?PackageId
+}
+
+#! owns source buffers, paths, the AST arena, package graph and one identifier pool.
+#! metadata borrows are invalidated by registration; IDs remain stable until deinit.
 Compilation :: struct {
     allocator mem.Allocator
     strings strpool.StringPool
+    builder ast.Builder
     sources arraylist.ArrayList(Source)
     packages arraylist.ArrayList(Package)
+    imports arraylist.ArrayList(Import)
+    paths hashmap.StringHashMap(PackageId)
+    module_root ?ast.NodeId = null
 }
 
-Error :: alias mem.AllocError | diagnostics.CapacityError | lexer.SourceError | strpool.InternError
+LoadError :: enum { empty_package }
+Error :: alias mem.AllocError | diagnostics.CapacityError | lexer.SourceError | strpool.InternError | loader.Error | LoadError
 
 #! retains `allocator`; release all compilation storage with deinit.
 init proc(allocator mem.Allocator) Compilation {
     return Compilation{
         allocator = allocator,
         strings = strpool.init(allocator),
+        builder = ast.init_builder(allocator, &[]),
         sources = arraylist.init(allocator),
         packages = arraylist.init(allocator),
+        imports = arraylist.init(allocator),
+        paths = hashmap.init(allocator),
     }
 }
 
 deinit proc(compilation @mut Compilation) void {
+    ast.deinit_builder(&compilation.builder)
+    hashmap.deinit(&compilation.paths)
     for (compilation.sources.items) |@file| {
-        ast.deinit(&file.tree)
         arraylist.deinit(&file.bytes)
         mem.free(compilation.allocator, file.path)
     }
-    for (compilation.packages.items) |@package| arraylist.deinit(&package.sources)
+    for (compilation.packages.items) |@package| {
+        arraylist.deinit(&package.sources)
+        mem.free(compilation.allocator, package.path)
+    }
     arraylist.deinit(&compilation.sources)
     arraylist.deinit(&compilation.packages)
+    arraylist.deinit(&compilation.imports)
     strpool.deinit(&compilation.strings)
 }
 
-#! creates a distinct package; directory discovery and path deduplication belong to the loader.
+#! creates an in-memory package; load registers filesystem packages by canonical directory.
 add_package proc(compilation @mut Compilation) PackageId ! mem.AllocError {
     id :: PackageId(compilation.packages.items.len)
     try arraylist.append(&compilation.packages, Package{
         sources = arraylist.init(compilation.allocator),
     })
+    compilation.module_root = null
     return id
 }
 
-#! copies `path` and scans/parses using the shared pool; takes `bytes` only on success.
-#! success resets `bytes` without copying its allocation. failure leaves it caller-owned;
-#! diagnostics and interned names produced before failure remain in their respective stores.
+#! copies `path`, scans/parses into the shared arena and takes `bytes` only on success.
+#! failure leaves bytes caller-owned and rolls back source diagnostics; interned names may remain.
 add_source proc(
     compilation @mut Compilation,
     package PackageId,
@@ -85,86 +109,248 @@ add_source proc(
     diagnostic_store @mut diagnostics.Store,
 ) SourceId ! Error {
     debug.assert(usize(package) < compilation.packages.items.len)
-
-    id :: SourceId(compilation.sources.items.len)
     try arraylist.reserve(&compilation.sources, compilation.sources.items.len + 1)
     members := &compilation.packages.items[usize(package)].sources
     try arraylist.reserve(members, members.items.len + 1)
     owned_path :: try mem.alloc(u8, compilation.allocator, path.len)
     errdefer mem.free(compilation.allocator, owned_path)
     memcopy!(owned_path, path)
-
+    id :: try ast.set_source(&compilation.builder, owned_path, bytes.items)
+    errdefer ast.remove_source(&compilation.builder)
+    diagnostic_count :: diagnostics.count(diagnostic_store)
+    errdefer diagnostic_store.list.items = diagnostic_store.list.items[..diagnostic_count]
+    debug.assert(usize(id) == compilation.sources.items.len)
     scan_state := lexer.init(compilation.allocator)
     defer lexer.deinit(&scan_state)
-    try lexer.scan(
-        &scan_state,
-        bytes.items,
-        &compilation.strings,
-        diagnostic_store,
-    )
-    tree := try parser.parse(
-        compilation.allocator,
-        scan_state.tokens.items,
-        bytes.items,
-        diagnostic_store,
-    )
-    errdefer ast.deinit(&tree)
-
+    scan_state.source_id = id
+    try lexer.scan(&scan_state, bytes.items, &compilation.strings, diagnostic_store)
+    root :: try parser.parse_source(&compilation.builder, scan_state.tokens.items, diagnostic_store)
     try arraylist.append(&compilation.sources, Source{
         package = package,
         path = owned_path,
         bytes = bytes^,
-        tree = tree,
+        root = root,
     })
     try arraylist.append(members, id)
     bytes^ = arraylist.init(bytes.allocator)
-
+    compilation.packages.items[usize(package)].root = null
+    compilation.module_root = null
     return id
 }
 
-#! borrows source metadata and its AST until add_source or deinit.
+#! borrows source metadata until registration or deinit.
 source proc(compilation @Compilation, id SourceId) @Source {
     debug.assert(usize(id) < compilation.sources.items.len)
     return &compilation.sources.items[usize(id)]
 }
 
-#! qualifies a top-level declaration's local AST identity with its owning source.
 declaration proc(compilation @Compilation, id SourceId, index usize) DeclarationId {
     return DeclarationId{
         source = id,
-        node = ast.package_decl(&source(compilation, id).tree, index),
+        node = ast.builder_package_decl(&compilation.builder, source(compilation, id).root, index),
     }
 }
 
-#! compiles one registered source; the returned IR owns its storage and borrows no frontend data.
-#! package scopes and cross-source resolution are added by the package-loading milestone.
+#! borrows source paths and bytes until registration or deinit; valid after loading a source.
+source_files proc(compilation @Compilation) []diagnostics.Source {
+    tree :: ast.view(&compilation.builder, compilation.sources.items[0].root)
+    return ast.source_files(&tree)
+}
+
+#! loads a file alone or every production source in a directory, then its dependency graph.
+#! directory identities resolve symlinks; file commands do not implicitly include siblings.
+load proc(
+    compilation @mut Compilation,
+    runtime io.Io,
+    path []u8,
+    diagnostic_store @mut diagnostics.Store,
+) PackageId ! Error {
+    canonical :: try loader.canonical(compilation.allocator, path)
+    defer mem.free(compilation.allocator, canonical)
+    directory :: try loader.directory(compilation.allocator, canonical)
+    transferred := false
+    defer { if (!transferred) mem.free(compilation.allocator, directory) }
+    project_root :: try loader.project_root(compilation.allocator, directory)
+    defer mem.free(compilation.allocator, project_root)
+    entry_file ?[]u8 := null
+    if (!try loader.is_directory(compilation.allocator, canonical)) entry_file = canonical
+    transferred = true
+    return try load_package(compilation, runtime, project_root, directory, entry_file, null, diagnostic_store)
+}
+
+@hide:file
+load_package proc(
+    compilation @mut Compilation,
+    runtime io.Io,
+    project_root []u8,
+    owned_directory []mut u8,
+    entry_file ?[]u8,
+    import_span ?diagnostics.Span,
+    diagnostic_store @mut diagnostics.Store,
+) PackageId ! Error {
+    registered := false
+    defer { if (!registered) mem.free(compilation.allocator, owned_directory) }
+    if (hashmap.get(&compilation.paths, owned_directory)) |existing| {
+        if (compilation.packages.items[usize(existing)].loading) {
+            if (import_span) |span| _ = try diagnostics.add(diagnostic_store, .import_cycle, span)
+        }
+        return existing
+    }
+    id :: try add_package(compilation)
+    compilation.packages.items[usize(id)].path = owned_directory
+    compilation.packages.items[usize(id)].loading = true
+    registered = true
+    hashmap.put(&compilation.paths, owned_directory, id) catch |err| {
+        match err {
+            .key_exists: unreachable
+            else: return err
+        }
+    }
+    read_failed := false
+    load_sources(compilation, runtime, id, entry_file, diagnostic_store) catch |err| {
+        match err {
+            .out_of_memory, .diagnostic_limit_reached, .storage_limit_reached, .out_of_space: return err
+            else: {
+                if (import_span) |span| {
+                    _ = try diagnostics.add(diagnostic_store, .package_unavailable, span)
+                    read_failed = true
+                } else return err
+            }
+        }
+    }
+    if (compilation.packages.items[usize(id)].sources.items.len == 0) {
+        if (import_span) |span| {
+            if (!read_failed) _ = try diagnostics.add(diagnostic_store, .package_unavailable, span)
+        } else return .empty_package
+    }
+    # Source and package arrays may grow during recursion; retain IDs, never their element addresses.
+    for (0..(compilation.packages.items[usize(id)].sources.items.len)) |index| {
+        file_id :: compilation.packages.items[usize(id)].sources.items[index]
+        root :: source(compilation, file_id).root
+        count :: ast.builder_package_decl_count(&compilation.builder, root)
+        for (0..count) |declaration_index| {
+            node :: ast.builder_package_decl(&compilation.builder, root, declaration_index)
+            tree :: ast.view(&compilation.builder, root)
+            if (ast.decl_kind(&tree, node) != .import) continue
+            imported :: ast.import_decl(&tree, node)
+            if (imported.test_only) continue
+            path :: ast.source_text(&tree, imported.path)
+            target ?PackageId := null
+            if (path.len == 0 or path[0] == '/' or (path.len == 1 and path[0] == '@')) {
+                _ = try diagnostics.add(diagnostic_store, .invalid_import_path, imported.path)
+            } else {
+                resolved := loader.resolve(
+                    compilation.allocator,
+                    project_root,
+                    compilation.packages.items[usize(id)].path,
+                    path,
+                ) catch |err| {
+                    match err {
+                        .out_of_memory: return err
+                        .invalid_path: {
+                            _ = try diagnostics.add(diagnostic_store, .invalid_import_path, imported.path)
+                            try arraylist.append(&compilation.imports, Import{ node = node, target = null })
+                            continue
+                        }
+                        else: {
+                            _ = try diagnostics.add(diagnostic_store, .package_unavailable, imported.path)
+                            try arraylist.append(&compilation.imports, Import{ node = node, target = null })
+                            continue
+                        }
+                    }
+                }
+                owned := true
+                defer { if (owned) mem.free(compilation.allocator, resolved) }
+                if (try loader.is_directory(compilation.allocator, resolved)) {
+                    owned = false
+                    target = try load_package(compilation, runtime, project_root, resolved, null, imported.path, diagnostic_store)
+                } else {
+                    _ = try diagnostics.add(diagnostic_store, .package_unavailable, imported.path)
+                }
+            }
+            try arraylist.append(&compilation.imports, Import{ node = node, target = target })
+        }
+    }
+    compilation.packages.items[usize(id)].loading = false
+    return id
+}
+
+@hide:file
+load_sources proc(
+    compilation @mut Compilation,
+    runtime io.Io,
+    package PackageId,
+    entry_file ?[]u8,
+    diagnostic_store @mut diagnostics.Store,
+) void ! Error {
+    if (entry_file) |path| {
+        try load_source(compilation, runtime, package, path, diagnostic_store)
+        return
+    }
+    files := try loader.files(compilation.allocator, compilation.packages.items[usize(package)].path)
+    defer {
+        for (files.items) |path| mem.free(compilation.allocator, path)
+        arraylist.deinit(&files)
+    }
+    for (files.items) |path| try load_source(compilation, runtime, package, path, diagnostic_store)
+}
+
+@hide:file
+load_source proc(
+    compilation @mut Compilation,
+    runtime io.Io,
+    package PackageId,
+    path []u8,
+    diagnostic_store @mut diagnostics.Store,
+) void ! Error {
+    bytes := try loader.read_source(compilation.allocator, runtime, path)
+    defer arraylist.deinit(&bytes)
+    _ = try add_source(compilation, package, path, &bytes, diagnostic_store)
+}
+
+@hide:file
+package_root proc(compilation @mut Compilation, package PackageId) ast.NodeId ! mem.AllocError {
+    if (compilation.packages.items[usize(package)].root) |root| return root
+    declarations := arraylist.init(ast.NodeId, compilation.allocator)
+    defer arraylist.deinit(&declarations)
+    span := diagnostics.Span{ start = 0, end = 0 }
+    for (compilation.packages.items[usize(package)].sources.items) |id, index| {
+        root :: source(compilation, id).root
+        tree :: ast.view(&compilation.builder, root)
+        if (index == 0) span = ast.node_span(&tree, root)
+        for (0..ast.package_decl_count(&tree)) |i| try arraylist.append(&declarations, ast.package_decl(&tree, i))
+    }
+    root :: try ast.add_package(&compilation.builder, span, declarations.items)
+    compilation.packages.items[usize(package)].root = root
+    return root
+}
+
+#! checks every package in a separate scope and lowers one module with root-only entry selection.
+#! returned IR borrows no frontend data; release it with ir.deinit.
 compile proc(
     compilation @mut Compilation,
-    id SourceId,
+    root_package PackageId,
     diagnostic_store @mut diagnostics.Store,
 ) ir.Module ! Error {
-    tree :: &source(compilation, id).tree
-    resolution := try sema.resolve(
-        compilation.allocator,
-        tree,
-        &compilation.strings,
-        diagnostic_store,
-    )
+    roots := arraylist.init(ast.NodeId, compilation.allocator)
+    defer arraylist.deinit(&roots)
+    declarations := arraylist.init(ast.NodeId, compilation.allocator)
+    defer arraylist.deinit(&declarations)
+    for (0..(compilation.packages.items.len)) |index| {
+        root :: try package_root(compilation, PackageId(index))
+        try arraylist.append(&roots, root)
+        if (compilation.module_root == null) {
+            tree :: ast.view(&compilation.builder, root)
+            for (0..ast.package_decl_count(&tree)) |i| try arraylist.append(&declarations, ast.package_decl(&tree, i))
+        }
+    }
+    if (compilation.module_root == null) {
+        compilation.module_root = try ast.add_package(&compilation.builder, diagnostics.Span{ start = 0, end = 0 }, declarations.items)
+    }
+    tree :: ast.view(&compilation.builder, compilation.module_root?)
+    resolution := try sema.resolve_packages(compilation.allocator, &tree, roots.items, &compilation.strings, diagnostic_store)
     defer sema.deinit(&resolution)
-
-    checked := try typecheck.check(
-        compilation.allocator,
-        tree,
-        &resolution,
-        diagnostic_store,
-    )
+    checked := try typecheck.check(compilation.allocator, &tree, &resolution, diagnostic_store)
     defer typecheck.deinit(&checked)
-
-    return try lower.lower(
-        compilation.allocator,
-        tree,
-        &resolution,
-        &checked,
-        diagnostic_store,
-    )
+    return try lower.lower_package(compilation.allocator, &tree, &resolution, &checked, roots.items[usize(root_package)], diagnostic_store)
 }

source/compilation/compilation.test.hon

@@ -15,7 +15,7 @@ owned_text proc(text []u8) arraylist.ArrayList(u8) ! mem.AllocError {
     return bytes
 }
 
-keeps_source_local_declarations_distinct_and_ir_independent test {
+keeps_package_declarations_distinct_and_ir_independent test {
     diagnostic_store := diagnostics.init(mem.c_allocator)
     defer diagnostics.deinit(&diagnostic_store)
     first := try owned_text("value :: 21\nmain proc() int { return value * 2 }")
@@ -47,16 +47,14 @@ keeps_source_local_declarations_distinct_and_ir_independent test {
         try testing.expect_equal(third, session.packages.items[usize(root)].sources.items[1])
         a :: declaration(&session, left, 0)
         b :: declaration(&session, right, 0)
-        try testing.expect_equal(a.node, b.node)
         try testing.expect(a.source != b.source)
-        left_tree :: &source(&session, a.source).tree
-        right_tree :: &source(&session, b.source).tree
-        try testing.expect_equal(ast.name_id(left_tree, a.node), ast.name_id(right_tree, b.node))
-        try testing.expect(mem.eql(ast.source_text(left_tree, ast.statement_decl(left_tree, a.node).span), "value :: 21"))
-        try testing.expect(mem.eql(ast.source_text(right_tree, ast.statement_decl(right_tree, b.node).span), "value :: 3"))
-        modules[0] = try compile(&session, left, &diagnostic_store)
+        tree :: ast.view(&session.builder, source(&session, left).root)
+        try testing.expect_equal(ast.name_id(&tree, a.node), ast.name_id(&tree, b.node))
+        try testing.expect(mem.eql(ast.source_text(&tree, ast.statement_decl(&tree, a.node).span), "value :: 21"))
+        try testing.expect(mem.eql(ast.source_text(&tree, ast.statement_decl(&tree, b.node).span), "value :: 3"))
+        modules[0] = try compile(&session, root, &diagnostic_store)
         errdefer ir.deinit(&modules[0])
-        modules[1] = try compile(&session, right, &diagnostic_store)
+        modules[1] = try compile(&session, dependency, &diagnostic_store)
     }
     defer ir.deinit(&modules[0])
     defer ir.deinit(&modules[1])

source/diagnostics/diagnostics.hon

@@ -2,58 +2,70 @@ import "@std/mem"
 import "@std/arraylist"
 import "@std/enummap"
 
+SourceId :: distinct usize
+
+#! borrows a source path and text owned by the compilation or caller.
+Source :: struct {
+    path []u8
+    text []u8
+}
+
 Span :: struct {
     start u32
     end u32
+    source SourceId = SourceId(0)
 }
 
 DiagnosticId :: distinct u32
 no_diagnostic :: maxval!(DiagnosticId)
 
 Code :: enum {
-    invalid_character,
-    float_must_end_with_digit,
-    unterminated_string,
-    unexpected_token,
-    duplicate_name,
-    unknown_name,
-    expected_type,
-    type_not_value,
-    invalid_void_type,
-    type_mismatch,
-    invalid_operand,
-    integer_division,
-    not_callable,
-    argument_count_mismatch,
-    cannot_infer_type,
-    missing_return,
-    return_type_mismatch,
-    condition_not_bool,
-    invalid_assignment_target,
-    break_outside_loop,
-    continue_outside_loop,
-    invalid_range_bound,
-    invalid_for_iterable,
-    range_value_not_supported,
-    range_pointer_capture,
-    range_index_capture,
-    invalid_array_length,
-    array_length_mismatch,
-    invalid_array_element_type,
-    array_size_overflow,
-    invalid_index_type,
-    not_indexable,
-    invalid_index_assignment,
-    index_out_of_bounds,
-    missing_entry,
-    invalid_entry,
-    literal_out_of_range,
-    inexact_integer_literal,
-    invalid_escape,
-    read_before_initialization,
-    write_before_initialization,
-    initialization_cycle,
-    integer_overflow,
+    invalid_character
+    float_must_end_with_digit
+    unterminated_string
+    unexpected_token
+    duplicate_name
+    unknown_name
+    expected_type
+    type_not_value
+    invalid_void_type
+    type_mismatch
+    invalid_operand
+    integer_division
+    not_callable
+    argument_count_mismatch
+    cannot_infer_type
+    missing_return
+    return_type_mismatch
+    condition_not_bool
+    invalid_assignment_target
+    break_outside_loop
+    continue_outside_loop
+    invalid_range_bound
+    invalid_for_iterable
+    range_value_not_supported
+    range_pointer_capture
+    range_index_capture
+    invalid_array_length
+    array_length_mismatch
+    invalid_array_element_type
+    array_size_overflow
+    invalid_index_type
+    not_indexable
+    invalid_index_assignment
+    index_out_of_bounds
+    missing_entry
+    invalid_entry
+    literal_out_of_range
+    inexact_integer_literal
+    invalid_escape
+    read_before_initialization
+    write_before_initialization
+    initialization_cycle
+    integer_overflow
+    import_cycle
+    invalid_import_path
+    package_unavailable
 }
 
 Diagnostic :: struct {
@@ -138,5 +150,8 @@ messages :: $enummap.init(Code, []u8, {
     write_before_initialization = "local value was assigned before initialization",
     initialization_cycle = "global initialization cycle",
     integer_overflow = "integer arithmetic overflow",
+    import_cycle = "package import cycle",
+    invalid_import_path = "invalid import path",
+    package_unavailable = "cannot load package directory",
 })
 

source/lexer/scan.hon

@@ -33,12 +33,15 @@ keywords strmap.StaticStringMap(TokenKind) :: strmap.init([
     { "and", .and },
     { "or", .or },
     { "mut", .mut },
+    { "import", .import },
+    { "test", .test },
 ])
 
 #! owns token storage and retains its allocator; release with deinit.
 #! identifier IDs belong to the compilation's separately owned string pool.
 State :: struct {
     tokens arraylist.ArrayList(Token)
+    source_id diagnostics.SourceId = diagnostics.SourceId(0)
 }
 
 #! retains `allocator` for scanned tokens; the caller owns the returned State.
@@ -53,6 +56,7 @@ deinit proc(state @mut State) void {
 
 #! borrows `program`, `pool`, and `diagnostic_store`; tokens remain state-owned.
 #! identifier storage uses the pool's allocator; keep the pool alive while using their IDs.
+#! `source_id` is copied into every token and scanner diagnostic span.
 scan proc(
     state @mut State,
     program []u8,
@@ -60,12 +64,16 @@ scan proc(
     diagnostic_store @mut diagnostics.Store,
 ) void ! (mem.AllocError | strpool.InternError | diagnostics.CapacityError | SourceError) {
     if (program.len > usize(maxval!(u32))) return .source_too_large
+    source :: state.source_id
     cursor := 0
     while cursor < program.len {
         char :: program[cursor]
 
         if char == '\n' {
-            try add_token(state, Token{ span = make_span(cursor, cursor + 1), kind = .newline })
+            try add_token(state, Token{
+                span = make_span(source, cursor, cursor + 1),
+                kind = .newline,
+            })
             cursor += 1
             continue
         }
@@ -84,7 +92,7 @@ scan proc(
             start :: cursor
             result :: try scan_ident(start, program, pool)
             try add_token(state, Token{
-                span = make_span(start, result.end),
+                span = make_span(source, start, result.end),
                 kind = result.kind,
                 data = TokenData{ str_id = result.str_id },
             })
@@ -98,10 +106,10 @@ scan proc(
                 id :: try diagnostics.add(
                     diagnostic_store,
                     diagnostic_code(err.code),
-                    make_span(start, err.end),
+                    make_span(source, start, err.end),
                 )
                 try add_token(state, Token{
-                    span = make_span(start, err.end),
+                    span = make_span(source, start, err.end),
                     kind = .invalid,
                     data = TokenData{ diagnostic = id },
                 })
@@ -109,7 +117,10 @@ scan proc(
                 continue
             }
             kind :: if (result.has_decimal) .float else .int
-            try add_token(state, Token{ span = make_span(start, result.end), kind = kind })
+            try add_token(state, Token{
+                span = make_span(source, start, result.end),
+                kind = kind,
+            })
             cursor = result.end
             continue
         }
@@ -120,17 +131,20 @@ scan proc(
                 id :: try diagnostics.add(
                     diagnostic_store,
                     diagnostic_code(err.code),
-                    make_span(start, err.end),
+                    make_span(source, start, err.end),
                 )
                 try add_token(state, Token{
-                    span = make_span(start, err.end),
+                    span = make_span(source, start, err.end),
                     kind = .invalid,
                     data = TokenData{ diagnostic = id },
                 })
                 cursor = err.end
                 continue
             }
-            try add_token(state, Token{ span = make_span(start, result.end), kind = .string })
+            try add_token(state, Token{
+                span = make_span(source, start, result.end),
+                kind = .string,
+            })
             cursor = result.end
             continue
         }
@@ -140,7 +154,7 @@ scan proc(
             width :: if (inclusive) 3 else 2
             range_kind :: if (inclusive) .range_inclusive else .range
             try add_token(state, Token{
-                span = make_span(cursor, cursor + width),
+                span = make_span(source, cursor, cursor + width),
                 kind = range_kind,
             })
             cursor += width
@@ -150,7 +164,7 @@ scan proc(
         if char == ':' {
             if cursor + 1 < program.len and program[cursor + 1] == ':' {
                 try add_token(state, Token{
-                    span = make_span(cursor, cursor + 2),
+                    span = make_span(source, cursor, cursor + 2),
                     kind = .double_colon,
                 })
                 cursor += 2
@@ -158,13 +172,16 @@ scan proc(
             }
             if cursor + 1 < program.len and program[cursor + 1] == '=' {
                 try add_token(state, Token{
-                    span = make_span(cursor, cursor + 2),
+                    span = make_span(source, cursor, cursor + 2),
                     kind = .colon_equal,
                 })
                 cursor += 2
                 continue
             }
-            try add_token(state, Token{ span = make_span(cursor, cursor + 1), kind = .colon })
+            try add_token(state, Token{
+                span = make_span(source, cursor, cursor + 1),
+                kind = .colon,
+            })
             cursor += 1
             continue
         }
@@ -172,52 +189,64 @@ scan proc(
         if char == '+' {
             if cursor + 1 < program.len and program[cursor + 1] == '=' {
                 try add_token(state, Token{
-                    span = make_span(cursor, cursor + 2),
+                    span = make_span(source, cursor, cursor + 2),
                     kind = .plus_equal,
                 })
                 cursor += 2
                 continue
             }
-            try add_token(state, Token{ span = make_span(cursor, cursor + 1), kind = .plus })
+            try add_token(state, Token{
+                span = make_span(source, cursor, cursor + 1),
+                kind = .plus,
+            })
             cursor += 1
             continue
         }
         if char == '-' {
             if cursor + 1 < program.len and program[cursor + 1] == '=' {
                 try add_token(state, Token{
-                    span = make_span(cursor, cursor + 2),
+                    span = make_span(source, cursor, cursor + 2),
                     kind = .minus_equal,
                 })
                 cursor += 2
                 continue
             }
-            try add_token(state, Token{ span = make_span(cursor, cursor + 1), kind = .minus })
+            try add_token(state, Token{
+                span = make_span(source, cursor, cursor + 1),
+                kind = .minus,
+            })
             cursor += 1
             continue
         }
         if char == '*' {
             if cursor + 1 < program.len and program[cursor + 1] == '=' {
                 try add_token(state, Token{
-                    span = make_span(cursor, cursor + 2),
+                    span = make_span(source, cursor, cursor + 2),
                     kind = .star_equal,
                 })
                 cursor += 2
                 continue
             }
-            try add_token(state, Token{ span = make_span(cursor, cursor + 1), kind = .star })
+            try add_token(state, Token{
+                span = make_span(source, cursor, cursor + 1),
+                kind = .star,
+            })
             cursor += 1
             continue
         }
         if char == '/' {
             if cursor + 1 < program.len and program[cursor + 1] == '=' {
                 try add_token(state, Token{
-                    span = make_span(cursor, cursor + 2),
+                    span = make_span(source, cursor, cursor + 2),
                     kind = .slash_equal,
                 })
                 cursor += 2
                 continue
             }
-            try add_token(state, Token{ span = make_span(cursor, cursor + 1), kind = .slash })
+            try add_token(state, Token{
+                span = make_span(source, cursor, cursor + 1),
+                kind = .slash,
+            })
             cursor += 1
             continue
         }
@@ -232,7 +261,7 @@ scan proc(
             }
             if (comparison) |value| {
                 try add_token(state, Token{
-                    span = make_span(cursor, cursor + 2),
+                    span = make_span(source, cursor, cursor + 2),
                     kind = value,
                  })
                 cursor += 2
@@ -257,7 +286,10 @@ scan proc(
             else: null
         }
         if (kind) |value| {
-            try add_token(state, Token{ span = make_span(cursor, cursor + 1), kind = value })
+            try add_token(state, Token{
+                span = make_span(source, cursor, cursor + 1),
+                kind = value,
+            })
             cursor += 1
             continue
         }
@@ -265,17 +297,20 @@ scan proc(
         id :: try diagnostics.add(
             diagnostic_store,
             .invalid_character,
-            make_span(cursor, cursor + 1),
+            make_span(source, cursor, cursor + 1),
         )
         try add_token(state, Token{
-            span = make_span(cursor, cursor + 1),
+            span = make_span(source, cursor, cursor + 1),
             kind = .invalid,
             data = TokenData{ diagnostic = id },
         })
         cursor += 1
     }
 
-    try add_token(state, Token{ span = make_span(cursor, cursor), kind = .eof })
+    try add_token(state, Token{
+        span = make_span(source, cursor, cursor),
+        kind = .eof,
+    })
 }
 
 @hide:file
@@ -389,8 +424,12 @@ diagnostic_code proc(code ErrorCode) diagnostics.Code {
 }
 
 @hide:file
-make_span proc(start, end usize) diagnostics.Span {
-    return diagnostics.Span{ start = u32(start), end = u32(end) }
+make_span proc(source diagnostics.SourceId, start, end usize) diagnostics.Span {
+    return diagnostics.Span{
+        start = u32(start),
+        end = u32(end),
+        source = source,
+    }
 }
 
 @hide:file

source/lexer/scan.test.hon

@@ -218,3 +218,23 @@ propagates_identifier_interning_allocation_failure test {
         try testing.expect_equal(usize(0), allocation.live)
     }
 }
+
+scans_import_keywords_and_preserves_source_in_tokens_and_diagnostics test {
+    pool := strpool.init(mem.c_allocator)
+    defer strpool.deinit(&pool)
+    diagnostic_store := diagnostics.init(mem.c_allocator)
+    defer diagnostics.deinit(&diagnostic_store)
+    state := init(mem.c_allocator)
+    defer deinit(&state)
+    state.source_id = diagnostics.SourceId(7)
+    try scan(&state, "import test $\n1.\n\"unfinished", &pool, &diagnostic_store)
+    try testing.expect_equal(TokenKind.import, state.tokens.items[0].kind)
+    try testing.expect_equal(TokenKind.test, state.tokens.items[1].kind)
+    try testing.expect_equal(null, strpool.get_id(&pool, "import"))
+    try testing.expect_equal(null, strpool.get_id(&pool, "test"))
+    for (state.tokens.items) |token| try testing.expect_equal(state.source_id, token.span.source)
+    try testing.expect_equal(3, diagnostics.count(&diagnostic_store))
+    for (diagnostic_store.list.items) |diagnostic| {
+        try testing.expect_equal(state.source_id, diagnostic.span.source)
+    }
+}

source/lexer/token.hon

@@ -25,6 +25,7 @@ TokenKind :: enum {
     true, false
     and, or
     mut
+    import, test
 
     # literals
     ident, string

source/llvm/instructions.hon

@@ -38,7 +38,7 @@ emit_instruction proc(state @mut State, block @ir.Block, instruction ir.Instruct
         .load_global: {
             pointer :: fresh(state)
             position :: location(state.line_starts, instruction.span)
-            try io.print("  %r{} = call ptr @global{}(i64 {}, i64 {})\n", output, {pointer, instruction.slot, position.line, position.column})
+            try io.print("  %r{} = call ptr @global{}(i64 {}, i64 {}, i64 {})\n", output, {pointer, instruction.slot, usize(instruction.span.source), position.line, position.column})
             try io.print("  %r{} = load ", output, {result})
             try write_type(output, state.module, type)
             try io.print(", ptr %r{}\n", output, {pointer})
@@ -93,7 +93,7 @@ emit_instruction proc(state @mut State, block @ir.Block, instruction ir.Instruct
             pointer :: fresh(state)
             address :: fresh(state)
             position :: location(state.line_starts, instruction.span)
-            try io.print("  %r{} = call ptr @global{}(i64 {}, i64 {})\n", output, {pointer, instruction.slot, position.line, position.column})
+            try io.print("  %r{} = call ptr @global{}(i64 {}, i64 {}, i64 {})\n", output, {pointer, instruction.slot, usize(instruction.span.source), position.line, position.column})
             try io.print("  %r{} = insertvalue %address poison, ptr %r{}, 0\n", output, {address, pointer})
             try io.print("  %r{} = insertvalue %address %r{}, ptr null, 1\n", output, {result, address})
         }
@@ -104,7 +104,7 @@ emit_instruction proc(state @mut State, block @ir.Block, instruction ir.Instruct
             position :: location(state.line_starts, instruction.span)
             try io.print("  %r{} = extractvalue %address %r{}, 0\n", output, {pointer, address})
             try io.print("  %r{} = extractvalue %address %r{}, 1\n", output, {flag, address})
-            try io.print("  call void @check_init(ptr %r{}, i1 {}, i64 {}, i64 {})\n", output, {flag, instruction.op == .load_address, position.line, position.column})
+            try io.print("  call void @check_init(ptr %r{}, i1 {}, i64 {}, i64 {}, i64 {})\n", output, {flag, instruction.op == .load_address, usize(instruction.span.source), position.line, position.column})
             if (instruction.op == .store_address) {
                 value :: try load(state, instruction.right)
                 try io.write_all(output, "  store ")

source/llvm/llvm.hon

@@ -15,7 +15,7 @@ Type :: struct {
 State :: struct {
     output io.Writer
     module @ir.Module
-    line_starts []usize
+    line_starts []arraylist.ArrayList(usize)
     values []mut Type
     next_register usize = 0
 }
@@ -26,19 +26,34 @@ emit proc(
     output io.Writer,
     module @ir.Module,
     diagnostic_store @diagnostics.Store,
-    source []u8,
-    path []u8,
+    sources []diagnostics.Source,
+    primary diagnostics.SourceId,
 ) void ! (mem.AllocError | io.WriteError) {
-    line_starts := arraylist.init(usize, allocator)
-    defer arraylist.deinit(&line_starts)
-    try arraylist.append(&line_starts, usize(0))
-    for (source) |byte, index| {
-        if (byte == '\n') try arraylist.append(&line_starts, index + 1)
+    line_starts := arraylist.init(arraylist.ArrayList(usize), allocator)
+    defer {
+        for (line_starts.items) |@starts| arraylist.deinit(starts)
+        arraylist.deinit(&line_starts)
+    }
+    for (sources) |source| {
+        starts := arraylist.init(usize, allocator)
+        errdefer arraylist.deinit(&starts)
+        try arraylist.append(&starts, usize(0))
+        for (source.text) |byte, index| {
+            if (byte == '\n') try arraylist.append(&starts, index + 1)
+        }
+        try arraylist.append(&line_starts, starts)
     }
     try io.write_all(output, runtime)
-    try io.write_all(output, "\n@path = private constant ")
-    try bytes_constant(output, path)
-    try io.write_all(output, "\n")
+    for (sources) |source, index| {
+        try io.print("\n@p{} = private constant ", output, {index})
+        try bytes_constant(output, source.path)
+    }
+    try io.print("\n@paths = private constant [{} x %string] [", output, {sources.len})
+    for (sources) |source, index| {
+        if (index != 0) try io.write_all(output, ", ")
+        try io.print("%string {{ ptr @p{}, i64 {} }}", output, {index, source.path.len})
+    }
+    try io.write_all(output, "]\n")
     for (module.strings.items) |_, index| {
         try io.print("@s{} = private constant ", output, {index})
         try bytes_constant(output, ir.string_bytes(module, ir.StringId(index)))
@@ -49,7 +64,7 @@ emit proc(
         try bytes_constant(output, enummap.get(diagnostics.Code, []u8, &diagnostics.messages, diagnostic.code)?)
         try io.write_all(output, "\n")
     }
-    try emit_reporting(output, diagnostic_store, line_starts.items, path.len)
+    try emit_reporting(output, diagnostic_store, line_starts.items, sources, primary)
     for (module.globals.items) |global, index| {
         try io.print("@g{} = internal global ", output, {index})
         storage_kind :: if (global.type == .void) ir.Type.invalid else global.type
@@ -89,7 +104,7 @@ emit proc(
         status i32 :: if (diagnostics.count(diagnostic_store) == 0) 0 else 1
         try io.print("  ret i32 {}\n", output, {status})
     } else {
-        try io.print("  call void @trap(i64 {}, i64 0, i64 0, ptr null, i64 0)\n  unreachable\n", output, {u32(module.entry_diagnostic)})
+        try io.print("  call void @trap(i64 {}, i64 0, i64 0, i64 0, ptr null, i64 0)\n  unreachable\n", output, {u32(module.entry_diagnostic)})
     }
     try io.write_all(output, "}\n")
 }
@@ -148,7 +163,8 @@ Location :: struct {
 }
 
 @hide
-location proc(line_starts []usize, span diagnostics.Span) Location {
+location proc(sources []arraylist.ArrayList(usize), span diagnostics.Span) Location {
+    line_starts :: sources[usize(span.source)].items
     offset :: usize(span.start)
     line := usize(0)
     end := line_starts.len
@@ -161,12 +177,19 @@ location proc(line_starts []usize, span diagnostics.Span) Location {
 }
 
 @hide:file
-emit_reporting proc(output io.Writer, diagnostic_store @diagnostics.Store, line_starts []usize, path_length usize) void ! io.WriteError {
+emit_reporting proc(
+    output io.Writer,
+    diagnostic_store @diagnostics.Store,
+    line_starts []arraylist.ArrayList(usize),
+    sources []diagnostics.Source,
+    primary diagnostics.SourceId,
+) void ! io.WriteError {
     try io.write_all(output, "define internal void @io_failure() noreturn {\nentry:\n")
-    try io.print("  %path_result = call i64 @write(i32 2, ptr @path, i64 {})\n", output, {path_length})
+    try io.print("  %path_result = call i64 @write(i32 2, ptr @p{}, i64 {})\n", output, {usize(primary), sources[usize(primary)].path.len})
     try io.write_all(output, "  %message_result = call i64 @write(i32 2, ptr @write_error, i64 44)\n  call void @exit(i32 2)\n  unreachable\n}\n")
-    try io.write_all(output, "define internal void @diagnose(i64 %line, i64 %column, ptr %message, i64 %length, i1 %trap) {\nentry:\n")
-    try io.print("  call void @out(i32 2, ptr @path, i64 {})\n", output, {path_length})
+    try io.write_all(output, "define internal void @diagnose(i64 %source, i64 %line, i64 %column, ptr %message, i64 %length, i1 %trap) {\nentry:\n")
+    try io.print("  %descriptor = getelementptr [{} x %string], ptr @paths, i64 0, i64 %source\n", output, {sources.len})
+    try io.write_all(output, "  %path = load %string, ptr %descriptor\n  %bytes = extractvalue %string %path, 0\n  %path_length = extractvalue %string %path, 1\n  call void @out(i32 2, ptr %bytes, i64 %path_length)\n")
     try io.write_all(output,
         `  call void @out(i32 2, ptr @colon, i64 1)
         `  call void @print_integer(i32 2, i64 %line, i1 true)
@@ -185,8 +208,8 @@ emit_reporting proc(output io.Writer, diagnostic_store @diagnostics.Store, line_
         position :: location(line_starts, diagnostic.span)
         message :: enummap.get(diagnostics.Code, []u8, &diagnostics.messages, diagnostic.code)?
         try io.print("  %t{} = icmp eq i64 %trap, {}\n", output, {index, index})
-        try io.print("  call void @diagnose(i64 {}, i64 {}, ptr @d{}, i64 {}, i1 %t{})\n", output, {
-            position.line, position.column, index, message.len, index,
+        try io.print("  call void @diagnose(i64 {}, i64 {}, i64 {}, ptr @d{}, i64 {}, i1 %t{})\n", output, {
+            usize(diagnostic.span.source), position.line, position.column, index, message.len, index,
         })
     }
     try io.write_all(output, "  ret void\n}\n")
@@ -195,12 +218,12 @@ emit_reporting proc(output io.Writer, diagnostic_store @diagnostics.Store, line_
 @hide:file
 emit_global proc(output io.Writer, module @ir.Module, global ir.Global, index usize) void ! io.WriteError {
     type :: Type{ kind = global.type, shape = global.array_type }
-    try io.print("define internal ptr @global{}(i64 %line, i64 %column) {{\nentry:\n", output, {index})
+    try io.print("define internal ptr @global{}(i64 %source, i64 %line, i64 %column) {{\nentry:\n", output, {index})
     try io.print("  %state = load i8, ptr @gs{}\n", output, {index})
     try io.write_all(output,
         `  switch i8 %state, label %initialize [ i8 1, label %cycle i8 2, label %ready ]
         `cycle:
-        `  call void @trap(i64 -1, i64 %line, i64 %column, ptr @cycle, i64 27)
+        `  call void @trap(i64 -1, i64 %source, i64 %line, i64 %column, ptr @cycle, i64 27)
         `  unreachable
         `initialize:
     )
@@ -284,7 +307,7 @@ store proc(state @State, destination ir.ValueId, register usize) void ! io.Write
 @hide
 check_local proc(state @State, slot u32, read bool, span diagnostics.Span) void ! io.WriteError {
     position :: location(state.line_starts, span)
-    try io.print("  call void @check_init(ptr %i{}, i1 {}, i64 {}, i64 {})\n", state.output, {slot, read, position.line, position.column})
+    try io.print("  call void @check_init(ptr %i{}, i1 {}, i64 {}, i64 {}, i64 {})\n", state.output, {slot, read, usize(span.source), position.line, position.column})
 }
 
 @hide:file
@@ -297,7 +320,7 @@ trap proc(state @State, code diagnostics.Code, span diagnostics.Span) void ! io.
     }
     message :: enummap.get(diagnostics.Code, []u8, &diagnostics.messages, code)?
     length :: message.len
-    try io.print("  call void @trap(i64 -1, i64 {}, i64 {}, ptr @{s}, i64 {})\n  unreachable\n", state.output, {position.line, position.column, symbol, length})
+    try io.print("  call void @trap(i64 -1, i64 {}, i64 {}, i64 {}, ptr @{s}, i64 {})\n  unreachable\n", state.output, {usize(span.source), position.line, position.column, symbol, length})
 }
 
 @hide
@@ -384,7 +407,7 @@ emit_function proc(state @mut State, function @ir.Function, id usize) void ! io.
                     try io.print(" %r{}\n", output, {result})
                 }
             }
-            .trap: try io.print("  call void @trap(i64 {}, i64 0, i64 0, ptr null, i64 0)\n  unreachable\n", output, {u32(terminator.diagnostic)})
+            .trap: try io.print("  call void @trap(i64 {}, i64 0, i64 0, i64 0, ptr null, i64 0)\n  unreachable\n", output, {u32(terminator.diagnostic)})
             .jump: try io.print("  br label %b{}\n", output, {u32(terminator.target)})
             .branch: {
                 condition :: try load(state, terminator.value)

source/llvm/runtime.hon

@@ -110,19 +110,19 @@ runtime ::
     `  call void @out(i32 1, ptr %bytes, i64 %length)
     `  ret void
     `}
-    `define internal void @trap(i64 %id, i64 %line, i64 %column, ptr %message, i64 %length) noreturn {
+    `define internal void @trap(i64 %id, i64 %source, i64 %line, i64 %column, ptr %message, i64 %length) noreturn {
     `entry:
     `  call void @report(i64 %id)
     `  %generated = icmp eq i64 %id, -1
     `  br i1 %generated, label %new, label %end
     `new:
-    `  call void @diagnose(i64 %line, i64 %column, ptr %message, i64 %length, i1 true)
+    `  call void @diagnose(i64 %source, i64 %line, i64 %column, ptr %message, i64 %length, i1 true)
     `  br label %end
     `end:
     `  call void @exit(i32 1)
     `  unreachable
     `}
-    `define internal void @check_init(ptr %flag, i1 %read, i64 %line, i64 %column) {
+    `define internal void @check_init(ptr %flag, i1 %read, i64 %source, i64 %line, i64 %column) {
     `entry:
     `  %global = icmp eq ptr %flag, null
     `  br i1 %global, label %end, label %check
@@ -132,7 +132,7 @@ runtime ::
     `fail:
     `  %message = select i1 %read, ptr @read, ptr @assign
     `  %length = select i1 %read, i64 42, i64 46
-    `  call void @trap(i64 -1, i64 %line, i64 %column, ptr %message, i64 %length)
+    `  call void @trap(i64 -1, i64 %source, i64 %line, i64 %column, ptr %message, i64 %length)
     `  unreachable
     `end:
     `  ret void

source/loader/loader.hon

@@ -0,0 +1,241 @@
+import "@ffi/c"
+import "@std/mem"
+import "@std/io"
+import "@std/arraylist"
+
+PathError :: enum { invalid_path, path_failed, directory_failed, source_too_large }
+Error :: alias mem.AllocError | PathError | io.OpenError | io.CloseError | io.ReadError
+
+@hide:file
+terminated proc(allocator mem.Allocator, text []u8) []mut u8 ! (mem.AllocError | PathError) {
+    if (text.len == 0 or text.len == maxval!(usize)) return .invalid_path
+    for (text) |byte| if (byte == 0) return .invalid_path
+    bytes :: try mem.alloc(u8, allocator, text.len + 1)
+    memcopy!(bytes[..text.len], text)
+    bytes[text.len] = 0
+    return bytes
+}
+
+@hide:file
+copy proc(allocator mem.Allocator, text []u8) []mut u8 ! mem.AllocError {
+    bytes :: try mem.alloc(u8, allocator, text.len)
+    memcopy!(bytes, text)
+    return bytes
+}
+
+#! caller owns the canonical absolute path; release it with mem.free using `allocator`.
+canonical proc(allocator mem.Allocator, path []u8) []mut u8 ! Error {
+    input :: try terminated(allocator, path)
+    defer mem.free(allocator, input)
+    # Darwin realpath requires a PATH_MAX-sized caller buffer (including its terminator).
+    buffer [1024]mut u8 := undefined
+    if (c.realpath(ptrcast!(input.ptr), ptrcast!(buffer[..].ptr)) == null) {
+        if (c.__error()?^ == c.ENOMEM) return .out_of_memory
+        return .path_failed
+    }
+    length := usize(0)
+    while (length < buffer.len and buffer[length] != 0) length += 1
+    if (length == buffer.len) return .path_failed
+    return try copy(allocator, buffer[..length])
+}
+
+@hide:file
+mode proc(allocator mem.Allocator, path []u8) c_ushort ! Error {
+    input :: try terminated(allocator, path)
+    defer mem.free(allocator, input)
+    result [1]mut c.Stat := undefined
+    if (c.stat(ptrcast!(input.ptr), result[..].ptr) == 0)
+        return result[0].st_mode
+
+    error :: c.__error()?^
+    if (error == c.ENOMEM) return .out_of_memory
+    if (error == c.ENOENT or error == c.ENOTDIR) return 0
+
+    return .path_failed
+}
+
+is_directory proc(allocator mem.Allocator, path []u8) bool ! Error {
+    return ((try mode(allocator, path)) & c_ushort(61440)) == 16384
+}
+
+#! returns an owned canonical package directory for either a file or directory input.
+directory proc(allocator mem.Allocator, path []u8) []mut u8 ! Error {
+    resolved :: try canonical(allocator, path)
+    errdefer mem.free(allocator, resolved)
+    if (try is_directory(allocator, resolved)) return resolved
+
+    end := resolved.len
+    while (end > 1 and resolved[end - 1] != '/') end -= 1
+    parent_text :: if (end == 1) resolved[..1] else resolved[..end - 1]
+    parent :: try copy(allocator, parent_text)
+    mem.free(allocator, resolved)
+
+    return parent
+}
+
+@hide:file
+join proc(allocator mem.Allocator, base, name []u8) []mut u8 ! mem.AllocError {
+    separator :: if (base.len == 1 and base[0] == '/') usize(0) else usize(1)
+    if (base.len > maxval!(usize) - separator or name.len > maxval!(usize) - base.len - separator)
+        return .out_of_memory
+
+    bytes :: try mem.alloc(u8, allocator, base.len + separator + name.len)
+    memcopy!(bytes[..base.len], base)
+
+    if (separator != 0) bytes[base.len] = '/'
+    memcopy!(bytes[base.len + separator..], name)
+
+    return bytes
+}
+
+@hide:file
+exists proc(allocator mem.Allocator, base, name []u8) bool ! Error {
+    path :: try join(allocator, base, name)
+    defer mem.free(allocator, path)
+    input :: try terminated(allocator, path)
+    defer mem.free(allocator, input)
+    if (c.access(ptrcast!(input.ptr), 0) == 0) return true
+
+    error :: c.__error()?^
+    if (error == c.ENOENT or error == c.ENOTDIR) return false
+
+    return .path_failed
+}
+
+#! finds the nearest ancestor build.hon/build.bro, otherwise uses the supplied directory.
+project_root proc(allocator mem.Allocator, package_dir []u8) []mut u8 ! Error {
+    end := package_dir.len
+    while true {
+        base :: package_dir[..end]
+        if (try exists(allocator, base, "build.hon") or try exists(allocator, base, "build.bro"))
+            return try copy(allocator, base)
+
+        if (end == 1) break
+
+        while (end > 1 and package_dir[end - 1] != '/') end -= 1
+        if (end > 1) end -= 1
+    }
+    return try copy(allocator, package_dir)
+}
+
+#! relative imports start at the importing package; @ imports start at the project root.
+resolve proc(allocator mem.Allocator, root_dir, package_dir, import_path []u8) []mut u8 ! Error {
+    if (import_path.len == 0 or import_path[0] == '/') return .invalid_path
+    for (import_path) |byte| if (byte == 0) return .invalid_path
+    base := package_dir
+    path := import_path
+    if (path[0] == '@') {
+        base = root_dir
+        path = path[1..]
+        if (path.len == 0 or path[0] == '/') return .invalid_path
+    }
+    joined :: try join(allocator, base, path)
+    defer mem.free(allocator, joined)
+    return try canonical(allocator, joined)
+}
+
+@hide:file
+ends_with proc(text, suffix []u8) bool {
+    return text.len >= suffix.len and mem.eql(text[text.len - suffix.len..], suffix)
+}
+
+@hide:file
+before proc(left, right []u8) bool {
+    count :: if (left.len < right.len) left.len else right.len
+    for 0..count |index| {
+        if (left[index] != right[index]) return left[index] < right[index]
+    }
+    return left.len < right.len
+}
+
+#! lists production sources in bytewise order; caller frees every path and the list.
+files proc(allocator mem.Allocator, package_dir []u8) arraylist.ArrayList([]u8) ! Error {
+    input :: try terminated(allocator, package_dir)
+    defer mem.free(allocator, input)
+    opened :: c.opendir(ptrcast!(input.ptr))
+    if (opened == null) {
+        if (c.__error()?^ == c.ENOMEM) return .out_of_memory
+        return .directory_failed
+    }
+    handle :: opened?
+    closing := false
+    defer { if (!closing) _ = c.closedir(handle) }
+    result := arraylist.init([]u8, allocator)
+    errdefer {
+        for (result.items) |path| mem.free(allocator, path)
+        arraylist.deinit(&result)
+    }
+    while true {
+        c.__error()?^ = 0
+        next :: c.readdir(handle)
+        if (next == null) {
+            if (c.__error()?^ != 0) return .directory_failed
+            break
+        }
+        entry :: next?
+        # DT_REG, DT_LNK and DT_UNKNOWN; never open directories, sockets or FIFOs as sources.
+        if (entry.d_type != 8 and entry.d_type != 10 and entry.d_type != 0) continue
+        length :: usize(entry.d_namlen)
+        if (length >= 1024) return .directory_failed
+        pointer *u8 :: ptrcast!(entry.d_name[..].ptr)
+        name :: pointer[..length]
+        if ((!ends_with(name, ".hon") and !ends_with(name, ".bro"))
+            or ends_with(name, ".test.hon") or ends_with(name, ".test.bro")) continue
+        path :: try join(allocator, package_dir, name)
+        errdefer mem.free(allocator, path)
+        if (entry.d_type != 8 and ((try mode(allocator, path)) & c_ushort(61440)) != 32768) {
+            mem.free(allocator, path)
+            continue
+        }
+        try arraylist.append(&result, path)
+    }
+    closing = true
+    if (c.closedir(handle) != 0) return .directory_failed
+    # in-place heapsort: deterministic O(n log n), no additional allocation.
+    count := result.items.len
+    start :: divtrunc!(count, 2)
+    for (0..start) |i| sift(result.items, start - i - 1, count)
+    while (count > 1) {
+        count -= 1
+        temporary :: result.items[0]
+        result.items[0] = result.items[count]
+        result.items[count] = temporary
+        sift(result.items, 0, count)
+    }
+    return result
+}
+
+@hide:file
+sift proc(items []mut []u8, start, count usize) void {
+    root := start
+    while (root < divtrunc!(count, 2)) {
+        child := root * 2 + 1
+        if (child + 1 < count and before(items[child], items[child + 1])) child += 1
+        if (!before(items[root], items[child])) break
+        temporary :: items[root]
+        items[root] = items[child]
+        items[child] = temporary
+        root = child
+    }
+}
+
+#! reads through the supplied IO provider; returned bytes belong to the caller.
+read_source proc(allocator mem.Allocator, runtime io.Io, path []u8) arraylist.ArrayList(u8) ! Error {
+    # source input must be a regular file, not a FIFO/device that can block before reading.
+    if (((try mode(allocator, path)) & c_ushort(61440)) != 32768) return .open_failed
+    input :: try terminated(allocator, path)
+    defer mem.free(allocator, input)
+    file :: try io.open(runtime, input.ptr[..path.len;0], .read_only)
+    closing := false
+    defer { if (!closing) io.close(file) catch |_| {} }
+    bytes := io.read_all(allocator, io.reader(file), usize(maxval!(u32))) catch |err| {
+        match err {
+            .limit_exceeded: return .source_too_large
+            else: return err
+        }
+    }
+    errdefer arraylist.deinit(&bytes)
+    closing = true
+    try io.close(file)
+    return bytes
+}

source/lower/lower.hon

@@ -47,6 +47,25 @@ lower proc(
     resolution @sema.Resolution,
     checked @typecheck.Result,
     diagnostic_store @mut diagnostics.Store,
+) ir.Module ! (mem.AllocError | diagnostics.CapacityError) {
+    return try lower_package(
+        allocator,
+        tree,
+        resolution,
+        checked,
+        tree.root,
+        diagnostic_store,
+    )
+}
+
+#! lowers the module arena, selecting an entry only from `root_package`.
+lower_package proc(
+    allocator mem.Allocator,
+    tree @ast.Tree,
+    resolution @sema.Resolution,
+    checked @typecheck.Result,
+    root_package ast.NodeId,
+    diagnostic_store @mut diagnostics.Store,
 ) ir.Module ! (mem.AllocError | diagnostics.CapacityError) {
     module := ir.init(allocator)
     errdefer ir.deinit(&module)
@@ -87,16 +106,16 @@ lower proc(
                 })
                 storage[usize(id)] = Storage{ kind = .global, index = global }
             }
-            .invalid: {}
+            .import, .invalid: {}
         }
     }
-    try find_entry(&state)
+    try find_entry(&state, root_package)
     for (0..ast.package_decl_count(tree)) |i| {
         id :: ast.package_decl(tree, i)
         match ast.decl_kind(tree, id) {
             .function: try lower_function(&state, id)
             .statement: try lower_global(&state, id)
-            .invalid: {}
+            .import, .invalid: {}
         }
     }
     return module
@@ -152,15 +171,17 @@ add_function proc(module @mut ir.Module, span diagnostics.Span) ir.FunctionId !
 }
 
 @hide:file
-find_entry proc(state @mut State) void ! (mem.AllocError | diagnostics.CapacityError) {
-    tree :: state.tree
+find_entry proc(state @mut State, root_package ast.NodeId) void ! (mem.AllocError | diagnostics.CapacityError) {
+    package_tree := state.tree^
+    package_tree.root = root_package
+    tree :: &package_tree
     for (0..ast.package_decl_count(tree)) |i| {
         id :: ast.package_decl(tree, i)
         name := diagnostics.Span{ start = 0, end = 0 }
         match ast.decl_kind(tree, id) {
             .function: name = ast.function_decl(tree, id).name
             .statement: name = ast.statement_decl(tree, id).name
-            .invalid: continue
+            .import, .invalid: continue
         }
         if (!mem.eql(ast.source_text(tree, name), "main")) continue
         if (ast.decl_kind(tree, id) != .function) {
@@ -176,7 +197,11 @@ find_entry proc(state @mut State) void ! (mem.AllocError | diagnostics.CapacityE
         state.module.entry = ir.FunctionId(state.storage[usize(id)].index)
         return
     }
-    state.module.entry_diagnostic = try diagnostics.add(state.diagnostics, .missing_entry, ast.node_span(tree, tree.root))
+    state.module.entry_diagnostic = try diagnostics.add(
+        state.diagnostics,
+        .missing_entry,
+        ast.node_span(tree, root_package),
+    )
 }
 
 @hide:file

source/main.hon

@@ -31,6 +31,7 @@ test import "@source/ir"
 test import "@source/lower"
 test import "@source/interpreter"
 test import "@source/compilation"
+test import "@source/loader"
 
 @hide:file
 DriverError :: alias (
@@ -80,6 +81,10 @@ infrastructure_messages :: $enummap.init(DriverError, []u8, {
     invalid_arguments = "invalid native compiler arguments",
     temporary_file_failed = "cannot create temporary LLVM file",
     compiler_failed = "native compiler failed",
+    empty_package = "package contains no production source files",
+    invalid_path = "invalid filesystem path",
+    path_failed = "cannot resolve filesystem path",
+    directory_failed = "cannot read package directory",
 })
 
 @hide:file
@@ -107,26 +112,24 @@ command proc(allocator mem.Allocator, runtime io.Io, args process.Args) i32 ! Dr
 
 @hide:file
 usage proc(output io.Writer) void ! io.WriteError {
-    try io.write_all(output, "usage: honey run <file>\n")
-    try io.write_all(output, "       honey emit-llvm <file>\n")
-    try io.write_all(output, "       honey build <file> -o <executable> [-O0|-O2]\n")
+    try io.write_all(output, "usage: honey run <file-or-directory>\n")
+    try io.write_all(output, "       honey emit-llvm <file-or-directory>\n")
+    try io.write_all(output, "       honey build <file-or-directory> -o <executable> [-O0|-O2]\n")
 }
 
 @hide:file
 emit_file proc(allocator mem.Allocator, runtime io.Io, path [;0]u8) i32 ! DriverError {
-    source := try read_source(allocator, runtime, path)
-    defer arraylist.deinit(&source)
     session := compilation.init(allocator)
     defer compilation.deinit(&session)
     diagnostic_store := diagnostics.init(allocator)
     defer diagnostics.deinit(&diagnostic_store)
-    package :: try compilation.add_package(&session)
-    root :: try compilation.add_source(&session, package, path, &source, &diagnostic_store)
+    root :: try compilation.load(&session, runtime, path, &diagnostic_store)
     module := try compilation.compile(&session, root, &diagnostic_store)
     defer ir.deinit(&module)
-    file :: compilation.source(&session, root)
-    try llvm.emit(allocator, io.stdout(runtime), &module, &diagnostic_store, file.bytes.items, file.path)
-    try report_diagnostics(allocator, io.stderr(runtime), file.path, file.bytes.items, &diagnostic_store, null)
+    sources :: compilation.source_files(&session)
+    primary :: session.packages.items[usize(root)].sources.items[0]
+    try llvm.emit(allocator, io.stdout(runtime), &module, &diagnostic_store, sources, primary)
+    try report_diagnostics(allocator, io.stderr(runtime), sources, &diagnostic_store, null)
     return if (diagnostics.count(&diagnostic_store) == 0) 0 else 1
 }
 
@@ -138,14 +141,11 @@ build_file proc(
     output [;0]u8,
     optimization [;0]u8,
 ) i32 ! DriverError {
-    source := try read_source(allocator, runtime, path)
-    defer arraylist.deinit(&source)
     session := compilation.init(allocator)
     defer compilation.deinit(&session)
     diagnostic_store := diagnostics.init(allocator)
     defer diagnostics.deinit(&diagnostic_store)
-    package :: try compilation.add_package(&session)
-    root :: try compilation.add_source(&session, package, path, &source, &diagnostic_store)
+    root :: try compilation.load(&session, runtime, path, &diagnostic_store)
     module := try compilation.compile(&session, root, &diagnostic_store)
     defer ir.deinit(&module)
 
@@ -159,8 +159,9 @@ build_file proc(
     defer {
         if (!closing) io.close(file) catch |_| {}
     }
-    root_source :: compilation.source(&session, root)
-    try llvm.emit(allocator, io.writer(file), &module, &diagnostic_store, root_source.bytes.items, root_source.path)
+    sources :: compilation.source_files(&session)
+    primary :: session.packages.items[usize(root)].sources.items[0]
+    try llvm.emit(allocator, io.writer(file), &module, &diagnostic_store, sources, primary)
     closing = true
     try io.close(file)
     temporary [;0]u8 :: template[..].ptr[..17;0]
@@ -170,46 +171,23 @@ build_file proc(
     ]
     status :: try process.run(allocator, arguments[..])
     if (status != 0) return .compiler_failed
-    try report_diagnostics(allocator, io.stderr(runtime), root_source.path, root_source.bytes.items, &diagnostic_store, null)
+    try report_diagnostics(allocator, io.stderr(runtime), sources, &diagnostic_store, null)
     return if (diagnostics.count(&diagnostic_store) == 0) 0 else 1
 }
 
-@hide:file
-read_source proc(allocator mem.Allocator, runtime io.Io, path [;0]u8) arraylist.ArrayList(u8) ! DriverError {
-    file :: try io.open(runtime, path, .read_only)
-    closing := false
-    defer {
-        if (!closing) io.close(file) catch |_| {}
-    }
-    bytes := io.read_all(allocator, io.reader(file), usize(maxval!(u32))) catch |err| {
-        match err {
-            .limit_exceeded: return .source_too_large
-            else: return err
-        }
-    }
-    errdefer arraylist.deinit(&bytes)
-    closing = true
-    try io.close(file)
-    return bytes
-}
-
 @hide:file
 run_file proc(allocator mem.Allocator, runtime io.Io, path [;0]u8) i32 ! DriverError {
-    source := try read_source(allocator, runtime, path)
-    defer arraylist.deinit(&source)
     session := compilation.init(allocator)
     defer compilation.deinit(&session)
     diagnostic_store := diagnostics.init(allocator)
     defer diagnostics.deinit(&diagnostic_store)
-    package :: try compilation.add_package(&session)
-    root :: try compilation.add_source(&session, package, path, &source, &diagnostic_store)
+    root :: try compilation.load(&session, runtime, path, &diagnostic_store)
     module := try compilation.compile(&session, root, &diagnostic_store)
     defer ir.deinit(&module)
     outcome := try interpreter.run(allocator, &module, &diagnostic_store)
     defer interpreter.deinit(&outcome)
     if (outcome.trap == null) try print_result(io.stdout(runtime), &module, outcome.value)
-    file :: compilation.source(&session, root)
-    try report_diagnostics(allocator, io.stderr(runtime), file.path, file.bytes.items, &diagnostic_store, outcome.trap)
+    try report_diagnostics(allocator, io.stderr(runtime), compilation.source_files(&session), &diagnostic_store, outcome.trap)
     if (outcome.trap != null or diagnostics.count(&diagnostic_store) != 0) return 1
     return 0
 }
@@ -261,26 +239,35 @@ print_value proc(output io.Writer, module @ir.Module, value ir.Value) void ! io.
 report_diagnostics proc(
     allocator mem.Allocator,
     output io.Writer,
-    path []u8,
-    source []u8,
+    sources []diagnostics.Source,
     diagnostic_store @diagnostics.Store,
     trap ?diagnostics.DiagnosticId,
 ) void ! (mem.AllocError | io.WriteError) {
     if (diagnostics.count(diagnostic_store) == 0) return
-    starts := arraylist.init(usize, allocator)
-    defer arraylist.deinit(&starts)
-    try arraylist.append(&starts, usize(0))
-    for (source) |byte, index| {
-        if (byte == '\n') try arraylist.append(&starts, index + 1)
+    line_starts := arraylist.init(arraylist.ArrayList(usize), allocator)
+    defer {
+        for (line_starts.items) |@starts| arraylist.deinit(starts)
+        arraylist.deinit(&line_starts)
+    }
+    for (sources) |source| {
+        starts := arraylist.init(usize, allocator)
+        errdefer arraylist.deinit(&starts)
+        try arraylist.append(&starts, usize(0))
+        for (source.text) |byte, index| {
+            if (byte == '\n') try arraylist.append(&starts, index + 1)
+        }
+        try arraylist.append(&line_starts, starts)
     }
     for (0..diagnostics.count(diagnostic_store)) |index| {
         if (diagnostics.get(diagnostic_store, diagnostics.DiagnosticId(index))) |diagnostic| {
+            source :: sources[usize(diagnostic.span.source)]
+            starts :: line_starts.items[usize(diagnostic.span.source)].items
             offset :: usize(diagnostic.span.start)
             line := usize(0)
-            end := starts.items.len
+            end := starts.len
             while (line + 1 < end) {
                 middle :: line + divtrunc!(end - line, 2)
-                if (starts.items[middle] <= offset) line = middle
+                if (starts[middle] <= offset) line = middle
                 else end = middle
             }
             label []u8 := "error"
@@ -288,9 +275,9 @@ report_diagnostics proc(
                 if (id == diagnostics.DiagnosticId(index)) label = "runtime trap"
             }
             try io.print("{s}:{}:{}: {s}: {s}\n", output, {
-                path,
+                source.path,
                 line + 1,
-                offset - starts.items[line] + 1,
+                offset - starts[line] + 1,
                 label,
                 enummap.get(diagnostics.Code, []u8, &diagnostics.messages, diagnostic.code)?,
             })

source/parser/parse.hon

@@ -4,6 +4,7 @@ import "@std/arraylist"
 import "@source/diagnostics"
 import "@source/lexer"
 import "@source/ast"
+import "@source/strpool"
 
 @hide:file
 SyntaxError :: enum { unexpected_token }
@@ -22,7 +23,7 @@ BindingPower :: struct {
 
 @hide:file
 State :: struct {
-    builder ast.Builder
+    builder @mut ast.Builder
     diagnostics @mut diagnostics.Store
     tokens []lexer.Token
     allocator mem.Allocator
@@ -39,16 +40,36 @@ parse proc(
     source []u8,
     diagnostic_store @mut diagnostics.Store,
 ) ast.Tree ! (mem.AllocError | diagnostics.CapacityError) {
+    builder := ast.init_builder(allocator, source)
+    defer ast.deinit_builder(&builder)
+    root :: try parse_source(&builder, tokens, diagnostic_store)
+    return ast.finish(&builder, root)
+}
+
+#! appends one source to shared node storage; token spans must identify the builder's current source.
+#! infrastructure failure rolls back all nodes, extra data, spans, and diagnostics added by this parse.
+parse_source proc(
+    builder @mut ast.Builder,
+    tokens []lexer.Token,
+    diagnostic_store @mut diagnostics.Store,
+) ast.NodeId ! (mem.AllocError | diagnostics.CapacityError) {
+    node_count :: builder.storage.nodes.items.len
+    extra_count :: builder.storage.extra.items.len
+    diagnostic_count :: diagnostic_store.list.items.len
+    errdefer {
+        builder.storage.nodes.items = builder.storage.nodes.items.ptr[..node_count]
+        builder.storage.spans.items = builder.storage.spans.items.ptr[..node_count]
+        builder.storage.extra.items = builder.storage.extra.items.ptr[..extra_count]
+        diagnostic_store.list.items = diagnostic_store.list.items.ptr[..diagnostic_count]
+    }
     state := State{
-        builder = ast.init_builder(allocator, source),
+        builder = builder,
         diagnostics = diagnostic_store,
         tokens = tokens,
-        allocator = allocator,
+        allocator = builder.storage.nodes.allocator,
         next_token = 0,
     }
-    defer ast.deinit_builder(&state.builder)
-    root :: try parse_package(&state)
-    return ast.finish(&state.builder, root)
+    return try parse_package(&state)
 }
 
 @hide:file
@@ -74,19 +95,50 @@ parse_package proc(state @mut State) ast.NodeId ! (mem.AllocError | diagnostics.
         skip_newlines(state)
     }
 
-    return try ast.add_package(
-        &state.builder,
-        ast.Span{ start = 0, end = u32(state.builder.source.len) },
-        declarations.items,
-    )
+    return try ast.add_package(state.builder, ast.Span{
+        start = 0,
+        end = u32(state.builder.source.len),
+        source = state.builder.source_id,
+    }, declarations.items)
 }
 
 @hide:file
 parse_decl proc(state @mut State) ast.NodeId ! ParseError {
+    if (
+        next_is(state, .import) or next_is(state, .test)
+        or (next_is(state, .ident) and next_n_is(state, 1, .double_colon) and next_n_is(state, 2, .import))
+    ) return try parse_import_decl(state)
     if (next_is(state, .ident) and next_n_is(state, 1, .proc)) return try parse_func_decl(state)
     return try parse_statement_decl(state)
 }
 
+@hide:file
+parse_import_decl proc(state @mut State) ast.NodeId ! ParseError {
+    start :: current(state).span
+    test_only :: next_is(state, .test)
+    if (test_only) state.next_token += 1
+    namespace ?strpool.StringId := null
+    if next_is(state, .ident) {
+        name :: try consume(state, .ident)
+        namespace = name.data.str_id
+        _ = try consume(state, .double_colon)
+    }
+    _ = try consume(state, .import)
+    token :: try consume(state, .string)
+    path :: ast.Span{ start = token.span.start + 1, end = token.span.end - 1, source = token.span.source }
+    # import paths are literal filesystem paths, not escaped string values.
+    valid := path.start != path.end
+    for (state.builder.source[usize(path.start)..usize(path.end)]) |byte| {
+        if (byte < 32 or byte == 127 or byte == '\\') valid = false
+    }
+    span :: ast.Span{ start = start.start, end = token.span.end, source = start.source }
+    if (!valid) {
+        diagnostic :: try diagnostics.add(state.diagnostics, .invalid_import_path, path)
+        return try ast.add_invalid_decl(state.builder, span, diagnostic)
+    }
+    return try ast.add_import_decl(state.builder, span, path, namespace, test_only)
+}
+
 @hide:file
 parse_func_decl proc(state @mut State) ast.NodeId ! ParseError {
     name :: try consume(state, .ident)
@@ -108,11 +160,11 @@ parse_func_decl proc(state @mut State) ast.NodeId ! ParseError {
         }
 
         type_id :: try parse_type(state)
-        type_span :: ast.builder_span(&state.builder, type_id)
+        type_span :: ast.builder_span(state.builder, type_id)
         for (names.items) |parameter_name| {
             param_id :: try ast.add_param(
-                &state.builder,
-                ast.Span{ start = parameter_name.span.start, end = type_span.end },
+                state.builder,
+                ast.Span{ start = parameter_name.span.start, end = type_span.end, source = parameter_name.span.source },
                 parameter_name.span,
                 parameter_name.data.str_id,
                 type_id,
@@ -125,10 +177,14 @@ parse_func_decl proc(state @mut State) ast.NodeId ! ParseError {
     _ = try consume(state, .close_paren)
     return_type :: try parse_type(state)
     body :: try parse_block(state)
-    body_span :: ast.builder_span(&state.builder, body)
+    body_span :: ast.builder_span(state.builder, body)
     return try ast.add_function_decl(
-        &state.builder,
-        ast.Span{ start = name.span.start, end = body_span.end },
+        state.builder,
+        ast.Span{
+            start = name.span.start,
+            end = body_span.end,
+            source = name.span.source,
+        },
         name.span,
         name.data.str_id,
         return_type,
@@ -163,8 +219,12 @@ parse_block proc(state @mut State) ast.NodeId ! ParseError {
 
     close :: try consume(state, .close_curly)
     return try ast.add_block(
-        &state.builder,
-        ast.Span{ start = open.span.start, end = close.span.end },
+        state.builder,
+        ast.Span{
+            start = open.span.start,
+            end = close.span.end,
+            source = open.span.source,
+        },
         statements.items,
     )
 }
@@ -191,7 +251,7 @@ parse_statement proc(state @mut State, allow_else bool) ast.NodeId ! ParseError
         .ident, .int, .float, .string, .true, .false, .minus, .bang, .open_paren, .open_bracket: {
             expression :: try parse_expr(state)
             if next_is(state, .equal) return try parse_assignment(state, expression, allow_else)
-            return try ast.add_expression_stmt(&state.builder, expression)
+            return try ast.add_expression_stmt(state.builder, expression)
         }
         else: return .unexpected_token
     }
@@ -204,7 +264,7 @@ parse_return proc(state @mut State, allow_else bool) ast.NodeId ! ParseError {
         next_is(state, .newline) or next_is(state, .close_curly) or next_is(state, .eof)
         or (allow_else and next_is(state, .else))
     ) {
-        return try ast.add_return_stmt(&state.builder, keyword.span, null)
+        return try ast.add_return_stmt(state.builder, keyword.span, null)
     }
     value :: try parse_expr(state)
     if (
@@ -216,8 +276,12 @@ parse_return proc(state @mut State, allow_else bool) ast.NodeId ! ParseError {
     # include consumed grouping punctuation, which has no separate expression node.
     end :: state.tokens[state.next_token - 1].span.end
     return try ast.add_return_stmt(
-        &state.builder,
-        ast.Span{ start = keyword.span.start, end = end },
+        state.builder,
+        ast.Span{
+            start = keyword.span.start,
+            end = end,
+            source = keyword.span.source,
+        },
         value,
     )
 }
@@ -234,9 +298,9 @@ parse_loop_control proc(state @mut State, allow_else bool) ast.NodeId ! ParseErr
     ) return .unexpected_token
 
     if (keyword.kind == .break)
-        return try ast.add_break_stmt(&state.builder, keyword.span)
+        return try ast.add_break_stmt(state.builder, keyword.span)
 
-    return try ast.add_continue_stmt(&state.builder, keyword.span)
+    return try ast.add_continue_stmt(state.builder, keyword.span)
 }
 
 @hide:file
@@ -253,8 +317,12 @@ parse_assignment proc(state @mut State, target ast.NodeId, allow_else bool) ast.
     }
     end :: state.tokens[state.next_token - 1].span.end
     return try ast.add_assignment_stmt(
-        &state.builder,
-        ast.Span{ start = ast.builder_span(&state.builder, target).start, end = end },
+        state.builder,
+        ast.Span{
+            start = ast.builder_span(state.builder, target).start,
+            end = end,
+            source = state.builder.source_id,
+        },
         target,
         value,
     )
@@ -277,12 +345,16 @@ parse_if proc(state @mut State) ast.NodeId ! ParseError {
         state.next_token = after_then
     }
     end :: if (alternative) |block|
-        ast.builder_span(&state.builder, block).end
+        ast.builder_span(state.builder, block).end
     else
-        ast.builder_span(&state.builder, then_block).end
+        ast.builder_span(state.builder, then_block).end
     return try ast.add_if_stmt(
-        &state.builder,
-        ast.Span{ start = keyword.span.start, end = end },
+        state.builder,
+        ast.Span{
+            start = keyword.span.start,
+            end = end,
+            source = keyword.span.source,
+        },
         condition,
         then_block,
         alternative,
@@ -295,8 +367,12 @@ parse_while proc(state @mut State, allow_else bool) ast.NodeId ! ParseError {
     condition :: try parse_guard(state)
     body :: try parse_arm(state, allow_else)
     return try ast.add_while_stmt(
-        &state.builder,
-        ast.Span{ start = keyword.span.start, end = ast.builder_span(&state.builder, body).end },
+        state.builder,
+        ast.Span{
+            start = keyword.span.start,
+            end = ast.builder_span(state.builder, body).end,
+            source = keyword.span.source,
+        },
         condition,
         body,
     )
@@ -317,7 +393,11 @@ parse_capture_name proc(state @mut State) ?ast.NodeId ! ParseError {
     if (
         name.span.end != name.span.start + 1
         or state.builder.source[usize(name.span.start)] != '_'
-    ) capture = try ast.add_identifier(&state.builder, name.span, name.data.str_id)
+    ) capture = try ast.add_identifier(
+        state.builder,
+        name.span,
+        name.data.str_id,
+    )
     return capture
 }
 
@@ -352,9 +432,15 @@ parse_for proc(state @mut State, allow_else bool) ast.NodeId ! ParseError {
             .unexpected_token: {}
             else: return err
         }
-        diagnostic :: try diagnostics.add(state.diagnostics, .unexpected_token, current(state).span)
+        diagnostic :: try diagnostics.add(
+            state.diagnostics,
+            .unexpected_token,
+            current(state).span,
+        )
         iterable = try ast.add_invalid_expr(
-            &state.builder, ast.builder_span(&state.builder, iterable), diagnostic,
+            state.builder,
+            ast.builder_span(state.builder, iterable),
+            diagnostic,
         )
         while (
             !next_is(state, .open_curly) and !next_is(state, .close_curly)
@@ -367,23 +453,27 @@ parse_for proc(state @mut State, allow_else bool) ast.NodeId ! ParseError {
     skip_newlines(state)
     if (
         !next_is(state, .open_curly) and !grouped
-        and ast.builder_expr_kind(&state.builder, iterable) != .call
-        and ast.builder_expr_kind(&state.builder, iterable) != .invalid
+        and ast.builder_expr_kind(state.builder, iterable) != .call
+        and ast.builder_expr_kind(state.builder, iterable) != .invalid
     ) {
         diagnostic :: try diagnostics.add(state.diagnostics, .unexpected_token, current(state).span)
         iterable = try ast.add_invalid_expr(
-            &state.builder, ast.builder_span(&state.builder, iterable), diagnostic,
+            state.builder, ast.builder_span(state.builder, iterable), diagnostic,
         )
     }
     # a malformed capture list must not swallow the next line's sibling.
     if (
-        ast.builder_expr_kind(&state.builder, iterable) == .invalid
+        ast.builder_expr_kind(state.builder, iterable) == .invalid
         and !next_is(state, .open_curly)
     ) state.next_token = after_captures
     body :: try parse_arm(state, allow_else)
     return try ast.add_for_stmt(
-        &state.builder,
-        ast.Span{ start = keyword.span.start, end = ast.builder_span(&state.builder, body).end },
+        state.builder,
+        ast.Span{
+            start = keyword.span.start,
+            end = ast.builder_span(state.builder, body).end,
+            source = keyword.span.source,
+        },
         iterable, captures.item, captures.index, captures.pointer, captures.indexed, body,
     )
 }
@@ -397,9 +487,9 @@ parse_guard proc(state @mut State) ast.NodeId ! ParseError {
             else: return err
         }
         diagnostic :: try diagnostics.add(state.diagnostics, .unexpected_token, current(state).span)
-        yield try ast.add_invalid_expr(&state.builder, current(state).span, diagnostic)
+        yield try ast.add_invalid_expr(state.builder, current(state).span, diagnostic)
     }
-    if (ast.builder_expr_kind(&state.builder, condition) == .invalid) {
+    if (ast.builder_expr_kind(state.builder, condition) == .invalid) {
         # discard a malformed guard's tail, but leave its body and enclosing block intact.
         while (
             !next_is(state, .open_curly) and !next_is(state, .close_curly)
@@ -416,13 +506,13 @@ parse_guard proc(state @mut State) ast.NodeId ! ParseError {
     if (
         !next_is(state, .open_curly)
         and !parenthesized_guard(state, condition_start)
-        and ast.builder_expr_kind(&state.builder, condition) != .call
-        and ast.builder_expr_kind(&state.builder, condition) != .invalid
+        and ast.builder_expr_kind(state.builder, condition) != .call
+        and ast.builder_expr_kind(state.builder, condition) != .invalid
     ) {
         diagnostic :: try diagnostics.add(state.diagnostics, .unexpected_token, current(state).span)
         condition = try ast.add_invalid_expr(
-            &state.builder,
-            ast.builder_span(&state.builder, condition),
+            state.builder,
+            ast.builder_span(state.builder, condition),
             diagnostic,
         )
     }
@@ -455,12 +545,12 @@ parse_arm proc(state @mut State, allow_else bool) ast.NodeId ! ParseError {
         }
         diagnostic :: try unexpected_diagnostic(state)
         span :: synchronize(state, start, true, true, false)
-        yield try ast.add_invalid_stmt(&state.builder, span, diagnostic)
+        yield try ast.add_invalid_stmt(state.builder, span, diagnostic)
     }
     statements [1]ast.NodeId :: [statement]
     return try ast.add_block(
-        &state.builder,
-        ast.builder_span(&state.builder, statement),
+        state.builder,
+        ast.builder_span(state.builder, statement),
         statements[..],
     )
 }
@@ -476,11 +566,15 @@ parse_statement_decl proc(state @mut State) ast.NodeId ! ParseError {
 
     assignment :: try consume_either(state, &[.double_colon, .colon_equal])
     initializer :: try parse_expr(state)
-    end :: ast.builder_span(&state.builder, initializer).end
+    end :: ast.builder_span(state.builder, initializer).end
     mutability :: if (assignment.kind == .double_colon) .constant else .variable
     return try ast.add_statement_decl(
-        &state.builder,
-        ast.Span{ start = name.span.start, end = end },
+        state.builder,
+        ast.Span{
+            start = name.span.start,
+            end = end,
+            source = name.span.source,
+        },
         name.span,
         name.data.str_id,
         type_id,
@@ -508,13 +602,17 @@ parse_type proc(state @mut State) ast.NodeId ! ParseError {
         if (mutable) state.next_token += 1
         element :: try parse_type(state)
         return try ast.add_array_type(
-            &state.builder,
-            ast.Span{ start = open.span.start, end = ast.builder_span(&state.builder, element).end },
+            state.builder,
+            ast.Span{
+                start = open.span.start,
+                end = ast.builder_span(state.builder, element).end,
+                source = open.span.source,
+            },
             count, element, mutable,
         )
     }
     token :: try consume(state, .ident)
-    return try ast.add_identifier(&state.builder, token.span, token.data.str_id)
+    return try ast.add_identifier(state.builder, token.span, token.data.str_id)
 }
 
 @hide:file
@@ -530,11 +628,15 @@ parse_expr_bp proc(state @mut State, min_bp BindingPowerScalar) ast.NodeId ! Par
     left := if (prefix_binding_power(token.kind)) |right_bp| {
         state.next_token += 1
         operand :: try parse_expr_bp(state, right_bp)
-        operand_span :: ast.builder_span(&state.builder, operand)
+        operand_span :: ast.builder_span(state.builder, operand)
         operator ast.UnaryOperator :: if (token.kind == .bang) .logical_not else .negate
         yield try ast.add_unary(
-            &state.builder,
-            ast.Span{ start = token.span.start, end = operand_span.end },
+            state.builder,
+            ast.Span{
+                start = token.span.start,
+                end = operand_span.end,
+                source = token.span.source,
+            },
             operator,
             operand,
         )
@@ -564,10 +666,11 @@ parse_expr_bp proc(state @mut State, min_bp BindingPowerScalar) ast.NodeId ! Par
             right := try parse_expr_bp(state, BindingPowerScalar(1))
             right = try validate_range_bound(state, right, parenthesized_guard(state, right_start))
             left = try ast.add_range(
-                &state.builder,
+                state.builder,
                 ast.Span{
                     start = state.tokens[expression_start].span.start,
                     end = state.tokens[state.next_token - 1].span.end,
+                    source = state.builder.source_id,
                 },
                 left, right, token.kind == .range_inclusive,
             )
@@ -578,11 +681,15 @@ parse_expr_bp proc(state @mut State, min_bp BindingPowerScalar) ast.NodeId ! Par
             if (power.left < min_bp) break
             state.next_token += 1
             right :: try parse_expr_bp(state, power.right)
-            left_span :: ast.builder_span(&state.builder, left)
-            right_span :: ast.builder_span(&state.builder, right)
+            left_span :: ast.builder_span(state.builder, left)
+            right_span :: ast.builder_span(state.builder, right)
             left = try ast.add_binary(
-                &state.builder,
-                ast.Span{ start = left_span.start, end = right_span.end },
+                state.builder,
+                ast.Span{
+                    start = left_span.start,
+                    end = right_span.end,
+                    source = left_span.source,
+                },
                 binary_operator(token.kind),
                 left,
                 right,
@@ -600,11 +707,11 @@ validate_range_bound proc(
     state @mut State, bound ast.NodeId, grouped bool,
 ) ast.NodeId ! (mem.AllocError | diagnostics.CapacityError) {
     if (grouped) return bound
-    match ast.builder_expr_kind(&state.builder, bound) {
+    match ast.builder_expr_kind(state.builder, bound) {
         .unary, .binary, .range: {
-            span :: ast.builder_span(&state.builder, bound)
+            span :: ast.builder_span(state.builder, bound)
             diagnostic :: try diagnostics.add(state.diagnostics, .unexpected_token, span)
-            return try ast.add_invalid_expr(&state.builder, span, diagnostic)
+            return try ast.add_invalid_expr(state.builder, span, diagnostic)
         }
         else: return bound
     }
@@ -628,10 +735,11 @@ parse_call proc(state @mut State, callee ast.NodeId, start u32) ast.NodeId ! Par
     }
     close :: try consume(state, .close_paren)
     return try ast.add_call(
-        &state.builder,
+        state.builder,
         ast.Span{
             start = start,
             end = close.span.end,
+            source = state.builder.source_id,
         },
         callee,
         arguments.items,
@@ -645,18 +753,23 @@ parse_index proc(state @mut State, base ast.NodeId, start u32) ast.NodeId ! Pars
     if !next_is(state, .close_bracket) {
         diagnostic :: try diagnostics.add(state.diagnostics, .unexpected_token, current(state).span)
         return try ast.add_invalid_expr(
-            &state.builder,
+            state.builder,
             ast.Span{
                 start = start,
-                end = ast.builder_span(&state.builder, index).end,
+                end = ast.builder_span(state.builder, index).end,
+                source = state.builder.source_id,
             },
             diagnostic,
         )
     }
     close :: try consume(state, .close_bracket)
     return try ast.add_index(
-        &state.builder,
-        ast.Span{ start = start, end = close.span.end },
+        state.builder,
+        ast.Span{
+            start = start,
+            end = close.span.end,
+            source = state.builder.source_id,
+        },
         base, index,
     )
 }
@@ -699,26 +812,32 @@ parse_array_literal proc(state @mut State) ast.NodeId ! ParseError {
         if next_is(state, .close_bracket) break
         if !next_is(state, .comma) {
             state.next_token = after_element
-            return try invalid_array_literal(state, open.span.start, ast.builder_span(&state.builder, element).end)
+            return try invalid_array_literal(state, open.span.start, ast.builder_span(state.builder, element).end)
         }
         state.next_token += 1
         after_comma :: state.next_token
         skip_newlines(state)
         if named_binding_starts(state) {
             state.next_token = after_comma
-            return try invalid_array_literal(state, open.span.start, ast.builder_span(&state.builder, element).end)
+            return try invalid_array_literal(state, open.span.start, ast.builder_span(state.builder, element).end)
         }
     }
     close :: try consume(state, .close_bracket)
-    return try ast.add_array_literal(
-        &state.builder, ast.Span{ start = open.span.start, end = close.span.end }, elements.items,
-    )
+    return try ast.add_array_literal(state.builder, ast.Span{
+        start = open.span.start,
+        end = close.span.end,
+        source = open.span.source,
+    }, elements.items)
 }
 
 @hide:file
 invalid_array_literal proc(state @mut State, start, end u32) ast.NodeId ! ParseError {
     diagnostic :: try diagnostics.add(state.diagnostics, .unexpected_token, current(state).span)
-    return try ast.add_invalid_expr(&state.builder, ast.Span{ start = start, end = end }, diagnostic)
+    return try ast.add_invalid_expr(state.builder, ast.Span{
+        start = start,
+        end = end,
+        source = state.builder.source_id,
+    }, diagnostic)
 }
 
 @hide:file
@@ -754,30 +873,31 @@ parse_primary proc(state @mut State) ast.NodeId ! ParseError {
                 .unexpected_token,
                 current(state).span,
             )
-            return try ast.add_invalid_expr(&state.builder, ast.Span{
+            return try ast.add_invalid_expr(state.builder, ast.Span{
                 start = token.span.start,
-                end = ast.builder_span(&state.builder, expression).end,
+                end = ast.builder_span(state.builder, expression).end,
+                source = token.span.source,
             }, diagnostic)
         }
         .int: {
             state.next_token += 1
-            return try ast.add_literal(&state.builder, .int, token.span)
+            return try ast.add_literal(state.builder, .int, token.span)
         }
         .float: {
             state.next_token += 1
-            return try ast.add_literal(&state.builder, .float, token.span)
+            return try ast.add_literal(state.builder, .float, token.span)
         }
         .string: {
             state.next_token += 1
-            return try ast.add_literal(&state.builder, .string, token.span)
+            return try ast.add_literal(state.builder, .string, token.span)
         }
         .true, .false: {
             state.next_token += 1
-            return try ast.add_literal(&state.builder, .bool, token.span)
+            return try ast.add_literal(state.builder, .bool, token.span)
         }
         .ident: {
             state.next_token += 1
-            return try ast.add_identifier(&state.builder, token.span, token.data.str_id)
+            return try ast.add_identifier(state.builder, token.span, token.data.str_id)
         }
         else: {
             diagnostic :: if (token.data.diagnostic != diagnostics.no_diagnostic)
@@ -789,7 +909,7 @@ parse_primary proc(state @mut State) ast.NodeId ! ParseError {
                 state.next_token += 1
             }
 
-            return try ast.add_invalid_expr(&state.builder, token.span, diagnostic)
+            return try ast.add_invalid_expr(state.builder, token.span, diagnostic)
         }
     }
 }
@@ -858,7 +978,7 @@ recover_decl proc(
 ) ast.NodeId ! (mem.AllocError | diagnostics.CapacityError) {
     diagnostic :: try unexpected_diagnostic(state)
     span :: synchronize(state, start, false, false, true)
-    return try ast.add_invalid_decl(&state.builder, span, diagnostic)
+    return try ast.add_invalid_decl(state.builder, span, diagnostic)
 }
 
 @hide:file
@@ -868,7 +988,7 @@ recover_statement proc(
 ) ast.NodeId ! (mem.AllocError | diagnostics.CapacityError) {
     diagnostic :: try unexpected_diagnostic(state)
     span :: synchronize(state, start, true, false, true)
-    return try ast.add_invalid_stmt(&state.builder, span, diagnostic)
+    return try ast.add_invalid_stmt(state.builder, span, diagnostic)
 }
 
 @hide:file
@@ -910,7 +1030,11 @@ synchronize proc(
         end = current(state).span.end
         state.next_token += 1
     }
-    return ast.Span{ start = first.start, end = end }
+    return ast.Span{
+        start = first.start,
+        end = end,
+        source = first.source,
+    }
 }
 
 @hide:file

source/parser/parse.test.hon

@@ -1423,3 +1423,177 @@ recovers_nested_square_and_parenthesis_delimiters_without_consuming_siblings tes
     try testing.expect_equal(0, ast.array_element_count(&tree, kept.initializer))
     try testing.expect_equal(1, ast.array_element_count(&tree, sibling.initializer))
 }
+
+parses_import_forms_with_shared_source_provenance test {
+    strings := strpool.init(mem.c_allocator)
+    defer strpool.deinit(&strings)
+    diagnostic_store := diagnostics.init(mem.c_allocator)
+    defer diagnostics.deinit(&diagnostic_store)
+    builder := ast.init_builder(mem.c_allocator, "")
+    defer ast.deinit_builder(&builder)
+    first_text :: "value :: 1\n"
+    first_source :: try ast.set_source(&builder, "first.hon", first_text)
+    first_scan := lexer.init(mem.c_allocator)
+    defer lexer.deinit(&first_scan)
+    first_scan.source_id = first_source
+    try lexer.scan(&first_scan, first_text, &strings, &diagnostic_store)
+    first_root :: try parse_source(&builder, first_scan.tokens.items, &diagnostic_store)
+    first_view :: ast.view(&builder, first_root)
+    first_node_count :: ast.node_count(&first_view)
+    second_text ::
+        `import "relative" # preserve imports after comments
+        `named :: import "@root/package"
+        `test import "test_only"
+        `test helper :: import "test_alias"
+        `run proc(parameter i64) i64 {
+        `    values [2]i64 :: [1, 2]
+        `    if (true) { values[0] = parameter + -1 } else { values[1] = 2 }
+        `    while (false) { break }
+        `    for 0..1 |i| { continue }
+        `    return values[0]
+        `}
+    second_source :: try ast.set_source(&builder, "second.hon", second_text)
+    second_scan := lexer.init(mem.c_allocator)
+    defer lexer.deinit(&second_scan)
+    second_scan.source_id = second_source
+    try lexer.scan(&second_scan, second_text, &strings, &diagnostic_store)
+    second_root :: try parse_source(&builder, second_scan.tokens.items, &diagnostic_store)
+    tree :: ast.view(&builder, second_root)
+    try testing.expect(first_root != second_root)
+    try testing.expect_equal(1, ast.package_decl_count_at(&tree, first_root))
+    first :: ast.package_decl_at(&tree, first_root, 0)
+    try testing.expect_equal(first_source, ast.statement_decl(&tree, first).name.source)
+    try testing.expect_equal("value", ast.source_text(&tree, ast.statement_decl(&tree, first).name))
+    try testing.expect_equal(5, ast.package_decl_count(&tree))
+    paths [4][]u8 :: ["relative", "@root/package", "test_only", "test_alias"]
+    for (paths) |path, index| {
+        declaration :: ast.package_decl(&tree, index)
+        try testing.expect_equal(ast.DeclKind.import, ast.decl_kind(&tree, declaration))
+        imported :: ast.import_decl(&tree, declaration)
+        try testing.expect_equal(path, ast.source_text(&tree, imported.path))
+        try testing.expect_equal(second_source, imported.path.source)
+        try testing.expect_equal(index >= 2, imported.test_only)
+        if (index == 0 or index == 2) {
+            try testing.expect_equal(null, imported.namespace)
+        } else {
+            expected :: if (index == 1) "named" else "helper"
+            try testing.expect_equal([]u8, expected, strpool.get_str(&strings, imported.namespace?)?)
+        }
+    }
+    function_id :: ast.package_decl(&tree, 4)
+    function :: ast.function_decl(&tree, function_id)
+    try testing.expect_equal(second_source, function.name.source)
+    try testing.expect_equal("run", ast.source_text(&tree, function.name))
+    parameter :: ast.param(&tree, ast.function_param(&tree, function_id, 0))
+    try testing.expect_equal(second_source, parameter.name.source)
+    try testing.expect_equal("parameter", ast.source_text(&tree, parameter.name))
+    for (usize(0)..ast.node_count(&tree)) |index| {
+        expected :: if (index < first_node_count) first_source else second_source
+        span :: ast.node_span(&tree, ast.NodeId(index))
+        try testing.expect_equal(expected, span.source)
+        _ = ast.source_text(&tree, span)
+    }
+    try testing.expect_equal(2, ast.source_files(&tree).len)
+    try testing.expect_equal("first.hon", ast.source_files(&tree)[usize(first_source)].path)
+    try testing.expect_equal("second.hon", ast.source_files(&tree)[usize(second_source)].path)
+    try testing.expect_equal(0, diagnostics.count(&diagnostic_store))
+}
+
+rejects_empty_escaped_and_control_import_paths_without_losing_siblings test {
+    strings := strpool.init(mem.c_allocator)
+    defer strpool.deinit(&strings)
+    diagnostic_store := diagnostics.init(mem.c_allocator)
+    defer diagnostics.deinit(&diagnostic_store)
+    source :: "import \"\"\nimport \"bad\\\\path\"\nimport \"bad\tpath\"\nkept :: 1\n"
+    scan_state := lexer.init(mem.c_allocator)
+    defer lexer.deinit(&scan_state)
+    try lexer.scan(&scan_state, source, &strings, &diagnostic_store)
+    tree := try parse(mem.c_allocator, scan_state.tokens.items, source, &diagnostic_store)
+    defer ast.deinit(&tree)
+    try testing.expect_equal(4, ast.package_decl_count(&tree))
+    try testing.expect_equal(3, diagnostics.count(&diagnostic_store))
+    for (usize(0)..usize(3)) |index| {
+        declaration :: ast.package_decl(&tree, index)
+        try testing.expect_equal(ast.DeclKind.invalid, ast.decl_kind(&tree, declaration))
+        diagnostic :: diagnostics.get(&diagnostic_store, ast.invalid(&tree, declaration).diagnostic)?
+        try testing.expect_equal(diagnostics.Code.invalid_import_path, diagnostic.code)
+    }
+    kept :: ast.statement_decl(&tree, ast.package_decl(&tree, 3))
+    try testing.expect_equal("kept", ast.source_text(&tree, kept.name))
+}
+
+@hide:file
+AllocationBudget :: struct { remaining usize }
+
+@hide:file
+budget_alloc proc(context ?@mut anyopaque, size usize, alignment usize) ?*mut u8 {
+    budget @mut AllocationBudget :: ptrcast!(context?)
+    if (budget.remaining == 0) return null
+    budget.remaining -= 1
+    return mem.raw_alloc(mem.c_allocator, size, alignment)
+}
+
+@hide:file
+budget_realloc proc(
+    context ?@mut anyopaque, memory ?*mut u8, old_size, new_size, alignment usize,
+) ?*mut u8 {
+    budget @mut AllocationBudget :: ptrcast!(context?)
+    if (budget.remaining == 0) return null
+    budget.remaining -= 1
+    return mem.raw_realloc(mem.c_allocator, memory, old_size, new_size, alignment)
+}
+
+@hide:file
+budget_free proc(context ?@mut anyopaque, memory ?*mut u8, size, alignment usize) void {
+    _ = context
+    mem.raw_free(mem.c_allocator, memory, size, alignment)
+}
+
+@hide:file
+budget_vtable :: mem.AllocatorVTable{ alloc = budget_alloc, realloc = budget_realloc, free = budget_free }
+
+rolls_back_shared_nodes_extra_spans_and_diagnostics_on_allocation_failure test {
+    failures := 0
+    for (usize(0)..usize(16)) |remaining| {
+        strings := strpool.init(mem.c_allocator)
+        defer strpool.deinit(&strings)
+        diagnostic_store := diagnostics.init(mem.c_allocator)
+        defer diagnostics.deinit(&diagnostic_store)
+        budget := AllocationBudget{ remaining = maxval!(usize) }
+        allocator :: mem.Allocator{ context = &budget, vtable = &budget_vtable }
+        builder := ast.init_builder(allocator, "")
+        defer ast.deinit_builder(&builder)
+        first_text :: "value :: 1\n"
+        first_source :: try ast.set_source(&builder, "first.hon", first_text)
+        first_scan := lexer.init(mem.c_allocator)
+        defer lexer.deinit(&first_scan)
+        first_scan.source_id = first_source
+        try lexer.scan(&first_scan, first_text, &strings, &diagnostic_store)
+        first_root :: try parse_source(&builder, first_scan.tokens.items, &diagnostic_store)
+        node_count :: builder.storage.nodes.items.len
+        extra_count :: builder.storage.extra.items.len
+        second_text :: "another :: [1, 2, 3]\n@ broken\nlast :: 4\n"
+        second_source :: try ast.set_source(&builder, "second.hon", second_text)
+        second_scan := lexer.init(mem.c_allocator)
+        defer lexer.deinit(&second_scan)
+        second_scan.source_id = second_source
+        try lexer.scan(&second_scan, second_text, &strings, &diagnostic_store)
+        budget.remaining = remaining
+        _ = parse_source(&builder, second_scan.tokens.items, &diagnostic_store) catch |error| {
+            try testing.expect(error == .out_of_memory)
+            failures += 1
+            try testing.expect_equal(node_count, builder.storage.nodes.items.len)
+            try testing.expect_equal(node_count, builder.storage.spans.items.len)
+            try testing.expect_equal(extra_count, builder.storage.extra.items.len)
+            try testing.expect_equal(0, diagnostics.count(&diagnostic_store))
+            ast.remove_source(&builder)
+            tree :: ast.view(&builder, first_root)
+            declaration :: ast.statement_decl(&tree, ast.package_decl(&tree, 0))
+            try testing.expect_equal("value", ast.source_text(&tree, declaration.name))
+            try testing.expect_equal(1, ast.source_files(&tree).len)
+            try testing.expect_equal(first_source, builder.source_id)
+            continue
+        }
+    }
+    try testing.expect(failures > 0)
+}

source/sema/check/check.hon

@@ -163,6 +163,7 @@ check proc(
             .statement: try collect_declaration(&state, id)
             .function: try collect_function(&state, id)
             .invalid: collect_invalid(&state, id)
+            .import: {}
         }
     }
     # Preserve binding errors before RHS errors propagate. Indexed writability

source/sema/resolve.hon

@@ -88,6 +88,18 @@ resolve proc(
     tree @ast.Tree,
     pool @mut strpool.StringPool,
     diagnostic_store @mut diagnostics.Store,
+) Resolution ! (mem.AllocError | strpool.InternError | diagnostics.CapacityError) {
+    roots [1]ast.NodeId :: [tree.root]
+    return try resolve_packages(allocator, tree, roots[..], pool, diagnostic_store)
+}
+
+#! resolves each package root in its own namespace; AST IDs share the compilation arena.
+resolve_packages proc(
+    allocator mem.Allocator,
+    tree @ast.Tree,
+    roots []ast.NodeId,
+    pool @mut strpool.StringPool,
+    diagnostic_store @mut diagnostics.Store,
 ) Resolution ! (mem.AllocError | strpool.InternError | diagnostics.CapacityError) {
     bindings :: try mem.alloc(Binding, allocator, ast.node_count(tree))
     errdefer mem.free(allocator, bindings)
@@ -102,30 +114,41 @@ resolve proc(
         allocator = allocator,
     }
 
+    for (roots) |root| try resolve_package(&state, root)
+
+    return Resolution{ storage = Storage{
+        bindings = bindings,
+        allocator = allocator,
+    } }
+}
+
+@hide:file
+resolve_package proc(state @mut State, root ast.NodeId) void ! (mem.AllocError | diagnostics.CapacityError) {
+    tree := state.tree^
+    tree.root = root
     package_scope := Scope{
-        names = hashmap.init(allocator),
+        names = hashmap.init(state.allocator),
         parent = null,
     }
     defer hashmap.deinit(&package_scope.names)
-    for (0..ast.package_decl_count(tree)) |i| {
-        id :: ast.package_decl(tree, i)
-        match ast.decl_kind(tree, id) {
-            .statement: try predeclare(&state, &package_scope, id, ast.statement_decl(tree, id).name)
-            .function: try predeclare(&state, &package_scope, id, ast.function_decl(tree, id).name)
-            .invalid: {}
+    for (0..ast.package_decl_count(&tree)) |i| {
+        id :: ast.package_decl(&tree, i)
+        match ast.decl_kind(&tree, id) {
+            .statement: try predeclare(state, &package_scope, id, ast.statement_decl(&tree, id).name)
+            .function: try predeclare(state, &package_scope, id, ast.function_decl(&tree, id).name)
+            .import, .invalid: {}
+        }
+    }
+    for (0..ast.package_decl_count(&tree)) |i| {
+        id :: ast.package_decl(&tree, i)
+        match ast.decl_kind(&tree, id) {
+            .statement: try resolve_statement_decl(state, &package_scope, id)
+            .function: try resolve_function(state, &package_scope, id)
+            .invalid: resolve_invalid(state, id)
+            .import: {}
         }
     }
 
-    for (0..ast.package_decl_count(tree)) |i| {
-        id :: ast.package_decl(tree, i)
-        match ast.decl_kind(tree, id) {
-            .statement: try resolve_statement_decl(&state, &package_scope, id)
-            .function: try resolve_function(&state, &package_scope, id)
-            .invalid: resolve_invalid(&state, id)
-        }
-    }
-
-    return Resolution{ storage = Storage{ bindings = bindings, allocator = allocator } }
 }
 
 #! releases binding storage with the allocator retained by resolve.