feat: retain compilation source and package identities

source/compilation/compilation.hon

@@ -0,0 +1,170 @@
+import "@std/mem"
+import "@std/debug"
+import "@std/arraylist"
+import "@source/strpool"
+import "@source/diagnostics"
+import "@source/lexer"
+import "@source/parser"
+import "@source/ast"
+import "@source/sema"
+import "@source/lower"
+import "@source/ir"
+typecheck :: import "@source/sema/check"
+
+#! identities are valid only within their owning compilation.
+SourceId :: distinct usize
+PackageId :: distinct usize
+
+#! AST node IDs are source-local; retain both fields when referring across sources.
+DeclarationId :: struct {
+    source SourceId
+    node ast.NodeId
+}
+
+Source :: struct {
+    package PackageId
+    path []u8
+    bytes arraylist.ArrayList(u8)
+    tree ast.Tree
+}
+
+Package :: struct {
+    sources arraylist.ArrayList(SourceId)
+}
+
+#! owns sources, paths, ASTs, package membership and one identifier pool.
+#! borrows returned by source are invalidated by add_source or deinit; IDs remain stable.
+Compilation :: struct {
+    allocator mem.Allocator
+    strings strpool.StringPool
+    sources arraylist.ArrayList(Source)
+    packages arraylist.ArrayList(Package)
+}
+
+Error :: alias mem.AllocError | diagnostics.CapacityError | lexer.SourceError | strpool.InternError
+
+#! retains `allocator`; release all compilation storage with deinit.
+init proc(allocator mem.Allocator) Compilation {
+    return Compilation{
+        allocator = allocator,
+        strings = strpool.init(allocator),
+        sources = arraylist.init(allocator),
+        packages = arraylist.init(allocator),
+    }
+}
+
+deinit proc(compilation @mut Compilation) void {
+    for (compilation.sources.items) |@file| {
+        ast.deinit(&file.tree)
+        arraylist.deinit(&file.bytes)
+        mem.free(compilation.allocator, file.path)
+    }
+    for (compilation.packages.items) |@package| arraylist.deinit(&package.sources)
+    arraylist.deinit(&compilation.sources)
+    arraylist.deinit(&compilation.packages)
+    strpool.deinit(&compilation.strings)
+}
+
+#! creates a distinct package; directory discovery and path deduplication belong to the loader.
+add_package proc(compilation @mut Compilation) PackageId ! mem.AllocError {
+    id :: PackageId(compilation.packages.items.len)
+    try arraylist.append(&compilation.packages, Package{
+        sources = arraylist.init(compilation.allocator),
+    })
+    return id
+}
+
+#! copies `path` and scans/parses using the shared pool; takes `bytes` only on success.
+#! success resets `bytes` without copying its allocation. failure leaves it caller-owned;
+#! diagnostics and interned names produced before failure remain in their respective stores.
+add_source proc(
+    compilation @mut Compilation,
+    package PackageId,
+    path []u8,
+    bytes @mut arraylist.ArrayList(u8),
+    diagnostic_store @mut diagnostics.Store,
+) SourceId ! Error {
+    debug.assert(usize(package) < compilation.packages.items.len)
+
+    id :: SourceId(compilation.sources.items.len)
+    try arraylist.reserve(&compilation.sources, compilation.sources.items.len + 1)
+    members := &compilation.packages.items[usize(package)].sources
+    try arraylist.reserve(members, members.items.len + 1)
+    owned_path :: try mem.alloc(u8, compilation.allocator, path.len)
+    errdefer mem.free(compilation.allocator, owned_path)
+    memcopy!(owned_path, path)
+
+    scan_state := lexer.init(compilation.allocator)
+    defer lexer.deinit(&scan_state)
+    try lexer.scan(
+        &scan_state,
+        bytes.items,
+        &compilation.strings,
+        diagnostic_store,
+    )
+    tree := try parser.parse(
+        compilation.allocator,
+        scan_state.tokens.items,
+        bytes.items,
+        diagnostic_store,
+    )
+    errdefer ast.deinit(&tree)
+
+    try arraylist.append(&compilation.sources, Source{
+        package = package,
+        path = owned_path,
+        bytes = bytes^,
+        tree = tree,
+    })
+    try arraylist.append(members, id)
+    bytes^ = arraylist.init(bytes.allocator)
+
+    return id
+}
+
+#! borrows source metadata and its AST until add_source or deinit.
+source proc(compilation @Compilation, id SourceId) @Source {
+    debug.assert(usize(id) < compilation.sources.items.len)
+    return &compilation.sources.items[usize(id)]
+}
+
+#! qualifies a top-level declaration's local AST identity with its owning source.
+declaration proc(compilation @Compilation, id SourceId, index usize) DeclarationId {
+    return DeclarationId{
+        source = id,
+        node = ast.package_decl(&source(compilation, id).tree, index),
+    }
+}
+
+#! compiles one registered source; the returned IR owns its storage and borrows no frontend data.
+#! package scopes and cross-source resolution are added by the package-loading milestone.
+compile proc(
+    compilation @mut Compilation,
+    id SourceId,
+    diagnostic_store @mut diagnostics.Store,
+) ir.Module ! Error {
+    tree :: &source(compilation, id).tree
+    resolution := try sema.resolve(
+        compilation.allocator,
+        tree,
+        &compilation.strings,
+        diagnostic_store,
+    )
+    defer sema.deinit(&resolution)
+
+    checked := try typecheck.check(
+        compilation.allocator,
+        tree,
+        &resolution,
+        diagnostic_store,
+    )
+    defer typecheck.deinit(&checked)
+
+    return try lower.lower(
+        compilation.allocator,
+        tree,
+        &resolution,
+        &checked,
+        diagnostic_store,
+    )
+}

source/compilation/compilation.test.hon

@@ -0,0 +1,71 @@
+import "@std/mem"
+import "@std/arraylist"
+import "@std/testing"
+import "@source/ast"
+import "@source/diagnostics"
+import "@source/ir"
+import "@source/interpreter"
+
+@hide:file
+owned_text proc(text []u8) arraylist.ArrayList(u8) ! mem.AllocError {
+    bytes := arraylist.init(u8, mem.c_allocator)
+    errdefer arraylist.deinit(&bytes)
+    try arraylist.reserve(&bytes, text.len)
+    for (text) |byte| try arraylist.append(&bytes, byte)
+    return bytes
+}
+
+keeps_source_local_declarations_distinct_and_ir_independent test {
+    diagnostic_store := diagnostics.init(mem.c_allocator)
+    defer diagnostics.deinit(&diagnostic_store)
+    first := try owned_text("value :: 21\nmain proc() int { return value * 2 }")
+    defer arraylist.deinit(&first)
+    second := try owned_text("value :: 3\nmain proc() int { return value * 2 }")
+    defer arraylist.deinit(&second)
+    sibling := try owned_text("helper proc() int { return 7 }")
+    defer arraylist.deinit(&sibling)
+    path [9]mut u8 := undefined
+    memcopy!(path[..], "first.hon")
+    session := init(mem.c_allocator)
+    # the IR must outlive the compilation, including source buffers and interned names.
+    modules [2]mut ir.Module := undefined
+    {
+        defer deinit(&session)
+        root :: try add_package(&session)
+        dependency :: try add_package(&session)
+        left :: try add_source(&session, root, path[..], &first, &diagnostic_store)
+        right :: try add_source(&session, dependency, "second.hon", &second, &diagnostic_store)
+        third :: try add_source(&session, root, "sibling.hon", &sibling, &diagnostic_store)
+        path[0] = 'X'
+        try testing.expect_equal(0, first.capacity)
+        try testing.expect_equal(0, first.items.len)
+        try testing.expect(mem.eql(source(&session, left).path, "first.hon"))
+        try testing.expect_equal(root, source(&session, left).package)
+        try testing.expect_equal(dependency, source(&session, right).package)
+        try testing.expect_equal(root, source(&session, third).package)
+        try testing.expect_equal(left, session.packages.items[usize(root)].sources.items[0])
+        try testing.expect_equal(third, session.packages.items[usize(root)].sources.items[1])
+        a :: declaration(&session, left, 0)
+        b :: declaration(&session, right, 0)
+        try testing.expect_equal(a.node, b.node)
+        try testing.expect(a.source != b.source)
+        left_tree :: &source(&session, a.source).tree
+        right_tree :: &source(&session, b.source).tree
+        try testing.expect_equal(ast.name_id(left_tree, a.node), ast.name_id(right_tree, b.node))
+        try testing.expect(mem.eql(ast.source_text(left_tree, ast.statement_decl(left_tree, a.node).span), "value :: 21"))
+        try testing.expect(mem.eql(ast.source_text(right_tree, ast.statement_decl(right_tree, b.node).span), "value :: 3"))
+        modules[0] = try compile(&session, left, &diagnostic_store)
+        errdefer ir.deinit(&modules[0])
+        modules[1] = try compile(&session, right, &diagnostic_store)
+    }
+    defer ir.deinit(&modules[0])
+    defer ir.deinit(&modules[1])
+    try testing.expect_equal(0, diagnostics.count(&diagnostic_store))
+    left_result := try interpreter.run(mem.c_allocator, &modules[0], &diagnostic_store)
+    defer interpreter.deinit(&left_result)
+    right_result := try interpreter.run(mem.c_allocator, &modules[1], &diagnostic_store)
+    defer interpreter.deinit(&right_result)
+    try testing.expect(left_result.trap == null and right_result.trap == null)
+    try testing.expect_equal(i64(42), left_result.value.data.integer)
+    try testing.expect_equal(i64(6), right_result.value.data.integer)
+}

source/main.hon

@@ -6,18 +6,11 @@ import "@std/io"
 import "@std/process"
 
 import "@source/diagnostics"
-import "@source/strpool"
-import "@source/lexer"
-import "@source/ast"
-import "@source/parser"
-import "@source/sema"
+import "@source/compilation"
 import "@source/ir"
-import "@source/lower"
 import "@source/interpreter"
 import "@source/llvm"
 
-typecheck :: import "@source/sema/check"
-
 test import "@std/mem"
 test import "@std/enummap"
 test import "@std/arraylist"
@@ -37,13 +30,11 @@ test import "@source/sema/check"
 test import "@source/ir"
 test import "@source/lower"
 test import "@source/interpreter"
+test import "@source/compilation"
 
 @hide:file
 DriverError :: alias (
-    mem.AllocError
-    | diagnostics.CapacityError
-    | lexer.SourceError
-    | strpool.InternError
+    compilation.Error
     | io.OpenError
     | io.CloseError
     | io.ReadError
@@ -125,12 +116,17 @@ usage proc(output io.Writer) void ! io.WriteError {
 emit_file proc(allocator mem.Allocator, runtime io.Io, path [;0]u8) i32 ! DriverError {
     source := try read_source(allocator, runtime, path)
     defer arraylist.deinit(&source)
+    session := compilation.init(allocator)
+    defer compilation.deinit(&session)
     diagnostic_store := diagnostics.init(allocator)
     defer diagnostics.deinit(&diagnostic_store)
-    module := try compile(allocator, source.items, &diagnostic_store)
+    package :: try compilation.add_package(&session)
+    root :: try compilation.add_source(&session, package, path, &source, &diagnostic_store)
+    module := try compilation.compile(&session, root, &diagnostic_store)
     defer ir.deinit(&module)
-    try llvm.emit(allocator, io.stdout(runtime), &module, &diagnostic_store, source.items, path)
-    try report_diagnostics(allocator, io.stderr(runtime), path, source.items, &diagnostic_store, null)
+    file :: compilation.source(&session, root)
+    try llvm.emit(allocator, io.stdout(runtime), &module, &diagnostic_store, file.bytes.items, file.path)
+    try report_diagnostics(allocator, io.stderr(runtime), file.path, file.bytes.items, &diagnostic_store, null)
     return if (diagnostics.count(&diagnostic_store) == 0) 0 else 1
 }
 
@@ -144,9 +140,13 @@ build_file proc(
 ) i32 ! DriverError {
     source := try read_source(allocator, runtime, path)
     defer arraylist.deinit(&source)
+    session := compilation.init(allocator)
+    defer compilation.deinit(&session)
     diagnostic_store := diagnostics.init(allocator)
     defer diagnostics.deinit(&diagnostic_store)
-    module := try compile(allocator, source.items, &diagnostic_store)
+    package :: try compilation.add_package(&session)
+    root :: try compilation.add_source(&session, package, path, &source, &diagnostic_store)
+    module := try compilation.compile(&session, root, &diagnostic_store)
     defer ir.deinit(&module)
 
     template [18]mut u8 := undefined
@@ -159,7 +159,8 @@ build_file proc(
     defer {
         if (!closing) io.close(file) catch |_| {}
     }
-    try llvm.emit(allocator, io.writer(file), &module, &diagnostic_store, source.items, path)
+    root_source :: compilation.source(&session, root)
+    try llvm.emit(allocator, io.writer(file), &module, &diagnostic_store, root_source.bytes.items, root_source.path)
     closing = true
     try io.close(file)
     temporary [;0]u8 :: template[..].ptr[..17;0]
@@ -169,7 +170,7 @@ build_file proc(
     ]
     status :: try process.run(allocator, arguments[..])
     if (status != 0) return .compiler_failed
-    try report_diagnostics(allocator, io.stderr(runtime), path, source.items, &diagnostic_store, null)
+    try report_diagnostics(allocator, io.stderr(runtime), root_source.path, root_source.bytes.items, &diagnostic_store, null)
     return if (diagnostics.count(&diagnostic_store) == 0) 0 else 1
 }
 
@@ -192,34 +193,23 @@ read_source proc(allocator mem.Allocator, runtime io.Io, path [;0]u8) arraylist.
     return bytes
 }
 
-@hide:file
-compile proc(allocator mem.Allocator, source []u8, diagnostic_store @mut diagnostics.Store) ir.Module ! DriverError {
-    strings := strpool.init(allocator)
-    defer strpool.deinit(&strings)
-    scan_state := lexer.init(allocator)
-    defer lexer.deinit(&scan_state)
-    try lexer.scan(&scan_state, source, &strings, diagnostic_store)
-    tree := try parser.parse(allocator, scan_state.tokens.items, source, diagnostic_store)
-    defer ast.deinit(&tree)
-    resolution := try sema.resolve(allocator, &tree, &strings, diagnostic_store)
-    defer sema.deinit(&resolution)
-    checked := try typecheck.check(allocator, &tree, &resolution, diagnostic_store)
-    defer typecheck.deinit(&checked)
-    return try lower.lower(allocator, &tree, &resolution, &checked, diagnostic_store)
-}
-
 @hide:file
 run_file proc(allocator mem.Allocator, runtime io.Io, path [;0]u8) i32 ! DriverError {
     source := try read_source(allocator, runtime, path)
     defer arraylist.deinit(&source)
+    session := compilation.init(allocator)
+    defer compilation.deinit(&session)
     diagnostic_store := diagnostics.init(allocator)
     defer diagnostics.deinit(&diagnostic_store)
-    module := try compile(allocator, source.items, &diagnostic_store)
+    package :: try compilation.add_package(&session)
+    root :: try compilation.add_source(&session, package, path, &source, &diagnostic_store)
+    module := try compilation.compile(&session, root, &diagnostic_store)
     defer ir.deinit(&module)
     outcome := try interpreter.run(allocator, &module, &diagnostic_store)
     defer interpreter.deinit(&outcome)
     if (outcome.trap == null) try print_result(io.stdout(runtime), &module, outcome.value)
-    try report_diagnostics(allocator, io.stderr(runtime), path, source.items, &diagnostic_store, outcome.trap)
+    file :: compilation.source(&session, root)
+    try report_diagnostics(allocator, io.stderr(runtime), file.path, file.bytes.items, &diagnostic_store, outcome.trap)
     if (outcome.trap != null or diagnostics.count(&diagnostic_store) != 0) return 1
     return 0
 }