diff --git a/README.md b/README.md index f9a63da..e9c909c 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,10 @@ range bounds that are compound or unary expressions require their own parenthese name resolution (`@source/sema`) runs after parsing. it predeclares package names and each block's direct locals, allowing forward references and mutually recursive functions. parameters and top-level function-body locals share a scope; each conditional arm and loop body introduces a child scope whose names do not escape it. signatures resolve names in the enclosing package scope. shadowing is forbidden, including reuse of builtin type names (`int`, `float`, `i64`, `f64`, `usize`, `bool`, `string`, `void`); sibling scopes may independently reuse names. +the driver uses `@source/compilation` to own loaded source buffers, copied paths, ASTs, package membership and a shared identifier pool. `add_source` scans/parses a source and transfers its `ArrayList(u8)` allocation only on success, resetting the caller's list; failure leaves the bytes caller-owned. `deinit` releases all retained frontend storage. source metadata borrows are invalidated by registration growth, but `SourceId` and `PackageId` remain stable within their compilation. + +AST `NodeId` values remain source-local. cross-source declarations use `compilation.DeclarationId { source, node }`, so identical local node numbers from different files cannot alias. `compilation.compile` currently checks/lowers one registered source into independently owned IR; package IDs and source membership do not yet implement directory loading, imports or cross-source name resolution. CLI commands still compile one file, and multi-source diagnostic reporting remains a later package milestone. + identifier interning is compilation-owned: initialize a `strpool.StringPool`, pass it to `lexer.scan` and `sema.resolve` before the diagnostic store, and release it after lowering. scanning interns identifier spellings once per pool; parsing preserves their IDs in compact AST storage, exposed through `ast.name_id`, while retaining source spans for diagnostics. name resolution keys scopes by those IDs and interns builtin names into the same pool, so identifier hashing and equality do not scan spelling bytes. IDs are meaningful only within their originating pool; independent compilations may assign the same numeric ID to different names. interning failures propagate as infrastructure errors rather than producing an invalid identifier ID. IR retains no pool dependency. resolver scopes use `@std/hashmap.AutoHashMap(K, V)`, which shares `init`, `put`, `get` and `deinit` with custom `HashMap` and `StringHashMap`. automatic keys support integers, booleans, enums, distinct types, fixed arrays and records recursively; record padding is ignored. slices, floating-point values, pointers, optionals and unions require explicit hashing and equality through `HashMap` (`StringHashMap` supplies content-based byte-string operations). keys and values are stored by value; referenced resources remain caller-owned. the builtin declaration count is inferred, and builtin ID storage derives its fixed stack-array length from that declaration array rather than repeating a count or initializer. diff --git a/TODO.md b/TODO.md index ecf4192..0b607b2 100644 --- a/TODO.md +++ b/TODO.md @@ -9,9 +9,9 @@ recovery ultimately reports a missing entry point. an isolated import followed by a supported `main` still prints `42` but exits `1`; recovery is not import support. -the driver compiles one source buffer (`source/main.hon:196`); AST declaration -identities and diagnostic spans currently belong to that single source. -the following order is a dependency plan, not implemented support. +the driver now retains frontend storage through `@source/compilation`; source IDs +qualify local AST declaration IDs, and package IDs retain source membership. +CLI commands still compile one source. the unchecked items below remain planned support. ## next milestone: scalar-function packages @@ -20,7 +20,7 @@ of scalar functions, through both the interpreter and native backend. using the existing standard library is a later acceptance gate: even `std/mem` needs records, pointers, generics and C calls. -- [ ] retain source/package identity and source ownership across the compilation; keep one identifier pool and unambiguous declaration IDs. +- [x] retain source/package identity and source ownership across the compilation; keep one identifier pool and unambiguous declaration IDs. - [ ] load directory packages once per canonical path, with deterministic file discovery, package scopes and root-only entry selection. follow Bro's existing relative and project-root `@` import rules; diagnose cycles rather than recurse indefinitely. - [ ] parse bare and aliased imports plus postfix namespace selection; resolve file-local import aliases without injecting dependency names into the importing package. - [ ] reuse ordinary scalar signature checking, function values and IR calls for selected declarations; namespaces have no runtime representation. diff --git a/source/compilation/compilation.hon b/source/compilation/compilation.hon new file mode 100644 index 0000000..ec2c3fd --- /dev/null +++ b/source/compilation/compilation.hon @@ -0,0 +1,170 @@ +import "@std/mem" +import "@std/debug" +import "@std/arraylist" +import "@source/strpool" +import "@source/diagnostics" +import "@source/lexer" +import "@source/parser" +import "@source/ast" +import "@source/sema" +import "@source/lower" +import "@source/ir" +typecheck :: import "@source/sema/check" + +#! identities are valid only within their owning compilation. +SourceId :: distinct usize +PackageId :: distinct usize + +#! AST node IDs are source-local; retain both fields when referring across sources. +DeclarationId :: struct { + source SourceId + node ast.NodeId +} + +Source :: struct { + package PackageId + path []u8 + bytes arraylist.ArrayList(u8) + tree ast.Tree +} + +Package :: struct { + sources arraylist.ArrayList(SourceId) +} + +#! owns sources, paths, ASTs, package membership and one identifier pool. +#! borrows returned by source are invalidated by add_source or deinit; IDs remain stable. +Compilation :: struct { + allocator mem.Allocator + strings strpool.StringPool + sources arraylist.ArrayList(Source) + packages arraylist.ArrayList(Package) +} + +Error :: alias mem.AllocError | diagnostics.CapacityError | lexer.SourceError | strpool.InternError + +#! retains `allocator`; release all compilation storage with deinit. +init proc(allocator mem.Allocator) Compilation { + return Compilation{ + allocator = allocator, + strings = strpool.init(allocator), + sources = arraylist.init(allocator), + packages = arraylist.init(allocator), + } +} + +deinit proc(compilation @mut Compilation) void { + for (compilation.sources.items) |@file| { + ast.deinit(&file.tree) + arraylist.deinit(&file.bytes) + mem.free(compilation.allocator, file.path) + } + for (compilation.packages.items) |@package| arraylist.deinit(&package.sources) + arraylist.deinit(&compilation.sources) + arraylist.deinit(&compilation.packages) + strpool.deinit(&compilation.strings) +} + +#! creates a distinct package; directory discovery and path deduplication belong to the loader. +add_package proc(compilation @mut Compilation) PackageId ! mem.AllocError { + id :: PackageId(compilation.packages.items.len) + try arraylist.append(&compilation.packages, Package{ + sources = arraylist.init(compilation.allocator), + }) + return id +} + +#! copies `path` and scans/parses using the shared pool; takes `bytes` only on success. +#! success resets `bytes` without copying its allocation. failure leaves it caller-owned; +#! diagnostics and interned names produced before failure remain in their respective stores. +add_source proc( + compilation @mut Compilation, + package PackageId, + path []u8, + bytes @mut arraylist.ArrayList(u8), + diagnostic_store @mut diagnostics.Store, +) SourceId ! Error { + debug.assert(usize(package) < compilation.packages.items.len) + + id :: SourceId(compilation.sources.items.len) + try arraylist.reserve(&compilation.sources, compilation.sources.items.len + 1) + members := &compilation.packages.items[usize(package)].sources + try arraylist.reserve(members, members.items.len + 1) + owned_path :: try mem.alloc(u8, compilation.allocator, path.len) + errdefer mem.free(compilation.allocator, owned_path) + memcopy!(owned_path, path) + + scan_state := lexer.init(compilation.allocator) + defer lexer.deinit(&scan_state) + try lexer.scan( + &scan_state, + bytes.items, + &compilation.strings, + diagnostic_store, + ) + tree := try parser.parse( + compilation.allocator, + scan_state.tokens.items, + bytes.items, + diagnostic_store, + ) + errdefer ast.deinit(&tree) + + try arraylist.append(&compilation.sources, Source{ + package = package, + path = owned_path, + bytes = bytes^, + tree = tree, + }) + try arraylist.append(members, id) + bytes^ = arraylist.init(bytes.allocator) + + return id +} + +#! borrows source metadata and its AST until add_source or deinit. +source proc(compilation @Compilation, id SourceId) @Source { + debug.assert(usize(id) < compilation.sources.items.len) + return &compilation.sources.items[usize(id)] +} + +#! qualifies a top-level declaration's local AST identity with its owning source. +declaration proc(compilation @Compilation, id SourceId, index usize) DeclarationId { + return DeclarationId{ + source = id, + node = ast.package_decl(&source(compilation, id).tree, index), + } +} + +#! compiles one registered source; the returned IR owns its storage and borrows no frontend data. +#! package scopes and cross-source resolution are added by the package-loading milestone. +compile proc( + compilation @mut Compilation, + id SourceId, + diagnostic_store @mut diagnostics.Store, +) ir.Module ! Error { + tree :: &source(compilation, id).tree + resolution := try sema.resolve( + compilation.allocator, + tree, + &compilation.strings, + diagnostic_store, + ) + defer sema.deinit(&resolution) + + checked := try typecheck.check( + compilation.allocator, + tree, + &resolution, + diagnostic_store, + ) + defer typecheck.deinit(&checked) + + return try lower.lower( + compilation.allocator, + tree, + &resolution, + &checked, + diagnostic_store, + ) +} diff --git a/source/compilation/compilation.test.hon b/source/compilation/compilation.test.hon new file mode 100644 index 0000000..9aaca53 --- /dev/null +++ b/source/compilation/compilation.test.hon @@ -0,0 +1,71 @@ +import "@std/mem" +import "@std/arraylist" +import "@std/testing" +import "@source/ast" +import "@source/diagnostics" +import "@source/ir" +import "@source/interpreter" + +@hide:file +owned_text proc(text []u8) arraylist.ArrayList(u8) ! mem.AllocError { + bytes := arraylist.init(u8, mem.c_allocator) + errdefer arraylist.deinit(&bytes) + try arraylist.reserve(&bytes, text.len) + for (text) |byte| try arraylist.append(&bytes, byte) + return bytes +} + +keeps_source_local_declarations_distinct_and_ir_independent test { + diagnostic_store := diagnostics.init(mem.c_allocator) + defer diagnostics.deinit(&diagnostic_store) + first := try owned_text("value :: 21\nmain proc() int { return value * 2 }") + defer arraylist.deinit(&first) + second := try owned_text("value :: 3\nmain proc() int { return value * 2 }") + defer arraylist.deinit(&second) + sibling := try owned_text("helper proc() int { return 7 }") + defer arraylist.deinit(&sibling) + path [9]mut u8 := undefined + memcopy!(path[..], "first.hon") + session := init(mem.c_allocator) + # the IR must outlive the compilation, including source buffers and interned names. + modules [2]mut ir.Module := undefined + { + defer deinit(&session) + root :: try add_package(&session) + dependency :: try add_package(&session) + left :: try add_source(&session, root, path[..], &first, &diagnostic_store) + right :: try add_source(&session, dependency, "second.hon", &second, &diagnostic_store) + third :: try add_source(&session, root, "sibling.hon", &sibling, &diagnostic_store) + path[0] = 'X' + try testing.expect_equal(0, first.capacity) + try testing.expect_equal(0, first.items.len) + try testing.expect(mem.eql(source(&session, left).path, "first.hon")) + try testing.expect_equal(root, source(&session, left).package) + try testing.expect_equal(dependency, source(&session, right).package) + try testing.expect_equal(root, source(&session, third).package) + try testing.expect_equal(left, session.packages.items[usize(root)].sources.items[0]) + try testing.expect_equal(third, session.packages.items[usize(root)].sources.items[1]) + a :: declaration(&session, left, 0) + b :: declaration(&session, right, 0) + try testing.expect_equal(a.node, b.node) + try testing.expect(a.source != b.source) + left_tree :: &source(&session, a.source).tree + right_tree :: &source(&session, b.source).tree + try testing.expect_equal(ast.name_id(left_tree, a.node), ast.name_id(right_tree, b.node)) + try testing.expect(mem.eql(ast.source_text(left_tree, ast.statement_decl(left_tree, a.node).span), "value :: 21")) + try testing.expect(mem.eql(ast.source_text(right_tree, ast.statement_decl(right_tree, b.node).span), "value :: 3")) + modules[0] = try compile(&session, left, &diagnostic_store) + errdefer ir.deinit(&modules[0]) + modules[1] = try compile(&session, right, &diagnostic_store) + } + defer ir.deinit(&modules[0]) + defer ir.deinit(&modules[1]) + try testing.expect_equal(0, diagnostics.count(&diagnostic_store)) + left_result := try interpreter.run(mem.c_allocator, &modules[0], &diagnostic_store) + defer interpreter.deinit(&left_result) + right_result := try interpreter.run(mem.c_allocator, &modules[1], &diagnostic_store) + defer interpreter.deinit(&right_result) + try testing.expect(left_result.trap == null and right_result.trap == null) + try testing.expect_equal(i64(42), left_result.value.data.integer) + try testing.expect_equal(i64(6), right_result.value.data.integer) +} diff --git a/source/main.hon b/source/main.hon index e995557..dfe4a1f 100644 --- a/source/main.hon +++ b/source/main.hon @@ -6,18 +6,11 @@ import "@std/io" import "@std/process" import "@source/diagnostics" -import "@source/strpool" -import "@source/lexer" -import "@source/ast" -import "@source/parser" -import "@source/sema" +import "@source/compilation" import "@source/ir" -import "@source/lower" import "@source/interpreter" import "@source/llvm" -typecheck :: import "@source/sema/check" - test import "@std/mem" test import "@std/enummap" test import "@std/arraylist" @@ -37,13 +30,11 @@ test import "@source/sema/check" test import "@source/ir" test import "@source/lower" test import "@source/interpreter" +test import "@source/compilation" @hide:file DriverError :: alias ( - mem.AllocError - | diagnostics.CapacityError - | lexer.SourceError - | strpool.InternError + compilation.Error | io.OpenError | io.CloseError | io.ReadError @@ -125,12 +116,17 @@ usage proc(output io.Writer) void ! io.WriteError { emit_file proc(allocator mem.Allocator, runtime io.Io, path [;0]u8) i32 ! DriverError { source := try read_source(allocator, runtime, path) defer arraylist.deinit(&source) + session := compilation.init(allocator) + defer compilation.deinit(&session) diagnostic_store := diagnostics.init(allocator) defer diagnostics.deinit(&diagnostic_store) - module := try compile(allocator, source.items, &diagnostic_store) + package :: try compilation.add_package(&session) + root :: try compilation.add_source(&session, package, path, &source, &diagnostic_store) + module := try compilation.compile(&session, root, &diagnostic_store) defer ir.deinit(&module) - try llvm.emit(allocator, io.stdout(runtime), &module, &diagnostic_store, source.items, path) - try report_diagnostics(allocator, io.stderr(runtime), path, source.items, &diagnostic_store, null) + file :: compilation.source(&session, root) + try llvm.emit(allocator, io.stdout(runtime), &module, &diagnostic_store, file.bytes.items, file.path) + try report_diagnostics(allocator, io.stderr(runtime), file.path, file.bytes.items, &diagnostic_store, null) return if (diagnostics.count(&diagnostic_store) == 0) 0 else 1 } @@ -144,9 +140,13 @@ build_file proc( ) i32 ! DriverError { source := try read_source(allocator, runtime, path) defer arraylist.deinit(&source) + session := compilation.init(allocator) + defer compilation.deinit(&session) diagnostic_store := diagnostics.init(allocator) defer diagnostics.deinit(&diagnostic_store) - module := try compile(allocator, source.items, &diagnostic_store) + package :: try compilation.add_package(&session) + root :: try compilation.add_source(&session, package, path, &source, &diagnostic_store) + module := try compilation.compile(&session, root, &diagnostic_store) defer ir.deinit(&module) template [18]mut u8 := undefined @@ -159,7 +159,8 @@ build_file proc( defer { if (!closing) io.close(file) catch |_| {} } - try llvm.emit(allocator, io.writer(file), &module, &diagnostic_store, source.items, path) + root_source :: compilation.source(&session, root) + try llvm.emit(allocator, io.writer(file), &module, &diagnostic_store, root_source.bytes.items, root_source.path) closing = true try io.close(file) temporary [;0]u8 :: template[..].ptr[..17;0] @@ -169,7 +170,7 @@ build_file proc( ] status :: try process.run(allocator, arguments[..]) if (status != 0) return .compiler_failed - try report_diagnostics(allocator, io.stderr(runtime), path, source.items, &diagnostic_store, null) + try report_diagnostics(allocator, io.stderr(runtime), root_source.path, root_source.bytes.items, &diagnostic_store, null) return if (diagnostics.count(&diagnostic_store) == 0) 0 else 1 } @@ -192,34 +193,23 @@ read_source proc(allocator mem.Allocator, runtime io.Io, path [;0]u8) arraylist. return bytes } -@hide:file -compile proc(allocator mem.Allocator, source []u8, diagnostic_store @mut diagnostics.Store) ir.Module ! DriverError { - strings := strpool.init(allocator) - defer strpool.deinit(&strings) - scan_state := lexer.init(allocator) - defer lexer.deinit(&scan_state) - try lexer.scan(&scan_state, source, &strings, diagnostic_store) - tree := try parser.parse(allocator, scan_state.tokens.items, source, diagnostic_store) - defer ast.deinit(&tree) - resolution := try sema.resolve(allocator, &tree, &strings, diagnostic_store) - defer sema.deinit(&resolution) - checked := try typecheck.check(allocator, &tree, &resolution, diagnostic_store) - defer typecheck.deinit(&checked) - return try lower.lower(allocator, &tree, &resolution, &checked, diagnostic_store) -} - @hide:file run_file proc(allocator mem.Allocator, runtime io.Io, path [;0]u8) i32 ! DriverError { source := try read_source(allocator, runtime, path) defer arraylist.deinit(&source) + session := compilation.init(allocator) + defer compilation.deinit(&session) diagnostic_store := diagnostics.init(allocator) defer diagnostics.deinit(&diagnostic_store) - module := try compile(allocator, source.items, &diagnostic_store) + package :: try compilation.add_package(&session) + root :: try compilation.add_source(&session, package, path, &source, &diagnostic_store) + module := try compilation.compile(&session, root, &diagnostic_store) defer ir.deinit(&module) outcome := try interpreter.run(allocator, &module, &diagnostic_store) defer interpreter.deinit(&outcome) if (outcome.trap == null) try print_result(io.stdout(runtime), &module, outcome.value) - try report_diagnostics(allocator, io.stderr(runtime), path, source.items, &diagnostic_store, outcome.trap) + file :: compilation.source(&session, root) + try report_diagnostics(allocator, io.stderr(runtime), file.path, file.bytes.items, &diagnostic_store, outcome.trap) if (outcome.trap != null or diagnostics.count(&diagnostic_store) != 0) return 1 return 0 }