intern identifiers

benchmarks/symbols/README.md

@@ -0,0 +1,27 @@
+# Identifier interning benchmark
+
+Run from the repository root:
+
+```sh
+odin run benchmarks/symbols -o:speed
+```
+
+The benchmark warms the pipeline once, then measures a fresh lex/parse/check
+run over a generated program with 5,000 repeated identifier-heavy statements.
+Timing is informational; token layout and allocation metrics are the stable
+comparison points.
+
+Results captured on 2026-06-10 with Odin `dev-2026-02:b942f72cb`:
+
+| Metric | Before interning | After interning |
+| --- | ---: | ---: |
+| Elapsed time | 15.723 ms | 9.248 ms |
+| Peak memory | 14,661,846 bytes | 13,220,507 bytes |
+| Allocations | 40,105 | 40,112 |
+| Token count | 65,032 | 65,032 |
+| Token size | 56 bytes | 48 bytes |
+| Unique symbols | n/a | 5 |
+| Stored symbol bytes | n/a | 21 |
+| Diagnostics | 0 | 0 |
+
+The measured run reduced token size by 14.3% and peak tracked memory by 9.8%.

benchmarks/symbols/main.odin

@@ -0,0 +1,78 @@
+package main
+
+import "../../compiler/ast"
+import "../../compiler/checker"
+import "../../compiler/hir"
+import "../../compiler/lexer"
+import "../../compiler/parser"
+import "../../compiler/source"
+import "../../compiler/symbol"
+import "../../compiler/token"
+import "core:fmt"
+import "core:mem"
+import "core:strings"
+import "core:time"
+
+Metrics :: struct {
+	token_count:         int,
+	unique_symbol_count: int,
+	stored_symbol_bytes: int,
+	diagnostic_count:    int,
+}
+
+make_source :: proc(repetitions: int, allocator := context.allocator) -> string {
+	builder := strings.builder_make(allocator)
+	defer strings.builder_destroy(&builder)
+	strings.write_string(&builder, "identity :: func(value int) int { return value }\n")
+	strings.write_string(&builder, "main :: func() i32 {\n\tacc i32 = 0\n")
+	for _ in 0 ..< repetitions {
+		strings.write_string(&builder, "\t_ = identity(acc)\n\tacc = acc + 1\n")
+	}
+	strings.write_string(&builder, "\treturn acc\n}\n")
+	return strings.clone(strings.to_string(builder), allocator)
+}
+
+run_pipeline :: proc(text: string, allocator := context.allocator) -> Metrics {
+	source_file := source.Source{path="benchmark.bro", text=text}
+	diagnostics := source.init_diagnostics(&source_file, allocator)
+	defer source.destroy_diagnostics(&diagnostics)
+	symbols := symbol.init_table(allocator)
+	defer symbol.destroy_table(&symbols)
+	stream := lexer.lex(&source_file, &diagnostics, &symbols, allocator)
+	defer delete(stream.items)
+	ast_module := parser.parse(&stream, &source_file, &diagnostics, allocator)
+	defer ast.destroy_module(&ast_module)
+	hir_module := checker.check(&ast_module, &diagnostics, &symbols, allocator)
+	defer hir.destroy_module(&hir_module)
+
+	return Metrics{
+		token_count=len(stream.items),
+		unique_symbol_count=len(symbols.items),
+		stored_symbol_bytes=symbols.stored_bytes,
+		diagnostic_count=len(diagnostics.items),
+	}
+}
+
+main :: proc() {
+	text := make_source(5_000)
+	defer delete(text)
+
+	_ = run_pipeline(text)
+
+	tracker: mem.Tracking_Allocator
+	mem.tracking_allocator_init(&tracker, context.allocator)
+	defer mem.tracking_allocator_destroy(&tracker)
+	allocator := mem.tracking_allocator(&tracker)
+	start := time.tick_now()
+	metrics := run_pipeline(text, allocator)
+	elapsed := time.tick_diff(start, time.tick_now())
+
+	fmt.printf("elapsed_ms: %.3f\n", time.duration_milliseconds(elapsed))
+	fmt.printf("peak_memory_bytes: %d\n", tracker.peak_memory_allocated)
+	fmt.printf("allocation_count: %d\n", tracker.total_allocation_count)
+	fmt.printf("token_count: %d\n", metrics.token_count)
+	fmt.printf("token_size_bytes: %d\n", size_of(token.Token))
+	fmt.printf("unique_symbol_count: %d\n", metrics.unique_symbol_count)
+	fmt.printf("stored_symbol_bytes: %d\n", metrics.stored_symbol_bytes)
+	fmt.printf("diagnostic_count: %d\n", metrics.diagnostic_count)
+}