From cdbe4fbc997662485ed99f63b3c32fb5ba325d6e Mon Sep 17 00:00:00 2001 From: hl-valdemar Date: Wed, 10 Jun 2026 21:27:00 +0200 Subject: [PATCH] intern identifiers --- TODO.md | 3 - benchmarks/symbols/README.md | 27 +++++ benchmarks/symbols/main.odin | 78 ++++++++++++ compiler/ast/ast.odin | 18 +-- compiler/checker/checker.odin | 142 ++++++++++++---------- compiler/compiler.odin | 8 +- compiler/hir/hir.odin | 7 +- compiler/ir/ir.odin | 6 +- compiler/lexer/lexer.odin | 20 +++- compiler/llvm/llvm.odin | 16 ++- compiler/loader/loader.odin | 23 ++-- compiler/lower/lower.odin | 3 +- compiler/parser/parser.odin | 50 +++++--- compiler/symbol/symbol.odin | 58 +++++++++ compiler/token/token.odin | 3 +- compiler_tests.odin | 217 +++++++++++++++++++++++++--------- 16 files changed, 497 insertions(+), 182 deletions(-) create mode 100644 benchmarks/symbols/README.md create mode 100644 benchmarks/symbols/main.odin create mode 100644 compiler/symbol/symbol.odin diff --git a/TODO.md b/TODO.md index f3fe42d..3187711 100644 --- a/TODO.md +++ b/TODO.md @@ -2,9 +2,6 @@ - for global initialization cycles, report also starting and ending lines -- intern strings across all phases of the compiler and reference strings by their hash/id - - makes string comparison and equality checks faster and takes up less memory - # milestones 1. get c interop working: diff --git a/benchmarks/symbols/README.md b/benchmarks/symbols/README.md new file mode 100644 index 0000000..6edd77f --- /dev/null +++ b/benchmarks/symbols/README.md @@ -0,0 +1,27 @@ +# Identifier interning benchmark + +Run from the repository root: + +```sh +odin run benchmarks/symbols -o:speed +``` + +The benchmark warms the pipeline once, then measures a fresh lex/parse/check +run over a generated program with 5,000 repeated identifier-heavy statements. +Timing is informational; token layout and allocation metrics are the stable +comparison points. + +Results captured on 2026-06-10 with Odin `dev-2026-02:b942f72cb`: + +| Metric | Before interning | After interning | +| --- | ---: | ---: | +| Elapsed time | 15.723 ms | 9.248 ms | +| Peak memory | 14,661,846 bytes | 13,220,507 bytes | +| Allocations | 40,105 | 40,112 | +| Token count | 65,032 | 65,032 | +| Token size | 56 bytes | 48 bytes | +| Unique symbols | n/a | 5 | +| Stored symbol bytes | n/a | 21 | +| Diagnostics | 0 | 0 | + +The measured run reduced token size by 14.3% and peak tracked memory by 9.8%. diff --git a/benchmarks/symbols/main.odin b/benchmarks/symbols/main.odin new file mode 100644 index 0000000..80b0c67 --- /dev/null +++ b/benchmarks/symbols/main.odin @@ -0,0 +1,78 @@ +package main + +import "../../compiler/ast" +import "../../compiler/checker" +import "../../compiler/hir" +import "../../compiler/lexer" +import "../../compiler/parser" +import "../../compiler/source" +import "../../compiler/symbol" +import "../../compiler/token" +import "core:fmt" +import "core:mem" +import "core:strings" +import "core:time" + +Metrics :: struct { + token_count: int, + unique_symbol_count: int, + stored_symbol_bytes: int, + diagnostic_count: int, +} + +make_source :: proc(repetitions: int, allocator := context.allocator) -> string { + builder := strings.builder_make(allocator) + defer strings.builder_destroy(&builder) + strings.write_string(&builder, "identity :: func(value int) int { return value }\n") + strings.write_string(&builder, "main :: func() i32 {\n\tacc i32 = 0\n") + for _ in 0 ..< repetitions { + strings.write_string(&builder, "\t_ = identity(acc)\n\tacc = acc + 1\n") + } + strings.write_string(&builder, "\treturn acc\n}\n") + return strings.clone(strings.to_string(builder), allocator) +} + +run_pipeline :: proc(text: string, allocator := context.allocator) -> Metrics { + source_file := source.Source{path="benchmark.bro", text=text} + diagnostics := source.init_diagnostics(&source_file, allocator) + defer source.destroy_diagnostics(&diagnostics) + symbols := symbol.init_table(allocator) + defer symbol.destroy_table(&symbols) + stream := lexer.lex(&source_file, &diagnostics, &symbols, allocator) + defer delete(stream.items) + ast_module := parser.parse(&stream, &source_file, &diagnostics, allocator) + defer ast.destroy_module(&ast_module) + hir_module := checker.check(&ast_module, &diagnostics, &symbols, allocator) + defer hir.destroy_module(&hir_module) + + return Metrics{ + token_count=len(stream.items), + unique_symbol_count=len(symbols.items), + stored_symbol_bytes=symbols.stored_bytes, + diagnostic_count=len(diagnostics.items), + } +} + +main :: proc() { + text := make_source(5_000) + defer delete(text) + + _ = run_pipeline(text) + + tracker: mem.Tracking_Allocator + mem.tracking_allocator_init(&tracker, context.allocator) + defer mem.tracking_allocator_destroy(&tracker) + allocator := mem.tracking_allocator(&tracker) + start := time.tick_now() + metrics := run_pipeline(text, allocator) + elapsed := time.tick_diff(start, time.tick_now()) + + fmt.printf("elapsed_ms: %.3f\n", time.duration_milliseconds(elapsed)) + fmt.printf("peak_memory_bytes: %d\n", tracker.peak_memory_allocated) + fmt.printf("allocation_count: %d\n", tracker.total_allocation_count) + fmt.printf("token_count: %d\n", metrics.token_count) + fmt.printf("token_size_bytes: %d\n", size_of(token.Token)) + fmt.printf("unique_symbol_count: %d\n", metrics.unique_symbol_count) + fmt.printf("stored_symbol_bytes: %d\n", metrics.stored_symbol_bytes) + fmt.printf("diagnostic_count: %d\n", metrics.diagnostic_count) +} diff --git a/compiler/ast/ast.odin b/compiler/ast/ast.odin index af33ff1..b724c7b 100644 --- a/compiler/ast/ast.odin +++ b/compiler/ast/ast.odin @@ -1,6 +1,7 @@ package ast import "../source" +import "../symbol" import "core:mem" INVALID_ID :: -1 @@ -26,8 +27,8 @@ Expr_Kind :: enum { Expr :: struct { kind: Expr_Kind, span: source.Span, - qualifier: string, - text: string, + qualifier: symbol.Id, + name: symbol.Id, integer: i64, left: int, right: int, @@ -36,7 +37,7 @@ Expr :: struct { } Param :: struct { - name: string, + name: symbol.Id, span: source.Span, type: Type_Syntax, } @@ -52,7 +53,7 @@ Stmt_Kind :: enum { Stmt :: struct { kind: Stmt_Kind, span: source.Span, - name: string, + name: symbol.Id, type: Type_Syntax, immutable: bool, expr: int, @@ -61,7 +62,7 @@ Stmt :: struct { Function :: struct { span: source.Span, - name: string, + name: symbol.Id, pkg: int, file: int, c_abi: bool, @@ -73,7 +74,7 @@ Function :: struct { Global :: struct { span: source.Span, - name: string, + name: symbol.Id, pkg: int, file: int, type: Type_Syntax, @@ -84,7 +85,7 @@ Global :: struct { Import :: struct { span: source.Span, - alias: string, + alias: symbol.Id, path: string, pkg: int, file: int, @@ -101,7 +102,7 @@ File :: struct { Package :: struct { path: string, - name: string, + name: symbol.Id, available: bool, } @@ -142,7 +143,6 @@ destroy_module :: proc(module: ^Module) { } for pkg in module.packages { delete(pkg.path, module.allocator) - delete(pkg.name, module.allocator) } delete(module.exprs) delete(module.statements) diff --git a/compiler/checker/checker.odin b/compiler/checker/checker.odin index 3c5959d..f00bec3 100644 --- a/compiler/checker/checker.odin +++ b/compiler/checker/checker.odin @@ -3,6 +3,7 @@ package checker import "../ast" import "../hir" import "../source" +import "../symbol" import "../types" import "base:intrinsics" import "core:fmt" @@ -18,12 +19,12 @@ Spec :: struct { } Infer_Local :: struct { - name: string, + name: symbol.Id, type: types.Type, } Build_Local :: struct { - name: string, + name: symbol.Id, type: types.Type, mutable: bool, id: int, @@ -43,13 +44,14 @@ Constant :: struct { Symbol_Index_Entry :: struct { scope: int, - name: string, + name: symbol.Id, id: int, } Checker :: struct { ast_module: ^ast.Module, diagnostics: ^source.Diagnostics, + symbols: ^symbol.Table, module: hir.Module, specs: [dynamic]Spec, function_index: []Symbol_Index_Entry, @@ -57,9 +59,15 @@ Checker :: struct { import_index: []Symbol_Index_Entry, global_types: []types.Type, constants: []Constant, + main_symbol: symbol.Id, + sink_symbol: symbol.Id, allocator: mem.Allocator, } +symbol_text :: proc(checker: ^Checker, id: symbol.Id) -> string { + return symbol.resolve(checker.symbols, id) +} + eval_constant :: proc(checker: ^Checker, expr_id: int) -> Constant { if expr_id < 0 || expr_id >= len(checker.ast_module.exprs) { return Constant{kind = .Not_Constant} @@ -131,18 +139,18 @@ symbol_index_less :: proc(left, right: Symbol_Index_Entry) -> bool { return left.scope < right.scope } if left.name != right.name { - return left.name < right.name + return int(left.name) < int(right.name) } return left.id < right.id } -find_symbol :: proc(index: []Symbol_Index_Entry, scope: int, name: string) -> int { +find_symbol :: proc(index: []Symbol_Index_Entry, scope: int, name: symbol.Id) -> int { low := 0 high := len(index) for low < high { middle := low + (high-low)/2 entry := index[middle] - if entry.scope < scope || entry.scope == scope && entry.name < name { + if entry.scope < scope || entry.scope == scope && int(entry.name) < int(name) { low = middle + 1 } else { high = middle @@ -174,15 +182,15 @@ build_symbol_indexes :: proc(checker: ^Checker) { slice.sort_by(checker.import_index, symbol_index_less) } -find_template :: proc(checker: ^Checker, name: string, pkg := 0) -> int { +find_template :: proc(checker: ^Checker, name: symbol.Id, pkg := 0) -> int { return find_symbol(checker.function_index, pkg, name) } -find_global :: proc(checker: ^Checker, name: string, pkg := 0) -> int { +find_global :: proc(checker: ^Checker, name: symbol.Id, pkg := 0) -> int { return find_symbol(checker.global_index, pkg, name) } -find_import :: proc(checker: ^Checker, file: int, alias: string, mark_used := false) -> int { +find_import :: proc(checker: ^Checker, file: int, alias: symbol.Id, mark_used := false) -> int { id := find_symbol(checker.import_index, file, alias) if id >= 0 && mark_used { checker.ast_module.imports[id].used = true @@ -191,7 +199,7 @@ find_import :: proc(checker: ^Checker, file: int, alias: string, mark_used := fa } expr_package :: proc(checker: ^Checker, expr: ast.Expr, pkg, file: int, mark_used := false) -> (int, bool) { - if expr.qualifier == "" { + if !symbol.is_valid(expr.qualifier) { return pkg, true } import_id := find_import(checker, file, expr.qualifier, mark_used) @@ -208,44 +216,44 @@ expr_package :: proc(checker: ^Checker, expr: ast.Expr, pkg, file: int, mark_use add_package_resolution_diagnostic :: proc(checker: ^Checker, expr: ast.Expr, file: int) -> int { if find_import(checker, file, expr.qualifier) < 0 { - return source.addf(checker.diagnostics, expr.span, "unknown package alias '%s'", expr.qualifier) + return source.addf(checker.diagnostics, expr.span, "unknown package alias '%s'", symbol_text(checker, expr.qualifier)) } - return source.addf(checker.diagnostics, expr.span, "unavailable imported package '%s'", expr.qualifier) + return source.addf(checker.diagnostics, expr.span, "unavailable imported package '%s'", symbol_text(checker, expr.qualifier)) } add_name_resolution_diagnostic :: proc(checker: ^Checker, expr: ast.Expr, target_pkg: int) -> int { - if find_template(checker, expr.text, target_pkg) >= 0 { - return source.addf(checker.diagnostics, expr.span, "'%s' is a function, not a global value", expr.text) + if find_template(checker, expr.name, target_pkg) >= 0 { + return source.addf(checker.diagnostics, expr.span, "'%s' is a function, not a global value", symbol_text(checker, expr.name)) } - if expr.qualifier != "" { + if symbol.is_valid(expr.qualifier) { return source.addf( checker.diagnostics, expr.span, "package '%s' has no member '%s'", - expr.qualifier, - expr.text, + symbol_text(checker, expr.qualifier), + symbol_text(checker, expr.name), ) } - return source.addf(checker.diagnostics, expr.span, "unresolved global '%s'", expr.text) + return source.addf(checker.diagnostics, expr.span, "unresolved global '%s'", symbol_text(checker, expr.name)) } add_call_resolution_diagnostic :: proc(checker: ^Checker, expr: ast.Expr, target_pkg: int) -> int { - if find_global(checker, expr.text, target_pkg) >= 0 { - return source.addf(checker.diagnostics, expr.span, "'%s' is a global, not a function", expr.text) + if find_global(checker, expr.name, target_pkg) >= 0 { + return source.addf(checker.diagnostics, expr.span, "'%s' is a global, not a function", symbol_text(checker, expr.name)) } - if expr.qualifier != "" { + if symbol.is_valid(expr.qualifier) { return source.addf( checker.diagnostics, expr.span, "package '%s' has no member '%s'", - expr.qualifier, - expr.text, + symbol_text(checker, expr.qualifier), + symbol_text(checker, expr.name), ) } - return source.addf(checker.diagnostics, expr.span, "unresolved function '%s'", expr.text) + return source.addf(checker.diagnostics, expr.span, "unresolved function '%s'", symbol_text(checker, expr.name)) } -contains_name :: proc(names: []string, name: string) -> bool { +contains_name :: proc(names: []symbol.Id, name: symbol.Id) -> bool { for existing in names { if existing == name { return true @@ -261,11 +269,11 @@ mark_expr_imports_used :: proc(checker: ^Checker, expr_id, file: int) { expr := checker.ast_module.exprs[expr_id] switch expr.kind { case .Name: - if expr.qualifier != "" { + if symbol.is_valid(expr.qualifier) { _ = find_import(checker, file, expr.qualifier, true) } case .Call: - if expr.qualifier != "" { + if symbol.is_valid(expr.qualifier) { _ = find_import(checker, file, expr.qualifier, true) } for arg in expr.args { @@ -280,7 +288,7 @@ mark_expr_imports_used :: proc(checker: ^Checker, expr_id, file: int) { validate_declarations :: proc(checker: ^Checker) { for function in checker.ast_module.functions { - locals: [dynamic]string + locals: [dynamic]symbol.Id locals.allocator = checker.allocator for param in function.params { if param.type == .Void { @@ -295,7 +303,7 @@ validate_declarations :: proc(checker: ^Checker) { checker.diagnostics, param.span, "duplicate parameter '%s'", - param.name, + symbol_text(checker, param.name), ) } append(&locals, param.name) @@ -312,7 +320,7 @@ validate_declarations :: proc(checker: ^Checker) { } } -find_infer_local :: proc(locals: []Infer_Local, name: string) -> types.Type { +find_infer_local :: proc(locals: []Infer_Local, name: symbol.Id) -> types.Type { for index := len(locals) - 1; index >= 0; index -= 1 { if locals[index].name == name { return locals[index].type @@ -372,7 +380,7 @@ ensure_spec :: proc(checker: ^Checker, template: int, actual_args: []types.Type) } } result := type_from_syntax(function.result) - if function.pkg == 0 && function.name == "main" && function.result == .Int { + if function.pkg == 0 && function.name == checker.main_symbol && function.result == .Int { result = types.I32 } index := len(checker.specs) @@ -401,8 +409,8 @@ infer_expr :: proc(checker: ^Checker, expr_id: int, locals: []Infer_Local, pkg : case .Integer: return types.smallest_signed_for_literal(expr.integer) case .Name: - if expr.qualifier == "" { - local_type := find_infer_local(locals, expr.text) + if !symbol.is_valid(expr.qualifier) { + local_type := find_infer_local(locals, expr.name) if types.is_valid(local_type) { return local_type } @@ -411,7 +419,7 @@ infer_expr :: proc(checker: ^Checker, expr_id: int, locals: []Infer_Local, pkg : if !available { return types.INVALID } - global := find_global(checker, expr.text, target_pkg) + global := find_global(checker, expr.name, target_pkg) if global >= 0 { return checker.global_types[global] } @@ -425,7 +433,7 @@ infer_expr :: proc(checker: ^Checker, expr_id: int, locals: []Infer_Local, pkg : if !available { return types.INVALID } - template := find_template(checker, expr.text, target_pkg) + template := find_template(checker, expr.name, target_pkg) if template < 0 { return types.INVALID } @@ -437,7 +445,7 @@ infer_expr :: proc(checker: ^Checker, expr_id: int, locals: []Infer_Local, pkg : if !can_specialize(function, args) { delete(args, checker.allocator) declared := type_from_syntax(function.result) - if function.pkg == 0 && function.name == "main" && function.result == .Int { + if function.pkg == 0 && function.name == checker.main_symbol && function.result == .Int { return types.I32 } if declared.kind == .Concrete || declared.kind == .Void { @@ -456,7 +464,7 @@ infer_spec_result :: proc(checker: ^Checker, spec_id: int) -> types.Type { spec := checker.specs[spec_id] function := checker.ast_module.functions[spec.template] declared := type_from_syntax(function.result) - if function.pkg == 0 && function.name == "main" && function.result == .Int { + if function.pkg == 0 && function.name == checker.main_symbol && function.result == .Int { declared = types.I32 } @@ -525,7 +533,7 @@ infer_all :: proc(checker: ^Checker) { } } - main_template := find_template(checker, "main", 0) + main_template := find_template(checker, checker.main_symbol, 0) if main_template >= 0 { ensure_spec(checker, main_template, nil) } @@ -588,7 +596,7 @@ add_unique :: proc(values: ^[dynamic]int, value: int) { append(values, value) } -find_build_local :: proc(locals: []Build_Local, name: string) -> (Build_Local, bool) { +find_build_local :: proc(locals: []Build_Local, name: symbol.Id) -> (Build_Local, bool) { for index := len(locals) - 1; index >= 0; index -= 1 { if locals[index].name == name { return locals[index], true @@ -708,8 +716,8 @@ build_expr :: proc( case .Integer: unreachable() case .Name: - if expr.qualifier == "" { - if local, ok := find_build_local(locals, expr.text); ok { + if !symbol.is_valid(expr.qualifier) { + if local, ok := find_build_local(locals, expr.name); ok { return add_hir_expr( checker, hir.Expr { @@ -729,7 +737,7 @@ build_expr :: proc( id := add_package_resolution_diagnostic(checker, expr, file) return invalid_hir_expr(checker, expr.span, id) } - global := find_global(checker, expr.text, target_pkg) + global := find_global(checker, expr.name, target_pkg) if global >= 0 { add_unique(global_reads, global) return add_hir_expr( @@ -781,7 +789,7 @@ build_expr :: proc( id := add_package_resolution_diagnostic(checker, expr, file) return invalid_hir_expr(checker, expr.span, id) } - template := find_template(checker, expr.text, target_pkg) + template := find_template(checker, expr.name, target_pkg) if template < 0 { id := add_call_resolution_diagnostic(checker, expr, target_pkg) return invalid_hir_expr(checker, expr.span, id) @@ -791,7 +799,7 @@ build_expr :: proc( checker.diagnostics, expr.span, "function '%s' expects %d arguments, got %d", - expr.text, + symbol_text(checker, expr.name), len(checker.ast_module.functions[template].params), len(expr.args), ) @@ -833,7 +841,7 @@ build_expr :: proc( checker.diagnostics, expr.span, "could not resolve result type for specialization of '%s'", - expr.text, + symbol_text(checker, expr.name), ) delete(built_args, checker.allocator) return invalid_hir_expr(checker, expr.span, id) @@ -862,14 +870,14 @@ build_expr :: proc( make_link_name :: proc(checker: ^Checker, spec_id: int) -> string { spec := checker.specs[spec_id] function := checker.ast_module.functions[spec.template] - if function.pkg == 0 && function.name == "main" { + if function.pkg == 0 && function.name == checker.main_symbol { return fmt.aprintf("main", allocator = checker.allocator) } builder := strings.builder_make(checker.allocator) defer strings.builder_destroy(&builder) strings.write_string(&builder, "bro_c__" if function.c_abi else "bro__") fmt.sbprintf(&builder, "p%d__", function.pkg) - strings.write_string(&builder, function.name) + strings.write_string(&builder, symbol_text(checker, function.name)) for arg in spec.args { strings.write_string(&builder, "__") strings.write_string(&builder, types.name(arg)) @@ -891,7 +899,7 @@ build_function :: proc(checker: ^Checker, spec_id: int) { checker.diagnostics, function.span, "could not resolve a concrete result type for '%s'", - function.name, + symbol_text(checker, function.name), ) } for arg in spec.args { @@ -900,7 +908,7 @@ build_function :: proc(checker: ^Checker, spec_id: int) { checker.diagnostics, function.span, "could not resolve a concrete parameter type for '%s'", - function.name, + symbol_text(checker, function.name), ) break } @@ -984,7 +992,7 @@ build_function :: proc(checker: ^Checker, spec_id: int) { checker.diagnostics, statement.span, "duplicate local '%s'", - statement.name, + symbol_text(checker, statement.name), ) append(&body, len(checker.module.statements)) append( @@ -1031,7 +1039,7 @@ build_function :: proc(checker: ^Checker, spec_id: int) { ) problematic = problematic || checker.module.exprs[value].kind == .Invalid case .Assignment: - if statement.name == "_" { + if statement.name == checker.sink_symbol { value := build_expr(checker, statement.expr, locals[:], &global_reads, &calls, types.INVALID, function.pkg, function.file) if checker.module.exprs[value].type.kind == .Void { id := source.add( @@ -1072,7 +1080,7 @@ build_function :: proc(checker: ^Checker, spec_id: int) { checker.diagnostics, statement.span, "cannot assign unresolved local '%s'", - statement.name, + symbol_text(checker, statement.name), ) append(&body, len(checker.module.statements)) append( @@ -1093,7 +1101,7 @@ build_function :: proc(checker: ^Checker, spec_id: int) { checker.diagnostics, statement.span, "cannot assign immutable local '%s'", - statement.name, + symbol_text(checker, statement.name), ) append(&body, len(checker.module.statements)) append( @@ -1265,7 +1273,7 @@ build_function :: proc(checker: ^Checker, spec_id: int) { checker.diagnostics, function.span, "function '%s' does not return a value", - function.name, + symbol_text(checker, function.name), ) append(&body, len(checker.module.statements)) append( @@ -1280,8 +1288,8 @@ build_function :: proc(checker: ^Checker, spec_id: int) { hir.Function { name = function.name, link_name = make_link_name(checker, spec_id), - c_abi = function.c_abi || (function.pkg == 0 && function.name == "main"), - is_main = function.pkg == 0 && function.name == "main", + c_abi = function.c_abi || (function.pkg == 0 && function.name == checker.main_symbol), + is_main = function.pkg == 0 && function.name == checker.main_symbol, params = params[:], result = spec.result, locals = hir_locals[:], @@ -1342,7 +1350,7 @@ build_globals :: proc(checker: ^Checker) { checker.diagnostics, global.span, "could not resolve a concrete type for global '%s'", - global.name, + symbol_text(checker, global.name), ) global_type = types.I64 expr = invalid_hir_expr(checker, global.span, diagnostic, global_type) @@ -1507,7 +1515,7 @@ detect_global_cycles_visit :: proc(checker: ^Checker, global_id: int, states: [] checker.diagnostics, checker.ast_module.globals[global_id].span, "global initialization cycle involving '%s'", - checker.module.globals[global_id].name, + symbol_text(checker, checker.module.globals[global_id].name), ) checker.module.globals[global_id].diagnostic = id checker.module.globals[global_id].problematic = true @@ -1537,7 +1545,7 @@ synthesize_trap_main :: proc(checker: ^Checker) { append( &checker.module.functions, hir.Function { - name = "main", + name = checker.main_symbol, link_name = fmt.aprintf("main", allocator = checker.allocator), c_abi = true, is_main = true, @@ -1581,12 +1589,16 @@ replace_main_with_trap :: proc(checker: ^Checker, diagnostic: int) { check :: proc( ast_module: ^ast.Module, diagnostics: ^source.Diagnostics, + symbols: ^symbol.Table, allocator := context.allocator, ) -> hir.Module { checker := Checker { ast_module = ast_module, diagnostics = diagnostics, + symbols = symbols, module = hir.init_module(allocator), + main_symbol = symbol.intern(symbols, "main"), + sink_symbol = symbol.intern(symbols, "_"), allocator = allocator, } checker.specs.allocator = allocator @@ -1608,19 +1620,19 @@ check :: proc( for function, index in ast_module.functions { for previous in ast_module.functions[:index] { if previous.pkg == function.pkg && previous.name == function.name { - source.addf(diagnostics, function.span, "duplicate function '%s'", function.name) + source.addf(diagnostics, function.span, "duplicate function '%s'", symbol_text(&checker, function.name)) } } for global in ast_module.globals { if global.pkg == function.pkg && global.name == function.name { - source.addf(diagnostics, function.span, "package declaration '%s' conflicts with a global", function.name) + source.addf(diagnostics, function.span, "package declaration '%s' conflicts with a global", symbol_text(&checker, function.name)) } } } for global, index in ast_module.globals { for previous in ast_module.globals[:index] { if previous.pkg == global.pkg && previous.name == global.name { - source.addf(diagnostics, global.span, "duplicate global '%s'", global.name) + source.addf(diagnostics, global.span, "duplicate global '%s'", symbol_text(&checker, global.name)) } } } @@ -1634,10 +1646,10 @@ check :: proc( resolve_call_targets(&checker) propagate_global_reads(&checker) - main_template := find_template(&checker, "main", 0) + main_template := find_template(&checker, checker.main_symbol, 0) main_declarations := 0 for function in ast_module.functions { - if function.pkg == 0 && function.name == "main" { + if function.pkg == 0 && function.name == checker.main_symbol { main_declarations += 1 } } @@ -1666,7 +1678,7 @@ check :: proc( propagate_problems(&checker) for import_item in ast_module.imports { if import_item.valid && !import_item.used { - source.addf(diagnostics, import_item.span, "unused import '%s'", import_item.alias) + source.addf(diagnostics, import_item.span, "unused import '%s'", symbol_text(&checker, import_item.alias)) } } return checker.module diff --git a/compiler/compiler.odin b/compiler/compiler.odin index 9f3084d..3ede0f6 100644 --- a/compiler/compiler.odin +++ b/compiler/compiler.odin @@ -7,6 +7,7 @@ import "./loader" import "./lower" import "./opt" import "./source" +import "./symbol" import "core:fmt" import vmem "core:mem/virtual" import "core:os" @@ -17,6 +18,8 @@ compile_package :: proc(input_path, output_path: string) -> int { defer source.destroy_store(&sources) diagnostics := source.init_store_diagnostics(&sources) defer source.destroy_diagnostics(&diagnostics) + symbols := symbol.init_table() + defer symbol.destroy_table(&symbols) lexer_arena: vmem.Arena if err := vmem.arena_init_growing(&lexer_arena); err != nil { @@ -47,6 +50,7 @@ compile_package :: proc(input_path, output_path: string) -> int { input_path, &sources, &diagnostics, + &symbols, vmem.arena_allocator(&lexer_arena), vmem.arena_allocator(&parser_arena), ) @@ -55,13 +59,13 @@ compile_package :: proc(input_path, output_path: string) -> int { return 2 } vmem.arena_free_all(&lexer_arena) - hir_module := checker.check(&ast_module, &diagnostics, vmem.arena_allocator(&checker_arena)) + hir_module := checker.check(&ast_module, &diagnostics, &symbols, vmem.arena_allocator(&checker_arena)) vmem.arena_free_all(&parser_arena) ir_module := lower.lower(&hir_module, vmem.arena_allocator(&lower_arena)) vmem.arena_free_all(&checker_arena) opt.run(&ir_module) - llvm_text := llvm.emit(&ir_module, &diagnostics) + llvm_text := llvm.emit(&ir_module, &diagnostics, &symbols) defer delete(llvm_text) vmem.arena_free_all(&lower_arena) llvm_path := fmt.tprintf("%s.brolang-%d.ll", output_path, os2.get_pid()) diff --git a/compiler/hir/hir.odin b/compiler/hir/hir.odin index 317e60c..8d88313 100644 --- a/compiler/hir/hir.odin +++ b/compiler/hir/hir.odin @@ -1,6 +1,7 @@ package hir import "../source" +import "../symbol" import "../types" import "core:mem" @@ -29,7 +30,7 @@ Expr :: struct { } Local :: struct { - name: string, + name: symbol.Id, type: types.Type, mutable: bool, parameter: bool, @@ -53,7 +54,7 @@ Stmt :: struct { } Function :: struct { - name: string, + name: symbol.Id, link_name: string, c_abi: bool, is_main: bool, @@ -68,7 +69,7 @@ Function :: struct { } Global :: struct { - name: string, + name: symbol.Id, type: types.Type, expr: int, static_value: i64, diff --git a/compiler/ir/ir.odin b/compiler/ir/ir.odin index 928dd3c..012af72 100644 --- a/compiler/ir/ir.odin +++ b/compiler/ir/ir.odin @@ -1,6 +1,7 @@ package ir import "../source" +import "../symbol" import "../types" import "core:mem" @@ -34,7 +35,6 @@ Instruction :: struct { } Function :: struct { - name: string, link_name: string, c_abi: bool, is_main: bool, @@ -45,7 +45,7 @@ Function :: struct { } Global :: struct { - name: string, + name: symbol.Id, type: types.Type, is_static: bool, static_value: i64, @@ -77,13 +77,11 @@ destroy_instructions :: proc(instructions: []Instruction, allocator: mem.Allocat destroy_module :: proc(module: ^Module) { for function in module.functions { - delete(function.name, module.allocator) delete(function.link_name, module.allocator) delete(function.param_types, module.allocator) destroy_instructions(function.instructions, module.allocator) } for global in module.globals { - delete(global.name, module.allocator) destroy_instructions(global.initializer, module.allocator) } delete(module.functions) diff --git a/compiler/lexer/lexer.odin b/compiler/lexer/lexer.odin index c7069f9..fed62d3 100644 --- a/compiler/lexer/lexer.odin +++ b/compiler/lexer/lexer.odin @@ -1,6 +1,7 @@ package lexer import "../source" +import "../symbol" import "../token" is_identifier_start :: proc(value: byte) -> bool { @@ -33,12 +34,13 @@ append_token :: proc( source_file: ^source.Source, kind: token.Kind, start, end: int, + id := symbol.INVALID, diagnostic := -1, ) { append(&stream.items, token.Token{ kind=kind, span=source.Span{file=source_file.id, start=start, end=end}, - text=source_file.text[start:end], + symbol=id, diagnostic=diagnostic, }) } @@ -46,6 +48,7 @@ append_token :: proc( lex :: proc( source_file: ^source.Source, diagnostics: ^source.Diagnostics, + symbols: ^symbol.Table, allocator := context.allocator, ) -> token.Stream { stream: token.Stream @@ -73,7 +76,7 @@ lex :: proc( append_token(&stream, source_file, .Colon_Colon, start, cursor) } else { id := source.add(diagnostics, source.Span{file=source_file.id, start=start, end=cursor}, "expected a second ':'") - append_token(&stream, source_file, .Invalid, start, cursor, id) + append_token(&stream, source_file, .Invalid, start, cursor, diagnostic=id) } case '=': append_token(&stream, source_file, .Equal, cursor, cursor+1) @@ -128,7 +131,7 @@ lex :: proc( source.Span{file=source_file.id, start=start, end=cursor}, "unterminated import string", ) - append_token(&stream, source_file, .Invalid, start, cursor, id) + append_token(&stream, source_file, .Invalid, start, cursor, diagnostic=id) } case ';': id := source.add( @@ -136,7 +139,7 @@ lex :: proc( source.Span{file=source_file.id, start=cursor, end=cursor+1}, "semicolons are invalid; terminate statements with a newline", ) - append_token(&stream, source_file, .Invalid, cursor, cursor+1, id) + append_token(&stream, source_file, .Invalid, cursor, cursor+1, diagnostic=id) cursor += 1 case: if value >= '0' && value <= '9' { @@ -151,7 +154,12 @@ lex :: proc( cursor += 1 } text := source_file.text[start:cursor] - append_token(&stream, source_file, keyword_kind(text), start, cursor) + kind := keyword_kind(text) + id := symbol.INVALID + if kind == .Identifier || kind == .Underscore { + id = symbol.intern(symbols, text) + } + append_token(&stream, source_file, kind, start, cursor, id) } else { id := source.addf( diagnostics, @@ -159,7 +167,7 @@ lex :: proc( "invalid source byte 0x%02x", value, ) - append_token(&stream, source_file, .Invalid, cursor, cursor+1, id) + append_token(&stream, source_file, .Invalid, cursor, cursor+1, diagnostic=id) cursor += 1 } } diff --git a/compiler/llvm/llvm.odin b/compiler/llvm/llvm.odin index 7d8d823..83e3040 100644 --- a/compiler/llvm/llvm.odin +++ b/compiler/llvm/llvm.odin @@ -2,6 +2,7 @@ package llvm import "../ir" import "../source" +import "../symbol" import "../types" import "core:fmt" import "core:mem" @@ -14,6 +15,7 @@ Trap_Message :: struct { Emitter :: struct { module: ^ir.Module, diagnostics: ^source.Diagnostics, + symbols: ^symbol.Table, builder: strings.Builder, messages: [dynamic]Trap_Message, allocator: mem.Allocator, @@ -271,7 +273,11 @@ emit_global_accessors :: proc(emitter: ^Emitter) { global_id, ) strings.write_string(&emitter.builder, "check:\n %visiting = icmp eq i8 %state, 1\n br i1 %visiting, label %cycle, label %initialize\ncycle:\n") - message_text := fmt.aprintf("runtime trap: global initialization cycle involving '%s'", global.name, allocator=emitter.allocator) + message_text := fmt.aprintf( + "runtime trap: global initialization cycle involving '%s'", + symbol.resolve(emitter.symbols, global.name), + allocator=emitter.allocator, + ) message := register_message(emitter, message_text) delete(message_text, emitter.allocator) emit_trap_call(emitter, message) @@ -366,10 +372,16 @@ emit_declarations :: proc(emitter: ^Emitter) { ) } -emit :: proc(module: ^ir.Module, diagnostics: ^source.Diagnostics, allocator := context.allocator) -> string { +emit :: proc( + module: ^ir.Module, + diagnostics: ^source.Diagnostics, + symbols: ^symbol.Table, + allocator := context.allocator, +) -> string { emitter := Emitter{ module=module, diagnostics=diagnostics, + symbols=symbols, builder=strings.builder_make(allocator), allocator=allocator, } diff --git a/compiler/loader/loader.odin b/compiler/loader/loader.odin index 13ff847..5438bd9 100644 --- a/compiler/loader/loader.odin +++ b/compiler/loader/loader.odin @@ -4,6 +4,7 @@ import "../ast" import "../lexer" import "../parser" import "../source" +import "../symbol" import "core:mem" import "core:os" import "core:path/filepath" @@ -14,6 +15,7 @@ State :: struct { module: ^ast.Module, sources: ^source.Store, diagnostics: ^source.Diagnostics, + symbols: ^symbol.Table, token_allocator: mem.Allocator, allocator: mem.Allocator, root_failed: bool, @@ -53,7 +55,7 @@ add_placeholder :: proc(state: ^State, path: string) -> int { id := len(state.module.packages) append(&state.module.packages, ast.Package{ path=strings.clone(path, state.allocator), - name=strings.clone(filepath.base(path), state.allocator), + name=symbol.intern(state.symbols, filepath.base(path)), available=false, }) return id @@ -130,7 +132,7 @@ load_package :: proc(state: ^State, path: string, import_span: source.Span, is_r pkg_id := len(state.module.packages) append(&state.module.packages, ast.Package{ path=canonical, - name=strings.clone(filepath.base(canonical), state.allocator), + name=symbol.intern(state.symbols, filepath.base(canonical)), available=true, }) files, files_ok := read_package_files(state, canonical) @@ -159,8 +161,8 @@ load_package :: proc(state: ^State, path: string, import_span: source.Span, is_r delete(bytes) file_id := len(state.module.files) append(&state.module.files, ast.File{source=source_id, pkg=pkg_id}) - stream := lexer.lex(&state.sources.items[source_id], state.diagnostics, state.token_allocator) - parser.parse_into(&stream, state.diagnostics, state.module, pkg_id, file_id) + stream := lexer.lex(&state.sources.items[source_id], state.diagnostics, state.symbols, state.token_allocator) + parser.parse_into(&stream, &state.sources.items[source_id], state.diagnostics, state.module, pkg_id, file_id) delete(stream.items) } os.file_info_slice_delete(files, state.allocator) @@ -188,7 +190,7 @@ load_package :: proc(state: ^State, path: string, import_span: source.Span, is_r return pkg_id } -declaration_conflicts :: proc(module: ^ast.Module, pkg: int, name: string) -> bool { +declaration_conflicts :: proc(module: ^ast.Module, pkg: int, name: symbol.Id) -> bool { for function in module.functions { if function.pkg == pkg && function.name == name { return true @@ -204,11 +206,12 @@ declaration_conflicts :: proc(module: ^ast.Module, pkg: int, name: string) -> bo validate_imports :: proc(state: ^State) { for import_item, import_id in state.module.imports { - if import_item.alias == "" && import_item.target >= 0 { + if !symbol.is_valid(import_item.alias) && import_item.target >= 0 { state.module.imports[import_id].alias = state.module.packages[import_item.target].name } alias := state.module.imports[import_id].alias - if !is_identifier(alias) { + alias_text := symbol.resolve(state.symbols, alias) + if !is_identifier(alias_text) { state.module.imports[import_id].diagnostic = source.add( state.diagnostics, import_item.span, @@ -221,7 +224,7 @@ validate_imports :: proc(state: ^State) { state.diagnostics, import_item.span, "import alias '%s' conflicts with a package declaration", - alias, + alias_text, ) state.module.imports[import_id].valid = false } @@ -231,7 +234,7 @@ validate_imports :: proc(state: ^State) { state.diagnostics, import_item.span, "duplicate import alias '%s' in the same file", - alias, + alias_text, ) state.module.imports[import_id].valid = false break @@ -244,6 +247,7 @@ load :: proc( root_path: string, sources: ^source.Store, diagnostics: ^source.Diagnostics, + symbols: ^symbol.Table, token_allocator := context.allocator, allocator := context.allocator, ) -> (ast.Module, bool) { @@ -252,6 +256,7 @@ load :: proc( module=&module, sources=sources, diagnostics=diagnostics, + symbols=symbols, token_allocator=token_allocator, allocator=allocator, } diff --git a/compiler/lower/lower.odin b/compiler/lower/lower.odin index c780dfa..f974115 100644 --- a/compiler/lower/lower.odin +++ b/compiler/lower/lower.odin @@ -327,7 +327,7 @@ lower :: proc(hir_module: ^hir.Module, allocator := context.allocator) -> ir.Mod module := ir.init_module(allocator) for global in hir_module.globals { append(&module.globals, ir.Global{ - name=fmt.aprintf("%s", global.name, allocator=allocator), + name=global.name, type=global.type, is_static=global.is_static, static_value=global.static_value, @@ -342,7 +342,6 @@ lower :: proc(hir_module: ^hir.Module, allocator := context.allocator) -> ir.Mod param_types[index] = function.locals[local_id].type } append(&module.functions, ir.Function{ - name=fmt.aprintf("%s", function.name, allocator=allocator), link_name=fmt.aprintf("%s", function.link_name, allocator=allocator), c_abi=function.c_abi, is_main=function.is_main, diff --git a/compiler/parser/parser.odin b/compiler/parser/parser.odin index edc2cac..1562f23 100644 --- a/compiler/parser/parser.odin +++ b/compiler/parser/parser.odin @@ -2,6 +2,7 @@ package parser import "../ast" import "../source" +import "../symbol" import "../token" import "core:fmt" import "core:strconv" @@ -9,6 +10,7 @@ import "core:strings" Parser :: struct { tokens: ^token.Stream, + source_file: ^source.Source, diagnostics: ^source.Diagnostics, module: ast.Module, pkg: int, @@ -17,6 +19,13 @@ Parser :: struct { delimiter_depth: int, } +token_text :: proc(parser: ^Parser, tok: token.Token) -> string { + if tok.span.start < 0 || tok.span.end < tok.span.start || tok.span.end > len(parser.source_file.text) { + return "" + } + return parser.source_file.text[tok.span.start:tok.span.end] +} + span_from :: proc(first, last: source.Span) -> source.Span { return source.Span{file=first.file, start=first.start, end=last.end} } @@ -101,7 +110,7 @@ parse_type :: proc(parser: ^Parser) -> ast.Type_Syntax { return .Invalid } -parse_call :: proc(parser: ^Parser, qualifier: string, first, name: token.Token) -> int { +parse_call :: proc(parser: ^Parser, qualifier: symbol.Id, first, name: token.Token) -> int { left_paren := advance(parser) parser.delimiter_depth += 1 defer parser.delimiter_depth -= 1 @@ -126,7 +135,7 @@ parse_call :: proc(parser: ^Parser, qualifier: string, first, name: token.Token) kind=.Call, span=source.Span{file=name.span.file, start=first.span.start, end=right_paren.span.end}, qualifier=qualifier, - text=name.text, + name=name.symbol, args=args[:], left=ast.INVALID_ID, right=ast.INVALID_ID, @@ -139,7 +148,7 @@ parse_primary :: proc(parser: ^Parser) -> int { #partial switch tok.kind { case .Integer: advance(parser) - value, ok := strconv.parse_i64(tok.text) + value, ok := strconv.parse_i64(token_text(parser, tok)) if !ok { return invalid_expr(parser, tok.span, "integer literal does not fit in i64") } @@ -154,12 +163,12 @@ parse_primary :: proc(parser: ^Parser) -> int { case .Identifier: first := advance(parser) name := first - qualifier := "" + qualifier := symbol.INVALID if _, ok := allow(parser, .Dot); ok { if current(parser).kind != .Identifier { return invalid_expr(parser, current(parser).span, "expected a package member after '.'") } - qualifier = first.text + qualifier = first.symbol name = advance(parser) } if current(parser).kind == .Left_Paren { @@ -169,7 +178,7 @@ parse_primary :: proc(parser: ^Parser) -> int { kind=.Name, span=span_from(first.span, name.span), qualifier=qualifier, - text=name.text, + name=name.symbol, left=ast.INVALID_ID, right=ast.INVALID_ID, diagnostic=-1, @@ -260,7 +269,7 @@ parse_return :: proc(parser: ^Parser) -> int { append(&parser.module.statements, ast.Stmt{ kind=.Return, span=span_from(start.span, end.span), - name="_", + name=end.symbol, expr=ast.INVALID_ID, diagnostic=-1, }) @@ -306,7 +315,7 @@ parse_statement :: proc(parser: ^Parser) -> int { append(&parser.module.statements, ast.Stmt{ kind=kind, span=span_from(name.span, parser.module.exprs[expr].span), - name=name.text, + name=name.symbol, type=type_syntax, immutable=immutable, expr=expr, @@ -352,7 +361,7 @@ parse_params :: proc(parser: ^Parser) -> []ast.Param { } type_syntax := parse_type(parser) for name in names { - append(¶ms, ast.Param{name=name.text, span=name.span, type=type_syntax}) + append(¶ms, ast.Param{name=name.symbol, span=name.span, type=type_syntax}) } delete(names) skip_newlines(parser) @@ -404,7 +413,7 @@ parse_function :: proc(parser: ^Parser, name: token.Token, c_abi: bool) { } append(&parser.module.functions, ast.Function{ span=span_from(name.span, end.span), - name=name.text, + name=name.symbol, pkg=parser.pkg, file=parser.file, c_abi=c_abi, @@ -416,16 +425,17 @@ parse_function :: proc(parser: ^Parser, name: token.Token, c_abi: bool) { } decode_import_path :: proc(parser: ^Parser, tok: token.Token) -> string { - if len(tok.text) < 2 { + text := token_text(parser, tok) + if len(text) < 2 { return fmt.aprintf("", allocator=parser.module.allocator) } builder := strings.builder_make(parser.module.allocator) defer strings.builder_destroy(&builder) - for index := 1; index < len(tok.text)-1; index += 1 { - value := tok.text[index] - if value == '\\' && index+1 < len(tok.text)-1 { + for index := 1; index < len(text)-1; index += 1 { + value := text[index] + if value == '\\' && index+1 < len(text)-1 { index += 1 - value = tok.text[index] + value = text[index] } strings.write_byte(&builder, value) } @@ -442,7 +452,7 @@ parse_import :: proc(parser: ^Parser, alias: token.Token, start: token.Token) { } append(&parser.module.imports, ast.Import{ span=start.span, - alias=alias.text, + alias=alias.symbol, pkg=parser.pkg, file=parser.file, target=-1, @@ -455,7 +465,7 @@ parse_import :: proc(parser: ^Parser, alias: token.Token, start: token.Token) { advance(parser) append(&parser.module.imports, ast.Import{ span=span_from(start.span, path_token.span), - alias=alias.text, + alias=alias.symbol, path=decode_import_path(parser, path_token), pkg=parser.pkg, file=parser.file, @@ -519,7 +529,7 @@ parse_top_level :: proc(parser: ^Parser) { expr := parse_expression(parser) append(&parser.module.globals, ast.Global{ span=span_from(name.span, parser.module.exprs[expr].span), - name=name.text, + name=name.symbol, pkg=parser.pkg, file=parser.file, type=type_syntax, @@ -532,11 +542,13 @@ parse_top_level :: proc(parser: ^Parser) { parse :: proc( stream: ^token.Stream, + source_file: ^source.Source, diagnostics: ^source.Diagnostics, allocator := context.allocator, ) -> ast.Module { parser := Parser{ tokens=stream, + source_file=source_file, diagnostics=diagnostics, module=ast.init_module(allocator), } @@ -550,12 +562,14 @@ parse :: proc( parse_into :: proc( stream: ^token.Stream, + source_file: ^source.Source, diagnostics: ^source.Diagnostics, module: ^ast.Module, pkg, file: int, ) { parser := Parser{ tokens=stream, + source_file=source_file, diagnostics=diagnostics, module=module^, pkg=pkg, diff --git a/compiler/symbol/symbol.odin b/compiler/symbol/symbol.odin new file mode 100644 index 0000000..95c7002 --- /dev/null +++ b/compiler/symbol/symbol.odin @@ -0,0 +1,58 @@ +package symbol + +import "core:mem" +import "core:strings" + +Id :: distinct u32 + +INVALID :: Id(0) + +Table :: struct { + items: [dynamic]string, + lookup: map[string]Id, + stored_bytes: int, + allocator: mem.Allocator, +} + +init_table :: proc(allocator := context.allocator) -> Table { + table: Table + table.items.allocator = allocator + table.lookup.allocator = allocator + table.allocator = allocator + return table +} + +destroy_table :: proc(table: ^Table) { + delete(table.lookup) + for item in table.items { + delete(item, table.allocator) + } + delete(table.items) +} + +intern :: proc(table: ^Table, text: string) -> Id { + if len(text) == 0 { + return INVALID + } + if id, ok := table.lookup[text]; ok { + return id + } + cloned := strings.clone(text, table.allocator) + id := Id(len(table.items) + 1) + append(&table.items, cloned) + table.lookup[cloned] = id + table.stored_bytes += len(cloned) + return id +} + +resolve :: proc(table: ^Table, id: Id) -> string { + index := int(id) - 1 + if index < 0 || index >= len(table.items) { + return "" + } + return table.items[index] +} + +is_valid :: proc(id: Id) -> bool { + return id != INVALID +} diff --git a/compiler/token/token.odin b/compiler/token/token.odin index 43593c9..5e218fd 100644 --- a/compiler/token/token.odin +++ b/compiler/token/token.odin @@ -1,6 +1,7 @@ package token import "../source" +import "../symbol" Kind :: enum { Invalid, @@ -34,7 +35,7 @@ Kind :: enum { Token :: struct { kind: Kind, span: source.Span, - text: string, + symbol: symbol.Id, diagnostic: int, } diff --git a/compiler_tests.odin b/compiler_tests.odin index 4e7b3c7..8c69896 100644 --- a/compiler_tests.odin +++ b/compiler_tests.odin @@ -12,6 +12,7 @@ import "./compiler/llvm" import "./compiler/lower" import "./compiler/parser" import "./compiler/source" +import "./compiler/symbol" import "./compiler/token" import "./compiler/types" import "core:fmt" @@ -20,12 +21,74 @@ import "core:os/os2" import "core:strings" import "core:testing" +@(test) +symbol_table_deduplicates_and_owns_spellings :: proc(t: ^testing.T) { + symbols := symbol.init_table() + defer symbol.destroy_table(&symbols) + + buffer := [5]byte{'a', 'l', 'p', 'h', 'a'} + alpha := symbol.intern(&symbols, string(buffer[:])) + duplicate := symbol.intern(&symbols, "alpha") + beta := symbol.intern(&symbols, "beta") + buffer[0] = 'x' + + testing.expect_value(t, alpha, duplicate) + testing.expect(t, alpha != beta) + testing.expect_value(t, symbol.resolve(&symbols, alpha), "alpha") + testing.expect_value(t, symbol.resolve(&symbols, beta), "beta") + testing.expect_value(t, symbol.intern(&symbols, ""), symbol.INVALID) + testing.expect_value(t, symbol.resolve(&symbols, symbol.INVALID), "") + testing.expect(t, !symbol.is_valid(symbol.INVALID)) + testing.expect(t, symbol.is_valid(alpha)) +} + +@(test) +compact_tokens_intern_only_identifiers_and_preserve_parser_text :: proc(t: ^testing.T) { + text := `other :: import "../math" +value :: 42 +main :: func() void { _ = value } +` + source_file := source.Source{path="test.bro", text=text} + diagnostics := source.init_diagnostics(&source_file) + defer source.destroy_diagnostics(&diagnostics) + symbols := symbol.init_table() + defer symbol.destroy_table(&symbols) + stream := lexer.lex(&source_file, &diagnostics, &symbols) + defer delete(stream.items) + module := parser.parse(&stream, &source_file, &diagnostics) + defer ast.destroy_module(&module) + + value_symbol := symbol.intern(&symbols, "value") + sink_symbol := symbol.intern(&symbols, "_") + value_count := 0 + for tok in stream.items { + #partial switch tok.kind { + case .Identifier: + if tok.symbol == value_symbol { + value_count += 1 + } + case .Underscore: + testing.expect_value(t, tok.symbol, sink_symbol) + case: + testing.expect_value(t, tok.symbol, symbol.INVALID) + } + } + + testing.expect_value(t, len(diagnostics.items), 0) + testing.expect_value(t, value_count, 2) + testing.expect_value(t, module.imports[0].path, "../math") + testing.expect_value(t, module.exprs[module.globals[0].expr].integer, i64(42)) + testing.expect_value(t, module.statements[module.functions[0].body[0]].name, sink_symbol) +} + @(test) lexer_preserves_newlines_and_skips_comments :: proc(t: ^testing.T) { source_file := source.Source{path="test.bro", text="# comment\nmain :: func() void {}\n"} diagnostics := source.init_diagnostics(&source_file) defer source.destroy_diagnostics(&diagnostics) - stream := lexer.lex(&source_file, &diagnostics) + symbols := symbol.init_table() + defer symbol.destroy_table(&symbols) + stream := lexer.lex(&source_file, &diagnostics, &symbols) defer delete(stream.items) testing.expect_value(t, len(diagnostics.items), 0) @@ -45,9 +108,11 @@ main :: func() void {} source_file := source.Source{path="test.bro", text=text} diagnostics := source.init_diagnostics(&source_file) defer source.destroy_diagnostics(&diagnostics) - stream := lexer.lex(&source_file, &diagnostics) + symbols := symbol.init_table() + defer symbol.destroy_table(&symbols) + stream := lexer.lex(&source_file, &diagnostics, &symbols) defer delete(stream.items) - module := parser.parse(&stream, &diagnostics) + module := parser.parse(&stream, &source_file, &diagnostics) defer ast.destroy_module(&module) testing.expect_value(t, len(diagnostics.items), 0) @@ -63,9 +128,11 @@ main :: func() void { _ = give() } source_file := source.Source{path="test.bro", text=text} diagnostics := source.init_diagnostics(&source_file) defer source.destroy_diagnostics(&diagnostics) - stream := lexer.lex(&source_file, &diagnostics) + symbols := symbol.init_table() + defer symbol.destroy_table(&symbols) + stream := lexer.lex(&source_file, &diagnostics, &symbols) defer delete(stream.items) - module := parser.parse(&stream, &diagnostics) + module := parser.parse(&stream, &source_file, &diagnostics) defer ast.destroy_module(&module) testing.expect_value(t, len(diagnostics.items), 0) @@ -87,16 +154,18 @@ main :: func() void {} source_file := source.Source{path="test.bro", text=text} diagnostics := source.init_diagnostics(&source_file) defer source.destroy_diagnostics(&diagnostics) - stream := lexer.lex(&source_file, &diagnostics) + symbols := symbol.init_table() + defer symbol.destroy_table(&symbols) + stream := lexer.lex(&source_file, &diagnostics, &symbols) defer delete(stream.items) - module := parser.parse(&stream, &diagnostics) + module := parser.parse(&stream, &source_file, &diagnostics) defer ast.destroy_module(&module) testing.expect_value(t, len(diagnostics.items), 0) testing.expect_value(t, len(module.imports), 3) - testing.expect_value(t, module.imports[0].alias, "") + testing.expect_value(t, symbol.resolve(&symbols, module.imports[0].alias), "") testing.expect_value(t, module.imports[0].path, "../math") - testing.expect_value(t, module.imports[1].alias, "other") + testing.expect_value(t, symbol.resolve(&symbols, module.imports[1].alias), "other") testing.expect_value(t, module.imports[2].path, "dir\"name\\tail") } @@ -109,9 +178,11 @@ parser_rejects_chained_package_access :: proc(t: ^testing.T) { source_file := source.Source{path="test.bro", text=text} diagnostics := source.init_diagnostics(&source_file) defer source.destroy_diagnostics(&diagnostics) - stream := lexer.lex(&source_file, &diagnostics) + symbols := symbol.init_table() + defer symbol.destroy_table(&symbols) + stream := lexer.lex(&source_file, &diagnostics, &symbols) defer delete(stream.items) - module := parser.parse(&stream, &diagnostics) + module := parser.parse(&stream, &source_file, &diagnostics) defer ast.destroy_module(&module) testing.expect(t, len(diagnostics.items) > 0) @@ -123,7 +194,9 @@ lexer_diagnoses_invalid_import_strings :: proc(t: ^testing.T) { source_file := source.Source{path="test.bro", text=text} diagnostics := source.init_diagnostics(&source_file) defer source.destroy_diagnostics(&diagnostics) - stream := lexer.lex(&source_file, &diagnostics) + symbols := symbol.init_table() + defer symbol.destroy_table(&symbols) + stream := lexer.lex(&source_file, &diagnostics, &symbols) defer delete(stream.items) testing.expect_value(t, len(diagnostics.items), 2) @@ -135,7 +208,9 @@ package_loader_discovers_lexical_immediate_bro_files :: proc(t: ^testing.T) { defer source.destroy_store(&sources) diagnostics := source.init_store_diagnostics(&sources) defer source.destroy_diagnostics(&diagnostics) - module, loaded := loader.load("examples/packages/basic/app", &sources, &diagnostics) + symbols := symbol.init_table() + defer symbol.destroy_table(&symbols) + module, loaded := loader.load("examples/packages/basic/app", &sources, &diagnostics, &symbols) defer ast.destroy_module(&module) testing.expect(t, loaded) @@ -153,9 +228,11 @@ multi_source_diagnostics_report_the_originating_file :: proc(t: ^testing.T) { defer source.destroy_store(&sources) diagnostics := source.init_store_diagnostics(&sources) defer source.destroy_diagnostics(&diagnostics) - module, loaded := loader.load("examples/packages/file_local/app", &sources, &diagnostics) + symbols := symbol.init_table() + defer symbol.destroy_table(&symbols) + module, loaded := loader.load("examples/packages/file_local/app", &sources, &diagnostics, &symbols) defer ast.destroy_module(&module) - hir_module := checker.check(&module, &diagnostics) + hir_module := checker.check(&module, &diagnostics, &symbols) defer hir.destroy_module(&hir_module) testing.expect(t, loaded) @@ -185,11 +262,13 @@ main :: func() void { source_file := source.Source{path="test.bro", text=text} diagnostics := source.init_diagnostics(&source_file) defer source.destroy_diagnostics(&diagnostics) - stream := lexer.lex(&source_file, &diagnostics) + symbols := symbol.init_table() + defer symbol.destroy_table(&symbols) + stream := lexer.lex(&source_file, &diagnostics, &symbols) defer delete(stream.items) - ast_module := parser.parse(&stream, &diagnostics) + ast_module := parser.parse(&stream, &source_file, &diagnostics) defer ast.destroy_module(&ast_module) - hir_module := checker.check(&ast_module, &diagnostics) + hir_module := checker.check(&ast_module, &diagnostics, &symbols) defer hir.destroy_module(&hir_module) found_global := false @@ -218,17 +297,19 @@ main :: func() void { source_file := source.Source{path="test.bro", text=text} diagnostics := source.init_diagnostics(&source_file) defer source.destroy_diagnostics(&diagnostics) - stream := lexer.lex(&source_file, &diagnostics) + symbols := symbol.init_table() + defer symbol.destroy_table(&symbols) + stream := lexer.lex(&source_file, &diagnostics, &symbols) defer delete(stream.items) - ast_module := parser.parse(&stream, &diagnostics) + ast_module := parser.parse(&stream, &source_file, &diagnostics) defer ast.destroy_module(&ast_module) - hir_module := checker.check(&ast_module, &diagnostics) + hir_module := checker.check(&ast_module, &diagnostics, &symbols) defer hir.destroy_module(&hir_module) ir_module := lower.lower(&hir_module) defer ir.destroy_module(&ir_module) - llvm_text := llvm.emit(&ir_module, &diagnostics) + llvm_text := llvm.emit(&ir_module, &diagnostics, &symbols) defer delete(llvm_text) - second_llvm_text := llvm.emit(&ir_module, &diagnostics) + second_llvm_text := llvm.emit(&ir_module, &diagnostics, &symbols) defer delete(second_llvm_text) testing.expect_value(t, len(diagnostics.items), 0) @@ -260,15 +341,17 @@ main :: func() void { source_file := source.Source{path="test.bro", text=text} diagnostics := source.init_diagnostics(&source_file) defer source.destroy_diagnostics(&diagnostics) - stream := lexer.lex(&source_file, &diagnostics) + symbols := symbol.init_table() + defer symbol.destroy_table(&symbols) + stream := lexer.lex(&source_file, &diagnostics, &symbols) defer delete(stream.items) - ast_module := parser.parse(&stream, &diagnostics) + ast_module := parser.parse(&stream, &source_file, &diagnostics) defer ast.destroy_module(&ast_module) - hir_module := checker.check(&ast_module, &diagnostics) + hir_module := checker.check(&ast_module, &diagnostics, &symbols) defer hir.destroy_module(&hir_module) ir_module := lower.lower(&hir_module) defer ir.destroy_module(&ir_module) - llvm_text := llvm.emit(&ir_module, &diagnostics) + llvm_text := llvm.emit(&ir_module, &diagnostics, &symbols) defer delete(llvm_text) testing.expect_value(t, len(diagnostics.items), 0) @@ -289,7 +372,7 @@ main :: func() void { if expr.kind == .Integer { testing.expect(t, types.equal(expr.type, types.I16)) } - if expr.kind == .Call && function.name == "main" && len(expr.args) > 0 { + if expr.kind == .Call && symbol.resolve(&symbols, function.name) == "main" && len(expr.args) > 0 { arg := hir_module.exprs[expr.args[0]] testing.expect_value(t, arg.kind, hir.Expr_Kind.Integer) testing.expect(t, types.equal(arg.type, types.I16)) @@ -310,17 +393,19 @@ main :: func() void { source_file := source.Source{path="test.bro", text=text} diagnostics := source.init_diagnostics(&source_file) defer source.destroy_diagnostics(&diagnostics) - stream := lexer.lex(&source_file, &diagnostics) + symbols := symbol.init_table() + defer symbol.destroy_table(&symbols) + stream := lexer.lex(&source_file, &diagnostics, &symbols) defer delete(stream.items) - ast_module := parser.parse(&stream, &diagnostics) + ast_module := parser.parse(&stream, &source_file, &diagnostics) defer ast.destroy_module(&ast_module) - hir_module := checker.check(&ast_module, &diagnostics) + hir_module := checker.check(&ast_module, &diagnostics, &symbols) defer hir.destroy_module(&hir_module) testing.expect_value(t, len(diagnostics.items), 0) found := false for function in hir_module.functions { - if function.name != "widen_after_add" { + if symbol.resolve(&symbols, function.name) != "widen_after_add" { continue } found = true @@ -343,14 +428,16 @@ main :: func() void {} source_file := source.Source{path="test.bro", text=text} diagnostics := source.init_diagnostics(&source_file) defer source.destroy_diagnostics(&diagnostics) - stream := lexer.lex(&source_file, &diagnostics) + symbols := symbol.init_table() + defer symbol.destroy_table(&symbols) + stream := lexer.lex(&source_file, &diagnostics, &symbols) defer delete(stream.items) - module := parser.parse(&stream, &diagnostics) + module := parser.parse(&stream, &source_file, &diagnostics) defer ast.destroy_module(&module) testing.expect(t, len(diagnostics.items) > 0) testing.expect_value(t, len(module.functions), 1) - testing.expect_value(t, module.functions[0].name, "main") + testing.expect_value(t, symbol.resolve(&symbols, module.functions[0].name), "main") } @(test) @@ -370,18 +457,20 @@ main :: func() void { source_file := source.Source{path="test.bro", text=text} diagnostics := source.init_diagnostics(&source_file) defer source.destroy_diagnostics(&diagnostics) - stream := lexer.lex(&source_file, &diagnostics) + symbols := symbol.init_table() + defer symbol.destroy_table(&symbols) + stream := lexer.lex(&source_file, &diagnostics, &symbols) defer delete(stream.items) - ast_module := parser.parse(&stream, &diagnostics) + ast_module := parser.parse(&stream, &source_file, &diagnostics) defer ast.destroy_module(&ast_module) - hir_module := checker.check(&ast_module, &diagnostics) + hir_module := checker.check(&ast_module, &diagnostics, &symbols) defer hir.destroy_module(&hir_module) done_id, main_id := -1, -1 for function, id in hir_module.functions { - if function.name == "done" { + if symbol.resolve(&symbols, function.name) == "done" { done_id = id - } else if function.name == "main" { + } else if symbol.resolve(&symbols, function.name) == "main" { main_id = id } } @@ -410,11 +499,13 @@ main :: func() void { source_file := source.Source{path="test.bro", text=text} diagnostics := source.init_diagnostics(&source_file) defer source.destroy_diagnostics(&diagnostics) - stream := lexer.lex(&source_file, &diagnostics) + symbols := symbol.init_table() + defer symbol.destroy_table(&symbols) + stream := lexer.lex(&source_file, &diagnostics, &symbols) defer delete(stream.items) - ast_module := parser.parse(&stream, &diagnostics) + ast_module := parser.parse(&stream, &source_file, &diagnostics) defer ast.destroy_module(&ast_module) - hir_module := checker.check(&ast_module, &diagnostics) + hir_module := checker.check(&ast_module, &diagnostics, &symbols) defer hir.destroy_module(&hir_module) testing.expect_value(t, len(diagnostics.items), 0) @@ -440,11 +531,13 @@ long_generic_call_chain_reaches_a_fixed_point :: proc(t: ^testing.T) { source_file := source.Source{path="test.bro", text=strings.to_string(builder)} diagnostics := source.init_diagnostics(&source_file) defer source.destroy_diagnostics(&diagnostics) - stream := lexer.lex(&source_file, &diagnostics) + symbols := symbol.init_table() + defer symbol.destroy_table(&symbols) + stream := lexer.lex(&source_file, &diagnostics, &symbols) defer delete(stream.items) - ast_module := parser.parse(&stream, &diagnostics) + ast_module := parser.parse(&stream, &source_file, &diagnostics) defer ast.destroy_module(&ast_module) - hir_module := checker.check(&ast_module, &diagnostics) + hir_module := checker.check(&ast_module, &diagnostics, &symbols) defer hir.destroy_module(&hir_module) testing.expect_value(t, len(diagnostics.items), 0) @@ -468,11 +561,13 @@ main :: func() i32 { source_file := source.Source{path="test.bro", text=text} diagnostics := source.init_diagnostics(&source_file) defer source.destroy_diagnostics(&diagnostics) - stream := lexer.lex(&source_file, &diagnostics) + symbols := symbol.init_table() + defer symbol.destroy_table(&symbols) + stream := lexer.lex(&source_file, &diagnostics, &symbols) defer delete(stream.items) - ast_module := parser.parse(&stream, &diagnostics) + ast_module := parser.parse(&stream, &source_file, &diagnostics) defer ast.destroy_module(&ast_module) - hir_module := checker.check(&ast_module, &diagnostics) + hir_module := checker.check(&ast_module, &diagnostics, &symbols) defer hir.destroy_module(&hir_module) testing.expect_value(t, len(diagnostics.items), 0) @@ -490,11 +585,13 @@ main :: func() void {} source_file := source.Source{path="test.bro", text=text} diagnostics := source.init_diagnostics(&source_file) defer source.destroy_diagnostics(&diagnostics) - stream := lexer.lex(&source_file, &diagnostics) + symbols := symbol.init_table() + defer symbol.destroy_table(&symbols) + stream := lexer.lex(&source_file, &diagnostics, &symbols) defer delete(stream.items) - ast_module := parser.parse(&stream, &diagnostics) + ast_module := parser.parse(&stream, &source_file, &diagnostics) defer ast.destroy_module(&ast_module) - hir_module := checker.check(&ast_module, &diagnostics) + hir_module := checker.check(&ast_module, &diagnostics, &symbols) defer hir.destroy_module(&hir_module) found_duplicate := false @@ -633,9 +730,11 @@ same_line_statements_are_diagnosed :: proc(t: ^testing.T) { source_file := source.Source{path="test.bro", text=text} diagnostics := source.init_diagnostics(&source_file) defer source.destroy_diagnostics(&diagnostics) - stream := lexer.lex(&source_file, &diagnostics) + symbols := symbol.init_table() + defer symbol.destroy_table(&symbols) + stream := lexer.lex(&source_file, &diagnostics, &symbols) defer delete(stream.items) - module := parser.parse(&stream, &diagnostics) + module := parser.parse(&stream, &source_file, &diagnostics) defer ast.destroy_module(&module) testing.expect(t, len(diagnostics.items) > 0) } @@ -1015,15 +1114,17 @@ package_llvm_is_deterministic_and_symbols_include_package_ids :: proc(t: ^testin defer source.destroy_store(&sources) diagnostics := source.init_store_diagnostics(&sources) defer source.destroy_diagnostics(&diagnostics) - ast_module, loaded := loader.load("examples/packages/c_symbols/app", &sources, &diagnostics) + symbols := symbol.init_table() + defer symbol.destroy_table(&symbols) + ast_module, loaded := loader.load("examples/packages/c_symbols/app", &sources, &diagnostics, &symbols) defer ast.destroy_module(&ast_module) - hir_module := checker.check(&ast_module, &diagnostics) + hir_module := checker.check(&ast_module, &diagnostics, &symbols) defer hir.destroy_module(&hir_module) ir_module := lower.lower(&hir_module) defer ir.destroy_module(&ir_module) - llvm_text := llvm.emit(&ir_module, &diagnostics) + llvm_text := llvm.emit(&ir_module, &diagnostics, &symbols) defer delete(llvm_text) - second_llvm_text := llvm.emit(&ir_module, &diagnostics) + second_llvm_text := llvm.emit(&ir_module, &diagnostics, &symbols) defer delete(second_llvm_text) testing.expect(t, loaded)