hl-valdemar/brolang
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Wiki Activity

intern identifiers

Browse Source
This commit is contained in:
Valdemar Høgsvig Lorenzen
2026-06-10 21:27:00 +02:00
parent d6e03b6f08
commit cdbe4fbc99
16 changed files with 497 additions and 182 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compiler_tests.odin
+159 -58
View File
@@ -12,6 +12,7 @@ import "./compiler/llvm"
import "./compiler/lower"
import "./compiler/parser"
import "./compiler/source"
import "./compiler/symbol"
import "./compiler/token"
import "./compiler/types"
import "core:fmt"
@@ -20,12 +21,74 @@ import "core:os/os2"
import "core:strings"
import "core:testing"
@(test)
symbol_table_deduplicates_and_owns_spellings :: proc(t: ^testing.T) {
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
buffer := [5]byte{'a', 'l', 'p', 'h', 'a'}
alpha := symbol.intern(&symbols, string(buffer[:]))
duplicate := symbol.intern(&symbols, "alpha")
beta := symbol.intern(&symbols, "beta")
buffer[0] = 'x'
testing.expect_value(t, alpha, duplicate)
testing.expect(t, alpha != beta)
testing.expect_value(t, symbol.resolve(&symbols, alpha), "alpha")
testing.expect_value(t, symbol.resolve(&symbols, beta), "beta")
testing.expect_value(t, symbol.intern(&symbols, ""), symbol.INVALID)
testing.expect_value(t, symbol.resolve(&symbols, symbol.INVALID), "")
testing.expect(t, !symbol.is_valid(symbol.INVALID))
testing.expect(t, symbol.is_valid(alpha))
}
@(test)
compact_tokens_intern_only_identifiers_and_preserve_parser_text :: proc(t: ^testing.T) {
text := `other :: import "../math"
value :: 42
main :: func() void { _ = value }
`
source_file := source.Source{path="test.bro", text=text}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
module := parser.parse(&stream, &source_file, &diagnostics)
defer ast.destroy_module(&module)
value_symbol := symbol.intern(&symbols, "value")
sink_symbol := symbol.intern(&symbols, "_")
value_count := 0
for tok in stream.items {
#partial switch tok.kind {
case .Identifier:
if tok.symbol == value_symbol {
value_count += 1
}
case .Underscore:
testing.expect_value(t, tok.symbol, sink_symbol)
case:
testing.expect_value(t, tok.symbol, symbol.INVALID)
}
}
testing.expect_value(t, len(diagnostics.items), 0)
testing.expect_value(t, value_count, 2)
testing.expect_value(t, module.imports[0].path, "../math")
testing.expect_value(t, module.exprs[module.globals[0].expr].integer, i64(42))
testing.expect_value(t, module.statements[module.functions[0].body[0]].name, sink_symbol)
}
@(test)
lexer_preserves_newlines_and_skips_comments :: proc(t: ^testing.T) {
source_file := source.Source{path="test.bro", text="# comment\nmain :: func() void {}\n"}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
stream := lexer.lex(&source_file, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
testing.expect_value(t, len(diagnostics.items), 0)
@@ -45,9 +108,11 @@ main :: func() void {}
source_file := source.Source{path="test.bro", text=text}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
stream := lexer.lex(&source_file, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
module := parser.parse(&stream, &diagnostics)
module := parser.parse(&stream, &source_file, &diagnostics)
defer ast.destroy_module(&module)
testing.expect_value(t, len(diagnostics.items), 0)
@@ -63,9 +128,11 @@ main :: func() void { _ = give() }
source_file := source.Source{path="test.bro", text=text}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
stream := lexer.lex(&source_file, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
module := parser.parse(&stream, &diagnostics)
module := parser.parse(&stream, &source_file, &diagnostics)
defer ast.destroy_module(&module)
testing.expect_value(t, len(diagnostics.items), 0)
@@ -87,16 +154,18 @@ main :: func() void {}
source_file := source.Source{path="test.bro", text=text}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
stream := lexer.lex(&source_file, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
module := parser.parse(&stream, &diagnostics)
module := parser.parse(&stream, &source_file, &diagnostics)
defer ast.destroy_module(&module)
testing.expect_value(t, len(diagnostics.items), 0)
testing.expect_value(t, len(module.imports), 3)
testing.expect_value(t, module.imports[0].alias, "")
testing.expect_value(t, symbol.resolve(&symbols, module.imports[0].alias), "")
testing.expect_value(t, module.imports[0].path, "../math")
testing.expect_value(t, module.imports[1].alias, "other")
testing.expect_value(t, symbol.resolve(&symbols, module.imports[1].alias), "other")
testing.expect_value(t, module.imports[2].path, "dir\"name\\tail")
}
@@ -109,9 +178,11 @@ parser_rejects_chained_package_access :: proc(t: ^testing.T) {
source_file := source.Source{path="test.bro", text=text}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
stream := lexer.lex(&source_file, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
module := parser.parse(&stream, &diagnostics)
module := parser.parse(&stream, &source_file, &diagnostics)
defer ast.destroy_module(&module)
testing.expect(t, len(diagnostics.items) > 0)
@@ -123,7 +194,9 @@ lexer_diagnoses_invalid_import_strings :: proc(t: ^testing.T) {
source_file := source.Source{path="test.bro", text=text}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
stream := lexer.lex(&source_file, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
testing.expect_value(t, len(diagnostics.items), 2)
@@ -135,7 +208,9 @@ package_loader_discovers_lexical_immediate_bro_files :: proc(t: ^testing.T) {
defer source.destroy_store(&sources)
diagnostics := source.init_store_diagnostics(&sources)
defer source.destroy_diagnostics(&diagnostics)
module, loaded := loader.load("examples/packages/basic/app", &sources, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
module, loaded := loader.load("examples/packages/basic/app", &sources, &diagnostics, &symbols)
defer ast.destroy_module(&module)
testing.expect(t, loaded)
@@ -153,9 +228,11 @@ multi_source_diagnostics_report_the_originating_file :: proc(t: ^testing.T) {
defer source.destroy_store(&sources)
diagnostics := source.init_store_diagnostics(&sources)
defer source.destroy_diagnostics(&diagnostics)
module, loaded := loader.load("examples/packages/file_local/app", &sources, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
module, loaded := loader.load("examples/packages/file_local/app", &sources, &diagnostics, &symbols)
defer ast.destroy_module(&module)
hir_module := checker.check(&module, &diagnostics)
hir_module := checker.check(&module, &diagnostics, &symbols)
defer hir.destroy_module(&hir_module)
testing.expect(t, loaded)
@@ -185,11 +262,13 @@ main :: func() void {
source_file := source.Source{path="test.bro", text=text}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
stream := lexer.lex(&source_file, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
ast_module := parser.parse(&stream, &diagnostics)
ast_module := parser.parse(&stream, &source_file, &diagnostics)
defer ast.destroy_module(&ast_module)
hir_module := checker.check(&ast_module, &diagnostics)
hir_module := checker.check(&ast_module, &diagnostics, &symbols)
defer hir.destroy_module(&hir_module)
found_global := false
@@ -218,17 +297,19 @@ main :: func() void {
source_file := source.Source{path="test.bro", text=text}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
stream := lexer.lex(&source_file, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
ast_module := parser.parse(&stream, &diagnostics)
ast_module := parser.parse(&stream, &source_file, &diagnostics)
defer ast.destroy_module(&ast_module)
hir_module := checker.check(&ast_module, &diagnostics)
hir_module := checker.check(&ast_module, &diagnostics, &symbols)
defer hir.destroy_module(&hir_module)
ir_module := lower.lower(&hir_module)
defer ir.destroy_module(&ir_module)
llvm_text := llvm.emit(&ir_module, &diagnostics)
llvm_text := llvm.emit(&ir_module, &diagnostics, &symbols)
defer delete(llvm_text)
second_llvm_text := llvm.emit(&ir_module, &diagnostics)
second_llvm_text := llvm.emit(&ir_module, &diagnostics, &symbols)
defer delete(second_llvm_text)
testing.expect_value(t, len(diagnostics.items), 0)
@@ -260,15 +341,17 @@ main :: func() void {
source_file := source.Source{path="test.bro", text=text}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
stream := lexer.lex(&source_file, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
ast_module := parser.parse(&stream, &diagnostics)
ast_module := parser.parse(&stream, &source_file, &diagnostics)
defer ast.destroy_module(&ast_module)
hir_module := checker.check(&ast_module, &diagnostics)
hir_module := checker.check(&ast_module, &diagnostics, &symbols)
defer hir.destroy_module(&hir_module)
ir_module := lower.lower(&hir_module)
defer ir.destroy_module(&ir_module)
llvm_text := llvm.emit(&ir_module, &diagnostics)
llvm_text := llvm.emit(&ir_module, &diagnostics, &symbols)
defer delete(llvm_text)
testing.expect_value(t, len(diagnostics.items), 0)
@@ -289,7 +372,7 @@ main :: func() void {
if expr.kind == .Integer {
testing.expect(t, types.equal(expr.type, types.I16))
}
if expr.kind == .Call && function.name == "main" && len(expr.args) > 0 {
if expr.kind == .Call && symbol.resolve(&symbols, function.name) == "main" && len(expr.args) > 0 {
arg := hir_module.exprs[expr.args[0]]
testing.expect_value(t, arg.kind, hir.Expr_Kind.Integer)
testing.expect(t, types.equal(arg.type, types.I16))
@@ -310,17 +393,19 @@ main :: func() void {
source_file := source.Source{path="test.bro", text=text}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
stream := lexer.lex(&source_file, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
ast_module := parser.parse(&stream, &diagnostics)
ast_module := parser.parse(&stream, &source_file, &diagnostics)
defer ast.destroy_module(&ast_module)
hir_module := checker.check(&ast_module, &diagnostics)
hir_module := checker.check(&ast_module, &diagnostics, &symbols)
defer hir.destroy_module(&hir_module)
testing.expect_value(t, len(diagnostics.items), 0)
found := false
for function in hir_module.functions {
if function.name != "widen_after_add" {
if symbol.resolve(&symbols, function.name) != "widen_after_add" {
continue
}
found = true
@@ -343,14 +428,16 @@ main :: func() void {}
source_file := source.Source{path="test.bro", text=text}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
stream := lexer.lex(&source_file, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
module := parser.parse(&stream, &diagnostics)
module := parser.parse(&stream, &source_file, &diagnostics)
defer ast.destroy_module(&module)
testing.expect(t, len(diagnostics.items) > 0)
testing.expect_value(t, len(module.functions), 1)
testing.expect_value(t, module.functions[0].name, "main")
testing.expect_value(t, symbol.resolve(&symbols, module.functions[0].name), "main")
}
@(test)
@@ -370,18 +457,20 @@ main :: func() void {
source_file := source.Source{path="test.bro", text=text}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
stream := lexer.lex(&source_file, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
ast_module := parser.parse(&stream, &diagnostics)
ast_module := parser.parse(&stream, &source_file, &diagnostics)
defer ast.destroy_module(&ast_module)
hir_module := checker.check(&ast_module, &diagnostics)
hir_module := checker.check(&ast_module, &diagnostics, &symbols)
defer hir.destroy_module(&hir_module)
done_id, main_id := -1, -1
for function, id in hir_module.functions {
if function.name == "done" {
if symbol.resolve(&symbols, function.name) == "done" {
done_id = id
} else if function.name == "main" {
} else if symbol.resolve(&symbols, function.name) == "main" {
main_id = id
}
}
@@ -410,11 +499,13 @@ main :: func() void {
source_file := source.Source{path="test.bro", text=text}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
stream := lexer.lex(&source_file, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
ast_module := parser.parse(&stream, &diagnostics)
ast_module := parser.parse(&stream, &source_file, &diagnostics)
defer ast.destroy_module(&ast_module)
hir_module := checker.check(&ast_module, &diagnostics)
hir_module := checker.check(&ast_module, &diagnostics, &symbols)
defer hir.destroy_module(&hir_module)
testing.expect_value(t, len(diagnostics.items), 0)
@@ -440,11 +531,13 @@ long_generic_call_chain_reaches_a_fixed_point :: proc(t: ^testing.T) {
source_file := source.Source{path="test.bro", text=strings.to_string(builder)}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
stream := lexer.lex(&source_file, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
ast_module := parser.parse(&stream, &diagnostics)
ast_module := parser.parse(&stream, &source_file, &diagnostics)
defer ast.destroy_module(&ast_module)
hir_module := checker.check(&ast_module, &diagnostics)
hir_module := checker.check(&ast_module, &diagnostics, &symbols)
defer hir.destroy_module(&hir_module)
testing.expect_value(t, len(diagnostics.items), 0)
@@ -468,11 +561,13 @@ main :: func() i32 {
source_file := source.Source{path="test.bro", text=text}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
stream := lexer.lex(&source_file, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
ast_module := parser.parse(&stream, &diagnostics)
ast_module := parser.parse(&stream, &source_file, &diagnostics)
defer ast.destroy_module(&ast_module)
hir_module := checker.check(&ast_module, &diagnostics)
hir_module := checker.check(&ast_module, &diagnostics, &symbols)
defer hir.destroy_module(&hir_module)
testing.expect_value(t, len(diagnostics.items), 0)
@@ -490,11 +585,13 @@ main :: func() void {}
source_file := source.Source{path="test.bro", text=text}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
stream := lexer.lex(&source_file, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
ast_module := parser.parse(&stream, &diagnostics)
ast_module := parser.parse(&stream, &source_file, &diagnostics)
defer ast.destroy_module(&ast_module)
hir_module := checker.check(&ast_module, &diagnostics)
hir_module := checker.check(&ast_module, &diagnostics, &symbols)
defer hir.destroy_module(&hir_module)
found_duplicate := false
@@ -633,9 +730,11 @@ same_line_statements_are_diagnosed :: proc(t: ^testing.T) {
source_file := source.Source{path="test.bro", text=text}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
stream := lexer.lex(&source_file, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
module := parser.parse(&stream, &diagnostics)
module := parser.parse(&stream, &source_file, &diagnostics)
defer ast.destroy_module(&module)
testing.expect(t, len(diagnostics.items) > 0)
}
@@ -1015,15 +1114,17 @@ package_llvm_is_deterministic_and_symbols_include_package_ids :: proc(t: ^testin
defer source.destroy_store(&sources)
diagnostics := source.init_store_diagnostics(&sources)
defer source.destroy_diagnostics(&diagnostics)
ast_module, loaded := loader.load("examples/packages/c_symbols/app", &sources, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
ast_module, loaded := loader.load("examples/packages/c_symbols/app", &sources, &diagnostics, &symbols)
defer ast.destroy_module(&ast_module)
hir_module := checker.check(&ast_module, &diagnostics)
hir_module := checker.check(&ast_module, &diagnostics, &symbols)
defer hir.destroy_module(&hir_module)
ir_module := lower.lower(&hir_module)
defer ir.destroy_module(&ir_module)
llvm_text := llvm.emit(&ir_module, &diagnostics)
llvm_text := llvm.emit(&ir_module, &diagnostics, &symbols)
defer delete(llvm_text)
second_llvm_text := llvm.emit(&ir_module, &diagnostics)
second_llvm_text := llvm.emit(&ir_module, &diagnostics, &symbols)
defer delete(second_llvm_text)
testing.expect(t, loaded)