hl-valdemar/brolang
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Wiki Activity

intern identifiers

Browse Source
This commit is contained in:
Valdemar Høgsvig Lorenzen
2026-06-10 21:27:00 +02:00
parent d6e03b6f08
commit cdbe4fbc99
16 changed files with 497 additions and 182 deletions
Download Patch File Download Diff File Expand all files Collapse all files
TODO.md
-3
View File
@@ -2,9 +2,6 @@
- for global initialization cycles, report also starting and ending lines
- intern strings across all phases of the compiler and reference strings by their hash/id
- makes string comparison and equality checks faster and takes up less memory
# milestones
1. get c interop working:
benchmarks/symbols/README.md
+27
View File
@@ -0,0 +1,27 @@
# Identifier interning benchmark
Run from the repository root:
```sh
odin run benchmarks/symbols -o:speed
```
The benchmark warms the pipeline once, then measures a fresh lex/parse/check
run over a generated program with 5,000 repeated identifier-heavy statements.
Timing is informational; token layout and allocation metrics are the stable
comparison points.
Results captured on 2026-06-10 with Odin `dev-2026-02:b942f72cb`:
| Metric | Before interning | After interning |
| --- | ---: | ---: |
| Elapsed time | 15.723 ms | 9.248 ms |
| Peak memory | 14,661,846 bytes | 13,220,507 bytes |
| Allocations | 40,105 | 40,112 |
| Token count | 65,032 | 65,032 |
| Token size | 56 bytes | 48 bytes |
| Unique symbols | n/a | 5 |
| Stored symbol bytes | n/a | 21 |
| Diagnostics | 0 | 0 |
The measured run reduced token size by 14.3% and peak tracked memory by 9.8%.
benchmarks/symbols/main.odin
+78
View File
@@ -0,0 +1,78 @@
package main
import "../../compiler/ast"
import "../../compiler/checker"
import "../../compiler/hir"
import "../../compiler/lexer"
import "../../compiler/parser"
import "../../compiler/source"
import "../../compiler/symbol"
import "../../compiler/token"
import "core:fmt"
import "core:mem"
import "core:strings"
import "core:time"
Metrics :: struct {
token_count: int,
unique_symbol_count: int,
stored_symbol_bytes: int,
diagnostic_count: int,
}
make_source :: proc(repetitions: int, allocator := context.allocator) -> string {
builder := strings.builder_make(allocator)
defer strings.builder_destroy(&builder)
strings.write_string(&builder, "identity :: func(value int) int { return value }\n")
strings.write_string(&builder, "main :: func() i32 {\n\tacc i32 = 0\n")
for _ in 0 ..< repetitions {
strings.write_string(&builder, "\t_ = identity(acc)\n\tacc = acc + 1\n")
}
strings.write_string(&builder, "\treturn acc\n}\n")
return strings.clone(strings.to_string(builder), allocator)
}
run_pipeline :: proc(text: string, allocator := context.allocator) -> Metrics {
source_file := source.Source{path="benchmark.bro", text=text}
diagnostics := source.init_diagnostics(&source_file, allocator)
defer source.destroy_diagnostics(&diagnostics)
symbols := symbol.init_table(allocator)
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols, allocator)
defer delete(stream.items)
ast_module := parser.parse(&stream, &source_file, &diagnostics, allocator)
defer ast.destroy_module(&ast_module)
hir_module := checker.check(&ast_module, &diagnostics, &symbols, allocator)
defer hir.destroy_module(&hir_module)
return Metrics{
token_count=len(stream.items),
unique_symbol_count=len(symbols.items),
stored_symbol_bytes=symbols.stored_bytes,
diagnostic_count=len(diagnostics.items),
}
}
main :: proc() {
text := make_source(5_000)
defer delete(text)
_ = run_pipeline(text)
tracker: mem.Tracking_Allocator
mem.tracking_allocator_init(&tracker, context.allocator)
defer mem.tracking_allocator_destroy(&tracker)
allocator := mem.tracking_allocator(&tracker)
start := time.tick_now()
metrics := run_pipeline(text, allocator)
elapsed := time.tick_diff(start, time.tick_now())
fmt.printf("elapsed_ms: %.3f\n", time.duration_milliseconds(elapsed))
fmt.printf("peak_memory_bytes: %d\n", tracker.peak_memory_allocated)
fmt.printf("allocation_count: %d\n", tracker.total_allocation_count)
fmt.printf("token_count: %d\n", metrics.token_count)
fmt.printf("token_size_bytes: %d\n", size_of(token.Token))
fmt.printf("unique_symbol_count: %d\n", metrics.unique_symbol_count)
fmt.printf("stored_symbol_bytes: %d\n", metrics.stored_symbol_bytes)
fmt.printf("diagnostic_count: %d\n", metrics.diagnostic_count)
}
compiler/ast/ast.odin
+9 -9
View File
@@ -1,6 +1,7 @@
package ast
import "../source"
import "../symbol"
import "core:mem"
INVALID_ID :: -1
@@ -26,8 +27,8 @@ Expr_Kind :: enum {
Expr :: struct {
kind: Expr_Kind,
span: source.Span,
qualifier: string,
text: string,
qualifier: symbol.Id,
name: symbol.Id,
integer: i64,
left: int,
right: int,
@@ -36,7 +37,7 @@ Expr :: struct {
}
Param :: struct {
name: string,
name: symbol.Id,
span: source.Span,
type: Type_Syntax,
}
@@ -52,7 +53,7 @@ Stmt_Kind :: enum {
Stmt :: struct {
kind: Stmt_Kind,
span: source.Span,
name: string,
name: symbol.Id,
type: Type_Syntax,
immutable: bool,
expr: int,
@@ -61,7 +62,7 @@ Stmt :: struct {
Function :: struct {
span: source.Span,
name: string,
name: symbol.Id,
pkg: int,
file: int,
c_abi: bool,
@@ -73,7 +74,7 @@ Function :: struct {
Global :: struct {
span: source.Span,
name: string,
name: symbol.Id,
pkg: int,
file: int,
type: Type_Syntax,
@@ -84,7 +85,7 @@ Global :: struct {
Import :: struct {
span: source.Span,
alias: string,
alias: symbol.Id,
path: string,
pkg: int,
file: int,
@@ -101,7 +102,7 @@ File :: struct {
Package :: struct {
path: string,
name: string,
name: symbol.Id,
available: bool,
}
@@ -142,7 +143,6 @@ destroy_module :: proc(module: ^Module) {
}
for pkg in module.packages {
delete(pkg.path, module.allocator)
delete(pkg.name, module.allocator)
}
delete(module.exprs)
delete(module.statements)
compiler/checker/checker.odin
+77 -65
View File
@@ -3,6 +3,7 @@ package checker
import "../ast"
import "../hir"
import "../source"
import "../symbol"
import "../types"
import "base:intrinsics"
import "core:fmt"
@@ -18,12 +19,12 @@ Spec :: struct {
}
Infer_Local :: struct {
name: string,
name: symbol.Id,
type: types.Type,
}
Build_Local :: struct {
name: string,
name: symbol.Id,
type: types.Type,
mutable: bool,
id: int,
@@ -43,13 +44,14 @@ Constant :: struct {
Symbol_Index_Entry :: struct {
scope: int,
name: string,
name: symbol.Id,
id: int,
}
Checker :: struct {
ast_module: ^ast.Module,
diagnostics: ^source.Diagnostics,
symbols: ^symbol.Table,
module: hir.Module,
specs: [dynamic]Spec,
function_index: []Symbol_Index_Entry,
@@ -57,9 +59,15 @@ Checker :: struct {
import_index: []Symbol_Index_Entry,
global_types: []types.Type,
constants: []Constant,
main_symbol: symbol.Id,
sink_symbol: symbol.Id,
allocator: mem.Allocator,
}
symbol_text :: proc(checker: ^Checker, id: symbol.Id) -> string {
return symbol.resolve(checker.symbols, id)
}
eval_constant :: proc(checker: ^Checker, expr_id: int) -> Constant {
if expr_id < 0 || expr_id >= len(checker.ast_module.exprs) {
return Constant{kind = .Not_Constant}
@@ -131,18 +139,18 @@ symbol_index_less :: proc(left, right: Symbol_Index_Entry) -> bool {
return left.scope < right.scope
}
if left.name != right.name {
return left.name < right.name
return int(left.name) < int(right.name)
}
return left.id < right.id
}
find_symbol :: proc(index: []Symbol_Index_Entry, scope: int, name: string) -> int {
find_symbol :: proc(index: []Symbol_Index_Entry, scope: int, name: symbol.Id) -> int {
low := 0
high := len(index)
for low < high {
middle := low + (high-low)/2
entry := index[middle]
if entry.scope < scope || entry.scope == scope && entry.name < name {
if entry.scope < scope || entry.scope == scope && int(entry.name) < int(name) {
low = middle + 1
} else {
high = middle
@@ -174,15 +182,15 @@ build_symbol_indexes :: proc(checker: ^Checker) {
slice.sort_by(checker.import_index, symbol_index_less)
}
find_template :: proc(checker: ^Checker, name: string, pkg := 0) -> int {
find_template :: proc(checker: ^Checker, name: symbol.Id, pkg := 0) -> int {
return find_symbol(checker.function_index, pkg, name)
}
find_global :: proc(checker: ^Checker, name: string, pkg := 0) -> int {
find_global :: proc(checker: ^Checker, name: symbol.Id, pkg := 0) -> int {
return find_symbol(checker.global_index, pkg, name)
}
find_import :: proc(checker: ^Checker, file: int, alias: string, mark_used := false) -> int {
find_import :: proc(checker: ^Checker, file: int, alias: symbol.Id, mark_used := false) -> int {
id := find_symbol(checker.import_index, file, alias)
if id >= 0 && mark_used {
checker.ast_module.imports[id].used = true
@@ -191,7 +199,7 @@ find_import :: proc(checker: ^Checker, file: int, alias: string, mark_used := fa
}
expr_package :: proc(checker: ^Checker, expr: ast.Expr, pkg, file: int, mark_used := false) -> (int, bool) {
if expr.qualifier == "" {
if !symbol.is_valid(expr.qualifier) {
return pkg, true
}
import_id := find_import(checker, file, expr.qualifier, mark_used)
@@ -208,44 +216,44 @@ expr_package :: proc(checker: ^Checker, expr: ast.Expr, pkg, file: int, mark_use
add_package_resolution_diagnostic :: proc(checker: ^Checker, expr: ast.Expr, file: int) -> int {
if find_import(checker, file, expr.qualifier) < 0 {
return source.addf(checker.diagnostics, expr.span, "unknown package alias '%s'", expr.qualifier)
return source.addf(checker.diagnostics, expr.span, "unknown package alias '%s'", symbol_text(checker, expr.qualifier))
}
return source.addf(checker.diagnostics, expr.span, "unavailable imported package '%s'", expr.qualifier)
return source.addf(checker.diagnostics, expr.span, "unavailable imported package '%s'", symbol_text(checker, expr.qualifier))
}
add_name_resolution_diagnostic :: proc(checker: ^Checker, expr: ast.Expr, target_pkg: int) -> int {
if find_template(checker, expr.text, target_pkg) >= 0 {
return source.addf(checker.diagnostics, expr.span, "'%s' is a function, not a global value", expr.text)
if find_template(checker, expr.name, target_pkg) >= 0 {
return source.addf(checker.diagnostics, expr.span, "'%s' is a function, not a global value", symbol_text(checker, expr.name))
}
if expr.qualifier != "" {
if symbol.is_valid(expr.qualifier) {
return source.addf(
checker.diagnostics,
expr.span,
"package '%s' has no member '%s'",
expr.qualifier,
expr.text,
symbol_text(checker, expr.qualifier),
symbol_text(checker, expr.name),
)
}
return source.addf(checker.diagnostics, expr.span, "unresolved global '%s'", expr.text)
return source.addf(checker.diagnostics, expr.span, "unresolved global '%s'", symbol_text(checker, expr.name))
}
add_call_resolution_diagnostic :: proc(checker: ^Checker, expr: ast.Expr, target_pkg: int) -> int {
if find_global(checker, expr.text, target_pkg) >= 0 {
return source.addf(checker.diagnostics, expr.span, "'%s' is a global, not a function", expr.text)
if find_global(checker, expr.name, target_pkg) >= 0 {
return source.addf(checker.diagnostics, expr.span, "'%s' is a global, not a function", symbol_text(checker, expr.name))
}
if expr.qualifier != "" {
if symbol.is_valid(expr.qualifier) {
return source.addf(
checker.diagnostics,
expr.span,
"package '%s' has no member '%s'",
expr.qualifier,
expr.text,
symbol_text(checker, expr.qualifier),
symbol_text(checker, expr.name),
)
}
return source.addf(checker.diagnostics, expr.span, "unresolved function '%s'", expr.text)
return source.addf(checker.diagnostics, expr.span, "unresolved function '%s'", symbol_text(checker, expr.name))
}
contains_name :: proc(names: []string, name: string) -> bool {
contains_name :: proc(names: []symbol.Id, name: symbol.Id) -> bool {
for existing in names {
if existing == name {
return true
@@ -261,11 +269,11 @@ mark_expr_imports_used :: proc(checker: ^Checker, expr_id, file: int) {
expr := checker.ast_module.exprs[expr_id]
switch expr.kind {
case .Name:
if expr.qualifier != "" {
if symbol.is_valid(expr.qualifier) {
_ = find_import(checker, file, expr.qualifier, true)
}
case .Call:
if expr.qualifier != "" {
if symbol.is_valid(expr.qualifier) {
_ = find_import(checker, file, expr.qualifier, true)
}
for arg in expr.args {
@@ -280,7 +288,7 @@ mark_expr_imports_used :: proc(checker: ^Checker, expr_id, file: int) {
validate_declarations :: proc(checker: ^Checker) {
for function in checker.ast_module.functions {
locals: [dynamic]string
locals: [dynamic]symbol.Id
locals.allocator = checker.allocator
for param in function.params {
if param.type == .Void {
@@ -295,7 +303,7 @@ validate_declarations :: proc(checker: ^Checker) {
checker.diagnostics,
param.span,
"duplicate parameter '%s'",
param.name,
symbol_text(checker, param.name),
)
}
append(&locals, param.name)
@@ -312,7 +320,7 @@ validate_declarations :: proc(checker: ^Checker) {
}
}
find_infer_local :: proc(locals: []Infer_Local, name: string) -> types.Type {
find_infer_local :: proc(locals: []Infer_Local, name: symbol.Id) -> types.Type {
for index := len(locals) - 1; index >= 0; index -= 1 {
if locals[index].name == name {
return locals[index].type
@@ -372,7 +380,7 @@ ensure_spec :: proc(checker: ^Checker, template: int, actual_args: []types.Type)
}
}
result := type_from_syntax(function.result)
if function.pkg == 0 && function.name == "main" && function.result == .Int {
if function.pkg == 0 && function.name == checker.main_symbol && function.result == .Int {
result = types.I32
}
index := len(checker.specs)
@@ -401,8 +409,8 @@ infer_expr :: proc(checker: ^Checker, expr_id: int, locals: []Infer_Local, pkg :
case .Integer:
return types.smallest_signed_for_literal(expr.integer)
case .Name:
if expr.qualifier == "" {
local_type := find_infer_local(locals, expr.text)
if !symbol.is_valid(expr.qualifier) {
local_type := find_infer_local(locals, expr.name)
if types.is_valid(local_type) {
return local_type
}
@@ -411,7 +419,7 @@ infer_expr :: proc(checker: ^Checker, expr_id: int, locals: []Infer_Local, pkg :
if !available {
return types.INVALID
}
global := find_global(checker, expr.text, target_pkg)
global := find_global(checker, expr.name, target_pkg)
if global >= 0 {
return checker.global_types[global]
}
@@ -425,7 +433,7 @@ infer_expr :: proc(checker: ^Checker, expr_id: int, locals: []Infer_Local, pkg :
if !available {
return types.INVALID
}
template := find_template(checker, expr.text, target_pkg)
template := find_template(checker, expr.name, target_pkg)
if template < 0 {
return types.INVALID
}
@@ -437,7 +445,7 @@ infer_expr :: proc(checker: ^Checker, expr_id: int, locals: []Infer_Local, pkg :
if !can_specialize(function, args) {
delete(args, checker.allocator)
declared := type_from_syntax(function.result)
if function.pkg == 0 && function.name == "main" && function.result == .Int {
if function.pkg == 0 && function.name == checker.main_symbol && function.result == .Int {
return types.I32
}
if declared.kind == .Concrete || declared.kind == .Void {
@@ -456,7 +464,7 @@ infer_spec_result :: proc(checker: ^Checker, spec_id: int) -> types.Type {
spec := checker.specs[spec_id]
function := checker.ast_module.functions[spec.template]
declared := type_from_syntax(function.result)
if function.pkg == 0 && function.name == "main" && function.result == .Int {
if function.pkg == 0 && function.name == checker.main_symbol && function.result == .Int {
declared = types.I32
}
@@ -525,7 +533,7 @@ infer_all :: proc(checker: ^Checker) {
}
}
main_template := find_template(checker, "main", 0)
main_template := find_template(checker, checker.main_symbol, 0)
if main_template >= 0 {
ensure_spec(checker, main_template, nil)
}
@@ -588,7 +596,7 @@ add_unique :: proc(values: ^[dynamic]int, value: int) {
append(values, value)
}
find_build_local :: proc(locals: []Build_Local, name: string) -> (Build_Local, bool) {
find_build_local :: proc(locals: []Build_Local, name: symbol.Id) -> (Build_Local, bool) {
for index := len(locals) - 1; index >= 0; index -= 1 {
if locals[index].name == name {
return locals[index], true
@@ -708,8 +716,8 @@ build_expr :: proc(
case .Integer:
unreachable()
case .Name:
if expr.qualifier == "" {
if local, ok := find_build_local(locals, expr.text); ok {
if !symbol.is_valid(expr.qualifier) {
if local, ok := find_build_local(locals, expr.name); ok {
return add_hir_expr(
checker,
hir.Expr {
@@ -729,7 +737,7 @@ build_expr :: proc(
id := add_package_resolution_diagnostic(checker, expr, file)
return invalid_hir_expr(checker, expr.span, id)
}
global := find_global(checker, expr.text, target_pkg)
global := find_global(checker, expr.name, target_pkg)
if global >= 0 {
add_unique(global_reads, global)
return add_hir_expr(
@@ -781,7 +789,7 @@ build_expr :: proc(
id := add_package_resolution_diagnostic(checker, expr, file)
return invalid_hir_expr(checker, expr.span, id)
}
template := find_template(checker, expr.text, target_pkg)
template := find_template(checker, expr.name, target_pkg)
if template < 0 {
id := add_call_resolution_diagnostic(checker, expr, target_pkg)
return invalid_hir_expr(checker, expr.span, id)
@@ -791,7 +799,7 @@ build_expr :: proc(
checker.diagnostics,
expr.span,
"function '%s' expects %d arguments, got %d",
expr.text,
symbol_text(checker, expr.name),
len(checker.ast_module.functions[template].params),
len(expr.args),
)
@@ -833,7 +841,7 @@ build_expr :: proc(
checker.diagnostics,
expr.span,
"could not resolve result type for specialization of '%s'",
expr.text,
symbol_text(checker, expr.name),
)
delete(built_args, checker.allocator)
return invalid_hir_expr(checker, expr.span, id)
@@ -862,14 +870,14 @@ build_expr :: proc(
make_link_name :: proc(checker: ^Checker, spec_id: int) -> string {
spec := checker.specs[spec_id]
function := checker.ast_module.functions[spec.template]
if function.pkg == 0 && function.name == "main" {
if function.pkg == 0 && function.name == checker.main_symbol {
return fmt.aprintf("main", allocator = checker.allocator)
}
builder := strings.builder_make(checker.allocator)
defer strings.builder_destroy(&builder)
strings.write_string(&builder, "bro_c__" if function.c_abi else "bro__")
fmt.sbprintf(&builder, "p%d__", function.pkg)
strings.write_string(&builder, function.name)
strings.write_string(&builder, symbol_text(checker, function.name))
for arg in spec.args {
strings.write_string(&builder, "__")
strings.write_string(&builder, types.name(arg))
@@ -891,7 +899,7 @@ build_function :: proc(checker: ^Checker, spec_id: int) {
checker.diagnostics,
function.span,
"could not resolve a concrete result type for '%s'",
function.name,
symbol_text(checker, function.name),
)
}
for arg in spec.args {
@@ -900,7 +908,7 @@ build_function :: proc(checker: ^Checker, spec_id: int) {
checker.diagnostics,
function.span,
"could not resolve a concrete parameter type for '%s'",
function.name,
symbol_text(checker, function.name),
)
break
}
@@ -984,7 +992,7 @@ build_function :: proc(checker: ^Checker, spec_id: int) {
checker.diagnostics,
statement.span,
"duplicate local '%s'",
statement.name,
symbol_text(checker, statement.name),
)
append(&body, len(checker.module.statements))
append(
@@ -1031,7 +1039,7 @@ build_function :: proc(checker: ^Checker, spec_id: int) {
)
problematic = problematic || checker.module.exprs[value].kind == .Invalid
case .Assignment:
if statement.name == "_" {
if statement.name == checker.sink_symbol {
value := build_expr(checker, statement.expr, locals[:], &global_reads, &calls, types.INVALID, function.pkg, function.file)
if checker.module.exprs[value].type.kind == .Void {
id := source.add(
@@ -1072,7 +1080,7 @@ build_function :: proc(checker: ^Checker, spec_id: int) {
checker.diagnostics,
statement.span,
"cannot assign unresolved local '%s'",
statement.name,
symbol_text(checker, statement.name),
)
append(&body, len(checker.module.statements))
append(
@@ -1093,7 +1101,7 @@ build_function :: proc(checker: ^Checker, spec_id: int) {
checker.diagnostics,
statement.span,
"cannot assign immutable local '%s'",
statement.name,
symbol_text(checker, statement.name),
)
append(&body, len(checker.module.statements))
append(
@@ -1265,7 +1273,7 @@ build_function :: proc(checker: ^Checker, spec_id: int) {
checker.diagnostics,
function.span,
"function '%s' does not return a value",
function.name,
symbol_text(checker, function.name),
)
append(&body, len(checker.module.statements))
append(
@@ -1280,8 +1288,8 @@ build_function :: proc(checker: ^Checker, spec_id: int) {
hir.Function {
name = function.name,
link_name = make_link_name(checker, spec_id),
c_abi = function.c_abi || (function.pkg == 0 && function.name == "main"),
is_main = function.pkg == 0 && function.name == "main",
c_abi = function.c_abi || (function.pkg == 0 && function.name == checker.main_symbol),
is_main = function.pkg == 0 && function.name == checker.main_symbol,
params = params[:],
result = spec.result,
locals = hir_locals[:],
@@ -1342,7 +1350,7 @@ build_globals :: proc(checker: ^Checker) {
checker.diagnostics,
global.span,
"could not resolve a concrete type for global '%s'",
global.name,
symbol_text(checker, global.name),
)
global_type = types.I64
expr = invalid_hir_expr(checker, global.span, diagnostic, global_type)
@@ -1507,7 +1515,7 @@ detect_global_cycles_visit :: proc(checker: ^Checker, global_id: int, states: []
checker.diagnostics,
checker.ast_module.globals[global_id].span,
"global initialization cycle involving '%s'",
checker.module.globals[global_id].name,
symbol_text(checker, checker.module.globals[global_id].name),
)
checker.module.globals[global_id].diagnostic = id
checker.module.globals[global_id].problematic = true
@@ -1537,7 +1545,7 @@ synthesize_trap_main :: proc(checker: ^Checker) {
append(
&checker.module.functions,
hir.Function {
name = "main",
name = checker.main_symbol,
link_name = fmt.aprintf("main", allocator = checker.allocator),
c_abi = true,
is_main = true,
@@ -1581,12 +1589,16 @@ replace_main_with_trap :: proc(checker: ^Checker, diagnostic: int) {
check :: proc(
ast_module: ^ast.Module,
diagnostics: ^source.Diagnostics,
symbols: ^symbol.Table,
allocator := context.allocator,
) -> hir.Module {
checker := Checker {
ast_module = ast_module,
diagnostics = diagnostics,
symbols = symbols,
module = hir.init_module(allocator),
main_symbol = symbol.intern(symbols, "main"),
sink_symbol = symbol.intern(symbols, "_"),
allocator = allocator,
}
checker.specs.allocator = allocator
@@ -1608,19 +1620,19 @@ check :: proc(
for function, index in ast_module.functions {
for previous in ast_module.functions[:index] {
if previous.pkg == function.pkg && previous.name == function.name {
source.addf(diagnostics, function.span, "duplicate function '%s'", function.name)
source.addf(diagnostics, function.span, "duplicate function '%s'", symbol_text(&checker, function.name))
}
}
for global in ast_module.globals {
if global.pkg == function.pkg && global.name == function.name {
source.addf(diagnostics, function.span, "package declaration '%s' conflicts with a global", function.name)
source.addf(diagnostics, function.span, "package declaration '%s' conflicts with a global", symbol_text(&checker, function.name))
}
}
}
for global, index in ast_module.globals {
for previous in ast_module.globals[:index] {
if previous.pkg == global.pkg && previous.name == global.name {
source.addf(diagnostics, global.span, "duplicate global '%s'", global.name)
source.addf(diagnostics, global.span, "duplicate global '%s'", symbol_text(&checker, global.name))
}
}
}
@@ -1634,10 +1646,10 @@ check :: proc(
resolve_call_targets(&checker)
propagate_global_reads(&checker)
main_template := find_template(&checker, "main", 0)
main_template := find_template(&checker, checker.main_symbol, 0)
main_declarations := 0
for function in ast_module.functions {
if function.pkg == 0 && function.name == "main" {
if function.pkg == 0 && function.name == checker.main_symbol {
main_declarations += 1
}
}
@@ -1666,7 +1678,7 @@ check :: proc(
propagate_problems(&checker)
for import_item in ast_module.imports {
if import_item.valid && !import_item.used {
source.addf(diagnostics, import_item.span, "unused import '%s'", import_item.alias)
source.addf(diagnostics, import_item.span, "unused import '%s'", symbol_text(&checker, import_item.alias))
}
}
return checker.module
compiler/compiler.odin
+6 -2
View File
@@ -7,6 +7,7 @@ import "./loader"
import "./lower"
import "./opt"
import "./source"
import "./symbol"
import "core:fmt"
import vmem "core:mem/virtual"
import "core:os"
@@ -17,6 +18,8 @@ compile_package :: proc(input_path, output_path: string) -> int {
defer source.destroy_store(&sources)
diagnostics := source.init_store_diagnostics(&sources)
defer source.destroy_diagnostics(&diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
lexer_arena: vmem.Arena
if err := vmem.arena_init_growing(&lexer_arena); err != nil {
@@ -47,6 +50,7 @@ compile_package :: proc(input_path, output_path: string) -> int {
input_path,
&sources,
&diagnostics,
&symbols,
vmem.arena_allocator(&lexer_arena),
vmem.arena_allocator(&parser_arena),
)
@@ -55,13 +59,13 @@ compile_package :: proc(input_path, output_path: string) -> int {
return 2
}
vmem.arena_free_all(&lexer_arena)
hir_module := checker.check(&ast_module, &diagnostics, vmem.arena_allocator(&checker_arena))
hir_module := checker.check(&ast_module, &diagnostics, &symbols, vmem.arena_allocator(&checker_arena))
vmem.arena_free_all(&parser_arena)
ir_module := lower.lower(&hir_module, vmem.arena_allocator(&lower_arena))
vmem.arena_free_all(&checker_arena)
opt.run(&ir_module)
llvm_text := llvm.emit(&ir_module, &diagnostics)
llvm_text := llvm.emit(&ir_module, &diagnostics, &symbols)
defer delete(llvm_text)
vmem.arena_free_all(&lower_arena)
llvm_path := fmt.tprintf("%s.brolang-%d.ll", output_path, os2.get_pid())
compiler/hir/hir.odin
+4 -3
View File
@@ -1,6 +1,7 @@
package hir
import "../source"
import "../symbol"
import "../types"
import "core:mem"
@@ -29,7 +30,7 @@ Expr :: struct {
}
Local :: struct {
name: string,
name: symbol.Id,
type: types.Type,
mutable: bool,
parameter: bool,
@@ -53,7 +54,7 @@ Stmt :: struct {
}
Function :: struct {
name: string,
name: symbol.Id,
link_name: string,
c_abi: bool,
is_main: bool,
@@ -68,7 +69,7 @@ Function :: struct {
}
Global :: struct {
name: string,
name: symbol.Id,
type: types.Type,
expr: int,
static_value: i64,
compiler/ir/ir.odin
+2 -4
View File
@@ -1,6 +1,7 @@
package ir
import "../source"
import "../symbol"
import "../types"
import "core:mem"
@@ -34,7 +35,6 @@ Instruction :: struct {
}
Function :: struct {
name: string,
link_name: string,
c_abi: bool,
is_main: bool,
@@ -45,7 +45,7 @@ Function :: struct {
}
Global :: struct {
name: string,
name: symbol.Id,
type: types.Type,
is_static: bool,
static_value: i64,
@@ -77,13 +77,11 @@ destroy_instructions :: proc(instructions: []Instruction, allocator: mem.Allocat
destroy_module :: proc(module: ^Module) {
for function in module.functions {
delete(function.name, module.allocator)
delete(function.link_name, module.allocator)
delete(function.param_types, module.allocator)
destroy_instructions(function.instructions, module.allocator)
}
for global in module.globals {
delete(global.name, module.allocator)
destroy_instructions(global.initializer, module.allocator)
}
delete(module.functions)
compiler/lexer/lexer.odin
+14 -6
View File
@@ -1,6 +1,7 @@
package lexer
import "../source"
import "../symbol"
import "../token"
is_identifier_start :: proc(value: byte) -> bool {
@@ -33,12 +34,13 @@ append_token :: proc(
source_file: ^source.Source,
kind: token.Kind,
start, end: int,
id := symbol.INVALID,
diagnostic := -1,
) {
append(&stream.items, token.Token{
kind=kind,
span=source.Span{file=source_file.id, start=start, end=end},
text=source_file.text[start:end],
symbol=id,
diagnostic=diagnostic,
})
}
@@ -46,6 +48,7 @@ append_token :: proc(
lex :: proc(
source_file: ^source.Source,
diagnostics: ^source.Diagnostics,
symbols: ^symbol.Table,
allocator := context.allocator,
) -> token.Stream {
stream: token.Stream
@@ -73,7 +76,7 @@ lex :: proc(
append_token(&stream, source_file, .Colon_Colon, start, cursor)
} else {
id := source.add(diagnostics, source.Span{file=source_file.id, start=start, end=cursor}, "expected a second ':'")
append_token(&stream, source_file, .Invalid, start, cursor, id)
append_token(&stream, source_file, .Invalid, start, cursor, diagnostic=id)
}
case '=':
append_token(&stream, source_file, .Equal, cursor, cursor+1)
@@ -128,7 +131,7 @@ lex :: proc(
source.Span{file=source_file.id, start=start, end=cursor},
"unterminated import string",
)
append_token(&stream, source_file, .Invalid, start, cursor, id)
append_token(&stream, source_file, .Invalid, start, cursor, diagnostic=id)
}
case ';':
id := source.add(
@@ -136,7 +139,7 @@ lex :: proc(
source.Span{file=source_file.id, start=cursor, end=cursor+1},
"semicolons are invalid; terminate statements with a newline",
)
append_token(&stream, source_file, .Invalid, cursor, cursor+1, id)
append_token(&stream, source_file, .Invalid, cursor, cursor+1, diagnostic=id)
cursor += 1
case:
if value >= '0' && value <= '9' {
@@ -151,7 +154,12 @@ lex :: proc(
cursor += 1
}
text := source_file.text[start:cursor]
append_token(&stream, source_file, keyword_kind(text), start, cursor)
kind := keyword_kind(text)
id := symbol.INVALID
if kind == .Identifier || kind == .Underscore {
id = symbol.intern(symbols, text)
}
append_token(&stream, source_file, kind, start, cursor, id)
} else {
id := source.addf(
diagnostics,
@@ -159,7 +167,7 @@ lex :: proc(
"invalid source byte 0x%02x",
value,
)
append_token(&stream, source_file, .Invalid, cursor, cursor+1, id)
append_token(&stream, source_file, .Invalid, cursor, cursor+1, diagnostic=id)
cursor += 1
}
}
compiler/llvm/llvm.odin
+14 -2
View File
@@ -2,6 +2,7 @@ package llvm
import "../ir"
import "../source"
import "../symbol"
import "../types"
import "core:fmt"
import "core:mem"
@@ -14,6 +15,7 @@ Trap_Message :: struct {
Emitter :: struct {
module: ^ir.Module,
diagnostics: ^source.Diagnostics,
symbols: ^symbol.Table,
builder: strings.Builder,
messages: [dynamic]Trap_Message,
allocator: mem.Allocator,
@@ -271,7 +273,11 @@ emit_global_accessors :: proc(emitter: ^Emitter) {
global_id,
)
strings.write_string(&emitter.builder, "check:\n %visiting = icmp eq i8 %state, 1\n br i1 %visiting, label %cycle, label %initialize\ncycle:\n")
message_text := fmt.aprintf("runtime trap: global initialization cycle involving '%s'", global.name, allocator=emitter.allocator)
message_text := fmt.aprintf(
"runtime trap: global initialization cycle involving '%s'",
symbol.resolve(emitter.symbols, global.name),
allocator=emitter.allocator,
)
message := register_message(emitter, message_text)
delete(message_text, emitter.allocator)
emit_trap_call(emitter, message)
@@ -366,10 +372,16 @@ emit_declarations :: proc(emitter: ^Emitter) {
)
}
emit :: proc(module: ^ir.Module, diagnostics: ^source.Diagnostics, allocator := context.allocator) -> string {
emit :: proc(
module: ^ir.Module,
diagnostics: ^source.Diagnostics,
symbols: ^symbol.Table,
allocator := context.allocator,
) -> string {
emitter := Emitter{
module=module,
diagnostics=diagnostics,
symbols=symbols,
builder=strings.builder_make(allocator),
allocator=allocator,
}
compiler/loader/loader.odin
+14 -9
View File
@@ -4,6 +4,7 @@ import "../ast"
import "../lexer"
import "../parser"
import "../source"
import "../symbol"
import "core:mem"
import "core:os"
import "core:path/filepath"
@@ -14,6 +15,7 @@ State :: struct {
module: ^ast.Module,
sources: ^source.Store,
diagnostics: ^source.Diagnostics,
symbols: ^symbol.Table,
token_allocator: mem.Allocator,
allocator: mem.Allocator,
root_failed: bool,
@@ -53,7 +55,7 @@ add_placeholder :: proc(state: ^State, path: string) -> int {
id := len(state.module.packages)
append(&state.module.packages, ast.Package{
path=strings.clone(path, state.allocator),
name=strings.clone(filepath.base(path), state.allocator),
name=symbol.intern(state.symbols, filepath.base(path)),
available=false,
})
return id
@@ -130,7 +132,7 @@ load_package :: proc(state: ^State, path: string, import_span: source.Span, is_r
pkg_id := len(state.module.packages)
append(&state.module.packages, ast.Package{
path=canonical,
name=strings.clone(filepath.base(canonical), state.allocator),
name=symbol.intern(state.symbols, filepath.base(canonical)),
available=true,
})
files, files_ok := read_package_files(state, canonical)
@@ -159,8 +161,8 @@ load_package :: proc(state: ^State, path: string, import_span: source.Span, is_r
delete(bytes)
file_id := len(state.module.files)
append(&state.module.files, ast.File{source=source_id, pkg=pkg_id})
stream := lexer.lex(&state.sources.items[source_id], state.diagnostics, state.token_allocator)
parser.parse_into(&stream, state.diagnostics, state.module, pkg_id, file_id)
stream := lexer.lex(&state.sources.items[source_id], state.diagnostics, state.symbols, state.token_allocator)
parser.parse_into(&stream, &state.sources.items[source_id], state.diagnostics, state.module, pkg_id, file_id)
delete(stream.items)
}
os.file_info_slice_delete(files, state.allocator)
@@ -188,7 +190,7 @@ load_package :: proc(state: ^State, path: string, import_span: source.Span, is_r
return pkg_id
}
declaration_conflicts :: proc(module: ^ast.Module, pkg: int, name: string) -> bool {
declaration_conflicts :: proc(module: ^ast.Module, pkg: int, name: symbol.Id) -> bool {
for function in module.functions {
if function.pkg == pkg && function.name == name {
return true
@@ -204,11 +206,12 @@ declaration_conflicts :: proc(module: ^ast.Module, pkg: int, name: string) -> bo
validate_imports :: proc(state: ^State) {
for import_item, import_id in state.module.imports {
if import_item.alias == "" && import_item.target >= 0 {
if !symbol.is_valid(import_item.alias) && import_item.target >= 0 {
state.module.imports[import_id].alias = state.module.packages[import_item.target].name
}
alias := state.module.imports[import_id].alias
if !is_identifier(alias) {
alias_text := symbol.resolve(state.symbols, alias)
if !is_identifier(alias_text) {
state.module.imports[import_id].diagnostic = source.add(
state.diagnostics,
import_item.span,
@@ -221,7 +224,7 @@ validate_imports :: proc(state: ^State) {
state.diagnostics,
import_item.span,
"import alias '%s' conflicts with a package declaration",
alias,
alias_text,
)
state.module.imports[import_id].valid = false
}
@@ -231,7 +234,7 @@ validate_imports :: proc(state: ^State) {
state.diagnostics,
import_item.span,
"duplicate import alias '%s' in the same file",
alias,
alias_text,
)
state.module.imports[import_id].valid = false
break
@@ -244,6 +247,7 @@ load :: proc(
root_path: string,
sources: ^source.Store,
diagnostics: ^source.Diagnostics,
symbols: ^symbol.Table,
token_allocator := context.allocator,
allocator := context.allocator,
) -> (ast.Module, bool) {
@@ -252,6 +256,7 @@ load :: proc(
module=&module,
sources=sources,
diagnostics=diagnostics,
symbols=symbols,
token_allocator=token_allocator,
allocator=allocator,
}
compiler/lower/lower.odin
+1 -2
View File
@@ -327,7 +327,7 @@ lower :: proc(hir_module: ^hir.Module, allocator := context.allocator) -> ir.Mod
module := ir.init_module(allocator)
for global in hir_module.globals {
append(&module.globals, ir.Global{
name=fmt.aprintf("%s", global.name, allocator=allocator),
name=global.name,
type=global.type,
is_static=global.is_static,
static_value=global.static_value,
@@ -342,7 +342,6 @@ lower :: proc(hir_module: ^hir.Module, allocator := context.allocator) -> ir.Mod
param_types[index] = function.locals[local_id].type
}
append(&module.functions, ir.Function{
name=fmt.aprintf("%s", function.name, allocator=allocator),
link_name=fmt.aprintf("%s", function.link_name, allocator=allocator),
c_abi=function.c_abi,
is_main=function.is_main,
compiler/parser/parser.odin
+32 -18
View File
@@ -2,6 +2,7 @@ package parser
import "../ast"
import "../source"
import "../symbol"
import "../token"
import "core:fmt"
import "core:strconv"
@@ -9,6 +10,7 @@ import "core:strings"
Parser :: struct {
tokens: ^token.Stream,
source_file: ^source.Source,
diagnostics: ^source.Diagnostics,
module: ast.Module,
pkg: int,
@@ -17,6 +19,13 @@ Parser :: struct {
delimiter_depth: int,
}
token_text :: proc(parser: ^Parser, tok: token.Token) -> string {
if tok.span.start < 0 || tok.span.end < tok.span.start || tok.span.end > len(parser.source_file.text) {
return ""
}
return parser.source_file.text[tok.span.start:tok.span.end]
}
span_from :: proc(first, last: source.Span) -> source.Span {
return source.Span{file=first.file, start=first.start, end=last.end}
}
@@ -101,7 +110,7 @@ parse_type :: proc(parser: ^Parser) -> ast.Type_Syntax {
return .Invalid
}
parse_call :: proc(parser: ^Parser, qualifier: string, first, name: token.Token) -> int {
parse_call :: proc(parser: ^Parser, qualifier: symbol.Id, first, name: token.Token) -> int {
left_paren := advance(parser)
parser.delimiter_depth += 1
defer parser.delimiter_depth -= 1
@@ -126,7 +135,7 @@ parse_call :: proc(parser: ^Parser, qualifier: string, first, name: token.Token)
kind=.Call,
span=source.Span{file=name.span.file, start=first.span.start, end=right_paren.span.end},
qualifier=qualifier,
text=name.text,
name=name.symbol,
args=args[:],
left=ast.INVALID_ID,
right=ast.INVALID_ID,
@@ -139,7 +148,7 @@ parse_primary :: proc(parser: ^Parser) -> int {
#partial switch tok.kind {
case .Integer:
advance(parser)
value, ok := strconv.parse_i64(tok.text)
value, ok := strconv.parse_i64(token_text(parser, tok))
if !ok {
return invalid_expr(parser, tok.span, "integer literal does not fit in i64")
}
@@ -154,12 +163,12 @@ parse_primary :: proc(parser: ^Parser) -> int {
case .Identifier:
first := advance(parser)
name := first
qualifier := ""
qualifier := symbol.INVALID
if _, ok := allow(parser, .Dot); ok {
if current(parser).kind != .Identifier {
return invalid_expr(parser, current(parser).span, "expected a package member after '.'")
}
qualifier = first.text
qualifier = first.symbol
name = advance(parser)
}
if current(parser).kind == .Left_Paren {
@@ -169,7 +178,7 @@ parse_primary :: proc(parser: ^Parser) -> int {
kind=.Name,
span=span_from(first.span, name.span),
qualifier=qualifier,
text=name.text,
name=name.symbol,
left=ast.INVALID_ID,
right=ast.INVALID_ID,
diagnostic=-1,
@@ -260,7 +269,7 @@ parse_return :: proc(parser: ^Parser) -> int {
append(&parser.module.statements, ast.Stmt{
kind=.Return,
span=span_from(start.span, end.span),
name="_",
name=end.symbol,
expr=ast.INVALID_ID,
diagnostic=-1,
})
@@ -306,7 +315,7 @@ parse_statement :: proc(parser: ^Parser) -> int {
append(&parser.module.statements, ast.Stmt{
kind=kind,
span=span_from(name.span, parser.module.exprs[expr].span),
name=name.text,
name=name.symbol,
type=type_syntax,
immutable=immutable,
expr=expr,
@@ -352,7 +361,7 @@ parse_params :: proc(parser: ^Parser) -> []ast.Param {
}
type_syntax := parse_type(parser)
for name in names {
append(&params, ast.Param{name=name.text, span=name.span, type=type_syntax})
append(&params, ast.Param{name=name.symbol, span=name.span, type=type_syntax})
}
delete(names)
skip_newlines(parser)
@@ -404,7 +413,7 @@ parse_function :: proc(parser: ^Parser, name: token.Token, c_abi: bool) {
}
append(&parser.module.functions, ast.Function{
span=span_from(name.span, end.span),
name=name.text,
name=name.symbol,
pkg=parser.pkg,
file=parser.file,
c_abi=c_abi,
@@ -416,16 +425,17 @@ parse_function :: proc(parser: ^Parser, name: token.Token, c_abi: bool) {
}
decode_import_path :: proc(parser: ^Parser, tok: token.Token) -> string {
if len(tok.text) < 2 {
text := token_text(parser, tok)
if len(text) < 2 {
return fmt.aprintf("", allocator=parser.module.allocator)
}
builder := strings.builder_make(parser.module.allocator)
defer strings.builder_destroy(&builder)
for index := 1; index < len(tok.text)-1; index += 1 {
value := tok.text[index]
if value == '\\' && index+1 < len(tok.text)-1 {
for index := 1; index < len(text)-1; index += 1 {
value := text[index]
if value == '\\' && index+1 < len(text)-1 {
index += 1
value = tok.text[index]
value = text[index]
}
strings.write_byte(&builder, value)
}
@@ -442,7 +452,7 @@ parse_import :: proc(parser: ^Parser, alias: token.Token, start: token.Token) {
}
append(&parser.module.imports, ast.Import{
span=start.span,
alias=alias.text,
alias=alias.symbol,
pkg=parser.pkg,
file=parser.file,
target=-1,
@@ -455,7 +465,7 @@ parse_import :: proc(parser: ^Parser, alias: token.Token, start: token.Token) {
advance(parser)
append(&parser.module.imports, ast.Import{
span=span_from(start.span, path_token.span),
alias=alias.text,
alias=alias.symbol,
path=decode_import_path(parser, path_token),
pkg=parser.pkg,
file=parser.file,
@@ -519,7 +529,7 @@ parse_top_level :: proc(parser: ^Parser) {
expr := parse_expression(parser)
append(&parser.module.globals, ast.Global{
span=span_from(name.span, parser.module.exprs[expr].span),
name=name.text,
name=name.symbol,
pkg=parser.pkg,
file=parser.file,
type=type_syntax,
@@ -532,11 +542,13 @@ parse_top_level :: proc(parser: ^Parser) {
parse :: proc(
stream: ^token.Stream,
source_file: ^source.Source,
diagnostics: ^source.Diagnostics,
allocator := context.allocator,
) -> ast.Module {
parser := Parser{
tokens=stream,
source_file=source_file,
diagnostics=diagnostics,
module=ast.init_module(allocator),
}
@@ -550,12 +562,14 @@ parse :: proc(
parse_into :: proc(
stream: ^token.Stream,
source_file: ^source.Source,
diagnostics: ^source.Diagnostics,
module: ^ast.Module,
pkg, file: int,
) {
parser := Parser{
tokens=stream,
source_file=source_file,
diagnostics=diagnostics,
module=module^,
pkg=pkg,
compiler/symbol/symbol.odin
+58
View File
@@ -0,0 +1,58 @@
package symbol
import "core:mem"
import "core:strings"
Id :: distinct u32
INVALID :: Id(0)
Table :: struct {
items: [dynamic]string,
lookup: map[string]Id,
stored_bytes: int,
allocator: mem.Allocator,
}
init_table :: proc(allocator := context.allocator) -> Table {
table: Table
table.items.allocator = allocator
table.lookup.allocator = allocator
table.allocator = allocator
return table
}
destroy_table :: proc(table: ^Table) {
delete(table.lookup)
for item in table.items {
delete(item, table.allocator)
}
delete(table.items)
}
intern :: proc(table: ^Table, text: string) -> Id {
if len(text) == 0 {
return INVALID
}
if id, ok := table.lookup[text]; ok {
return id
}
cloned := strings.clone(text, table.allocator)
id := Id(len(table.items) + 1)
append(&table.items, cloned)
table.lookup[cloned] = id
table.stored_bytes += len(cloned)
return id
}
resolve :: proc(table: ^Table, id: Id) -> string {
index := int(id) - 1
if index < 0 || index >= len(table.items) {
return ""
}
return table.items[index]
}
is_valid :: proc(id: Id) -> bool {
return id != INVALID
}
compiler/token/token.odin
+2 -1
View File
@@ -1,6 +1,7 @@
package token
import "../source"
import "../symbol"
Kind :: enum {
Invalid,
@@ -34,7 +35,7 @@ Kind :: enum {
Token :: struct {
kind: Kind,
span: source.Span,
text: string,
symbol: symbol.Id,
diagnostic: int,
}
compiler_tests.odin
+159 -58
View File
@@ -12,6 +12,7 @@ import "./compiler/llvm"
import "./compiler/lower"
import "./compiler/parser"
import "./compiler/source"
import "./compiler/symbol"
import "./compiler/token"
import "./compiler/types"
import "core:fmt"
@@ -20,12 +21,74 @@ import "core:os/os2"
import "core:strings"
import "core:testing"
@(test)
symbol_table_deduplicates_and_owns_spellings :: proc(t: ^testing.T) {
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
buffer := [5]byte{'a', 'l', 'p', 'h', 'a'}
alpha := symbol.intern(&symbols, string(buffer[:]))
duplicate := symbol.intern(&symbols, "alpha")
beta := symbol.intern(&symbols, "beta")
buffer[0] = 'x'
testing.expect_value(t, alpha, duplicate)
testing.expect(t, alpha != beta)
testing.expect_value(t, symbol.resolve(&symbols, alpha), "alpha")
testing.expect_value(t, symbol.resolve(&symbols, beta), "beta")
testing.expect_value(t, symbol.intern(&symbols, ""), symbol.INVALID)
testing.expect_value(t, symbol.resolve(&symbols, symbol.INVALID), "")
testing.expect(t, !symbol.is_valid(symbol.INVALID))
testing.expect(t, symbol.is_valid(alpha))
}
@(test)
compact_tokens_intern_only_identifiers_and_preserve_parser_text :: proc(t: ^testing.T) {
text := `other :: import "../math"
value :: 42
main :: func() void { _ = value }
`
source_file := source.Source{path="test.bro", text=text}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
module := parser.parse(&stream, &source_file, &diagnostics)
defer ast.destroy_module(&module)
value_symbol := symbol.intern(&symbols, "value")
sink_symbol := symbol.intern(&symbols, "_")
value_count := 0
for tok in stream.items {
#partial switch tok.kind {
case .Identifier:
if tok.symbol == value_symbol {
value_count += 1
}
case .Underscore:
testing.expect_value(t, tok.symbol, sink_symbol)
case:
testing.expect_value(t, tok.symbol, symbol.INVALID)
}
}
testing.expect_value(t, len(diagnostics.items), 0)
testing.expect_value(t, value_count, 2)
testing.expect_value(t, module.imports[0].path, "../math")
testing.expect_value(t, module.exprs[module.globals[0].expr].integer, i64(42))
testing.expect_value(t, module.statements[module.functions[0].body[0]].name, sink_symbol)
}
@(test)
lexer_preserves_newlines_and_skips_comments :: proc(t: ^testing.T) {
source_file := source.Source{path="test.bro", text="# comment\nmain :: func() void {}\n"}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
stream := lexer.lex(&source_file, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
testing.expect_value(t, len(diagnostics.items), 0)
@@ -45,9 +108,11 @@ main :: func() void {}
source_file := source.Source{path="test.bro", text=text}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
stream := lexer.lex(&source_file, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
module := parser.parse(&stream, &diagnostics)
module := parser.parse(&stream, &source_file, &diagnostics)
defer ast.destroy_module(&module)
testing.expect_value(t, len(diagnostics.items), 0)
@@ -63,9 +128,11 @@ main :: func() void { _ = give() }
source_file := source.Source{path="test.bro", text=text}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
stream := lexer.lex(&source_file, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
module := parser.parse(&stream, &diagnostics)
module := parser.parse(&stream, &source_file, &diagnostics)
defer ast.destroy_module(&module)
testing.expect_value(t, len(diagnostics.items), 0)
@@ -87,16 +154,18 @@ main :: func() void {}
source_file := source.Source{path="test.bro", text=text}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
stream := lexer.lex(&source_file, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
module := parser.parse(&stream, &diagnostics)
module := parser.parse(&stream, &source_file, &diagnostics)
defer ast.destroy_module(&module)
testing.expect_value(t, len(diagnostics.items), 0)
testing.expect_value(t, len(module.imports), 3)
testing.expect_value(t, module.imports[0].alias, "")
testing.expect_value(t, symbol.resolve(&symbols, module.imports[0].alias), "")
testing.expect_value(t, module.imports[0].path, "../math")
testing.expect_value(t, module.imports[1].alias, "other")
testing.expect_value(t, symbol.resolve(&symbols, module.imports[1].alias), "other")
testing.expect_value(t, module.imports[2].path, "dir\"name\\tail")
}
@@ -109,9 +178,11 @@ parser_rejects_chained_package_access :: proc(t: ^testing.T) {
source_file := source.Source{path="test.bro", text=text}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
stream := lexer.lex(&source_file, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
module := parser.parse(&stream, &diagnostics)
module := parser.parse(&stream, &source_file, &diagnostics)
defer ast.destroy_module(&module)
testing.expect(t, len(diagnostics.items) > 0)
@@ -123,7 +194,9 @@ lexer_diagnoses_invalid_import_strings :: proc(t: ^testing.T) {
source_file := source.Source{path="test.bro", text=text}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
stream := lexer.lex(&source_file, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
testing.expect_value(t, len(diagnostics.items), 2)
@@ -135,7 +208,9 @@ package_loader_discovers_lexical_immediate_bro_files :: proc(t: ^testing.T) {
defer source.destroy_store(&sources)
diagnostics := source.init_store_diagnostics(&sources)
defer source.destroy_diagnostics(&diagnostics)
module, loaded := loader.load("examples/packages/basic/app", &sources, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
module, loaded := loader.load("examples/packages/basic/app", &sources, &diagnostics, &symbols)
defer ast.destroy_module(&module)
testing.expect(t, loaded)
@@ -153,9 +228,11 @@ multi_source_diagnostics_report_the_originating_file :: proc(t: ^testing.T) {
defer source.destroy_store(&sources)
diagnostics := source.init_store_diagnostics(&sources)
defer source.destroy_diagnostics(&diagnostics)
module, loaded := loader.load("examples/packages/file_local/app", &sources, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
module, loaded := loader.load("examples/packages/file_local/app", &sources, &diagnostics, &symbols)
defer ast.destroy_module(&module)
hir_module := checker.check(&module, &diagnostics)
hir_module := checker.check(&module, &diagnostics, &symbols)
defer hir.destroy_module(&hir_module)
testing.expect(t, loaded)
@@ -185,11 +262,13 @@ main :: func() void {
source_file := source.Source{path="test.bro", text=text}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
stream := lexer.lex(&source_file, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
ast_module := parser.parse(&stream, &diagnostics)
ast_module := parser.parse(&stream, &source_file, &diagnostics)
defer ast.destroy_module(&ast_module)
hir_module := checker.check(&ast_module, &diagnostics)
hir_module := checker.check(&ast_module, &diagnostics, &symbols)
defer hir.destroy_module(&hir_module)
found_global := false
@@ -218,17 +297,19 @@ main :: func() void {
source_file := source.Source{path="test.bro", text=text}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
stream := lexer.lex(&source_file, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
ast_module := parser.parse(&stream, &diagnostics)
ast_module := parser.parse(&stream, &source_file, &diagnostics)
defer ast.destroy_module(&ast_module)
hir_module := checker.check(&ast_module, &diagnostics)
hir_module := checker.check(&ast_module, &diagnostics, &symbols)
defer hir.destroy_module(&hir_module)
ir_module := lower.lower(&hir_module)
defer ir.destroy_module(&ir_module)
llvm_text := llvm.emit(&ir_module, &diagnostics)
llvm_text := llvm.emit(&ir_module, &diagnostics, &symbols)
defer delete(llvm_text)
second_llvm_text := llvm.emit(&ir_module, &diagnostics)
second_llvm_text := llvm.emit(&ir_module, &diagnostics, &symbols)
defer delete(second_llvm_text)
testing.expect_value(t, len(diagnostics.items), 0)
@@ -260,15 +341,17 @@ main :: func() void {
source_file := source.Source{path="test.bro", text=text}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
stream := lexer.lex(&source_file, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
ast_module := parser.parse(&stream, &diagnostics)
ast_module := parser.parse(&stream, &source_file, &diagnostics)
defer ast.destroy_module(&ast_module)
hir_module := checker.check(&ast_module, &diagnostics)
hir_module := checker.check(&ast_module, &diagnostics, &symbols)
defer hir.destroy_module(&hir_module)
ir_module := lower.lower(&hir_module)
defer ir.destroy_module(&ir_module)
llvm_text := llvm.emit(&ir_module, &diagnostics)
llvm_text := llvm.emit(&ir_module, &diagnostics, &symbols)
defer delete(llvm_text)
testing.expect_value(t, len(diagnostics.items), 0)
@@ -289,7 +372,7 @@ main :: func() void {
if expr.kind == .Integer {
testing.expect(t, types.equal(expr.type, types.I16))
}
if expr.kind == .Call && function.name == "main" && len(expr.args) > 0 {
if expr.kind == .Call && symbol.resolve(&symbols, function.name) == "main" && len(expr.args) > 0 {
arg := hir_module.exprs[expr.args[0]]
testing.expect_value(t, arg.kind, hir.Expr_Kind.Integer)
testing.expect(t, types.equal(arg.type, types.I16))
@@ -310,17 +393,19 @@ main :: func() void {
source_file := source.Source{path="test.bro", text=text}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
stream := lexer.lex(&source_file, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
ast_module := parser.parse(&stream, &diagnostics)
ast_module := parser.parse(&stream, &source_file, &diagnostics)
defer ast.destroy_module(&ast_module)
hir_module := checker.check(&ast_module, &diagnostics)
hir_module := checker.check(&ast_module, &diagnostics, &symbols)
defer hir.destroy_module(&hir_module)
testing.expect_value(t, len(diagnostics.items), 0)
found := false
for function in hir_module.functions {
if function.name != "widen_after_add" {
if symbol.resolve(&symbols, function.name) != "widen_after_add" {
continue
}
found = true
@@ -343,14 +428,16 @@ main :: func() void {}
source_file := source.Source{path="test.bro", text=text}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
stream := lexer.lex(&source_file, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
module := parser.parse(&stream, &diagnostics)
module := parser.parse(&stream, &source_file, &diagnostics)
defer ast.destroy_module(&module)
testing.expect(t, len(diagnostics.items) > 0)
testing.expect_value(t, len(module.functions), 1)
testing.expect_value(t, module.functions[0].name, "main")
testing.expect_value(t, symbol.resolve(&symbols, module.functions[0].name), "main")
}
@(test)
@@ -370,18 +457,20 @@ main :: func() void {
source_file := source.Source{path="test.bro", text=text}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
stream := lexer.lex(&source_file, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
ast_module := parser.parse(&stream, &diagnostics)
ast_module := parser.parse(&stream, &source_file, &diagnostics)
defer ast.destroy_module(&ast_module)
hir_module := checker.check(&ast_module, &diagnostics)
hir_module := checker.check(&ast_module, &diagnostics, &symbols)
defer hir.destroy_module(&hir_module)
done_id, main_id := -1, -1
for function, id in hir_module.functions {
if function.name == "done" {
if symbol.resolve(&symbols, function.name) == "done" {
done_id = id
} else if function.name == "main" {
} else if symbol.resolve(&symbols, function.name) == "main" {
main_id = id
}
}
@@ -410,11 +499,13 @@ main :: func() void {
source_file := source.Source{path="test.bro", text=text}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
stream := lexer.lex(&source_file, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
ast_module := parser.parse(&stream, &diagnostics)
ast_module := parser.parse(&stream, &source_file, &diagnostics)
defer ast.destroy_module(&ast_module)
hir_module := checker.check(&ast_module, &diagnostics)
hir_module := checker.check(&ast_module, &diagnostics, &symbols)
defer hir.destroy_module(&hir_module)
testing.expect_value(t, len(diagnostics.items), 0)
@@ -440,11 +531,13 @@ long_generic_call_chain_reaches_a_fixed_point :: proc(t: ^testing.T) {
source_file := source.Source{path="test.bro", text=strings.to_string(builder)}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
stream := lexer.lex(&source_file, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
ast_module := parser.parse(&stream, &diagnostics)
ast_module := parser.parse(&stream, &source_file, &diagnostics)
defer ast.destroy_module(&ast_module)
hir_module := checker.check(&ast_module, &diagnostics)
hir_module := checker.check(&ast_module, &diagnostics, &symbols)
defer hir.destroy_module(&hir_module)
testing.expect_value(t, len(diagnostics.items), 0)
@@ -468,11 +561,13 @@ main :: func() i32 {
source_file := source.Source{path="test.bro", text=text}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
stream := lexer.lex(&source_file, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
ast_module := parser.parse(&stream, &diagnostics)
ast_module := parser.parse(&stream, &source_file, &diagnostics)
defer ast.destroy_module(&ast_module)
hir_module := checker.check(&ast_module, &diagnostics)
hir_module := checker.check(&ast_module, &diagnostics, &symbols)
defer hir.destroy_module(&hir_module)
testing.expect_value(t, len(diagnostics.items), 0)
@@ -490,11 +585,13 @@ main :: func() void {}
source_file := source.Source{path="test.bro", text=text}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
stream := lexer.lex(&source_file, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
ast_module := parser.parse(&stream, &diagnostics)
ast_module := parser.parse(&stream, &source_file, &diagnostics)
defer ast.destroy_module(&ast_module)
hir_module := checker.check(&ast_module, &diagnostics)
hir_module := checker.check(&ast_module, &diagnostics, &symbols)
defer hir.destroy_module(&hir_module)
found_duplicate := false
@@ -633,9 +730,11 @@ same_line_statements_are_diagnosed :: proc(t: ^testing.T) {
source_file := source.Source{path="test.bro", text=text}
diagnostics := source.init_diagnostics(&source_file)
defer source.destroy_diagnostics(&diagnostics)
stream := lexer.lex(&source_file, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
stream := lexer.lex(&source_file, &diagnostics, &symbols)
defer delete(stream.items)
module := parser.parse(&stream, &diagnostics)
module := parser.parse(&stream, &source_file, &diagnostics)
defer ast.destroy_module(&module)
testing.expect(t, len(diagnostics.items) > 0)
}
@@ -1015,15 +1114,17 @@ package_llvm_is_deterministic_and_symbols_include_package_ids :: proc(t: ^testin
defer source.destroy_store(&sources)
diagnostics := source.init_store_diagnostics(&sources)
defer source.destroy_diagnostics(&diagnostics)
ast_module, loaded := loader.load("examples/packages/c_symbols/app", &sources, &diagnostics)
symbols := symbol.init_table()
defer symbol.destroy_table(&symbols)
ast_module, loaded := loader.load("examples/packages/c_symbols/app", &sources, &diagnostics, &symbols)
defer ast.destroy_module(&ast_module)
hir_module := checker.check(&ast_module, &diagnostics)
hir_module := checker.check(&ast_module, &diagnostics, &symbols)
defer hir.destroy_module(&hir_module)
ir_module := lower.lower(&hir_module)
defer ir.destroy_module(&ir_module)
llvm_text := llvm.emit(&ir_module, &diagnostics)
llvm_text := llvm.emit(&ir_module, &diagnostics, &symbols)
defer delete(llvm_text)
second_llvm_text := llvm.emit(&ir_module, &diagnostics)
second_llvm_text := llvm.emit(&ir_module, &diagnostics, &symbols)
defer delete(second_llvm_text)
testing.expect(t, loaded)