hl-valdemar/brolang
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Wiki Activity

compact compiler ids and spans to reduce memory usage

Browse Source
This commit is contained in:
Valdemar Høgsvig Lorenzen
2026-06-12 16:55:16 +02:00
parent 99ba907f59
commit 4112b79c6b
16 changed files with 972 additions and 573 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compiler/parser/parser.odin
+58 -52
View File
@@ -13,8 +13,8 @@ Parser :: struct {
source_file: ^source.Source,
diagnostics: ^source.Diagnostics,
module: ast.Module,
pkg: int,
file: int,
pkg: ast.Package_Id,
file: ast.File_Id,
cursor: int,
delimiter_depth: int,
}
@@ -22,10 +22,10 @@ Parser :: struct {
MAX_EXPRESSION_NESTING :: 256
token_text :: proc(parser: ^Parser, tok: token.Token) -> string {
if tok.span.start < 0 || tok.span.end < tok.span.start || tok.span.end > len(parser.source_file.text) {
if tok.span.end < tok.span.start || int(tok.span.end) > len(parser.source_file.text) {
return ""
}
return parser.source_file.text[tok.span.start:tok.span.end]
return parser.source_file.text[int(tok.span.start):int(tok.span.end)]
}
span_from :: proc(first, last: source.Span) -> source.Span {
@@ -61,19 +61,19 @@ skip_newlines :: proc(parser: ^Parser) {
}
}
add_expr :: proc(parser: ^Parser, expr: ast.Expr) -> int {
id := len(parser.module.exprs)
add_expr :: proc(parser: ^Parser, expr: ast.Expr) -> ast.Expr_Id {
id := ast.expr_id(len(parser.module.exprs))
append(&parser.module.exprs, expr)
return id
}
invalid_expr :: proc(parser: ^Parser, span: source.Span, message: string) -> int {
invalid_expr :: proc(parser: ^Parser, span: source.Span, message: string) -> ast.Expr_Id {
id := source.add(parser.diagnostics, span, message)
return add_expr(parser, ast.Expr{
kind=.Invalid,
span=span,
left=ast.INVALID_ID,
right=ast.INVALID_ID,
left=ast.INVALID_EXPR,
right=ast.INVALID_EXPR,
diagnostic=id,
})
}
@@ -131,7 +131,7 @@ skip_parenthesized :: proc(parser: ^Parser) -> source.Span {
return span_from(start.span, end.span)
}
parse_call :: proc(parser: ^Parser, qualifier: symbol.Id, first, name: token.Token, nesting: int) -> int {
parse_call :: proc(parser: ^Parser, qualifier: symbol.Id, first, name: token.Token, nesting: int) -> ast.Expr_Id {
if nesting >= MAX_EXPRESSION_NESTING {
span := skip_parenthesized(parser)
return invalid_expr(parser, span, "expression nesting exceeds 256 levels")
@@ -139,7 +139,7 @@ parse_call :: proc(parser: ^Parser, qualifier: symbol.Id, first, name: token.Tok
left_paren := advance(parser)
parser.delimiter_depth += 1
defer parser.delimiter_depth -= 1
args: [dynamic]int
args: [dynamic]ast.Expr_Id
args.allocator = parser.module.allocator
skip_newlines(parser)
for current(parser).kind != .Right_Paren && current(parser).kind != .Eof {
@@ -162,13 +162,13 @@ parse_call :: proc(parser: ^Parser, qualifier: symbol.Id, first, name: token.Tok
qualifier=qualifier,
name=name.symbol,
args=args[:],
left=ast.INVALID_ID,
right=ast.INVALID_ID,
diagnostic=-1,
left=ast.INVALID_EXPR,
right=ast.INVALID_EXPR,
diagnostic=source.INVALID_DIAGNOSTIC,
})
}
parse_primary :: proc(parser: ^Parser, nesting: int) -> int {
parse_primary :: proc(parser: ^Parser, nesting: int) -> ast.Expr_Id {
tok := current(parser)
#partial switch tok.kind {
case .Integer:
@@ -181,9 +181,9 @@ parse_primary :: proc(parser: ^Parser, nesting: int) -> int {
kind=.Integer,
span=tok.span,
integer=value,
left=ast.INVALID_ID,
right=ast.INVALID_ID,
diagnostic=-1,
left=ast.INVALID_EXPR,
right=ast.INVALID_EXPR,
diagnostic=source.INVALID_DIAGNOSTIC,
})
case .Identifier:
first := advance(parser)
@@ -204,9 +204,9 @@ parse_primary :: proc(parser: ^Parser, nesting: int) -> int {
span=span_from(first.span, name.span),
qualifier=qualifier,
name=name.symbol,
left=ast.INVALID_ID,
right=ast.INVALID_ID,
diagnostic=-1,
left=ast.INVALID_EXPR,
right=ast.INVALID_EXPR,
diagnostic=source.INVALID_DIAGNOSTIC,
})
case .Underscore:
advance(parser)
@@ -231,8 +231,8 @@ parse_primary :: proc(parser: ^Parser, nesting: int) -> int {
return add_expr(parser, ast.Expr{
kind=.Invalid,
span=tok.span,
left=ast.INVALID_ID,
right=ast.INVALID_ID,
left=ast.INVALID_EXPR,
right=ast.INVALID_EXPR,
diagnostic=tok.diagnostic,
})
}
@@ -250,7 +250,7 @@ infix_binding_power :: proc(kind: token.Kind) -> (left, right: int, ok: bool) {
return 0, 0, false
}
parse_expression_bp :: proc(parser: ^Parser, minimum_binding_power, nesting: int) -> int {
parse_expression_bp :: proc(parser: ^Parser, minimum_binding_power, nesting: int) -> ast.Expr_Id {
if nesting > MAX_EXPRESSION_NESTING {
tok := current(parser)
if tok.kind != .Newline && tok.kind != .Right_Brace && tok.kind != .Eof {
@@ -277,7 +277,7 @@ parse_expression_bp :: proc(parser: ^Parser, minimum_binding_power, nesting: int
span=span_from(left_expr.span, right_expr.span),
left=left,
right=right,
diagnostic=-1,
diagnostic=source.INVALID_DIAGNOSTIC,
})
if parser.delimiter_depth > 0 {
skip_newlines(parser)
@@ -286,17 +286,17 @@ parse_expression_bp :: proc(parser: ^Parser, minimum_binding_power, nesting: int
return left
}
parse_expression :: proc(parser: ^Parser) -> int {
parse_expression :: proc(parser: ^Parser) -> ast.Expr_Id {
return parse_expression_bp(parser, 0, 0)
}
finish_statement :: proc(parser: ^Parser, allow_closing_brace := false) -> int {
finish_statement :: proc(parser: ^Parser, allow_closing_brace := false) -> source.Diagnostic_Id {
if current(parser).kind == .Newline {
skip_newlines(parser)
return -1
return source.INVALID_DIAGNOSTIC
}
if current(parser).kind == .Eof || allow_closing_brace && current(parser).kind == .Right_Brace {
return -1
return source.INVALID_DIAGNOSTIC
}
diagnostic := source.add(
parser.diagnostics,
@@ -312,33 +312,33 @@ finish_statement :: proc(parser: ^Parser, allow_closing_brace := false) -> int {
return diagnostic
}
parse_return :: proc(parser: ^Parser) -> int {
parse_return :: proc(parser: ^Parser) -> ast.Stmt_Id {
start := advance(parser)
skip_newlines(parser)
if current(parser).kind == .Underscore {
end := advance(parser)
id := len(parser.module.statements)
id := ast.stmt_id(len(parser.module.statements))
append(&parser.module.statements, ast.Stmt{
kind=.Return,
span=span_from(start.span, end.span),
name=end.symbol,
expr=ast.INVALID_ID,
diagnostic=-1,
expr=ast.INVALID_EXPR,
diagnostic=source.INVALID_DIAGNOSTIC,
})
return id
}
expr := parse_expression(parser)
id := len(parser.module.statements)
id := ast.stmt_id(len(parser.module.statements))
append(&parser.module.statements, ast.Stmt{
kind=.Return,
span=span_from(start.span, parser.module.exprs[expr].span),
expr=expr,
diagnostic=-1,
diagnostic=source.INVALID_DIAGNOSTIC,
})
return id
}
parse_statement :: proc(parser: ^Parser) -> int {
parse_statement :: proc(parser: ^Parser) -> ast.Stmt_Id {
if current(parser).kind == .Keyword_Return {
return parse_return(parser)
}
@@ -363,7 +363,7 @@ parse_statement :: proc(parser: ^Parser) -> int {
kind = .Declaration
immutable = operator.kind == .Colon_Colon
}
id := len(parser.module.statements)
id := ast.stmt_id(len(parser.module.statements))
append(&parser.module.statements, ast.Stmt{
kind=kind,
span=span_from(name.span, parser.module.exprs[expr].span),
@@ -371,7 +371,7 @@ parse_statement :: proc(parser: ^Parser) -> int {
type=type_syntax,
immutable=immutable,
expr=expr,
diagnostic=-1,
diagnostic=source.INVALID_DIAGNOSTIC,
})
return id
}
@@ -379,12 +379,12 @@ parse_statement :: proc(parser: ^Parser) -> int {
}
expr := parse_expression(parser)
id := len(parser.module.statements)
id := ast.stmt_id(len(parser.module.statements))
append(&parser.module.statements, ast.Stmt{
kind=.Expression,
span=parser.module.exprs[expr].span,
expr=expr,
diagnostic=-1,
diagnostic=source.INVALID_DIAGNOSTIC,
})
return id
}
@@ -446,6 +446,7 @@ parse_function :: proc(parser: ^Parser, name: token.Token, c_abi: bool) {
if !ended_by_newline && current(parser).kind != .Eof {
_ = finish_statement(parser)
}
_ = ast.function_id(len(parser.module.functions))
append(&parser.module.functions, ast.Function{
span=span_from(name.span, end.span),
name=name.symbol,
@@ -455,23 +456,23 @@ parse_function :: proc(parser: ^Parser, name: token.Token, c_abi: bool) {
has_body=false,
params=params,
result=result,
diagnostic=-1,
diagnostic=source.INVALID_DIAGNOSTIC,
})
return
}
advance(parser)
body: [dynamic]int
body: [dynamic]ast.Stmt_Id
body.allocator = parser.module.allocator
skip_newlines(parser)
for current(parser).kind != .Right_Brace && current(parser).kind != .Eof {
append(&body, parse_statement(parser))
if diagnostic := finish_statement(parser, true); diagnostic >= 0 {
statement_id := len(parser.module.statements)
if diagnostic := finish_statement(parser, true); diagnostic != source.INVALID_DIAGNOSTIC {
statement_id := ast.stmt_id(len(parser.module.statements))
append(&parser.module.statements, ast.Stmt{
kind=.Invalid,
span=current(parser).span,
expr=ast.INVALID_ID,
expr=ast.INVALID_EXPR,
diagnostic=diagnostic,
})
append(&body, statement_id)
@@ -482,6 +483,7 @@ parse_function :: proc(parser: ^Parser, name: token.Token, c_abi: bool) {
source.add(parser.diagnostics, current(parser).span, "expected '}' after function body")
end = func_token
}
_ = ast.function_id(len(parser.module.functions))
append(&parser.module.functions, ast.Function{
span=span_from(name.span, end.span),
name=name.symbol,
@@ -492,7 +494,7 @@ parse_function :: proc(parser: ^Parser, name: token.Token, c_abi: bool) {
params=params,
result=result,
body=body[:],
diagnostic=-1,
diagnostic=source.INVALID_DIAGNOSTIC,
})
}
@@ -522,28 +524,30 @@ parse_import :: proc(parser: ^Parser, alias: token.Token, start: token.Token) {
if path_token.kind != .Newline && path_token.kind != .Eof {
advance(parser)
}
_ = ast.import_id(len(parser.module.imports))
append(&parser.module.imports, ast.Import{
span=start.span,
alias=alias.symbol,
pkg=parser.pkg,
file=parser.file,
target=-1,
target=ast.INVALID_PACKAGE,
valid=false,
diagnostic=-1,
diagnostic=source.INVALID_DIAGNOSTIC,
})
_ = finish_statement(parser)
return
}
advance(parser)
_ = ast.import_id(len(parser.module.imports))
append(&parser.module.imports, ast.Import{
span=span_from(start.span, path_token.span),
alias=alias.symbol,
path=decode_import_path(parser, path_token),
pkg=parser.pkg,
file=parser.file,
target=-1,
target=ast.INVALID_PACKAGE,
valid=true,
diagnostic=-1,
diagnostic=source.INVALID_DIAGNOSTIC,
})
_ = finish_statement(parser)
}
@@ -606,6 +610,7 @@ parse_top_level :: proc(parser: ^Parser) {
}
expr := parse_expression(parser)
_ = ast.global_id(len(parser.module.globals))
append(&parser.module.globals, ast.Global{
span=span_from(name.span, parser.module.exprs[expr].span),
name=name.symbol,
@@ -614,7 +619,7 @@ parse_top_level :: proc(parser: ^Parser) {
type=type_syntax,
immutable=operator.kind == .Colon_Colon,
expr=expr,
diagnostic=-1,
diagnostic=source.INVALID_DIAGNOSTIC,
})
_ = finish_statement(parser)
}
@@ -644,7 +649,8 @@ parse_into :: proc(
source_file: ^source.Source,
diagnostics: ^source.Diagnostics,
module: ^ast.Module,
pkg, file: int,
pkg: ast.Package_Id,
file: ast.File_Id,
) {
parser := Parser{
tokens=stream,