hl-valdemar/brolang
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Wiki Activity

intern identifiers

Browse Source
This commit is contained in:
Valdemar Høgsvig Lorenzen
2026-06-10 21:27:00 +02:00
parent d6e03b6f08
commit cdbe4fbc99
16 changed files with 497 additions and 182 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compiler/lexer/lexer.odin
+14 -6
View File
@@ -1,6 +1,7 @@
package lexer
import "../source"
import "../symbol"
import "../token"
is_identifier_start :: proc(value: byte) -> bool {
@@ -33,12 +34,13 @@ append_token :: proc(
source_file: ^source.Source,
kind: token.Kind,
start, end: int,
id := symbol.INVALID,
diagnostic := -1,
) {
append(&stream.items, token.Token{
kind=kind,
span=source.Span{file=source_file.id, start=start, end=end},
text=source_file.text[start:end],
symbol=id,
diagnostic=diagnostic,
})
}
@@ -46,6 +48,7 @@ append_token :: proc(
lex :: proc(
source_file: ^source.Source,
diagnostics: ^source.Diagnostics,
symbols: ^symbol.Table,
allocator := context.allocator,
) -> token.Stream {
stream: token.Stream
@@ -73,7 +76,7 @@ lex :: proc(
append_token(&stream, source_file, .Colon_Colon, start, cursor)
} else {
id := source.add(diagnostics, source.Span{file=source_file.id, start=start, end=cursor}, "expected a second ':'")
append_token(&stream, source_file, .Invalid, start, cursor, id)
append_token(&stream, source_file, .Invalid, start, cursor, diagnostic=id)
}
case '=':
append_token(&stream, source_file, .Equal, cursor, cursor+1)
@@ -128,7 +131,7 @@ lex :: proc(
source.Span{file=source_file.id, start=start, end=cursor},
"unterminated import string",
)
append_token(&stream, source_file, .Invalid, start, cursor, id)
append_token(&stream, source_file, .Invalid, start, cursor, diagnostic=id)
}
case ';':
id := source.add(
@@ -136,7 +139,7 @@ lex :: proc(
source.Span{file=source_file.id, start=cursor, end=cursor+1},
"semicolons are invalid; terminate statements with a newline",
)
append_token(&stream, source_file, .Invalid, cursor, cursor+1, id)
append_token(&stream, source_file, .Invalid, cursor, cursor+1, diagnostic=id)
cursor += 1
case:
if value >= '0' && value <= '9' {
@@ -151,7 +154,12 @@ lex :: proc(
cursor += 1
}
text := source_file.text[start:cursor]
append_token(&stream, source_file, keyword_kind(text), start, cursor)
kind := keyword_kind(text)
id := symbol.INVALID
if kind == .Identifier || kind == .Underscore {
id = symbol.intern(symbols, text)
}
append_token(&stream, source_file, kind, start, cursor, id)
} else {
id := source.addf(
diagnostics,
@@ -159,7 +167,7 @@ lex :: proc(
"invalid source byte 0x%02x",
value,
)
append_token(&stream, source_file, .Invalid, cursor, cursor+1, id)
append_token(&stream, source_file, .Invalid, cursor, cursor+1, diagnostic=id)
cursor += 1
}
}