intern identifiers

compiler/parser/parser.odin

@@ -2,6 +2,7 @@ package parser
 
 import "../ast"
 import "../source"
+import "../symbol"
 import "../token"
 import "core:fmt"
 import "core:strconv"
@@ -9,6 +10,7 @@ import "core:strings"
 
 Parser :: struct {
 	tokens:      ^token.Stream,
+	source_file: ^source.Source,
 	diagnostics: ^source.Diagnostics,
 	module:      ast.Module,
 	pkg:         int,
@@ -17,6 +19,13 @@ Parser :: struct {
 	delimiter_depth: int,
 }
 
+token_text :: proc(parser: ^Parser, tok: token.Token) -> string {
+	if tok.span.start < 0 || tok.span.end < tok.span.start || tok.span.end > len(parser.source_file.text) {
+		return ""
+	}
+	return parser.source_file.text[tok.span.start:tok.span.end]
+}
+
 span_from :: proc(first, last: source.Span) -> source.Span {
 	return source.Span{file=first.file, start=first.start, end=last.end}
 }
@@ -101,7 +110,7 @@ parse_type :: proc(parser: ^Parser) -> ast.Type_Syntax {
 	return .Invalid
 }
 
-parse_call :: proc(parser: ^Parser, qualifier: string, first, name: token.Token) -> int {
+parse_call :: proc(parser: ^Parser, qualifier: symbol.Id, first, name: token.Token) -> int {
 	left_paren := advance(parser)
 	parser.delimiter_depth += 1
 	defer parser.delimiter_depth -= 1
@@ -126,7 +135,7 @@ parse_call :: proc(parser: ^Parser, qualifier: string, first, name: token.Token)
 		kind=.Call,
 		span=source.Span{file=name.span.file, start=first.span.start, end=right_paren.span.end},
 		qualifier=qualifier,
-		text=name.text,
+		name=name.symbol,
 		args=args[:],
 		left=ast.INVALID_ID,
 		right=ast.INVALID_ID,
@@ -139,7 +148,7 @@ parse_primary :: proc(parser: ^Parser) -> int {
 	#partial switch tok.kind {
 	case .Integer:
 		advance(parser)
-		value, ok := strconv.parse_i64(tok.text)
+		value, ok := strconv.parse_i64(token_text(parser, tok))
 		if !ok {
 			return invalid_expr(parser, tok.span, "integer literal does not fit in i64")
 		}
@@ -154,12 +163,12 @@ parse_primary :: proc(parser: ^Parser) -> int {
 	case .Identifier:
 		first := advance(parser)
 		name := first
-		qualifier := ""
+		qualifier := symbol.INVALID
 		if _, ok := allow(parser, .Dot); ok {
 			if current(parser).kind != .Identifier {
 				return invalid_expr(parser, current(parser).span, "expected a package member after '.'")
 			}
-			qualifier = first.text
+			qualifier = first.symbol
 			name = advance(parser)
 		}
 		if current(parser).kind == .Left_Paren {
@@ -169,7 +178,7 @@ parse_primary :: proc(parser: ^Parser) -> int {
 			kind=.Name,
 			span=span_from(first.span, name.span),
 			qualifier=qualifier,
-			text=name.text,
+			name=name.symbol,
 			left=ast.INVALID_ID,
 			right=ast.INVALID_ID,
 			diagnostic=-1,
@@ -260,7 +269,7 @@ parse_return :: proc(parser: ^Parser) -> int {
 		append(&parser.module.statements, ast.Stmt{
 			kind=.Return,
 			span=span_from(start.span, end.span),
-			name="_",
+			name=end.symbol,
 			expr=ast.INVALID_ID,
 			diagnostic=-1,
 		})
@@ -306,7 +315,7 @@ parse_statement :: proc(parser: ^Parser) -> int {
 			append(&parser.module.statements, ast.Stmt{
 				kind=kind,
 				span=span_from(name.span, parser.module.exprs[expr].span),
-				name=name.text,
+				name=name.symbol,
 				type=type_syntax,
 				immutable=immutable,
 				expr=expr,
@@ -352,7 +361,7 @@ parse_params :: proc(parser: ^Parser) -> []ast.Param {
 		}
 		type_syntax := parse_type(parser)
 		for name in names {
-			append(&params, ast.Param{name=name.text, span=name.span, type=type_syntax})
+			append(&params, ast.Param{name=name.symbol, span=name.span, type=type_syntax})
 		}
 		delete(names)
 		skip_newlines(parser)
@@ -404,7 +413,7 @@ parse_function :: proc(parser: ^Parser, name: token.Token, c_abi: bool) {
 	}
 	append(&parser.module.functions, ast.Function{
 		span=span_from(name.span, end.span),
-		name=name.text,
+		name=name.symbol,
 		pkg=parser.pkg,
 		file=parser.file,
 		c_abi=c_abi,
@@ -416,16 +425,17 @@ parse_function :: proc(parser: ^Parser, name: token.Token, c_abi: bool) {
 }
 
 decode_import_path :: proc(parser: ^Parser, tok: token.Token) -> string {
-	if len(tok.text) < 2 {
+	text := token_text(parser, tok)
+	if len(text) < 2 {
 		return fmt.aprintf("", allocator=parser.module.allocator)
 	}
 	builder := strings.builder_make(parser.module.allocator)
 	defer strings.builder_destroy(&builder)
-	for index := 1; index < len(tok.text)-1; index += 1 {
-		value := tok.text[index]
-		if value == '\\' && index+1 < len(tok.text)-1 {
+	for index := 1; index < len(text)-1; index += 1 {
+		value := text[index]
+		if value == '\\' && index+1 < len(text)-1 {
 			index += 1
-			value = tok.text[index]
+			value = text[index]
 		}
 		strings.write_byte(&builder, value)
 	}
@@ -442,7 +452,7 @@ parse_import :: proc(parser: ^Parser, alias: token.Token, start: token.Token) {
 		}
 		append(&parser.module.imports, ast.Import{
 			span=start.span,
-			alias=alias.text,
+			alias=alias.symbol,
 			pkg=parser.pkg,
 			file=parser.file,
 			target=-1,
@@ -455,7 +465,7 @@ parse_import :: proc(parser: ^Parser, alias: token.Token, start: token.Token) {
 	advance(parser)
 	append(&parser.module.imports, ast.Import{
 		span=span_from(start.span, path_token.span),
-		alias=alias.text,
+		alias=alias.symbol,
 		path=decode_import_path(parser, path_token),
 		pkg=parser.pkg,
 		file=parser.file,
@@ -519,7 +529,7 @@ parse_top_level :: proc(parser: ^Parser) {
 	expr := parse_expression(parser)
 	append(&parser.module.globals, ast.Global{
 		span=span_from(name.span, parser.module.exprs[expr].span),
-		name=name.text,
+		name=name.symbol,
 		pkg=parser.pkg,
 		file=parser.file,
 		type=type_syntax,
@@ -532,11 +542,13 @@ parse_top_level :: proc(parser: ^Parser) {
 
 parse :: proc(
 	stream: ^token.Stream,
+	source_file: ^source.Source,
 	diagnostics: ^source.Diagnostics,
 	allocator := context.allocator,
 ) -> ast.Module {
 	parser := Parser{
 		tokens=stream,
+		source_file=source_file,
 		diagnostics=diagnostics,
 		module=ast.init_module(allocator),
 	}
@@ -550,12 +562,14 @@ parse :: proc(
 
 parse_into :: proc(
 	stream: ^token.Stream,
+	source_file: ^source.Source,
 	diagnostics: ^source.Diagnostics,
 	module: ^ast.Module,
 	pkg, file: int,
 ) {
 	parser := Parser{
 		tokens=stream,
+		source_file=source_file,
 		diagnostics=diagnostics,
 		module=module^,
 		pkg=pkg,