brolang/compiler/lexer/lexer.odin

package lexer

import "../source"
import "../symbol"
import "../token"

is_identifier_start :: proc(value: byte) -> bool {
	return value == '_' || value >= 'a' && value <= 'z' || value >= 'A' && value <= 'Z'
}

is_identifier_continue :: proc(value: byte) -> bool {
	return is_identifier_start(value) || value >= '0' && value <= '9'
}

keyword_kind :: proc(text: string) -> token.Kind {
	switch text {
	case "func":   return .Keyword_Func
	case "import": return .Keyword_Import
	case "return": return .Keyword_Return
	case "void":   return .Keyword_Void
	case "int":    return .Keyword_Int
	case "i8":     return .Keyword_I8
	case "i16":    return .Keyword_I16
	case "i32":    return .Keyword_I32
	case "i64":    return .Keyword_I64
	case "_":      return .Underscore
	}
	return .Identifier
}

append_token :: proc(
	stream: ^token.Stream,
	source_file: ^source.Source,
	kind: token.Kind,
	start, end: int,
	id := symbol.INVALID,
	diagnostic := source.INVALID_DIAGNOSTIC,
) {
	append(&stream.items, token.Token{
		kind=kind,
		span=source.Span{file=source_file.id, start=source.Offset(start), end=source.Offset(end)},
		symbol=id,
		diagnostic=diagnostic,
	})
}

lex :: proc(
	source_file: ^source.Source,
	diagnostics: ^source.Diagnostics,
	symbols: ^symbol.Table,
	allocator := context.allocator,
) -> token.Stream {
	stream: token.Stream
	stream.items.allocator = allocator
	bytes := transmute([]byte)source_file.text
	cursor := 0

	for cursor < len(bytes) {
		value := bytes[cursor]
		switch value {
		case ' ', '\t', '\r':
			cursor += 1
		case '\n':
			append_token(&stream, source_file, .Newline, cursor, cursor+1)
			cursor += 1
		case '#':
			for cursor < len(bytes) && bytes[cursor] != '\n' {
				cursor += 1
			}
		case ':':
			start := cursor
			cursor += 1
			if cursor < len(bytes) && bytes[cursor] == ':' {
				cursor += 1
				append_token(&stream, source_file, .Colon_Colon, start, cursor)
			} else {
				id := source.add(diagnostics, source.Span{file=source_file.id, start=source.Offset(start), end=source.Offset(cursor)}, "expected a second ':'")
				append_token(&stream, source_file, .Invalid, start, cursor, diagnostic=id)
			}
		case '=':
			append_token(&stream, source_file, .Equal, cursor, cursor+1)
			cursor += 1
		case '+':
			append_token(&stream, source_file, .Plus, cursor, cursor+1)
			cursor += 1
		case '.':
			append_token(&stream, source_file, .Dot, cursor, cursor+1)
			cursor += 1
		case '(':
			append_token(&stream, source_file, .Left_Paren, cursor, cursor+1)
			cursor += 1
		case ')':
			append_token(&stream, source_file, .Right_Paren, cursor, cursor+1)
			cursor += 1
		case '{':
			append_token(&stream, source_file, .Left_Brace, cursor, cursor+1)
			cursor += 1
		case '}':
			append_token(&stream, source_file, .Right_Brace, cursor, cursor+1)
			cursor += 1
		case ',':
			append_token(&stream, source_file, .Comma, cursor, cursor+1)
			cursor += 1
		case '"':
			start := cursor
			cursor += 1
			valid := true
			for cursor < len(bytes) && bytes[cursor] != '"' && bytes[cursor] != '\n' {
				if bytes[cursor] == '\\' {
					cursor += 1
					if cursor >= len(bytes) || (bytes[cursor] != '\\' && bytes[cursor] != '"') {
						source.add(
							diagnostics,
							source.Span{file=source_file.id, start=source.Offset(max(cursor-1, start)), end=source.Offset(min(cursor+1, len(bytes)))},
							"import strings only support '\\\\' and '\\\"' escapes",
						)
						valid = false
					}
				}
				if cursor < len(bytes) && bytes[cursor] != '\n' {
					cursor += 1
				}
			}
			if cursor < len(bytes) && bytes[cursor] == '"' {
				cursor += 1
				append_token(&stream, source_file, .String if valid else .Invalid, start, cursor)
			} else {
				id := source.add(
					diagnostics,
					source.Span{file=source_file.id, start=source.Offset(start), end=source.Offset(cursor)},
					"unterminated import string",
				)
				append_token(&stream, source_file, .Invalid, start, cursor, diagnostic=id)
			}
		case ';':
			id := source.add(
				diagnostics,
				source.Span{file=source_file.id, start=source.Offset(cursor), end=source.Offset(cursor+1)},
				"semicolons are invalid; terminate statements with a newline",
			)
			append_token(&stream, source_file, .Invalid, cursor, cursor+1, diagnostic=id)
			cursor += 1
		case:
			if value >= '0' && value <= '9' {
				start := cursor
				for cursor < len(bytes) && bytes[cursor] >= '0' && bytes[cursor] <= '9' {
					cursor += 1
				}
				append_token(&stream, source_file, .Integer, start, cursor)
			} else if is_identifier_start(value) {
				start := cursor
				for cursor < len(bytes) && is_identifier_continue(bytes[cursor]) {
					cursor += 1
				}
				text := source_file.text[start:cursor]
				kind := keyword_kind(text)
				id := symbol.INVALID
				if kind == .Identifier || kind == .Underscore {
					id = symbol.intern(symbols, text)
				}
				append_token(&stream, source_file, kind, start, cursor, id)
			} else {
				id := source.addf(
					diagnostics,
					source.Span{file=source_file.id, start=source.Offset(cursor), end=source.Offset(cursor+1)},
					"invalid source byte 0x%02x",
					value,
				)
				append_token(&stream, source_file, .Invalid, cursor, cursor+1, diagnostic=id)
				cursor += 1
			}
		}
	}

	append_token(&stream, source_file, .Eof, len(bytes), len(bytes))
	return stream
}