initial draft

2026-06-09 17:28:28 +02:00
commit 087fdb45d5
40 changed files with 4371 additions and 0 deletions
@@ -0,0 +1,135 @@
+package lexer
+
+import "../source"
+import "../token"
+
+is_identifier_start :: proc(value: byte) -> bool {
+	return value == '_' || value >= 'a' && value <= 'z' || value >= 'A' && value <= 'Z'
+}
+
+is_identifier_continue :: proc(value: byte) -> bool {
+	return is_identifier_start(value) || value >= '0' && value <= '9'
+}
+
+keyword_kind :: proc(text: string) -> token.Kind {
+	switch text {
+	case "c":      return .Keyword_C
+	case "func":   return .Keyword_Func
+	case "return": return .Keyword_Return
+	case "void":   return .Keyword_Void
+	case "int":    return .Keyword_Int
+	case "i8":     return .Keyword_I8
+	case "i16":    return .Keyword_I16
+	case "i32":    return .Keyword_I32
+	case "i64":    return .Keyword_I64
+	case "_":      return .Underscore
+	}
+	return .Identifier
+}
+
+append_token :: proc(
+	stream: ^token.Stream,
+	source_file: ^source.Source,
+	kind: token.Kind,
+	start, end: int,
+	diagnostic := -1,
+) {
+	append(&stream.items, token.Token{
+		kind=kind,
+		span=source.Span{start=start, end=end},
+		text=source_file.text[start:end],
+		diagnostic=diagnostic,
+	})
+}
+
+lex :: proc(
+	source_file: ^source.Source,
+	diagnostics: ^source.Diagnostics,
+	allocator := context.allocator,
+) -> token.Stream {
+	stream: token.Stream
+	stream.items.allocator = allocator
+	bytes := transmute([]byte)source_file.text
+	cursor := 0
+
+	for cursor < len(bytes) {
+		value := bytes[cursor]
+		switch value {
+		case ' ', '\t', '\r':
+			cursor += 1
+		case '\n':
+			append_token(&stream, source_file, .Newline, cursor, cursor+1)
+			cursor += 1
+		case '#':
+			for cursor < len(bytes) && bytes[cursor] != '\n' {
+				cursor += 1
+			}
+		case ':':
+			start := cursor
+			cursor += 1
+			if cursor < len(bytes) && bytes[cursor] == ':' {
+				cursor += 1
+				append_token(&stream, source_file, .Colon_Colon, start, cursor)
+			} else {
+				id := source.add(diagnostics, source.Span{start=start, end=cursor}, "expected a second ':'")
+				append_token(&stream, source_file, .Invalid, start, cursor, id)
+			}
+		case '=':
+			append_token(&stream, source_file, .Equal, cursor, cursor+1)
+			cursor += 1
+		case '+':
+			append_token(&stream, source_file, .Plus, cursor, cursor+1)
+			cursor += 1
+		case '(':
+			append_token(&stream, source_file, .Left_Paren, cursor, cursor+1)
+			cursor += 1
+		case ')':
+			append_token(&stream, source_file, .Right_Paren, cursor, cursor+1)
+			cursor += 1
+		case '{':
+			append_token(&stream, source_file, .Left_Brace, cursor, cursor+1)
+			cursor += 1
+		case '}':
+			append_token(&stream, source_file, .Right_Brace, cursor, cursor+1)
+			cursor += 1
+		case ',':
+			append_token(&stream, source_file, .Comma, cursor, cursor+1)
+			cursor += 1
+		case ';':
+			id := source.add(
+				diagnostics,
+				source.Span{start=cursor, end=cursor+1},
+				"semicolons are invalid; terminate statements with a newline",
+			)
+			append_token(&stream, source_file, .Invalid, cursor, cursor+1, id)
+			cursor += 1
+		case:
+			if value >= '0' && value <= '9' {
+				start := cursor
+				for cursor < len(bytes) && bytes[cursor] >= '0' && bytes[cursor] <= '9' {
+					cursor += 1
+				}
+				append_token(&stream, source_file, .Integer, start, cursor)
+			} else if is_identifier_start(value) {
+				start := cursor
+				for cursor < len(bytes) && is_identifier_continue(bytes[cursor]) {
+					cursor += 1
+				}
+				text := source_file.text[start:cursor]
+				append_token(&stream, source_file, keyword_kind(text), start, cursor)
+			} else {
+				id := source.addf(
+					diagnostics,
+					source.Span{start=cursor, end=cursor+1},
+					"invalid source byte 0x%02x",
+					value,
+				)
+				append_token(&stream, source_file, .Invalid, cursor, cursor+1, id)
+				cursor += 1
+			}
+		}
+	}
+
+	append_token(&stream, source_file, .Eof, len(bytes), len(bytes))
+	return stream
+}