tiny lexer test (testbed)

testbed/lexer/main.bro

@@ -0,0 +1,154 @@
+arraylist :: import "@std/arraylist"
+mem :: import "@std/mem"
+c :: import "@ffi/c"
+
+Kind :: enum(u8) {
+	invalid
+	eof
+	newline
+	identifier
+	keyword
+	integer
+	string
+	punctuation
+}
+
+Token :: struct {
+	start usize
+	length usize
+	kind Kind
+}
+
+_is_alpha func(value u8) bool {
+	return value == '_' or
+	       value >= 'a' and value <= 'z' or
+	       value >= 'A' and value <= 'Z'
+}
+
+_is_digit func(value u8) bool {
+	return value >= '0' and value <= '9'
+}
+
+_word_kind func(word []u8) Kind {
+	# ponytail: enough keywords for the demo; add the full language set when a parser needs it.
+	if mem.eql(word, "func") or mem.eql(word, "void") {
+		return .keyword
+	}
+	return .identifier
+}
+
+_append func(tokens @mut arraylist.ArrayList(Token), kind Kind, start, end usize) void ! mem.AllocError {
+	try arraylist.append(tokens, Token {
+		start = start,
+		length = end - start,
+		kind = kind,
+	})
+	return _
+}
+
+lex func(source []u8, tokens @mut arraylist.ArrayList(Token)) void ! mem.AllocError {
+	cursor usize = 0
+	while cursor < source.len {
+		value u8 :: source[cursor]
+		if value == ' ' or value == '\t' or value == '\r' {
+			cursor += 1
+		} else if value == '\n' {
+			try _append(tokens, .newline, cursor, cursor + 1)
+			cursor += 1
+		} else if value == '#' {
+			while cursor < source.len and source[cursor] != '\n' : cursor += 1 {}
+		} else if _is_alpha(value) {
+			start usize :: cursor
+			cursor += 1
+			while cursor < source.len and (_is_alpha(source[cursor]) or _is_digit(source[cursor])) : cursor += 1 {}
+			try _append(tokens, _word_kind(source[start..cursor]), start, cursor)
+		} else if _is_digit(value) {
+			start usize :: cursor
+			while cursor < source.len and _is_digit(source[cursor]) : cursor += 1 {}
+			try _append(tokens, .integer, start, cursor)
+		} else if value == '"' {
+			start usize :: cursor
+			cursor += 1
+			while cursor < source.len and source[cursor] != '"' and source[cursor] != '\n' {
+				if source[cursor] == '\\' and cursor + 1 < source.len {
+					cursor += 1
+				}
+				cursor += 1
+			}
+			if cursor < source.len and source[cursor] == '"' {
+				cursor += 1
+				try _append(tokens, .string, start, cursor)
+			} else {
+				try _append(tokens, .invalid, start, cursor)
+			}
+		} else {
+			start usize :: cursor
+			cursor += 1
+			if value == ':' and cursor < source.len and source[cursor] == ':' {
+				cursor += 1
+			}
+			try _append(tokens, .punctuation, start, cursor)
+		}
+	}
+	try _append(tokens, .eof, cursor, cursor)
+	return _
+}
+
+_kind_name func(kind Kind) *c_char {
+	return match kind {
+		.invalid: "invalid"
+		.eof: "eof"
+		.newline: "newline"
+		.identifier: "identifier"
+		.keyword: "keyword"
+		.integer: "integer"
+		.string: "string"
+		.punctuation: "punctuation"
+	}
+}
+
+_print_token func(source []u8, token Token) void {
+	_ = c.printf("%-11s", _kind_name(token.kind))
+	if token.length != 0 {
+		_ = c.printf(" `")
+		i usize = 0
+		while i < token.length : i += 1 {
+			value u8 :: source[token.start + i]
+			if value == '\n' {
+				_ = c.printf("\\n")
+			} else {
+				_ = c.putchar(c_int(value))
+			}
+		}
+		_ = c.putchar('`')
+	}
+	_ = c.putchar('\n')
+}
+
+main func() i32 {
+	source ::
+		`main func() void {
+		`   hello()
+		`}
+
+	tokens arraylist.ArrayList(Token) = arraylist.init(mem.c_allocator)
+	defer arraylist.deinit(&tokens)
+
+	lex(source, &tokens) catch |_| {
+		_ = c.printf("out of memory\n")
+		return 1
+	}
+
+	# Small executable self-check for the lexer and ArrayList path.
+	if (tokens.items.len != 13 or
+	    tokens.items[0].kind != .identifier or
+	    tokens.items[1].kind != .keyword or
+	    tokens.items[12].kind != .eof) {
+		return 2
+	}
+
+	for tokens.items |token| {
+		_print_token(source, token)
+	}
+	return 0
+}