brolang/testbed/lexer/main.bro

import "@ffi/c"
import "@std"
import "@std/mem"
import "@std/arraylist"

Kind :: enum(u8) {
	invalid
	eof
	newline
	identifier
	keyword
	integer
	string
	punctuation
}

Token :: struct {
	start usize
	length usize
	kind Kind
}

@hide is_alpha func(value u8) bool {
	return value == '_' or
	       value >= 'a' and value <= 'z' or
	       value >= 'A' and value <= 'Z'
}

@hide is_digit func(value u8) bool {
	return value >= '0' and value <= '9'
}

@hide word_kind func(word []u8) Kind {
	# ponytail: enough keywords for the demo; add the full language set when a parser needs it.
	if mem.eql(word, "func") or mem.eql(word, "void") {
		return .keyword
	}
	return .identifier
}

@hide append_token func(tokens @mut std.ArrayList(Token), kind Kind, start, end usize) void ! mem.AllocError {
	try arraylist.append(tokens, Token {
		start = start,
		length = end - start,
		kind = kind,
	})
	return
}

lex func(source []u8, tokens @mut std.ArrayList(Token)) void ! mem.AllocError {
	cursor usize := 0
	while cursor < source.len {
		value u8 :: source[cursor]
		if value == ' ' or value == '\t' or value == '\r' {
			cursor += 1
		} else if value == '\n' {
			try append_token(tokens, .newline, cursor, cursor + 1)
			cursor += 1
		} else if value == '#' {
			while cursor < source.len and source[cursor] != '\n' : cursor += 1 {}
		} else if is_alpha(value) {
			start usize :: cursor
			cursor += 1
			while cursor < source.len and (is_alpha(source[cursor]) or is_digit(source[cursor])) : cursor += 1 {}
			try append_token(tokens, word_kind(source[start..cursor]), start, cursor)
		} else if is_digit(value) {
			start usize :: cursor
			while cursor < source.len and is_digit(source[cursor]) : cursor += 1 {}
			try append_token(tokens, .integer, start, cursor)
		} else if value == '"' {
			start usize :: cursor
			cursor += 1
			while cursor < source.len and source[cursor] != '"' and source[cursor] != '\n' {
				if source[cursor] == '\\' and cursor + 1 < source.len {
					cursor += 1
				}
				cursor += 1
			}
			if cursor < source.len and source[cursor] == '"' {
				cursor += 1
				try append_token(tokens, .string, start, cursor)
			} else {
				try append_token(tokens, .invalid, start, cursor)
			}
		} else {
			start usize :: cursor
			cursor += 1
			if value == ':' and cursor < source.len and source[cursor] == ':' {
				cursor += 1
			}
			try append_token(tokens, .punctuation, start, cursor)
		}
	}
	try append_token(tokens, .eof, cursor, cursor)
	return
}

@hide kind_name func(kind Kind) *c_char {
	return match kind {
		.invalid: "invalid"
		.eof: "eof"
		.newline: "newline"
		.identifier: "identifier"
		.keyword: "keyword"
		.integer: "integer"
		.string: "string"
		.punctuation: "punctuation"
	}
}

@hide print_token func(source []u8, token Token) void {
	_ = c.printf("%-11s", kind_name(token.kind))
	if token.length != 0 {
		_ = c.printf(" `")
		i usize := 0
		while i < token.length : i += 1 {
			value u8 :: source[token.start + i]
			if value == '\n' {
				_ = c.printf("\\n")
			} else {
				_ = c.putchar(c_int(value))
			}
		}
		_ = c.putchar('`')
	}
	_ = c.putchar('\n')
}

main func() i32 {
	source ::
		`main func() void {
		`   hello()
		`}

	tokens std.ArrayList(Token) := arraylist.init(mem.c_allocator)
	defer arraylist.deinit(&tokens)

	lex(source, &tokens) catch |_| {
		_ = c.printf("out of memory\n")
		return 1
	}

	# Small executable self-check for the lexer and ArrayList path.
	if (tokens.items.len != 13 or
	    tokens.items[0].kind != .identifier or
	    tokens.items[1].kind != .keyword or
	    tokens.items[12].kind != .eof) {
		return 2
	}

	for tokens.items |token| {
		print_token(source, token)
	}
	return 0
}