toy lexer

2026-07-13 15:02:42 +02:00
commit 0b93cd1c90
11 changed files with 1015 additions and 0 deletions
@@ -0,0 +1,67 @@
+mem :: import "@std/mem"
+
+ArrayList func($T type) type {
+	return struct {
+		items []mut T
+		capacity usize
+		allocator mem.Allocator
+	}
+}
+
+init func($T type, allocator mem.Allocator) ArrayList(T) {
+	return ArrayList(T) {
+		items = mem.empty(T),
+		capacity = 0,
+		allocator = allocator,
+	}
+}
+
+deinit func($T type, list @mut ArrayList(T)) void {
+	allocation []mut T :: list.items.ptr[..list.capacity]
+	mem.free(list.allocator, allocation)
+	list.items = mem.empty(T)
+	list.capacity = 0
+}
+
+reserve func($T type, list @mut ArrayList(T), minimum_capacity usize) void ! mem.AllocError {
+	if minimum_capacity <= list.capacity {
+		return _
+	}
+
+	new_capacity usize = 8
+	if list.capacity >= 8 {
+		half usize :: div_trunc(list.capacity, 2)
+		if list.capacity > max_value(usize) - half {
+			new_capacity = minimum_capacity
+		} else {
+			new_capacity = list.capacity + half
+		}
+	}
+	if new_capacity < minimum_capacity {
+		new_capacity = minimum_capacity
+	}
+
+	length usize :: list.items.len
+	allocation []mut T :: list.items.ptr[..list.capacity]
+	grown []mut T :: mem.realloc(list.allocator, allocation, new_capacity) catch |_| {
+		return .out_of_memory
+	}
+	list.items = grown.ptr[..length]
+	list.capacity = new_capacity
+	return _
+}
+
+append func($T type, list @mut ArrayList(T), value T) void ! mem.AllocError {
+	length usize :: list.items.len
+	if length == max_value(usize) {
+		return .out_of_memory
+	}
+	try reserve(list, length + 1)
+	list.items = list.items.ptr[..length + 1]
+	list.items[length] = value
+	return _
+}
+
+clear func($T type, list @mut ArrayList(T)) void {
+	list.items = list.items.ptr[..0]
+}
@@ -0,0 +1,17 @@
+# Build configuration surface for `brolang build` (v0).
+#
+# A project's `build.bro` imports this module and declares a top-level constant
+# named `config` of type `BuildConfig`. `brolang build [root]` type-checks
+# build.bro, reads the config, and writes root/build/name.
+#
+# Declarative and literal-only: one executable per build. List fields take an
+# address-of an array literal (`&["raylib"]`); empty lists are written `&[]`.
+BuildConfig :: struct {
+	name      []u8      # output executable name under root/build
+	source    []u8      # program package directory, relative to build.bro
+	libraries [][]u8    # library names to link (-l)
+	lib_paths [][]u8    # library search directories (-L)
+	includes  [][]u8    # C include directories (-I)
+	defines   [][]u8    # C preprocessor defines (name or name=value)
+	links     [][]u8    # extra linker inputs (object/source files, -framework pairs)
+}
@@ -0,0 +1,122 @@
+c :: import "@ffi/c"
+
+ReadError :: enum {
+	read_failed
+}
+
+WriteError :: enum {
+	write_failed
+	no_progress
+}
+
+Io :: struct {
+	context ?*mut anyopaque
+	vtable @IoVTable
+}
+
+IoVTable :: struct {
+	read @func(context ?*mut anyopaque, stream ReadStream, buffer []mut u8) usize ! ReadError
+	write @func(context ?*mut anyopaque, stream WriteStream, bytes []u8) usize ! WriteError
+}
+
+ReadStream :: enum(c_int) {
+	stdin = 0
+}
+
+WriteStream :: enum(c_int) {
+	stdout = 1
+	stderr = 2
+}
+
+Reader :: struct {
+	impl Io
+	stream ReadStream
+}
+
+Writer :: struct {
+	impl Io
+	stream WriteStream
+}
+
+read func(reader Reader, buffer []mut u8) usize ! ReadError {
+	if buffer.len == 0 {
+		return 0
+	}
+	count usize :: try reader.impl.vtable.read(reader.impl.context, reader.stream, buffer)
+	if count > buffer.len {
+		return .read_failed
+	}
+	return count
+}
+
+write func(writer Writer, bytes []u8) usize ! WriteError {
+	if bytes.len == 0 {
+		return 0
+	}
+	count usize :: try writer.impl.vtable.write(writer.impl.context, writer.stream, bytes)
+	if count > bytes.len {
+		return .write_failed
+	}
+	return count
+}
+
+write_all func(writer Writer, bytes []u8) void ! WriteError {
+	offset usize = 0
+	while offset < bytes.len {
+		count usize :: write(writer, bytes[offset..]) catch |err| {
+			return err
+		}
+		if count == 0 {
+			return .no_progress
+		}
+		offset += count
+	}
+	return _
+}
+
+_system_read func(_ ?*mut anyopaque, stream ReadStream, buffer []mut u8) usize ! ReadError {
+	request usize = buffer.len
+	maximum usize :: usize(max_value(c_long))
+	if request > maximum {
+		request = maximum
+	}
+	while true {
+		count c_long :: c.read(c_int(stream), buffer.ptr, c_ulong(request))
+		if count >= 0 {
+			return usize(count)
+		}
+		if c.__error()^ != 4 {
+			return .read_failed
+		}
+	}
+}
+
+_system_write func(_ ?*mut anyopaque, stream WriteStream, bytes []u8) usize ! WriteError {
+	fd c_int :: c_int(stream)
+	request usize = bytes.len
+	maximum usize :: usize(max_value(c_long))
+	if request > maximum {
+		request = maximum
+	}
+	while true {
+		count c_long :: c.write(fd, bytes.ptr, c_ulong(request))
+		if count >= 0 {
+			return usize(count)
+		}
+		if c.__error()^ != 4 {
+			return .write_failed
+		}
+	}
+}
+
+_system_vtable IoVTable :: IoVTable {
+	read = _system_read,
+	write = _system_write,
+}
+
+_system func() Io {
+	return Io {
+		context = none,
+		vtable = &_system_vtable,
+	}
+}
@@ -0,0 +1,202 @@
+c :: import "@ffi/c"
+
+AllocError :: enum {
+	out_of_memory
+}
+
+Allocator :: struct {
+	context ?*mut anyopaque
+	vtable @AllocatorVTable
+}
+
+AllocatorVTable :: struct {
+	alloc @func(context ?*mut anyopaque, size usize, alignment usize) ?*mut u8
+	realloc @func(context ?*mut anyopaque, memory ?*mut u8, old_size usize, new_size usize, alignment usize) ?*mut u8
+	free @func(context ?*mut anyopaque, memory ?*mut u8, size usize, alignment usize) void
+}
+
+raw_alloc func(allocator Allocator, size usize, alignment usize) ?*mut u8 {
+	return allocator.vtable.alloc(allocator.context, size, alignment)
+}
+
+raw_realloc func(allocator Allocator, memory ?*mut u8, old_size usize, new_size usize, alignment usize) ?*mut u8 {
+	return allocator.vtable.realloc(allocator.context, memory, old_size, new_size, alignment)
+}
+
+raw_free func(allocator Allocator, memory ?*mut u8, size usize, alignment usize) void {
+	allocator.vtable.free(allocator.context, memory, size, alignment)
+}
+
+eql func($T type, left, right []T) bool {
+	if left.len != right.len {
+		return false
+	}
+
+	i usize = 0
+	while i < left.len : i += 1 {
+		if left[i] != right[i] {
+			return false
+		}
+	}
+	return true
+}
+
+_empty_storage [1]mut u64 = [0]
+
+_empty_slice func($T type, count usize) []mut T {
+	pointer *mut T :: ptr_cast(T, (&_empty_storage).ptr)
+	return pointer[..count]
+}
+
+empty func($T type) []mut T {
+	return _empty_slice(T, 0)
+}
+
+alloc func($T type, allocator Allocator, count usize) []mut T ! AllocError {
+	if count == 0 {
+		return _empty_slice(T, 0)
+	}
+
+	element_size usize :: size_of(T)
+	if element_size == 0 {
+		return _empty_slice(T, count)
+	}
+	if count > div_trunc(max_value(usize), element_size) {
+		return .out_of_memory
+	}
+
+	memory ?*mut u8 = raw_alloc(allocator, count * element_size, align_of(T))
+	if memory |bytes| {
+		pointer *mut T :: ptr_cast(T, bytes)
+		return pointer[..count]
+	}
+	return .out_of_memory
+}
+
+realloc func($T type, allocator Allocator, memory []mut T, new_count usize) []mut T ! AllocError {
+	if new_count == memory.len {
+		return memory
+	}
+	if new_count == 0 {
+		free(allocator, memory)
+		return _empty_slice(T, 0)
+	}
+
+	element_size usize :: size_of(T)
+	if element_size == 0 {
+		return _empty_slice(T, new_count)
+	}
+	if new_count > div_trunc(max_value(usize), element_size) {
+		return .out_of_memory
+	}
+
+	old_memory ?*mut u8 = none
+	old_size usize = 0
+	if memory.len != 0 {
+		old_memory = ptr_cast(u8, memory.ptr)
+		old_size = memory.len * element_size
+	}
+	resized ?*mut u8 = raw_realloc(
+		allocator,
+		old_memory,
+		old_size,
+		new_count * element_size,
+		align_of(T),
+	)
+	if resized |bytes| {
+		pointer *mut T :: ptr_cast(T, bytes)
+		return pointer[..new_count]
+	}
+	return .out_of_memory
+}
+
+free func($T type, allocator Allocator, memory []mut T) void {
+	if memory.len != 0 and size_of(T) != 0 {
+		raw_free(allocator, ptr_cast(u8, memory.ptr), memory.len * size_of(T), align_of(T))
+	}
+}
+
+_malloc_alignment usize :: 16 # ponytail: aarch64-macos libc malloc alignment assumption.
+
+_power_of_two func(value usize) bool {
+	if value == 0 {
+		return false
+	}
+
+	current usize = value
+	while current > 1 {
+		half usize = div_trunc(current, 2)
+		if half * 2 != current {
+			return false
+		}
+		current = half
+	}
+
+	return true
+}
+
+_c_alloc func(_ ?*mut anyopaque, size usize, alignment usize) ?*mut u8 {
+	if _power_of_two(alignment) == false {
+		return none
+	}
+
+	if alignment <= _malloc_alignment {
+		return ptr_cast(u8, c.malloc(c_ulong(size)))
+	}
+
+	memory [1]mut ?*mut anyopaque = [none]
+	status c_int = c.posix_memalign((&memory).ptr, c_ulong(alignment), c_ulong(size))
+	if status != 0 {
+		return none
+	}
+
+	return ptr_cast(u8, memory[0])
+}
+
+_c_realloc func(_ ?*mut anyopaque, memory ?*mut u8, old_size usize, new_size usize, alignment usize) ?*mut u8 {
+	if _power_of_two(alignment) == false {
+		return none
+	}
+
+	if new_size == 0 {
+		c.free(memory)
+		return none
+	}
+
+	if memory |old_memory| {
+		if alignment <= _malloc_alignment {
+			return ptr_cast(u8, c.realloc(old_memory, c_ulong(new_size)))
+		}
+
+		new_memory ?*mut u8 = _c_alloc(none, new_size, alignment)
+		if new_memory |new_bytes| {
+			copy_size usize = old_size
+			if new_size < copy_size {
+				copy_size = new_size
+			}
+			i usize = 0
+			while i < copy_size : i += 1 {
+				new_bytes[i] = old_memory[i]
+			}
+			c.free(old_memory)
+		}
+		return new_memory
+	}
+
+	return _c_alloc(none, new_size, alignment)
+}
+
+_c_free func(_ ?*mut anyopaque, memory ?*mut u8, _ usize, _ usize) void {
+	c.free(memory)
+}
+
+_c_vtable AllocatorVTable :: AllocatorVTable {
+	alloc = _c_alloc,
+	realloc = _c_realloc,
+	free = _c_free,
+}
+
+c_allocator Allocator :: Allocator {
+	context = none,
+	vtable = &_c_vtable,
+}
@@ -0,0 +1,3 @@
+import "arraylist"
+
+ArrayList :: alias arraylist.ArrayList