Compare commits

..

33 Commits

Author SHA1 Message Date
hl-valdemar cfd668b729 parse func decls 2026-08-16 00:50:11 +02:00
hl-valdemar 518037e7b7 cleanup on isle 4 2026-08-12 22:52:04 +02:00
hl-valdemar 753609f916 improve ast printing 2026-08-12 22:49:18 +02:00
hl-valdemar 7fa761d3f0 add parser test for unary prefix expressions 2026-08-12 22:21:15 +02:00
hl-valdemar 8fbf137c6d parse unary prefix and postfix expressions 2026-08-12 22:18:55 +02:00
hl-valdemar a384949ddf hide parser utility functions 2026-08-11 22:42:28 +02:00
hl-valdemar 4f142b72f2 improve std/mem doc comments 2026-08-11 22:40:44 +02:00
hl-valdemar 101795c0b8 cleanup (isle 6 again?) 2026-08-11 22:14:21 +02:00
hl-valdemar 8dd06afde2 parse arithmetic binary expressions 2026-08-11 21:24:00 +02:00
hl-valdemar 891dc6516e align with new @hide syntax 2026-08-10 23:20:52 +02:00
hl-valdemar 8de7ec341e cleanup on isle 6 2026-08-09 23:35:55 +02:00
hl-valdemar 6293c93c3c basic ast renderer 2026-08-09 23:26:41 +02:00
hl-valdemar 425ce38011 lexer refactor 2026-08-08 22:56:04 +02:00
hl-valdemar 410246faee migrate to new mutable decl syntax 2026-08-05 22:09:21 +02:00
hl-valdemar 1689c4db3c lexer touchups 2026-08-05 22:07:55 +02:00
hl-valdemar a2ccf60eb1 move ast definitions into its own package 2026-08-03 08:40:15 +02:00
hl-valdemar 939043dd8b refactor parse decl impl 2026-08-02 22:52:55 +02:00
hl-valdemar 469415a5b4 general decl parsing 2026-08-02 22:32:17 +02:00
hl-valdemar a0ba0865ff remove outdated todo 2026-08-02 21:17:09 +02:00
hl-valdemar 7616b4dcff align on inline (as opposed to expand) 2026-08-02 21:14:30 +02:00
hl-valdemar d693711a59 minor cleanup 2026-08-02 20:59:10 +02:00
hl-valdemar 90c7195d4b parse const decls 2026-08-02 20:51:39 +02:00
hl-valdemar c7f527e3c3 minor refactoring 2026-07-25 00:24:16 +02:00
hl-valdemar ad54a00893 refactor and func -> proc rename 2026-07-25 00:03:54 +02:00
hl-valdemar 0338e35e75 checkpoint 2026-07-23 10:56:53 +02:00
hl-valdemar a4e6f96951 static string map 2026-07-23 09:42:00 +02:00
hl-valdemar 41c326c961 fiddling with presentation 2026-07-22 02:58:25 +02:00
hl-valdemar cbccf03648 fix doc comments 2026-07-22 02:54:40 +02:00
hl-valdemar 596183b9a0 string pool 2026-07-22 02:42:25 +02:00
hl-valdemar 1d719c6a42 allow freeing immutable allocations 2026-07-22 01:10:33 +02:00
hl-valdemar 77696e19c9 rename none to null 2026-07-22 00:49:05 +02:00
hl-valdemar 979a8e5f07 update stdlib memory and reflection helpers 2026-07-22 00:47:48 +02:00
hl-valdemar c1225f4ac2 stdlib enum map 2026-07-20 15:51:32 +02:00
32 changed files with 2889 additions and 1022 deletions
+112
View File
@@ -0,0 +1,112 @@
#! GRAMMAR:
#! * type ..... -> identifier
#! * assignment -> :: | :=
#! * literal .. -> int | float | string
#! * expr ..... -> identifier | literal
#! * decl ..... -> identifier [type] assignment expr
#! * stmt ..... -> decl
import "@std/debug"
import "@source/lexer"
@hide:file
TokenId :: alias lexer.TokenId
@hide:file
NO_TOKEN :: alias lexer.NO_TOKEN
NodeId :: distinct u32
ExtraId :: distinct u32
NO_NODE :: maxval!(NodeId)
NO_EXTRA :: maxval!(ExtraId)
# unsafe access on its own; NodeKind serves as a the tag
# that determines the variant.
NodeData :: union {
token_id TokenId
node_id NodeId
extra_id ExtraId
}
Node :: struct {
kind NodeKind
main_token TokenId = NO_TOKEN
data0 NodeData = NodeData{ node_id = NO_NODE }
data1 NodeData = NodeData{ node_id = NO_NODE }
}
NodeKind :: enum {
# LITERALS:
# * main_token: token literal
literal_int
literal_float
literal_string
# IDENTIFIER EXPR:
# * main_token: token literal
expr_identifier
# UNARY:
# * main_token: operator
# * data0: NodeId - expression
expr_unary
# BINARY:
# * main_token: operator
# * data0: NodeId - left-hand side
# * data1: NodeId - right-hand side
expr_binary
# STATEMENT DECLS:
# * main_token: symbol name (identifier)
# * data0: ExtraId
# + extra[data0]: NodeId - type expr or NO_NODE
# + extra[data0 + 1]: NodeId - initializer expression
# * data1: TokenId of either `::` or `:=` indicating mutability
stmt_decl
# FUNC DECLS:
# * main_token: function name (identifier)
# * data0: NodeId - body block (or NO_NODE for extern)
# * data1: ExtraId
# + extra[data1]: NodeId - return type
# + extra[data1 + 1]: u32 - param count
# + extra[data1 + 2..]: NodeId - param decls
func_decl
# PARAM DECLS:
# * main_token: param name (identifier)
# * data0: NodeId - param type
param_decl
# BLOCK:
# * data0: ExtraId - statement range start
# * data1: ExtraId - statement range end
block
# PACKAGE:
# * data0: ExtraId
# * data1: ExtraId
#
# extra[data0..data1]: NodeId - all statement node ids in package
package
invalid
}
node_id proc(idx uint) NodeId {
debug.assert(u64(idx) < u64(NO_NODE))
return NodeId(idx)
}
maybe_node_id proc(idx uint) ?NodeId {
if (u64(idx) >= u64(NO_NODE)) return null
return NodeId(idx)
}
extra_id proc(idx uint) ExtraId {
debug.assert(u64(idx) < u64(NO_EXTRA))
return ExtraId(idx)
}
+319
View File
@@ -0,0 +1,319 @@
import "@std"
import "@std/mem"
import "@std/arraylist"
import "@source/lexer"
import "@source/parser"
import "@source/ast"
@hide:file TokenId :: alias lexer.TokenId
@hide:file Token :: alias lexer.Token
@hide:file Node :: alias ast.Node
@hide:file NodeId :: alias ast.NodeId
@hide:file ParseState :: alias parser.State
Renderer :: struct { tokens std.ArrayList(Token) }
init proc(allocator mem.Allocator) Renderer {
return Renderer{ tokens = arraylist.init(allocator) }
}
deinit proc(renderer @mut Renderer) void {
arraylist.deinit(&renderer.tokens)
}
render_token_stream proc(
renderer @mut Renderer,
node_id NodeId,
parse_state @ParseState,
tokens []Token
) void ! mem.AllocError {
node :: parse_state.nodes.items[usize(node_id)]
match node.kind {
.literal_int, .literal_float, .literal_string:
_ = try render_lit(renderer, node_id, parse_state, tokens)
.stmt_decl:
try render_stmt_decl(renderer, node_id, parse_state, tokens)
.func_decl:
try render_func_decl(renderer, node_id, parse_state, tokens)
.package:
try render_pkg(renderer, node_id, parse_state, tokens)
else: unreachable
}
}
@hide:file
render_pkg proc(
renderer @mut Renderer,
node_id NodeId,
parse_state @ParseState,
tokens []Token,
) void ! mem.AllocError {
node :: parse_state.nodes.items[usize(node_id)]
extra_start :: usize(node.data0.extra_id)
extra_end :: usize(node.data1.extra_id)
for extra_start..extra_end |i| {
stmt_node_id :: ast.node_id(parse_state.extra.items[i])
stmt_node :: parse_state.nodes.items[usize(stmt_node_id)]
match stmt_node.kind {
.stmt_decl:
try render_stmt_decl(renderer, stmt_node_id, parse_state, tokens)
.func_decl:
try render_func_decl(renderer, stmt_node_id, parse_state, tokens)
else: unreachable
}
}
}
@hide:file
render_stmt_decl proc(
renderer @mut Renderer,
node_id NodeId,
parse_state @ParseState,
tokens []Token,
) void ! mem.AllocError {
node :: parse_state.nodes.items[usize(node_id)]
tok_ident :: tokens[usize(node.main_token)]
try arraylist.append(&renderer.tokens, tok_ident)
raw_node_id_type :: parse_state.extra.items[usize(node.data0.extra_id)]
node_id_type :: ast.maybe_node_id(raw_node_id_type)
if (node_id_type) |id| try render_type(renderer, id, parse_state, tokens)
tok_assign :: tokens[usize(node.data1.token_id)]
try arraylist.append(&renderer.tokens, tok_assign)
node_id_expr :: ast.node_id(parse_state.extra.items[usize(node.data0.extra_id + 1)])
last_expr_token :: try render_expr(renderer, node_id_expr, parse_state, tokens)
terminator_idx :: usize(last_expr_token) + 1
tok_terminator :: if (terminator_idx < tokens.len) tokens[terminator_idx] else return
if (tok_terminator.kind != .newline and tok_terminator.kind != .eof) return
try arraylist.append(&renderer.tokens, tok_terminator)
eof_idx :: terminator_idx + 1
if (tok_terminator.kind == .newline and
eof_idx < tokens.len and
tokens[eof_idx].kind == .eof
) try arraylist.append(&renderer.tokens, tokens[eof_idx])
}
@hide:file
render_func_decl proc(
renderer @mut Renderer,
node_id NodeId,
parse_state @ParseState,
tokens []Token,
) void ! mem.AllocError {
node :: parse_state.nodes.items[usize(node_id)]
ident_idx :: usize(node.main_token)
try arraylist.append(&renderer.tokens, tokens[ident_idx])
try arraylist.append(&renderer.tokens, tokens[ident_idx + 1])
try arraylist.append(&renderer.tokens, tokens[ident_idx + 2])
payload_start :: usize(node.data1.extra_id)
param_count :: usize(parse_state.extra.items[payload_start + 1])
first_param :: payload_start + 2
for 0..param_count |i| {
param_id :: ast.node_id(parse_state.extra.items[first_param + i])
param_node :: parse_state.nodes.items[usize(param_id)]
type_id :: param_node.data0.node_id
type_node :: parse_state.nodes.items[usize(type_id)]
try arraylist.append(&renderer.tokens, tokens[usize(param_node.main_token)])
has_next :: i + 1 < param_count
if has_next {
next_param_id :: ast.node_id(parse_state.extra.items[first_param + i + 1])
next_param :: parse_state.nodes.items[usize(next_param_id)]
if (next_param.data0.node_id == type_id) {
try arraylist.append(
&renderer.tokens,
tokens[usize(param_node.main_token) + 1],
)
continue
}
}
try render_type(renderer, type_id, parse_state, tokens)
if has_next {
try arraylist.append(
&renderer.tokens,
tokens[usize(type_node.main_token) + 1],
)
}
}
return_type_id :: ast.node_id(parse_state.extra.items[payload_start])
return_type_node :: parse_state.nodes.items[usize(return_type_id)]
try arraylist.append(
&renderer.tokens,
tokens[usize(return_type_node.main_token) - 1],
)
try render_type(renderer, return_type_id, parse_state, tokens)
open_curly_id :: TokenId(usize(return_type_node.main_token) + 1)
close_curly_id :: try render_block(
renderer,
node.data0.node_id,
open_curly_id,
parse_state,
tokens,
)
terminator_idx :: usize(close_curly_id) + 1
if (terminator_idx >= tokens.len) return
tok_terminator :: tokens[terminator_idx]
if (tok_terminator.kind != .newline and tok_terminator.kind != .eof) return
try arraylist.append(&renderer.tokens, tok_terminator)
eof_idx :: terminator_idx + 1
if (tok_terminator.kind == .newline and
eof_idx < tokens.len and
tokens[eof_idx].kind == .eof
) try arraylist.append(&renderer.tokens, tokens[eof_idx])
}
@hide:file
render_block proc(
renderer @mut Renderer,
node_id NodeId,
open_curly_id TokenId,
parse_state @ParseState,
tokens []Token,
) TokenId ! mem.AllocError {
node :: parse_state.nodes.items[usize(node_id)]
open_curly_idx :: usize(open_curly_id)
try arraylist.append(&renderer.tokens, tokens[open_curly_idx])
leading_token := open_curly_idx + 1
while (leading_token < tokens.len and tokens[leading_token].kind == .newline) {
try arraylist.append(&renderer.tokens, tokens[leading_token])
leading_token += 1
}
extra_start :: usize(node.data0.extra_id)
extra_end :: usize(node.data1.extra_id)
for extra_start..extra_end |i| {
stmt_id :: ast.node_id(parse_state.extra.items[i])
stmt :: parse_state.nodes.items[usize(stmt_id)]
match stmt.kind {
.stmt_decl:
try render_stmt_decl(renderer, stmt_id, parse_state, tokens)
else: unreachable
}
}
# ponytail: linear delimiter scan; store brace tokens if nested blocks make rendering hot.
depth usize := 1
close_curly_idx := open_curly_idx + 1
while close_curly_idx < tokens.len {
match tokens[close_curly_idx].kind {
.open_curly: depth += 1
.close_curly: {
depth -= 1
if (depth == 0) {
try arraylist.append(&renderer.tokens, tokens[close_curly_idx])
return TokenId(close_curly_idx)
}
}
else: {}
}
close_curly_idx += 1
}
unreachable
}
@hide:file
render_expr proc(
renderer @mut Renderer,
node_id NodeId,
parse_state @ParseState,
tokens []Token,
) TokenId ! mem.AllocError {
# todo: support unary expressions
node :: parse_state.nodes.items[usize(node_id)]
return match node.kind {
.expr_identifier: {
tok :: tokens[usize(node.main_token)]
try arraylist.append(&renderer.tokens, tok)
yield node.main_token
}
.expr_unary:
try render_expr_unary(renderer, node_id, parse_state, tokens)
.expr_binary:
try render_expr_binary(renderer, node_id, parse_state, tokens)
.literal_int, .literal_float, .literal_string:
try render_lit(renderer, node_id, parse_state, tokens)
else: unreachable
}
}
@hide:file
render_expr_unary proc(
renderer @mut Renderer,
node_id NodeId,
parse_state @ParseState,
tokens []Token,
) TokenId ! mem.AllocError {
node :: parse_state.nodes.items[usize(node_id)]
operator :: node.main_token
operand :: node.data0.node_id
# render operator operand
try arraylist.append(&renderer.tokens, tokens[usize(operator)])
last_token :: try render_expr(renderer, operand, parse_state, tokens)
return last_token
}
@hide:file
render_expr_binary proc(
renderer @mut Renderer,
node_id NodeId,
parse_state @ParseState,
tokens []Token,
) TokenId ! mem.AllocError {
node :: parse_state.nodes.items[usize(node_id)]
operator :: node.main_token
lhs :: node.data0.node_id
rhs :: node.data1.node_id
# render lhs operator rhs
_ = try render_expr(renderer, lhs, parse_state, tokens)
try arraylist.append(&renderer.tokens, tokens[usize(operator)])
last_token :: try render_expr(renderer, rhs, parse_state, tokens)
return last_token
}
@hide:file
render_type proc(
renderer @mut Renderer,
node_id NodeId,
parse_state @ParseState,
tokens []Token,
) void ! mem.AllocError {
# todo: support more expressive type expressions...
node :: parse_state.nodes.items[usize(node_id)]
match node.kind {
.expr_identifier: {
tok :: tokens[usize(node.main_token)]
try arraylist.append(&renderer.tokens, tok)
}
else: unreachable
}
}
@hide:file
render_lit proc(
renderer @mut Renderer,
node_id NodeId,
parse_state @ParseState,
tokens []Token,
) TokenId ! mem.AllocError {
node :: parse_state.nodes.items[usize(node_id)]
tok :: tokens[usize(node.main_token)]
try arraylist.append(&renderer.tokens, tok)
return node.main_token
}
-106
View File
@@ -1,106 +0,0 @@
import "@std"
import "@std/mem"
import "@std/arraylist"
scan func(tokens @mut std.ArrayList(Token), input []u8) void ! mem.AllocError {
cursor usize = 0
while cursor < input.len {
char :: input[cursor]
# whitespace
if char == '\n' {
try arraylist.append(tokens, Token{ kind = .newline, start = cursor })
cursor += 1
continue
} else if is_whitespace(char) {
cursor += 1
continue
}
# comments
if char == '#' {
while cursor < input.len and input[cursor] != '\n' : cursor += 1 {}
cursor += 1
continue
}
# identifiers and keywords
if is_alpha(char) or char == '_' {
start :: cursor
cursor += 1
while cursor < input.len and (is_alpha(input[cursor]) or is_digit(input[cursor]) or input[cursor] == '_') {
cursor += 1
}
try arraylist.append(tokens, Token{ kind = .ident, start = start })
continue
}
# integers literals
if is_digit(char) {
start :: cursor
cursor += 1
while cursor < input.len and is_digit(input[cursor]) {
cursor += 1
}
try arraylist.append(tokens, Token{ kind = .int, start = start })
continue
}
# string literals
if char == '"' {
start :: cursor
cursor += 1
while cursor < input.len and input[cursor] != '"' and input[cursor] != '\n' : cursor += 1 {
# ignore escaped characters
if (input[cursor] == '\\' and cursor + 1 < input.len) cursor += 1
}
if cursor < input.len and input[cursor] == '"' {
cursor += 1
try arraylist.append(tokens, Token{ kind = .string, start = start })
} else {
try arraylist.append(tokens, Token{ kind = .invalid, start = start })
}
continue
}
# mutable assignment
if char == '=' {
try arraylist.append(tokens, Token{ kind = .equal, start = cursor })
cursor += 1
continue
}
# immutable assignment
cursor += 1
if cursor < input.len and char == ':' and input[cursor] == ':' {
try arraylist.append(tokens, Token{ kind = .double_colon, start = cursor })
cursor += 1
continue
}
# invalid character
try arraylist.append(tokens, Token{ kind = .invalid, start = cursor })
}
try arraylist.append(tokens, Token{ kind = .eof, start = cursor })
}
hide is_whitespace func(char u8) bool {
return char == ' ' or char == '\t' or char == '\n' or char == '\r'
}
hide is_alpha func(char u8) bool {
return match char {
'a'..'z', 'A'..'Z': true
else: false
}
}
hide is_digit func(char u8) bool {
return match char {
'0'..'9': true
else: false
}
}
+342
View File
@@ -0,0 +1,342 @@
import "@std"
import "@std/mem"
import "@std/strmap"
import "@std/enums/enummap"
import "@std/arraylist"
import "@source/strpool"
ScanError :: struct {
code ErrorCode
end usize
}
ErrorCode :: enum {
invalid_character,
float_must_end_with_digit,
unterminated_string,
}
ErrorDetails :: struct {
name []u8
message []u8
}
error_msg_map std.EnumMap(ErrorCode, ErrorDetails) :: enummap.init({
invalid_character = ErrorDetails {
name = "L0",
message = "invalid character",
},
float_must_end_with_digit = ErrorDetails {
name = "L1",
message = "float must end with a digit",
},
})
keywords std.StringMap(TokenKind) :: strmap.init([
{ "proc", .proc },
{ "return", .return },
{ "if", .if },
{ "for", .for },
{ "else", .else },
{ "while", .while },
])
Diagnostic :: struct {
token TokenId
code ErrorCode
}
State :: struct {
tokens std.ArrayList(Token)
diagnostics std.ArrayList(Diagnostic)
}
init proc(allocator mem.Allocator) State {
return State {
tokens = arraylist.init(allocator),
diagnostics = arraylist.init(allocator),
}
}
deinit proc(state @mut State) void {
arraylist.deinit(&state.tokens)
arraylist.deinit(&state.diagnostics)
}
scan proc(state @mut State, program []u8) void ! (mem.AllocError | strpool.InternError) {
diagnostics :: &state.diagnostics
cursor := 0
while cursor < program.len {
char :: program[cursor]
# whitespace
if char == '\n' {
try add_token(state, Token{ start = cursor, kind = .newline })
cursor += 1
continue
} else if is_whitespace(char) {
cursor += 1
continue
}
# comments
if char == '#' {
while (cursor < program.len and program[cursor] != '\n') cursor += 1
cursor += 1 # also skip newline
continue
}
# identifiers and keywords
if is_alpha(char) or char == '_' {
start :: cursor
result :: scan_ident(start, program)
try add_token(state, Token{
start = start,
kind = result.kind,
str_id = result.str_id,
})
cursor = result.end
continue
}
# numeric literals
if is_digit(char) {
start :: cursor
result :: scan_number(start, program) catch |err| {
token :: token_id(state.tokens.items.len)
try add_token(state, Token{ start = start, kind = .invalid })
try arraylist.append(diagnostics, Diagnostic{ token = token, code = err.code })
cursor = err.end
continue
}
kind :: if (result.has_decimal) .float else .int
try add_token(state, Token{ start = start, kind = kind })
cursor = result.end
continue
}
# string literals
if char == '"' {
start :: cursor
result :: scan_string(start, program) catch |err| {
token :: token_id(state.tokens.items.len)
try add_token(state, Token{ start = start, kind = .invalid })
try arraylist.append(diagnostics, Diagnostic{ token = token, code = err.code })
cursor = err.end
continue
}
try add_token(state, Token{ start = start, kind = .string })
cursor = result.end
continue
}
# immutable decl, mutable decl, or single colon
if char == ':' {
if cursor + 1 < program.len and program[cursor + 1] == ':' {
try add_token(state, Token{ start = cursor, kind = .double_colon })
cursor += 2
continue
}
if cursor + 1 < program.len and program[cursor + 1] == '=' {
try add_token(state, Token{ start = cursor, kind = .colon_equal })
cursor += 2
continue
}
try add_token(state, Token{ start = cursor, kind = .colon })
cursor += 1
continue
}
# compound arithmetic assignment
if char == '+' {
if cursor + 1 < program.len and program[cursor + 1] == '=' {
try add_token(state, Token{ start = cursor, kind = .plus_equal })
cursor += 2
continue
}
try add_token(state, Token{ start = cursor, kind = .plus })
cursor += 1
continue
}
if char == '-' {
if cursor + 1 < program.len and program[cursor + 1] == '=' {
try add_token(state, Token{ start = cursor, kind = .minus_equal })
cursor += 2
continue
}
try add_token(state, Token{ start = cursor, kind = .minus })
cursor += 1
continue
}
if char == '*' {
if cursor + 1 < program.len and program[cursor + 1] == '=' {
try add_token(state, Token{ start = cursor, kind = .star_equal })
cursor += 2
continue
}
try add_token(state, Token{ start = cursor, kind = .star })
cursor += 1
continue
}
if char == '/' {
if cursor + 1 < program.len and program[cursor + 1] == '=' {
try add_token(state, Token{ start = cursor, kind = .slash_equal })
cursor += 2
continue
}
try add_token(state, Token{ start = cursor, kind = .slash })
cursor += 1
continue
}
# single character tokens
kind ?TokenKind :: match char {
'=': .equal
',': .comma
'(': .open_paren
')': .close_paren
'{': .open_curly
'}': .close_curly
else: null
}
if (kind) |k| {
try add_token(state, Token{ start = cursor, kind = k })
cursor += 1
continue
}
# invalid character
token :: token_id(state.tokens.items.len)
try add_token(state, Token{ start = cursor, kind = .invalid })
try arraylist.append(diagnostics, Diagnostic{ token = token, code = .invalid_character })
cursor += 1
}
try add_token(state, Token{ start = cursor, kind = .eof })
}
ScanIdentResult :: struct {
end usize
kind TokenKind
str_id strpool.StringId
}
scan_ident proc(start usize, program []u8) ScanIdentResult {
cursor := start + 1
# scan whole identifier
while (cursor < program.len and (
is_alpha(program[cursor]) or
is_digit(program[cursor]) or
program[cursor] == '_'
)) cursor += 1
kind :: strmap.get(&keywords, program[start..cursor]) orelse .ident
# don't intern keywords (already O(1) lookup via token kind)
str_id :: if (kind == .ident)
strpool.intern(&strpool.STRINGS, program[start..cursor]) catch strpool.NO_STR
else
strpool.NO_STR
return ScanIdentResult{
end = cursor,
kind = kind,
str_id = str_id,
}
}
ScanNumResult :: struct {
end usize
has_decimal bool
}
scan_number proc(start usize, program []u8) ScanNumResult ! ScanError {
cursor := start
has_decimal := false
# scan integer part
while (cursor < program.len and is_digit(program[cursor])) cursor += 1
# check for decimal
if cursor < program.len and program[cursor] == '.' {
has_decimal = true
cursor += 1
}
# assert non-terminating decimal
if has_decimal and (cursor >= program.len or !is_digit(program[cursor])) {
return ScanError{
code = .float_must_end_with_digit,
end = cursor,
}
}
# scan fractional part
while (cursor < program.len and is_digit(program[cursor])) cursor += 1
return ScanNumResult{
end = cursor,
has_decimal = has_decimal,
}
}
ScanStrResult :: struct { end usize }
scan_string proc(start usize, program []u8) ScanStrResult ! ScanError {
cursor := start + 1 # skip first `"`
# scan entire string
while cursor < program.len and program[cursor] != '"' and program[cursor] != '\n' : cursor += 1 {
# ignore escaped characters
if (program[cursor] == '\\' and cursor + 1 < program.len) cursor += 1
}
# assert string terminal
if (cursor >= program.len or program[cursor] != '"') return ScanError{
code = .unterminated_string,
end = cursor,
}
return ScanStrResult{
end = cursor + 1, # skip last `"`
}
}
@hide
is_whitespace proc(char u8) bool {
return char == ' ' or char == '\t' or char == '\n' or char == '\r'
}
@hide
is_alpha proc(char u8) bool {
return match char {
'a'..='z', 'A'..='Z': true
else: false
}
}
@hide
is_digit proc(char u8) bool {
return match char {
'0'..='9': true
else: false
}
}
@hide
add_token proc(state @mut State, token Token) void ! mem.AllocError {
_ = token_id(state.tokens.items.len)
try arraylist.append(&state.tokens, token)
}
+24
View File
@@ -0,0 +1,24 @@
import "@source/strpool"
import "@std/mem"
import "@std/testing"
handles_keywords_identifiers_and_error_progress test {
strpool.STRINGS = strpool.init(mem.c_allocator)
defer strpool.deinit(&strpool.STRINGS)
state State := init(mem.c_allocator)
defer deinit(&state)
try scan(&state, "if name # comment\n@1. 2 3.5 \"hi\"")
try testing.expect_equal(8, state.tokens.items.len)
try testing.expect_equal(TokenKind.if, state.tokens.items[0].kind)
try testing.expect_equal(TokenKind.ident, state.tokens.items[1].kind)
try testing.expect_equal(TokenKind.invalid, state.tokens.items[2].kind)
try testing.expect_equal(TokenKind.invalid, state.tokens.items[3].kind)
try testing.expect_equal(TokenKind.int, state.tokens.items[4].kind)
try testing.expect_equal(TokenKind.float, state.tokens.items[5].kind)
try testing.expect_equal(TokenKind.string, state.tokens.items[6].kind)
try testing.expect_equal(TokenKind.eof, state.tokens.items[7].kind)
try testing.expect_equal(2, state.diagnostics.items.len)
try testing.expect_equal("name", strpool.get_str(&strpool.STRINGS, state.tokens.items[1].str_id)?)
}
+86
View File
@@ -0,0 +1,86 @@
import "@std/debug"
import "@source/strpool"
TokenId :: distinct u32
NO_TOKEN :: maxval!(TokenId)
Token :: struct {
start uint
kind TokenKind
str_id strpool.StringId = strpool.NO_STR
}
TokenKind :: enum {
# keywords
if, else
for, while
proc, return
# literals
ident, string
int, float
# comparison
equal_equal, not_equal
less, less_equal
greater, greater_equal
# assignment
double_colon, colon_equal
equal
# arithmetic
plus, minus
star, slash
# assignment & arithmetic
plus_equal, minus_equal
star_equal, slash_equal
# delimiters
open_paren, close_paren
open_bracket, close_bracket
open_curly, close_curly
comma
colon
newline
# special
eof
invalid
}
render_token proc(tok Token, program []u8) void {
match tok.kind {
.ident: {
if (strpool.get_str(&strpool.STRINGS, tok.str_id)) |str| {
debug.print("{}({})\n", {tok.kind, str})
} else {
res :: scan_ident(tok.start, program)
debug.print("{}({})\n", {tok.kind, program[tok.start..res.end]})
}
}
.string: {
res :: scan_string(tok.start, program) catch |_| {
debug.print("{}({})\n", {tok.kind, "null"})
return
}
debug.print("{}({})\n", {tok.kind, program[tok.start..res.end]})
}
.int, .float: {
res :: scan_number(tok.start, program) catch |_| {
debug.print("{}({})\n", {tok.kind, "null"})
return
}
debug.print("{}({})\n", {tok.kind, program[tok.start..res.end]})
}
else: debug.print("{}\n", {tok.kind})
}
}
token_id proc(idx uint) TokenId {
debug.assert(u64(idx) < u64(NO_TOKEN))
return TokenId(idx)
}
+94 -16
View File
@@ -1,33 +1,111 @@
import "@std"
import "@std/debug"
import "@std/mem"
import "@std/arraylist"
test import "@std/enums/enummap"
test import "@std/arraylist"
test import "@std/hashmap"
test import "@std/strmap"
test import "@std/meta"
import "@source/strpool"
import "@source/lexer"
import "@source/parser"
ast_renderer :: import "@source/ast/renderer"
test import "@source/strpool"
test import "@source/lexer"
test import "@source/parser"
program ::
`# these are immutable
`x :: 32
`y :: 3.2
`
`# these are mutable
`z u32 = 54
`# literals
`x float :: 123.9
`y int := 43
`z :: "hello"
`
`a := 1 + 2 * 3
`b :: -3 / 2 + 1
`
`empty proc() void {}
`grouped proc(a, b T, c U) void {
` value int :: 1
`}
main proc() void! {
strpool.STRINGS = strpool.init(mem.c_allocator)
defer strpool.deinit(&strpool.STRINGS)
main func() void {
debug.print("PROGRAM::[[\n{}\n]]\n\n", {program})
tokens std.ArrayList(Token) = arraylist.init(mem.c_allocator)
defer arraylist.deinit(&tokens)
scan(&tokens, program) catch |_| {
debug.print("failed to scan: out of memory\n", {})
scan_state := lexer.init(mem.c_allocator)
defer lexer.deinit(&scan_state)
lexer.scan(&scan_state, program) catch |err| {
debug.print("failed to scan: {}\n", {err})
return
}
debug.print("TOKENS::[[\n", {})
for tokens.items |token| {
debug.print("{}\n", { token.kind })
for (scan_state.tokens.items) |tok| lexer.render_token(tok, program)
debug.print("]]\n\n", {})
parse_state := parser.init(mem.c_allocator)
defer parser.deinit(&parse_state)
root :: parser.parse(&parse_state, scan_state.tokens.items) catch |err| {
debug.print("failed to parse: {}\n", {err})
return
}
debug.print("AST::[[\n", {})
for parse_state.nodes.items |node, i| {
match node.kind {
.expr_identifier: {
ident :: scan_state.tokens.items[usize(node.main_token)]
res :: lexer.scan_ident(ident.start, program)
debug.print("{}: {}({})\n", { i, node.kind, program[ident.start..res.end] })
}
.literal_int, .literal_float: {
lit :: scan_state.tokens.items[usize(node.main_token)]
res :: lexer.scan_number(lit.start, program) catch |_| {
debug.print("{}: {}\n", { i, node.kind })
continue
}
debug.print("{}: {}({})\n", { i, node.kind, program[lit.start..res.end] })
}
.literal_string: {
str :: scan_state.tokens.items[usize(node.main_token)]
res :: lexer.scan_string(str.start, program) catch |_| {
debug.print("{}: {}\n", { i, node.kind })
continue
}
debug.print("{}: {}({})\n", { i, node.kind, program[str.start..res.end] })
}
.expr_unary: {
operator :: scan_state.tokens.items[usize(node.main_token)]
operand :: node.data0.node_id
debug.print("{}: {}({}, operand={})\n", { i, node.kind, operator.kind, operand })
}
.expr_binary: {
operator :: scan_state.tokens.items[usize(node.main_token)]
lhs :: node.data0.node_id
rhs :: node.data1.node_id
debug.print("{}: {}({}, lhs={}, rhs={})\n", { i, node.kind, operator.kind, lhs, rhs })
}
.stmt_decl: {
ident :: scan_state.tokens.items[usize(node.main_token)]
init :: parse_state.extra.items[usize(node.data0.extra_id) + 1]
res :: lexer.scan_ident(ident.start, program)
debug.print("{}: {}({}, init={})\n", { i, node.kind, program[ident.start..res.end], init })
}
else: debug.print("{}: {}\n", { i, node.kind })
}
}
debug.print("]]\n\n", {})
renderer := ast_renderer.init(mem.c_allocator)
defer ast_renderer.deinit(&renderer)
try ast_renderer.render_token_stream(&renderer, root, &parse_state, scan_state.tokens.items)
debug.print("AST Render::[[\n", {})
for (renderer.tokens.items) |tok| lexer.render_token(tok, program)
debug.print("]]\n", {})
}
+401
View File
@@ -0,0 +1,401 @@
import "@std"
import "@std/mem"
import "@std/arraylist"
import "@std/enums/enummap"
import "@std/debug"
import "@source/lexer"
import "@source/ast"
# todo: move error stuff into a separate file
Diagnostic :: struct {
code ErrorCode
node NodeId
}
ParseError :: alias ErrorCode | mem.AllocError
ErrorCode :: enum {
unexpected_token,
}
ErrorDetails :: struct {
name []u8
message []u8
}
error_msg_map std.EnumMap(ErrorCode, ErrorDetails) :: enummap.init({
unexpected_token = ErrorDetails {
name = "P0",
message = "unexpected token",
},
})
@hide:file Token :: alias lexer.Token
@hide:file TokenId :: alias lexer.TokenId
@hide:file TokenKind :: alias lexer.TokenKind
@hide:file Node :: alias ast.Node
@hide:file NodeId :: alias ast.NodeId
@hide:file ExtraId :: alias ast.ExtraId
@hide:file NodeData :: alias ast.NodeData
@hide:file NO_NODE :: alias ast.NO_NODE
@hide:file NO_EXTRA :: alias ast.NO_EXTRA
@hide:file token_id :: alias lexer.token_id
BindingPowerScalar :: distinct u32
BindingPower :: struct {
left BindingPowerScalar
right BindingPowerScalar
}
prefix_binding_power proc(token_kind TokenKind) ?BindingPowerScalar {
return match token_kind {
.minus: BindingPowerScalar(30)
else: null
}
}
infix_binding_power proc(token_kind TokenKind) ?BindingPower {
return match token_kind {
.plus, .minus: BindingPower{ left = BindingPowerScalar(10), right = BindingPowerScalar(11) }
.star, .slash: BindingPower{ left = BindingPowerScalar(20), right = BindingPowerScalar(21) }
else: null
}
}
postfix_binding_power proc(token_kind TokenKind) ?BindingPowerScalar {
return match token_kind {
.open_paren, .open_bracket: BindingPowerScalar(40)
else: null
}
}
State :: struct {
nodes std.ArrayList(Node)
#! array of raw u32's indexing nodes, tokens, etc.
extra std.ArrayList(u32)
tokens []Token = &[]
next_token usize = 0
}
init proc(allocator mem.Allocator) State {
return State{
nodes = arraylist.init(allocator),
extra = arraylist.init(allocator),
}
}
deinit proc(state @mut State) void {
arraylist.deinit(&state.nodes)
arraylist.deinit(&state.extra)
}
parse proc(state @mut State, tokens []Token) NodeId ! ParseError {
state.tokens = tokens
return try parse_pkg(state)
}
parse_pkg proc(state @mut State) NodeId ! ParseError {
if (state.next_token < state.tokens.len
and state.tokens[state.next_token].kind == .eof
) return NO_NODE
stmt_ids std.ArrayList(NodeId) := arraylist.init(state.nodes.allocator)
defer arraylist.deinit(&stmt_ids)
while (state.next_token < state.tokens.len
and state.tokens[state.next_token].kind != .eof
) {
stmt_id :: try parse_decl(state)
try arraylist.append(&stmt_ids, stmt_id)
}
pkg_start :: ast.extra_id(state.extra.items.len)
for (stmt_ids.items) |stmt_id| _ = try add_extra(state, u32(stmt_id))
pkg_end :: ast.extra_id(state.extra.items.len)
return try add_node(state, Node{
kind = .package,
data0 = NodeData{ extra_id = pkg_start },
data1 = NodeData{ extra_id = pkg_end },
})
}
parse_decl proc(state @mut State) NodeId ! ParseError {
if next_n_is(state, 1, .proc)
return try parse_func_decl(state)
else
return try parse_stmt_decl(state)
}
parse_func_decl proc(state @mut State) NodeId ! ParseError {
ident_tok_id :: try consume(state, .ident)
_ = try consume(state, .proc)
_ = try consume(state, .open_paren)
param_decls std.ArrayList(NodeId) := arraylist.init(state.nodes.allocator)
defer arraylist.deinit(&param_decls)
ident_ids std.ArrayList(TokenId) := arraylist.init(state.nodes.allocator)
defer arraylist.deinit(&ident_ids)
while !next_is(state, .close_paren) {
arraylist.clear(&ident_ids)
# collect param group
while true {
param_tok_id :: try consume(state, .ident)
try arraylist.append(&ident_ids, param_tok_id)
if next_is(state, .ident) break
_ = try consume(state, .comma)
}
# collect type
try expect(state, .ident)
type_id :: try parse_primary(state)
for (ident_ids.items) |id| {
param_decl :: try add_node(state, Node{
kind = .param_decl,
main_token = id,
data0 = NodeData{ node_id = type_id },
})
try arraylist.append(&param_decls, param_decl)
}
if next_is(state, .comma) _ = try consume(state, .comma)
}
_ = try consume(state, .close_paren)
try expect(state, .ident)
return_type_id :: try parse_primary(state)
block_id :: try parse_block(state)
while next_is(state, .newline) _ = try consume(state, .newline)
extra_id :: try add_extra(state, u32(return_type_id))
_ = try add_extra(state, u32(param_decls.items.len))
for (param_decls.items) |param| _ = try add_extra(state, u32(param))
return try add_node(state, Node{
kind = .func_decl,
main_token = ident_tok_id,
data0 = NodeData{ node_id = block_id },
data1 = NodeData{ extra_id = extra_id },
})
}
parse_block proc(state @mut State) NodeId ! ParseError {
stmt_ids std.ArrayList(NodeId) := arraylist.init(state.nodes.allocator)
defer arraylist.deinit(&stmt_ids)
_ = try consume(state, .open_curly)
while next_is(state, .newline) _ = try consume(state, .newline)
while !next_is(state, .close_curly) {
stmt_id :: try parse_stmt(state)
try arraylist.append(&stmt_ids, stmt_id)
while next_is(state, .newline) _ = try consume(state, .newline)
}
_ = try consume(state, .close_curly)
block_start :: ast.extra_id(state.extra.items.len)
for (stmt_ids.items) |id| _ = try add_extra(state, u32(id))
block_end :: ast.extra_id(state.extra.items.len)
return try add_node(state, Node{
kind = .block,
data0 = NodeData{ extra_id = block_start },
data1 = NodeData{ extra_id = block_end },
})
}
parse_stmt proc(state @mut State) NodeId ! ParseError {
if next_is(state, .ident) return try parse_stmt_decl(state)
return .unexpected_token
}
parse_stmt_decl proc(state @mut State) NodeId ! ParseError {
# expect identifier
ident_tok_id :: try consume(state, .ident)
# check for type
# todo: make a parse_type proc for this
type_id :: if (next_is(state, .ident))
try parse_primary(state)
else
NO_NODE
# expect `::` or `:=` (immutable or mutable assignment)
mutability_tok_id :: try consume_either(state, &[.double_colon, .colon_equal])
# expect expression
expr_id :: try parse_expr(state)
# expect statement terminator (newline)
while (next_is(state, .newline)) _ = try consume(state, .newline)
extra_start_id :: try add_extra(state, u32(type_id))
_ = try add_extra(state, u32(expr_id))
return try add_node(state, Node{
kind = .stmt_decl,
main_token = ident_tok_id,
data0 = NodeData{ extra_id = extra_start_id },
data1 = NodeData{ token_id = mutability_tok_id },
})
}
parse_expr proc(state @mut State) NodeId ! mem.AllocError {
return try parse_expr_bp(state, BindingPowerScalar(0))
}
parse_expr_bp proc(state @mut State, min_bp BindingPowerScalar) NodeId ! mem.AllocError {
# parse left-hand side
kind := state.tokens[state.next_token].kind
lhs := if (prefix_binding_power(kind)) |right_bp| {
operator :: token_id(state.next_token)
state.next_token += 1
operand :: try parse_expr_bp(state, right_bp)
yield try add_node(state, Node{
kind = .expr_unary,
main_token = operator,
data0 = NodeData{ node_id = operand },
})
} else {
yield try parse_primary(state)
}
# parse right-hand side
while true {
kind = state.tokens[state.next_token].kind
# try postfix (higher precedence)
if postfix_binding_power(kind) |left_bp| {
if (left_bp < min_bp) break
lhs = match state.tokens[state.next_token].kind {
.open_paren: debug.unimplemented() # todo: parse call
.open_bracket: debug.unimplemented() # todo: parse index
else: unreachable
}
continue
}
# try infex (lower precedence)
if infix_binding_power(kind) |bp| {
if (bp.left < min_bp) break
operator :: token_id(state.next_token)
state.next_token += 1
rhs :: try parse_expr_bp(state, bp.right)
lhs = try add_node(state, Node{
kind = .expr_binary,
main_token = operator,
data0 = NodeData{ node_id = lhs },
data1 = NodeData{ node_id = rhs },
})
continue
}
break
}
return lhs
}
parse_primary proc(state @mut State) NodeId ! mem.AllocError {
start_idx :: state.next_token
start_tok_id :: TokenId(start_idx)
state.next_token += 1
return match state.tokens[start_idx].kind {
.int: try add_node(state, Node{
kind = .literal_int,
main_token = start_tok_id,
})
.float: try add_node(state, Node{
kind = .literal_float,
main_token = start_tok_id,
})
.string: try add_node(state, Node{
kind = .literal_string,
main_token = start_tok_id,
})
.ident: try add_node(state, Node{
kind = .expr_identifier,
main_token = start_tok_id,
})
else: try add_node(state, Node{
kind = .invalid,
main_token = start_tok_id,
})
}
}
@hide
next_is proc(state @mut State, token_kind TokenKind) bool {
return state.tokens[state.next_token].kind == token_kind
}
@hide
next_is_either proc(state @mut State, token_kinds []TokenKind) bool {
for (token_kinds) |kind| if next_is(state, kind) return true
return false
}
@hide
next_n_is proc(state @mut State, n usize, token_kind TokenKind) bool {
return state.tokens[state.next_token + n].kind == token_kind
}
@hide
expect proc(state @mut State, token_kind TokenKind) void ! ErrorCode {
if (state.tokens[state.next_token].kind != token_kind) return .unexpected_token
}
@hide
expect_either proc(state @mut State, token_kinds []TokenKind) void ! ErrorCode {
for (token_kinds) |kind| if (state.tokens[state.next_token].kind == kind) return
return .unexpected_token
}
@hide
consume proc(state @mut State, token_kind TokenKind) TokenId ! ErrorCode {
try expect(state, token_kind)
id :: TokenId(state.next_token)
state.next_token += 1
return id
}
@hide
consume_either proc(state @mut State, token_kinds []TokenKind) TokenId ! ErrorCode {
try expect_either(state, token_kinds)
id :: TokenId(state.next_token)
state.next_token += 1
return id
}
@hide
add_node proc(state @mut State, node Node) NodeId ! mem.AllocError {
id :: ast.node_id(state.nodes.items.len)
try arraylist.append(&state.nodes, node)
return id
}
@hide
add_extra proc(state @mut State, data u32) ExtraId ! mem.AllocError {
id :: ast.extra_id(state.extra.items.len)
try arraylist.append(&state.extra, data)
return id
}
+202
View File
@@ -0,0 +1,202 @@
import "@std/mem"
import "@std/testing"
import "@source/strpool"
import "@source/lexer"
import "@source/ast"
ast_renderer :: import "@source/ast/renderer"
handles_package_declarations test {
strpool.STRINGS = strpool.init(mem.c_allocator)
defer strpool.deinit(&strpool.STRINGS)
source ::
`first int :: 42
`second float :: 2.15 + 3
`message :: "hello"
scan_state := lexer.init(mem.c_allocator)
defer lexer.deinit(&scan_state)
try lexer.scan(&scan_state, source)
parse_state := init(mem.c_allocator)
defer deinit(&parse_state)
root :: try parse(&parse_state, scan_state.tokens.items)
renderer := ast_renderer.init(mem.c_allocator)
defer ast_renderer.deinit(&renderer)
try ast_renderer.render_token_stream(&renderer, root, &parse_state, scan_state.tokens.items)
try testing.expect_equal(scan_state.tokens.items.len, renderer.tokens.items.len)
for scan_state.tokens.items |expected, i| {
actual :: renderer.tokens.items[i]
try testing.expect_equal(expected.kind, actual.kind)
try testing.expect_equal(expected.start, actual.start)
try testing.expect_equal(expected.str_id, actual.str_id)
}
}
handles_unary_arithmetic_prefix_expression test {
strpool.STRINGS = strpool.init(mem.c_allocator)
defer strpool.deinit(&strpool.STRINGS)
source ::
`first :: -42 + 1
scan_state := lexer.init(mem.c_allocator)
defer lexer.deinit(&scan_state)
try lexer.scan(&scan_state, source)
parse_state := init(mem.c_allocator)
defer deinit(&parse_state)
root :: try parse(&parse_state, scan_state.tokens.items)
renderer := ast_renderer.init(mem.c_allocator)
defer ast_renderer.deinit(&renderer)
try ast_renderer.render_token_stream(&renderer, root, &parse_state, scan_state.tokens.items)
try testing.expect_equal(scan_state.tokens.items.len, renderer.tokens.items.len)
for scan_state.tokens.items |expected, i| {
actual :: renderer.tokens.items[i]
try testing.expect_equal(expected.kind, actual.kind)
try testing.expect_equal(expected.start, actual.start)
try testing.expect_equal(expected.str_id, actual.str_id)
}
}
handles_binary_arithmetic_expression test {
strpool.STRINGS = strpool.init(mem.c_allocator)
defer strpool.deinit(&strpool.STRINGS)
source ::
`first int :: 42 + 3
`second :: 134 + 5
`third float :: 1.35 + 2
`fourth :: first + second
scan_state := lexer.init(mem.c_allocator)
defer lexer.deinit(&scan_state)
try lexer.scan(&scan_state, source)
parse_state := init(mem.c_allocator)
defer deinit(&parse_state)
root :: try parse(&parse_state, scan_state.tokens.items)
renderer := ast_renderer.init(mem.c_allocator)
defer ast_renderer.deinit(&renderer)
try ast_renderer.render_token_stream(&renderer, root, &parse_state, scan_state.tokens.items)
try testing.expect_equal(scan_state.tokens.items.len, renderer.tokens.items.len)
for scan_state.tokens.items |expected, i| {
actual :: renderer.tokens.items[i]
try testing.expect_equal(expected.kind, actual.kind)
try testing.expect_equal(expected.start, actual.start)
try testing.expect_equal(expected.str_id, actual.str_id)
}
}
handles_statement_declaration test {
strpool.STRINGS = strpool.init(mem.c_allocator)
defer strpool.deinit(&strpool.STRINGS)
source ::
`value int :: 42
scan_state := lexer.init(mem.c_allocator)
defer lexer.deinit(&scan_state)
try lexer.scan(&scan_state, source)
parse_state := init(mem.c_allocator)
defer deinit(&parse_state)
root :: try parse(&parse_state, scan_state.tokens.items)
renderer := ast_renderer.init(mem.c_allocator)
defer ast_renderer.deinit(&renderer)
try ast_renderer.render_token_stream(&renderer, root, &parse_state, scan_state.tokens.items)
try testing.expect_equal(scan_state.tokens.items.len, renderer.tokens.items.len)
for scan_state.tokens.items |expected, i| {
actual :: renderer.tokens.items[i]
try testing.expect_equal(expected.kind, actual.kind)
try testing.expect_equal(expected.start, actual.start)
try testing.expect_equal(expected.str_id, actual.str_id)
}
}
handles_function_declarations test {
strpool.STRINGS = strpool.init(mem.c_allocator)
defer strpool.deinit(&strpool.STRINGS)
source ::
`empty proc() void {}
`grouped proc(a, b T, c U) void {
` value int :: 1
`}
scan_state := lexer.init(mem.c_allocator)
defer lexer.deinit(&scan_state)
try lexer.scan(&scan_state, source)
parse_state := init(mem.c_allocator)
defer deinit(&parse_state)
root_id :: try parse(&parse_state, scan_state.tokens.items)
root :: parse_state.nodes.items[usize(root_id)]
package_start :: usize(root.data0.extra_id)
package_end :: usize(root.data1.extra_id)
try testing.expect_equal(usize(2), package_end - package_start)
empty_id :: ast.node_id(parse_state.extra.items[package_start])
empty :: parse_state.nodes.items[usize(empty_id)]
empty_payload :: usize(empty.data1.extra_id)
empty_block :: parse_state.nodes.items[usize(empty.data0.node_id)]
try testing.expect_equal(ast.NodeKind.func_decl, empty.kind)
try testing.expect_equal(u32(0), parse_state.extra.items[empty_payload + 1])
try testing.expect_equal(
usize(empty_block.data0.extra_id),
usize(empty_block.data1.extra_id),
)
grouped_id :: ast.node_id(parse_state.extra.items[package_start + 1])
grouped :: parse_state.nodes.items[usize(grouped_id)]
grouped_payload :: usize(grouped.data1.extra_id)
try testing.expect_equal(ast.NodeKind.func_decl, grouped.kind)
try testing.expect_equal(u32(3), parse_state.extra.items[grouped_payload + 1])
param_a_id :: ast.node_id(parse_state.extra.items[grouped_payload + 2])
param_b_id :: ast.node_id(parse_state.extra.items[grouped_payload + 3])
param_c_id :: ast.node_id(parse_state.extra.items[grouped_payload + 4])
param_a :: parse_state.nodes.items[usize(param_a_id)]
param_b :: parse_state.nodes.items[usize(param_b_id)]
param_c :: parse_state.nodes.items[usize(param_c_id)]
try testing.expect_equal(ast.NodeKind.param_decl, param_a.kind)
try testing.expect_equal(ast.NodeKind.param_decl, param_b.kind)
try testing.expect_equal(ast.NodeKind.param_decl, param_c.kind)
try testing.expect(param_a.data0.node_id == param_b.data0.node_id)
try testing.expect(param_a.data0.node_id != param_c.data0.node_id)
grouped_block :: parse_state.nodes.items[usize(grouped.data0.node_id)]
try testing.expect_equal(ast.NodeKind.block, grouped_block.kind)
try testing.expect_equal(
usize(1),
usize(grouped_block.data1.extra_id) - usize(grouped_block.data0.extra_id),
)
renderer := ast_renderer.init(mem.c_allocator)
defer ast_renderer.deinit(&renderer)
try ast_renderer.render_token_stream(
&renderer,
root_id,
&parse_state,
scan_state.tokens.items,
)
try testing.expect_equal(scan_state.tokens.items.len, renderer.tokens.items.len)
for scan_state.tokens.items |expected, i| {
actual :: renderer.tokens.items[i]
try testing.expect_equal(expected.kind, actual.kind)
try testing.expect_equal(expected.start, actual.start)
try testing.expect_equal(expected.str_id, actual.str_id)
}
}
+66
View File
@@ -0,0 +1,66 @@
import "@std/mem"
import "@std/arraylist"
import "@std/hashmap"
InternError :: enum { out_of_space }
StringId :: alias u32
NO_STR :: maxval!(StringId)
# cross-cutting concern, hence global singleton (owned by main.hon)
STRINGS StringPool := undefined
StringPool :: struct {
ids hashmap.StringHashMap(StringId) # string → id
strings arraylist.ArrayList([]u8) # id → owned string
allocator mem.Allocator
}
init proc(allocator mem.Allocator) StringPool {
return StringPool{
ids = hashmap.init(allocator),
strings = arraylist.init(allocator),
allocator = allocator,
}
}
deinit proc(pool @mut StringPool) void {
hashmap.deinit(&pool.ids)
for pool.strings.items |str| {
mem.free(pool.allocator, str)
}
arraylist.deinit(&pool.strings)
}
intern proc(pool @mut StringPool, str []u8) StringId ! (mem.AllocError | InternError) {
if hashmap.get(&pool.ids, str) |id| return id
if (pool.strings.items.len >= usize(NO_STR)) return .out_of_space
id StringId :: StringId(pool.strings.items.len)
owned_str []mut u8 :: try mem.alloc(u8, pool.allocator, str.len)
errdefer mem.free(pool.allocator, owned_str)
memcopy!(owned_str, str)
try arraylist.append(&pool.strings, owned_str)
errdefer _ = arraylist.pop(&pool.strings)
hashmap.put(&pool.ids, owned_str, id) catch |err| {
match err {
.key_exists: unreachable
else: return err
}
}
return id
}
get_str proc(pool @StringPool, id StringId) ?[]u8 {
if (usize(id) >= pool.strings.items.len) return null
return pool.strings.items[usize(id)]
}
get_id proc(pool @StringPool, str []u8) ?StringId {
return hashmap.get(&pool.ids, str)
}
+24
View File
@@ -0,0 +1,24 @@
import "@std/mem"
import "@std/testing"
handles_interning test {
pool StringPool := init(mem.c_allocator)
defer deinit(&pool)
input [5]mut u8 := ['h', 'e', 'l', 'l', 'o']
id :: try intern(&pool, input[..])
duplicate :: try intern(&pool, "hello")
input[0] = 'j'
same_id :: get_id(&pool, "hello")
mutated_id :: get_id(&pool, input[..])
hello_str :: get_str(&pool, id)
invalid_str :: get_str(&pool, id + 1)
try testing.expect_equal(0, id)
try testing.expect_equal(id, duplicate)
try testing.expect_equal(id, same_id?)
try testing.expect_equal(null, mutated_id)
try testing.expect_equal(null, invalid_str)
try testing.expect_equal("hello", hello_str?)
}
-24
View File
@@ -1,24 +0,0 @@
TokenKind :: enum {
ident
int
float
string
equal
double_colon
left_paren
right_paren
left_curly
right_curly
newline
invalid
eof
}
Token :: struct {
kind TokenKind
start int
}
+59 -52
View File
@@ -1,67 +1,74 @@
import "@std/mem"
ArrayList func($T type) type {
return struct {
items []mut T
capacity usize
allocator mem.Allocator
}
ArrayList proc($T type) type {
return struct {
items []mut T
capacity usize
allocator mem.Allocator
}
}
init func($T type, allocator mem.Allocator) ArrayList(T) {
return ArrayList(T) {
items = mem.empty(T),
capacity = 0,
allocator = allocator,
}
init proc($T type, allocator mem.Allocator) ArrayList(T) {
return ArrayList(T) {
items = mem.empty(T),
capacity = 0,
allocator = allocator,
}
}
deinit func($T type, list @mut ArrayList(T)) void {
allocation []mut T :: list.items.ptr[..list.capacity]
mem.free(list.allocator, allocation)
list.items = mem.empty(T)
list.capacity = 0
deinit proc($T type, list @mut ArrayList(T)) void {
allocation :: list.items.ptr[..list.capacity]
mem.free(list.allocator, allocation)
list.items = mem.empty(T)
list.capacity = 0
}
reserve func($T type, list @mut ArrayList(T), min_capacity usize) void ! mem.AllocError {
if min_capacity <= list.capacity {
return
}
reserve proc($T type, list @mut ArrayList(T), min_capacity usize) void ! mem.AllocError {
if min_capacity <= list.capacity {
return
}
new_capacity usize = 8
if list.capacity >= 8 {
half usize :: divtrunc!(list.capacity, 2)
if list.capacity > maxval!(usize) - half {
new_capacity = min_capacity
} else {
new_capacity = list.capacity + half
}
}
if new_capacity < min_capacity {
new_capacity = min_capacity
}
new_capacity := 8
if list.capacity >= 8 {
half :: divtrunc!(list.capacity, 2)
if list.capacity > maxval!(usize) - half {
new_capacity = min_capacity
} else {
new_capacity = list.capacity + half
}
}
if new_capacity < min_capacity {
new_capacity = min_capacity
}
length usize :: list.items.len
allocation []mut T :: list.items.ptr[..list.capacity]
grown []mut T :: mem.realloc(list.allocator, allocation, new_capacity) catch |_| {
return .out_of_memory
}
list.items = grown.ptr[..length]
list.capacity = new_capacity
return
length usize :: list.items.len
allocation []mut T :: list.items.ptr[..list.capacity]
grown []mut T :: mem.realloc(list.allocator, allocation, new_capacity) catch |_| {
return .out_of_memory
}
list.items = grown.ptr[..length]
list.capacity = new_capacity
return
}
append func($T type, list @mut ArrayList(T), value T) void ! mem.AllocError {
length usize :: list.items.len
if length == maxval!(usize) {
return .out_of_memory
}
try reserve(list, length + 1)
list.items = list.items.ptr[..length + 1]
list.items[length] = value
return
append proc($T type, list @mut ArrayList(T), value T) void ! mem.AllocError {
length usize :: list.items.len
if length == maxval!(usize) {
return .out_of_memory
}
try reserve(list, length + 1)
list.items = list.items.ptr[..length + 1]
list.items[length] = value
return
}
clear func($T type, list @mut ArrayList(T)) void {
list.items = list.items.ptr[..0]
pop proc($T type, list @mut ArrayList(T)) ?T {
if (list.items.len == 0) return null
value :: list.items[list.items.len - 1]
list.items = list.items.ptr[..list.items.len - 1]
return value
}
clear proc($T type, list @mut ArrayList(T)) void {
list.items = list.items.ptr[..0]
}
+15 -15
View File
@@ -2,31 +2,31 @@ import "@std/mem"
import "@std/testing"
handles_append test {
list ArrayList(i32) = init(mem.c_allocator)
defer deinit(&list)
list ArrayList(i32) := init(mem.c_allocator)
defer deinit(&list)
try append(&list, 42)
try append(&list, 42)
try testing.expect_equal(1, list.items.len)
try testing.expect_equal(42, list.items[0])
try testing.expect_equal(1, list.items.len)
try testing.expect_equal(42, list.items[0])
}
handles_clear test {
list ArrayList(i32) = init(mem.c_allocator)
defer deinit(&list)
list ArrayList(i32) := init(mem.c_allocator)
defer deinit(&list)
try append(&list, 42)
clear(&list)
try append(&list, 42)
clear(&list)
try testing.expect_equal(0, list.items.len)
try testing.expect_equal(0, list.items.len)
}
handles_reserve test {
list ArrayList(i32) = init(mem.c_allocator)
defer deinit(&list)
list ArrayList(i32) := init(mem.c_allocator)
defer deinit(&list)
try reserve(&list, 10)
try reserve(&list, 10)
try testing.expect_equal(10, list.capacity)
try testing.expect_equal(0, list.items.len)
try testing.expect_equal(10, list.capacity)
try testing.expect_equal(0, list.items.len)
}
+7 -7
View File
@@ -7,11 +7,11 @@
# Declarative and literal-only: one executable per build. List fields take an
# address-of an array literal (`&["raylib"]`) and default to empty.
BuildConfig :: struct {
name []u8 # output executable name under root/build
source []u8 # program package directory, relative to build.bro
libraries [][]u8 = &[] # library names to link (-l)
lib_paths [][]u8 = &[] # library search directories (-L)
includes [][]u8 = &[] # C include directories (-I)
defines [][]u8 = &[] # C preprocessor defines (name or name=value)
links [][]u8 = &[] # extra linker inputs (object/source files, -framework pairs)
name []u8 # output executable name under root/build
source []u8 # program package directory, relative to build.bro
libraries [][]u8 = &[] # library names to link (-l)
lib_paths [][]u8 = &[] # library search directories (-L)
includes [][]u8 = &[] # C include directories (-I)
defines [][]u8 = &[] # C preprocessor defines (name or name=value)
links [][]u8 = &[] # extra linker inputs (object/source files, -framework pairs)
}
+33 -22
View File
@@ -1,28 +1,39 @@
import "@ffi/c"
import "@std/io"
print func($format []u8, $Args type, args Args) void {
writer io.Writer :: io.Writer{
context = none,
handle = io.Handle{ file_desc = c_int(io.Stream.stderr) },
write = write,
}
io.print(writer, format, Args, args) catch |_| {}
assert proc(ok bool) void {
if (!ok) unreachable
}
hide write func(_ ?@mut anyopaque, handle io.Handle, bytes []u8) usize ! io.WriteError {
request usize = bytes.len
maximum usize :: usize(maxval!(c_long))
if request > maximum {
request = maximum
}
while true {
count c_long :: c.write(handle.file_desc, bytes.ptr, c_ulong(request))
if count >= 0 {
return usize(count)
}
if c.__error()?^ != c.EINTR {
return .write_failed
}
}
unimplemented proc() noreturn {
print("not yet implemented\n", {})
c.abort()
unreachable
}
print proc($format []u8, $Args type, args Args) void {
writer :: io.Writer{
context = null,
handle = io.Handle{ file_desc = c_int(io.Stream.stderr) },
write = write,
}
io.print(writer, format, Args, args) catch |_| {}
}
@hide
write proc(_ ?@mut anyopaque, handle io.Handle, bytes []u8) usize ! io.WriteError {
request := bytes.len
maximum :: usize(maxval!(c_long))
if request > maximum {
request = maximum
}
while true {
count :: c.write(handle.file_desc, bytes.ptr, c_ulong(request))
if count >= 0 {
return usize(count)
}
if c.__error()?^ != c.EINTR {
return .write_failed
}
}
}
+45
View File
@@ -0,0 +1,45 @@
import "@std/meta"
EnumMap proc($E, $V type) type {
match typeinfo!(E) {
.enum |info|: return struct {
present [info.fields.len]mut bool # fixme: replace with bitset
values [info.fields.len]mut V
}
else: compile_error!("EnumMap key must be an enum")
}
}
init proc(
$E, $V type,
values meta.EnumFieldStruct(E, ?V, some!(null)),
) EnumMap(E, V) {
map EnumMap(E, V) := undefined
match typeinfo!(E) {
.enum |info|: inline for info.fields |field, i| {
map.present[i] = false
if field!(values, field.name) |value| {
map.present[i] = true
map.values[i] = value
}
}
else: compile_error!("EnumMap key must be an enum")
}
return map
}
get proc($E, $V type, map @EnumMap(E, V), key E) ?V {
# fixme: linear lookup; implement an enum index/discriminant map for O(1) lookup
match typeinfo!(E) {
.enum |info|: inline for info.fields |field, i| {
if key == field!(E, field.name) {
if (map.present[i]) return map.values[i]
return null
}
}
else: compile_error!("EnumMap key must be an enum")
}
}
+25
View File
@@ -0,0 +1,25 @@
import "@std/mem"
import "@std/testing"
TestEnum :: enum(u8) {
ident = 3
int = 8
eof = 21
}
handles_sparse_enum_get test {
names EnumMap(TestEnum, []u8) := init({
ident = "identifier",
int = "integer",
})
ident :: get(&names, TestEnum.ident)
try testing.expect_type(?[]u8, ident)
try testing.expect(mem.eql("identifier", ident?))
eof :: get(&names, TestEnum.eof)
try testing.expect_type(?[]u8, eof)
try testing.expect_equal(null, eof)
}
View File
+116 -115
View File
@@ -2,157 +2,158 @@ import "@std/mem"
PutError :: enum { key_exists }
Entry func($K, $V type) type {
return struct {
# hash = 0 means empty
hash usize = 0
key K
value V
}
Entry proc($K, $V type) type {
return struct {
# hash = 0 means empty
hash usize = 0
key K
value V
}
}
HashMap func(
$K, $V type,
$hash_key func(key K) usize,
$keys_eql func(a, b K) bool,
HashMap proc(
$K, $V type,
$hash_key proc(key K) usize,
$keys_eql proc(a, b K) bool,
) type {
return struct {
entries []mut Entry(K, V)
count usize
allocator mem.Allocator
}
return struct {
entries []mut Entry(K, V)
count usize
allocator mem.Allocator
}
}
StringHashMap func($V type) type {
return HashMap([]u8, V, str_hash, str_eql)
StringHashMap proc($V type) type {
return HashMap([]u8, V, str_hash, str_eql)
}
init func(
$K, $V type,
$hash_key func(key K) usize,
$keys_eql func(a, b K) bool,
allocator mem.Allocator,
init proc(
$K, $V type,
$hash_key proc(key K) usize,
$keys_eql proc(a, b K) bool,
allocator mem.Allocator,
) HashMap(K, V, hash_key, keys_eql) {
return HashMap(K, V, hash_key, keys_eql){
entries = mem.empty(Entry(K, V)),
count = 0,
allocator = allocator,
}
return HashMap(K, V, hash_key, keys_eql){
entries = mem.empty(Entry(K, V)),
count = 0,
allocator = allocator,
}
}
# frees the entries in the hash map and invalidates it.
deinit func(
$K, $V type,
$hash_key func(key K) usize,
$keys_eql func(a, b K) bool,
map @HashMap(K, V, hash_key, keys_eql),
#! free the entries in the hash map.
#! note: this operation invalidates the map.
deinit proc(
$K, $V type,
$hash_key proc(key K) usize,
$keys_eql proc(a, b K) bool,
map @HashMap(K, V, hash_key, keys_eql),
) void { mem.free(map.allocator, map.entries) }
get func(
$K, $V type,
$hash_key func(key K) usize,
$keys_eql func(a, b K) bool,
map @HashMap(K, V, hash_key, keys_eql),
key K,
get proc(
$K, $V type,
$hash_key proc(key K) usize,
$keys_eql proc(a, b K) bool,
map @HashMap(K, V, hash_key, keys_eql),
key K,
) ?V {
if (map.count == 0) return none
if (map.count == 0) return null
hash :: normalize(hash_key(key))
idx usize = hash & (map.entries.len - 1)
hash :: normalize(hash_key(key))
idx := hash & (map.entries.len - 1)
while true {
entry :: map.entries[idx]
if (entry.hash == 0) return none
while true {
entry :: map.entries[idx]
if (entry.hash == 0) return null
if (entry.hash == hash and keys_eql(entry.key, key)) {
return entry.value
}
if (entry.hash == hash and keys_eql(entry.key, key)) {
return entry.value
}
idx = (idx + 1) & (map.entries.len - 1)
}
idx = (idx + 1) & (map.entries.len - 1)
}
}
put func(
$K, $V type,
$hash_key func(key K) usize,
$keys_eql func(a, b K) bool,
map @mut HashMap(K, V, hash_key, keys_eql),
key K,
value V,
put proc(
$K, $V type,
$hash_key proc(key K) usize,
$keys_eql proc(a, b K) bool,
map @mut HashMap(K, V, hash_key, keys_eql),
key K,
value V,
) void ! (PutError | mem.AllocError) {
threshold :: map.entries.len - divtrunc!(map.entries.len, 4)
if (map.entries.len == 0 or map.count + 1 > threshold) {
# grow entries array
old_entries :: map.entries
new_size :: if (old_entries.len > 0) old_entries.len * 2 else 8
new_entries :: try mem.alloc(Entry(K, V), map.allocator, new_size)
# check grow
threshold :: map.entries.len - divtrunc!(map.entries.len, 4)
if (map.entries.len == 0 or map.count + 1 > threshold) {
old_entries :: map.entries
new_size :: if (old_entries.len > 0) old_entries.len * 2 else 8
new_entries :: try mem.alloc(Entry(K, V), map.allocator, new_size)
# zero new entries
for 0..new_entries.len |i| {
new_entries[i].hash = 0
}
# zero new entries
for (0..new_entries.len) |i| new_entries[i].hash = 0
# move old entries
for old_entries |entry| {
if (entry.hash == 0) continue
# move old entries
for old_entries |entry| {
if (entry.hash == 0) continue
# find an empty slot
idx usize = entry.hash & (new_entries.len - 1)
while new_entries[idx].hash != 0 {
idx = (idx + 1) & (new_entries.len - 1)
}
# find an empty slot
idx := entry.hash & (new_entries.len - 1)
while new_entries[idx].hash != 0 {
idx = (idx + 1) & (new_entries.len - 1)
}
new_entries[idx] = entry
}
new_entries[idx] = entry
}
map.entries = new_entries
mem.free(map.allocator, old_entries)
}
map.entries = new_entries
mem.free(map.allocator, old_entries)
}
hash :: normalize(hash_key(key))
idx usize = hash & (map.entries.len - 1)
# put new entry
hash :: normalize(hash_key(key))
idx := hash & (map.entries.len - 1)
while true {
entry :: map.entries[idx]
if entry.hash == 0 {
map.entries[idx] = Entry(K, V){
hash = hash,
key = key,
value = value,
}
map.count += 1
return
}
while true {
entry :: map.entries[idx]
if entry.hash == 0 {
map.entries[idx] = Entry(K, V){
hash = hash,
key = key,
value = value,
}
map.count += 1
return
}
if (entry.hash == hash and keys_eql(entry.key, key)) {
return .key_exists
}
if entry.hash == hash and keys_eql(entry.key, key) {
return .key_exists
}
idx = (idx + 1) & (map.entries.len - 1)
}
idx = (idx + 1) & (map.entries.len - 1)
}
}
hide normalize func(hash usize) usize {
# mapping both 0 and 1 to 1 is safe because equality resolves
# collisions (since hash and key must both be equal).
if (hash == 0) return 1
return hash
@hide
normalize proc(hash usize) usize {
# mapping both 0 and 1 to 1 is safe because equality resolves
# collisions (since hash and key must both be equal).
if (hash == 0) return 1
return hash
}
#! FNV-1a hash implementation.
#! note: vulnerable to collision attacks.
hide str_hash func(key []u8) usize {
hash u32 = 2166136261 # offset basis
prime u32 = 16777619
@hide
str_hash proc(key []u8) usize {
hash u32 := 2166136261 # offset basis
prime u32 := 16777619
for key |byte| {
product u64 :: u64(hash xor u32(byte)) * prime
hash = u32(product & u64(maxval!(u32)))
}
for key |byte| {
product u64 :: u64(hash xor u32(byte)) * prime
hash = u32(product & u64(maxval!(u32)))
}
return usize(hash)
return usize(hash)
}
hide str_eql func(a, b []u8) bool {
return mem.eql(a, b)
}
@hide
str_eql proc(a, b []u8) bool { return mem.eql(a, b) }
+6 -6
View File
@@ -2,12 +2,12 @@ import "@std/mem"
import "@std/testing"
handles_put_and_get test {
map StringHashMap(u32) = init(mem.c_allocator)
defer deinit(&map)
map StringHashMap(u32) := init(mem.c_allocator)
defer deinit(&map)
try put(&map, "key", 42)
value :: get(&map, "key")
try put(&map, "key", 42)
value :: get(&map, "key")
try testing.expect_type(?u32, value)
try testing.expect_equal(42, value?)
try testing.expect_type(?u32, value)
try testing.expect_equal(42, value?)
}
+85 -105
View File
@@ -1,144 +1,124 @@
import "@ffi/c"
File :: struct {
io Io
handle Handle
io Io
handle Handle
}
FileMode :: enum {
read_only
write_only
read_write
read_only
write_only
read_write
}
OpenError :: enum {
open_failed
open_failed
}
CloseError :: enum {
close_failed
close_failed
}
open func(io Io, path [;0]u8, mode FileMode) File ! OpenError {
handle Handle :: io.vtable.open(io.context, path, mode) catch |err| {
return err
}
return File {io = io, handle = handle}
open proc(io Io, path [;0]u8, mode FileMode) File ! OpenError {
handle Handle :: try io.vtable.open(io.context, path, mode)
return File{ io = io, handle = handle }
}
close func(file File) void ! CloseError {
try file.io.vtable.close(file.io.context, file.handle)
close proc(file File) void ! CloseError {
try file.io.vtable.close(file.io.context, file.handle)
}
reader func(file File) Reader {
return Reader {
context = file.io.context,
handle = file.handle,
read = file.io.vtable.read,
}
reader proc(file File) Reader {
return Reader {
context = file.io.context,
handle = file.handle,
read = file.io.vtable.read,
}
}
writer func(file File) Writer {
return Writer {
context = file.io.context,
handle = file.handle,
write = file.io.vtable.write,
}
writer proc(file File) Writer {
return Writer {
context = file.io.context,
handle = file.handle,
write = file.io.vtable.write,
}
}
hide system_read func(_ ?@mut anyopaque, handle Handle, buffer []mut u8) usize ! ReadError {
request usize = buffer.len
maximum usize :: usize(maxval!(c_long))
if request > maximum {
request = maximum
}
while true {
count c_long :: c.read(handle.file_desc, buffer.ptr, c_ulong(request))
if count >= 0 {
return usize(count)
}
errno c_int :: c.__error()?^
if errno == c.EINTR {
continue
}
if errno == c.EBADF {
return .not_open_for_reading
}
return .read_failed
}
@hide system_read proc(_ ?@mut anyopaque, handle Handle, buffer []mut u8) usize ! ReadError {
request := buffer.len
maximum usize :: usize(maxval!(c_long))
if (request > maximum) request = maximum
while true {
count c_long :: c.read(handle.file_desc, buffer.ptr, c_ulong(request))
if (count >= 0) return usize(count)
errno c_int :: c.__error()?^
if (errno == c.EINTR) continue
if (errno == c.EBADF) return .not_open_for_reading
return .read_failed
}
}
hide system_write func(_ ?@mut anyopaque, handle Handle, bytes []u8) usize ! WriteError {
request usize = bytes.len
maximum usize :: usize(maxval!(c_long))
if request > maximum {
request = maximum
}
while true {
count c_long :: c.write(handle.file_desc, bytes.ptr, c_ulong(request))
if count >= 0 {
return usize(count)
}
errno c_int :: c.__error()?^
if errno == c.EINTR {
continue
}
if errno == c.EBADF {
return .not_open_for_writing
}
return .write_failed
}
@hide system_write proc(_ ?@mut anyopaque, handle Handle, bytes []u8) usize ! WriteError {
request := bytes.len
maximum usize :: usize(maxval!(c_long))
if (request > maximum) request = maximum
while true {
count c_long :: c.write(handle.file_desc, bytes.ptr, c_ulong(request))
if (count >= 0) return usize(count)
errno c_int :: c.__error()?^
if (errno == c.EINTR) continue
if (errno == c.EBADF) return .not_open_for_writing
return .write_failed
}
}
hide system_open func(_ ?@mut anyopaque, path [;0]u8, mode FileMode) Handle ! OpenError {
flags c_int = c.O_RDONLY
match mode {
.read_only: flags = c.O_RDONLY
.write_only: flags = c.O_WRONLY
.read_write: flags = c.O_RDWR
}
while true {
fd c_int :: c.open(ptrcast!(c_char, path.ptr), flags)
if fd >= 0 {
return Handle {file_desc = fd}
}
if c.__error()?^ != c.EINTR {
return .open_failed
}
}
@hide system_open proc(_ ?@mut anyopaque, path [;0]u8, mode FileMode) Handle ! OpenError {
flags := c.O_RDONLY
match mode {
.read_only: flags = c.O_RDONLY
.write_only: flags = c.O_WRONLY
.read_write: flags = c.O_RDWR
}
while true {
fd c_int :: c.open(ptrcast!(c_char, path.ptr), flags)
if (fd >= 0) return Handle{ file_desc = fd }
if (c.__error()?^ != c.EINTR) return .open_failed
}
}
hide system_close func(_ ?@mut anyopaque, handle Handle) void ! CloseError {
if c.close(handle.file_desc) != 0 {
return .close_failed
}
@hide system_close proc(_ ?@mut anyopaque, handle Handle) void ! CloseError {
if (c.close(handle.file_desc) != 0) return .close_failed
}
hide system_stdin func(_ ?@mut anyopaque) Handle {
return Handle {file_desc = c_int(Stream.stdin)}
@hide system_stdin proc(_ ?@mut anyopaque) Handle {
return Handle{ file_desc = c_int(Stream.stdin) }
}
hide system_stdout func(_ ?@mut anyopaque) Handle {
return Handle {file_desc = c_int(Stream.stdout)}
@hide system_stdout proc(_ ?@mut anyopaque) Handle {
return Handle{ file_desc = c_int(Stream.stdout) }
}
hide system_stderr func(_ ?@mut anyopaque) Handle {
return Handle {file_desc = c_int(Stream.stderr)}
@hide system_stderr proc(_ ?@mut anyopaque) Handle {
return Handle{ file_desc = c_int(Stream.stderr) }
}
hide system_vtable IoVTable :: IoVTable {
read = system_read,
write = system_write,
open = system_open,
close = system_close,
stdin = system_stdin,
stdout = system_stdout,
stderr = system_stderr,
@hide system_vtable IoVTable :: IoVTable {
read = system_read,
write = system_write,
open = system_open,
close = system_close,
stdin = system_stdin,
stdout = system_stdout,
stderr = system_stderr,
}
hide system func() Io {
return Io {
context = none,
vtable = &system_vtable,
}
@hide system proc() Io {
return Io {
context = null,
vtable = &system_vtable,
}
}
+396 -343
View File
@@ -2,406 +2,459 @@ import "@ffi/c"
import "@std/meta"
ReadError :: enum {
not_open_for_reading
read_failed
not_open_for_reading
read_failed
}
WriteError :: enum {
not_open_for_writing
write_failed
no_progress
not_open_for_writing
write_failed
no_progress
}
Handle :: union {
file_desc c_int
ptr @mut anyopaque
file_desc c_int
ptr @mut anyopaque
}
Stream :: enum(c_int) {
stdin = c.STDIN_FILENO
stdout = c.STDOUT_FILENO
stderr = c.STDERR_FILENO
stdin = c.STDIN_FILENO
stdout = c.STDOUT_FILENO
stderr = c.STDERR_FILENO
}
Io :: struct {
context ?@mut anyopaque
vtable @IoVTable
context ?@mut anyopaque
vtable @IoVTable
}
IoVTable :: struct {
read @func(context ?@mut anyopaque, handle Handle, buffer []mut u8) usize ! ReadError
write @func(context ?@mut anyopaque, handle Handle, bytes []u8) usize ! WriteError
open @func(context ?@mut anyopaque, path [;0]u8, mode FileMode) Handle ! OpenError
close @func(context ?@mut anyopaque, handle Handle) void ! CloseError
stdin @func(context ?@mut anyopaque) Handle
stdout @func(context ?@mut anyopaque) Handle
stderr @func(context ?@mut anyopaque) Handle
read @proc(context ?@mut anyopaque, handle Handle, buffer []mut u8) usize ! ReadError
write @proc(context ?@mut anyopaque, handle Handle, bytes []u8) usize ! WriteError
open @proc(context ?@mut anyopaque, path [;0]u8, mode FileMode) Handle ! OpenError
close @proc(context ?@mut anyopaque, handle Handle) void ! CloseError
stdin @proc(context ?@mut anyopaque) Handle
stdout @proc(context ?@mut anyopaque) Handle
stderr @proc(context ?@mut anyopaque) Handle
}
Reader :: struct {
context ?@mut anyopaque
handle Handle
read @func(context ?@mut anyopaque, handle Handle, buffer []mut u8) usize ! ReadError
context ?@mut anyopaque
handle Handle
read @proc(context ?@mut anyopaque, handle Handle, buffer []mut u8) usize ! ReadError
}
Writer :: struct {
context ?@mut anyopaque
handle Handle
write @func(context ?@mut anyopaque, handle Handle, bytes []u8) usize ! WriteError
context ?@mut anyopaque
handle Handle
write @proc(context ?@mut anyopaque, handle Handle, bytes []u8) usize ! WriteError
}
read func(input Reader, buffer []mut u8) usize ! ReadError {
if buffer.len == 0 {
return 0
}
count usize :: try input.read(input.context, input.handle, buffer)
if count > buffer.len {
return .read_failed
}
return count
read proc(input Reader, buffer []mut u8) usize ! ReadError {
if (buffer.len == 0) return 0
count usize :: try input.read(input.context, input.handle, buffer)
if (count > buffer.len) return .read_failed
return count
}
write func(output Writer, bytes []u8) usize ! WriteError {
if bytes.len == 0 {
return 0
}
count usize :: try output.write(output.context, output.handle, bytes)
if count > bytes.len {
return .write_failed
}
return count
write proc(output Writer, bytes []u8) usize ! WriteError {
if (bytes.len == 0) return 0
count usize :: try output.write(output.context, output.handle, bytes)
if (count > bytes.len) return .write_failed
return count
}
write_all func(output Writer, bytes []u8) void ! WriteError {
offset usize = 0
while offset < bytes.len {
count usize :: write(output, bytes[offset..]) catch |err| {
return err
}
if count == 0 {
return .no_progress
}
offset += count
}
return
write_all proc(output Writer, bytes []u8) void ! WriteError {
offset usize := 0
while offset < bytes.len {
count usize :: try write(output, bytes[offset..])
if (count == 0) return .no_progress
offset += count
}
}
stdin func(io Io) Reader {
return Reader {
context = io.context,
handle = io.vtable.stdin(io.context),
read = io.vtable.read,
}
stdin proc(io Io) Reader {
return Reader {
context = io.context,
handle = io.vtable.stdin(io.context),
read = io.vtable.read,
}
}
stdout func(io Io) Writer {
return Writer {
context = io.context,
handle = io.vtable.stdout(io.context),
write = io.vtable.write,
}
stdout proc(io Io) Writer {
return Writer {
context = io.context,
handle = io.vtable.stdout(io.context),
write = io.vtable.write,
}
}
stderr func(io Io) Writer {
return Writer {
context = io.context,
handle = io.vtable.stderr(io.context),
write = io.vtable.write,
}
stderr proc(io Io) Writer {
return Writer {
context = io.context,
handle = io.vtable.stderr(io.context),
write = io.vtable.write,
}
}
print func(output Writer, $format []u8, $Args type, args Args) void ! WriteError {
expand for parse_format(format.len, format, Args) |token| {
match token.kind {
.unused: break
.literal: try write_all(output, format[token.start..token.end])
.string: try write_all(output, field!(args, token.field))
.default: try write_default(output, field!(args, token.field))
.decimal: try write_decimal(output, field!(args, token.field))
.binary: try write_integer(output, field!(args, token.field), 2, false)
.octal: try write_integer(output, field!(args, token.field), 8, false)
.hex_lower: try write_integer(output, field!(args, token.field), 16, false)
.hex_upper: try write_integer(output, field!(args, token.field), 16, true)
.character: try write_character(output, field!(args, token.field))
else: try write_float(output, field!(args, token.field), true)
}
}
print proc(output Writer, $format []u8, $Args type, args Args) void ! WriteError {
inline for parse_format(format.len, format, Args) |token| {
match token.kind {
.unused: break
.literal: try write_all(output, format[token.start..token.end])
.string: try write_all(output, field!(args, token.field))
.default: try write_default(output, field!(args, token.field))
.decimal: try write_decimal(output, field!(args, token.field))
.binary: try write_integer(output, field!(args, token.field), 2, false)
.octal: try write_integer(output, field!(args, token.field), 8, false)
.hex_lower: try write_integer(output, field!(args, token.field), 16, false)
.hex_upper: try write_integer(output, field!(args, token.field), 16, true)
.character: try write_character(output, field!(args, token.field))
else: try write_float(output, field!(args, token.field), true)
}
}
}
hide write_integer_signed func(output Writer, value i64, base u64, uppercase bool) void ! WriteError {
buffer [65]mut u8 = undefined
end usize = buffer.len
current i64 = value
while true {
digit_value i64 :: rem!(current, i64(base))
digit u8 = 0
if digit_value < 0 {
digit = u8(-digit_value)
} else {
digit = u8(digit_value)
}
end -= 1
if digit < 10 {
buffer[end] = '0' + digit
} else if uppercase {
buffer[end] = 'A' + digit - 10
} else {
buffer[end] = 'a' + digit - 10
}
current = divtrunc!(current, i64(base))
if current == 0 {
break
}
}
if value < 0 {
end -= 1
buffer[end] = '-'
}
try write_all(output, buffer[end..])
return
@hide:file
write_integer_signed proc(output Writer, value i64, base u64, uppercase bool) void ! WriteError {
buffer [65]mut u8 := undefined
end := buffer.len
current := value
while true {
digit_value i64 :: rem!(current, i64(base))
digit := if (digit_value < 0)
u8(-digit_value)
else
u8(digit_value)
end -= 1
buffer[end] = if (digit < 10)
'0' + digit
else if (uppercase)
'A' + digit - 10
else
'a' + digit - 10
current = divtrunc!(current, i64(base))
if (current == 0) break
}
if value < 0 {
end -= 1
buffer[end] = '-'
}
try write_all(output, buffer[end..])
}
hide write_integer_unsigned func(output Writer, value u64, base u64, uppercase bool) void ! WriteError {
buffer [65]mut u8 = undefined
end usize = buffer.len
current u64 = value
while true {
digit u8 :: u8(rem!(current, base))
end -= 1
if digit < 10 {
buffer[end] = '0' + digit
} else if uppercase {
buffer[end] = 'A' + digit - 10
} else {
buffer[end] = 'a' + digit - 10
}
current = divtrunc!(current, base)
if current == 0 {
break
}
}
try write_all(output, buffer[end..])
return
@hide:file
write_integer_unsigned proc(output Writer, value u64, base u64, uppercase bool) void ! WriteError {
buffer [65]mut u8 := undefined
end := buffer.len
current := value
while true {
digit u8 :: u8(rem!(current, base))
end -= 1
buffer[end] = if (digit < 10)
'0' + digit
else if (uppercase)
'A' + digit - 10
else
'a' + digit - 10
current = divtrunc!(current, base)
if (current == 0) break
}
try write_all(output, buffer[end..])
}
hide FormatTokenKind :: enum {
unused
literal
default
string
decimal
binary
octal
hex_lower
hex_upper
character
scientific
@hide:file
FormatTokenKind :: enum {
unused
literal
default
string
decimal
binary
octal
hex_lower
hex_upper
character
scientific
}
hide FormatToken :: struct {
kind FormatTokenKind
start usize
end usize
field []u8
@hide:file
FormatToken :: struct {
kind FormatTokenKind
start usize
end usize
field []u8
}
hide parse_format func($N usize, $format []u8, $Args type) [N]mut FormatToken {
tokens [N]mut FormatToken = undefined
for (usize(0))..format.len |index| {
tokens[index] = FormatToken {kind = .unused, start = 0, end = 0, field = ""}
}
field_count usize = 0
match typeinfo!(Args) {
.record |record|: {
if !record.is_tuple {
compile_error!("io.print arguments must be a tuple")
}
field_count = record.fields.len
}
else: compile_error!("io.print arguments must be a tuple")
}
@hide:file
parse_format proc($N usize, $format []u8, $Args type) [N]mut FormatToken {
tokens [N]mut FormatToken := undefined
for (usize(0))..format.len |index| {
tokens[index] = FormatToken{ kind = .unused, start = 0, end = 0, field = "" }
}
token_count usize = 0
argument_count usize = 0
literal_start usize = 0
cursor usize = 0
while cursor < format.len {
byte :: format[cursor]
if byte == '{' {
if cursor + 1 >= format.len {
compile_error!("io.print format has an unmatched '{'")
}
if cursor > literal_start {
tokens[token_count] = FormatToken {kind = .literal, start = literal_start, end = cursor, field = ""}
token_count += 1
}
next :: format[cursor + 1]
if next == '{' {
tokens[token_count] = FormatToken {kind = .literal, start = cursor, end = cursor + 1, field = ""}
token_count += 1
cursor += 2
literal_start = cursor
continue
}
kind FormatTokenKind = .default
width usize = 2
if next != '}' {
if cursor + 2 >= format.len or format[cursor + 2] != '}' {
compile_error!("io.print format expects a one-character specifier")
}
width = 3
if next == 's' {
kind = .string
} else if next == 'd' {
kind = .decimal
} else if next == 'b' {
kind = .binary
} else if next == 'o' {
kind = .octal
} else if next == 'x' {
kind = .hex_lower
} else if next == 'X' {
kind = .hex_upper
} else if next == 'c' {
kind = .character
} else if next == 'e' {
kind = .scientific
} else {
compile_error!("io.print format has an unknown specifier")
}
}
if argument_count >= field_count {
compile_error!("io.print format argument count does not match the tuple")
}
tokens[token_count] = FormatToken {
kind = kind,
start = 0,
end = 0,
field = format_field_name(Args, argument_count),
}
token_count += 1
argument_count += 1
cursor += width
literal_start = cursor
continue
}
if byte == '}' {
if cursor + 1 >= format.len or format[cursor + 1] != '}' {
compile_error!("io.print format has an unmatched '}'")
}
if cursor > literal_start {
tokens[token_count] = FormatToken {kind = .literal, start = literal_start, end = cursor, field = ""}
token_count += 1
}
tokens[token_count] = FormatToken {kind = .literal, start = cursor, end = cursor + 1, field = ""}
token_count += 1
cursor += 2
literal_start = cursor
continue
}
cursor += 1
}
if literal_start < format.len {
tokens[token_count] = FormatToken {kind = .literal, start = literal_start, end = format.len, field = ""}
}
if argument_count != field_count {
compile_error!("io.print format argument count does not match the tuple")
}
return tokens
field_count := 0
match typeinfo!(Args) {
.record |r|: {
if (!r.is_tuple) compile_error!("io.print arguments must be a tuple")
field_count = r.fields.len
}
else: compile_error!("io.print arguments must be a tuple")
}
token_count := 0
argument_count := 0
literal_start := 0
cursor := 0
while cursor < format.len {
byte :: format[cursor]
if byte == '{' {
if cursor + 1 >= format.len {
compile_error!("io.print format has an unmatched '{'")
}
if cursor > literal_start {
tokens[token_count] = FormatToken{
kind = .literal,
start = literal_start,
end = cursor,
field = "",
}
token_count += 1
}
next :: format[cursor + 1]
if next == '{' {
tokens[token_count] = FormatToken{
kind = .literal,
start = cursor,
end = cursor + 1,
field = "",
}
token_count += 1
cursor += 2
literal_start = cursor
continue
}
kind FormatTokenKind := .default
width := 2
if next != '}' {
if cursor + 2 >= format.len or format[cursor + 2] != '}' {
compile_error!("io.print format expects a one-character specifier")
}
width = 3
if (next == 's') kind = .string
else if (next == 'd') kind = .decimal
else if (next == 'b') kind = .binary
else if (next == 'o') kind = .octal
else if (next == 'x') kind = .hex_lower
else if (next == 'X') kind = .hex_upper
else if (next == 'c') kind = .character
else if (next == 'e') kind = .scientific
else compile_error!("io.print format has an unknown specifier")
}
if argument_count >= field_count {
compile_error!("io.print format argument count does not match the tuple")
}
tokens[token_count] = FormatToken{
kind = kind,
start = 0,
end = 0,
field = format_field_name(Args, argument_count),
}
token_count += 1
argument_count += 1
cursor += width
literal_start = cursor
continue
}
if byte == '}' {
if cursor + 1 >= format.len or format[cursor + 1] != '}' {
compile_error!("io.print format has an unmatched '}'")
}
if cursor > literal_start {
tokens[token_count] = FormatToken{
kind = .literal,
start = literal_start,
end = cursor,
field = "",
}
token_count += 1
}
tokens[token_count] = FormatToken{
kind = .literal,
start = cursor,
end = cursor + 1,
field = "",
}
token_count += 1
cursor += 2
literal_start = cursor
continue
}
cursor += 1
}
if literal_start < format.len {
tokens[token_count] = FormatToken{
kind = .literal,
start = literal_start,
end = format.len,
field = "",
}
}
if argument_count != field_count {
compile_error!("io.print format argument count does not match the tuple")
}
return tokens
}
hide format_field_name func($T type, index usize) []u8 {
match typeinfo!(T) {
.record |record|: return record.fields[index].name
else: compile_error!("io.print arguments must be a tuple")
}
@hide:file
format_field_name proc($T type, index usize) []u8 {
match typeinfo!(T) {
.record |r|: return r.fields[index].name
else: compile_error!("io.print arguments must be a tuple")
}
}
hide write_integer func(output Writer, $T type, value T, base u64, uppercase bool) void ! WriteError {
match typeinfo!(T) {
.integer: if minval!(T) < 0 {
try write_integer_signed(output, i64(value), base, uppercase)
} else {
try write_integer_unsigned(output, u64(value), base, uppercase)
}
else: compile_error!("io.print integer format requires an integer argument")
}
return
@hide:file
distinct_value proc($Backing, $Distinct type, value Distinct) Backing {
return ptrcast!(Backing, &value)^
}
@hide:file
scalar_or_distinct_type proc($T type) bool {
match typeinfo!(T) {
.bool: return true
.integer: return true
.float: return true
.distinct: return true
else: return false
}
}
@hide:file
write_integer proc(output Writer, $T type, value T, base u64, uppercase bool) void ! WriteError {
match typeinfo!(T) {
.integer: if minval!(T) < 0 {
try write_integer_signed(output, i64(value), base, uppercase)
} else {
try write_integer_unsigned(output, u64(value), base, uppercase)
}
.distinct |backing|: if scalar_or_distinct_type(backing) {
try write_integer(output, distinct_value(backing, T, value), base, uppercase)
} else {
compile_error!("io.print integer format requires an integer argument")
}
else: compile_error!("io.print integer format requires an integer argument")
}
}
# note: libc keeps float formatting small; replace it with a native shortest-roundtrip writer if locale independence matters.
hide write_float func(output Writer, $T type, value T, scientific bool) void ! WriteError {
match typeinfo!(T) {
.float: {
buffer [64]mut u8 = undefined
count c_int = 0
if sizeof!(T) == 4 {
if scientific {
count = c.snprintf(ptrcast!(c_char, (&buffer).ptr), c_ulong(buffer.len), "%.8e", value)
} else {
count = c.snprintf(ptrcast!(c_char, (&buffer).ptr), c_ulong(buffer.len), "%.9g", value)
}
} else if scientific {
count = c.snprintf(ptrcast!(c_char, (&buffer).ptr), c_ulong(buffer.len), "%.16e", value)
} else {
count = c.snprintf(ptrcast!(c_char, (&buffer).ptr), c_ulong(buffer.len), "%.17g", value)
}
if count < 0 or usize(count) >= buffer.len {
return .write_failed
}
try write_all(output, buffer[0..usize(count)])
}
else: compile_error!("io.print float format requires a float argument")
}
return
@hide:file
write_float proc(output Writer, $T type, value T, scientific bool) void ! WriteError {
match typeinfo!(T) {
.float: {
buffer [64]mut u8 := undefined
count := 0
if sizeof!(T) == 4 {
if (scientific) count = c.snprintf(ptrcast!(c_char, (&buffer).ptr), c_ulong(buffer.len), "%.8e", value)
else count = c.snprintf(ptrcast!(c_char, (&buffer).ptr), c_ulong(buffer.len), "%.9g", value)
} else if scientific {
count = c.snprintf(ptrcast!(c_char, (&buffer).ptr), c_ulong(buffer.len), "%.16e", value)
} else {
count = c.snprintf(ptrcast!(c_char, (&buffer).ptr), c_ulong(buffer.len), "%.17g", value)
}
if (count < 0 or usize(count) >= buffer.len) return .write_failed
try write_all(output, buffer[0..usize(count)])
}
.distinct |backing|: if scalar_or_distinct_type(backing) {
try write_float(output, distinct_value(backing, T, value), scientific)
} else {
compile_error!("io.print float format requires a float argument")
}
else: compile_error!("io.print float format requires a float argument")
}
}
hide write_decimal func(output Writer, $T type, value T) void ! WriteError {
match typeinfo!(T) {
.integer: try write_integer(output, value, 10, false)
.float: try write_float(output, value, false)
else: compile_error!("io.print '{d}' requires an integer or float argument")
}
return
@hide:file
write_decimal proc(output Writer, $T type, value T) void ! WriteError {
match typeinfo!(T) {
.integer: try write_integer(output, value, 10, false)
.float: try write_float(output, value, false)
.distinct |backing|: if scalar_or_distinct_type(backing) {
try write_decimal(output, distinct_value(backing, T, value))
} else {
compile_error!("io.print '{d}' requires an integer or float argument")
}
else: compile_error!("io.print '{d}' requires an integer or float argument")
}
}
hide write_character func(output Writer, $T type, value T) void ! WriteError {
match typeinfo!(T) {
.integer: {
if minval!(T) < 0 or maxval!(T) > 255 {
compile_error!("io.print '{c}' requires an unsigned integer that fits in u8")
}
buffer [1]u8 = [u8(value)]
try write_all(output, buffer[..])
}
else: compile_error!("io.print '{c}' requires an unsigned integer that fits in u8")
}
return
@hide:file
write_character proc(output Writer, $T type, value T) void ! WriteError {
match typeinfo!(T) {
.integer: {
if minval!(T) < 0 or maxval!(T) > 255 {
compile_error!("io.print '{c}' requires an unsigned integer that fits in u8")
}
buffer [1]u8 := [u8(value)]
try write_all(output, buffer[..])
}
.distinct |backing|: if scalar_or_distinct_type(backing) {
try write_character(output, distinct_value(backing, T, value))
} else {
compile_error!("io.print '{c}' requires an unsigned integer that fits in u8")
}
else: compile_error!("io.print '{c}' requires an unsigned integer that fits in u8")
}
}
hide write_default func(output Writer, $T type, value T) void ! WriteError {
match typeinfo!(T) {
.bool: if value {
try write_all(output, "true")
} else {
try write_all(output, "false")
}
.integer: try write_integer(output, value, 10, false)
.float: try write_float(output, value, false)
.array: try write_all(output, value)
.pointer: try write_all(output, value)
.slice: try write_all(output, value)
.enum |enum_info|: {
expand for enum_info.fields |field| {
if value == field!(T, field.name) {
try write_all(output, ".")
try write_all(output, field.name)
return
}
}
return .write_failed
}
else: compile_error!("io.print '{}' does not support this argument type")
}
return
@hide:file
write_default proc(output Writer, $T type, value T) void ! WriteError {
match typeinfo!(T) {
.bool: if value {
try write_all(output, "true")
} else {
try write_all(output, "false")
}
.integer: try write_integer(output, value, 10, false)
.float: try write_float(output, value, false)
.array: try write_all(output, value)
.pointer: try write_all(output, value)
.slice: try write_all(output, value)
.enum |enum_info|: {
inline for enum_info.fields |field| {
if value == field!(T, field.name) {
try write_all(output, ".")
try write_all(output, field.name)
return
}
}
return .write_failed
}
.distinct |backing|: if scalar_or_distinct_type(backing) {
try write_default(output, distinct_value(backing, T, value))
} else {
compile_error!("io.print '{}' does not support this argument type")
}
else: compile_error!("io.print '{}' does not support this argument type")
}
}
+147 -157
View File
@@ -1,212 +1,202 @@
import "@ffi/c"
AllocError :: enum {
out_of_memory
out_of_memory
}
Allocator :: struct {
context ?@mut anyopaque
vtable @AllocatorVTable
context ?@mut anyopaque
vtable @AllocatorVTable
}
AllocatorVTable :: struct {
alloc @func(context ?@mut anyopaque, size usize, alignment usize) ?*mut u8
realloc @func(context ?@mut anyopaque, memory ?*mut u8, old_size usize, new_size usize, alignment usize) ?*mut u8
free @func(context ?@mut anyopaque, memory ?*mut u8, size usize, alignment usize) void
alloc @proc(context ?@mut anyopaque, size usize, alignment usize) ?*mut u8
realloc @proc(context ?@mut anyopaque, memory ?*mut u8, old_size usize, new_size usize, alignment usize) ?*mut u8
free @proc(context ?@mut anyopaque, memory ?*mut u8, size usize, alignment usize) void
}
raw_alloc func(allocator Allocator, size usize, alignment usize) ?*mut u8 {
return allocator.vtable.alloc(allocator.context, size, alignment)
raw_alloc proc(allocator Allocator, size usize, alignment usize) ?*mut u8 {
return allocator.vtable.alloc(allocator.context, size, alignment)
}
raw_realloc func(allocator Allocator, memory ?*mut u8, old_size usize, new_size usize, alignment usize) ?*mut u8 {
return allocator.vtable.realloc(allocator.context, memory, old_size, new_size, alignment)
raw_realloc proc(allocator Allocator, memory ?*mut u8, old_size usize, new_size usize, alignment usize) ?*mut u8 {
return allocator.vtable.realloc(allocator.context, memory, old_size, new_size, alignment)
}
raw_free func(allocator Allocator, memory ?*mut u8, size usize, alignment usize) void {
allocator.vtable.free(allocator.context, memory, size, alignment)
raw_free proc(allocator Allocator, memory ?*mut u8, size usize, alignment usize) void {
allocator.vtable.free(allocator.context, memory, size, alignment)
}
eql func($T type, left, right []T) bool {
if left.len != right.len {
return false
}
i usize = 0
while i < left.len : i += 1 {
if left[i] != right[i] {
return false
}
}
return true
eql proc($T type, left, right []T) bool {
if (left.len != right.len) return false
for (0..left.len) |i| if (left[i] != right[i]) {
return false
}
return true
}
# allocate memory for a slice of type `T` with `count` elements.
alloc func($T type, allocator Allocator, count usize) []mut T ! AllocError {
if count == 0 {
return empty_slice(T, 0)
}
#! allocate memory for a slice of type `T` with `count` elements.
alloc proc($T type, allocator Allocator, count usize) []mut T ! AllocError {
if (count == 0) return empty_slice(T, 0)
element_size usize :: sizeof!(T)
if element_size == 0 {
return empty_slice(T, count)
}
if count > divtrunc!(maxval!(usize), element_size) {
return .out_of_memory
}
element_size usize :: sizeof!(T)
if (element_size == 0) return empty_slice(T, count)
memory ?*mut u8 = raw_alloc(allocator, count * element_size, alignof!(T))
if memory |bytes| {
pointer *mut T :: ptrcast!(T, bytes)
return pointer[..count]
}
return .out_of_memory
if count > divtrunc!(maxval!(usize), element_size) {
return .out_of_memory
}
memory := raw_alloc(allocator, count * element_size, alignof!(T))
if memory |bytes| {
pointer *mut T :: ptrcast!(T, bytes)
return pointer[..count]
}
return .out_of_memory
}
# reallocate memory for a slice of type `T` with `new_count` elements.
# reallocating with `new_count == 0` will free the memory and return an empty slice.
# note: memory must be reallocated with the same allocator that was used to allocate it.
realloc func($T type, allocator Allocator, memory []mut T, new_count usize) []mut T ! AllocError {
if new_count == memory.len {
return memory
}
#! reallocate memory for a slice of type `T` with `new_count` elements.
#! reallocating with `new_count == 0` will free the memory and return an empty slice.
#! note: memory must be reallocated with the same allocator that was used to allocate it.
realloc proc($T type, allocator Allocator, memory []mut T, new_count usize) []mut T ! AllocError {
if new_count == memory.len {
return memory
}
if new_count == 0 {
free(allocator, memory)
return empty_slice(T, 0)
}
if new_count == 0 {
free(allocator, memory)
return empty_slice(T, 0)
}
element_size usize :: sizeof!(T)
if element_size == 0 {
return empty_slice(T, new_count)
}
if new_count > divtrunc!(maxval!(usize), element_size) {
return .out_of_memory
}
element_size usize :: sizeof!(T)
if (element_size == 0) return empty_slice(T, new_count)
if (new_count > divtrunc!(maxval!(usize), element_size)) return .out_of_memory
old_memory ?*mut u8 = none
old_size usize = 0
if memory.len != 0 {
old_memory = ptrcast!(u8, memory.ptr)
old_size = memory.len * element_size
}
resized ?*mut u8 = raw_realloc(
allocator,
old_memory,
old_size,
new_count * element_size,
alignof!(T),
)
if resized |bytes| {
pointer *mut T :: ptrcast!(T, bytes)
return pointer[..new_count]
}
old_memory ?*mut u8 := null
old_size := 0
if memory.len != 0 {
old_memory = ptrcast!(u8, memory.ptr)
old_size = memory.len * element_size
}
return .out_of_memory
resized := raw_realloc(
allocator,
old_memory,
old_size,
new_count * element_size,
alignof!(T),
)
if resized |bytes| {
pointer *mut T :: ptrcast!(T, bytes)
return pointer[..new_count]
}
return .out_of_memory
}
# free memory allocated for a slice of type `T`.
# note: memory must be freed with the same allocator that was used to allocate it.
free func($T type, allocator Allocator, memory []mut T) void {
if memory.len != 0 and sizeof!(T) != 0 {
raw_free(allocator, ptrcast!(u8, memory.ptr), memory.len * sizeof!(T), alignof!(T))
}
#! free memory allocated for a slice of type `T`.
#! note: memory must be freed with the same allocator that was used to allocate it.
free proc($T type, allocator Allocator, memory []T) void {
if (memory.len == 0 or sizeof!(T) == 0) return
raw_free(allocator, ptrcast!(
u8,
constcast!(memory).ptr),
memory.len * sizeof!(T),
alignof!(T),
)
}
# get an empty slice of type `T` with `count` elements.
empty_slice func($T type, count usize) []mut T {
pointer *mut T :: ptrcast!(T, (&empty_storage).ptr)
return pointer[..count]
#! create a slice backed by `empty_storage` without allocating.
#! precondition: `count` must be 0 or `sizeof!(T)` must be 0; otherwise the slice
#! exceeds its backing storage.
@hide:file
empty_slice proc($T type, count usize) []mut T {
pointer *mut T :: ptrcast!(T, (&empty_storage).ptr)
return pointer[..count]
}
# get an empty slice of type `T` with 0 elements.
empty func($T type) []mut T {
return empty_slice(T, 0)
#! get an empty slice of type `T` with 0 elements.
empty proc($T type) []mut T {
return empty_slice(T, 0)
}
hide empty_storage [1]mut u64 = [0]
@hide
empty_storage [1]mut u64 := [0]
hide malloc_alignment usize :: 16 # ponytail: aarch64-macos libc malloc alignment assumption.
@hide
malloc_alignment usize :: 16 # note: aarch64-macos libc malloc alignment assumption.
hide power_of_two func(value usize) bool {
if value == 0 {
return false
}
current usize = value
while current > 1 {
half usize = divtrunc!(current, 2)
if half * 2 != current {
return false
}
current = half
}
return true
@hide:file
power_of_two proc(value usize) bool {
if (value == 0) return false
current := value
while current > 1 {
half := divtrunc!(current, 2)
if (half * 2 != current) return false
current = half
}
return true
}
hide c_alloc func(_ ?@mut anyopaque, size usize, alignment usize) ?*mut u8 {
if power_of_two(alignment) == false {
return none
}
@hide:file
c_alloc proc(_ ?@mut anyopaque, size usize, alignment usize) ?*mut u8 {
if (power_of_two(alignment) == false) return null
if (alignment <= malloc_alignment) return ptrcast!(u8, c.malloc(c_ulong(size)))
if alignment <= malloc_alignment {
return ptrcast!(u8, c.malloc(c_ulong(size)))
}
memory [1]mut ?*mut anyopaque := [null]
status := c.posix_memalign((&memory).ptr, c_ulong(alignment), c_ulong(size))
if (status != 0) return null
memory [1]mut ?*mut anyopaque = [none]
status c_int = c.posix_memalign((&memory).ptr, c_ulong(alignment), c_ulong(size))
if status != 0 {
return none
}
return ptrcast!(u8, memory[0])
return ptrcast!(u8, memory[0])
}
hide c_realloc func(_ ?@mut anyopaque, memory ?*mut u8, old_size usize, new_size usize, alignment usize) ?*mut u8 {
if power_of_two(alignment) == false {
return none
}
@hide:file
c_realloc proc(
_ ?@mut anyopaque,
memory ?*mut u8,
old_size usize,
new_size usize,
alignment usize,
) ?*mut u8 {
if (power_of_two(alignment) == false) return null
if new_size == 0 {
c.free(memory)
return none
}
if new_size == 0 {
c.free(memory)
return null
}
if memory |old_memory| {
if alignment <= malloc_alignment {
return ptrcast!(u8, c.realloc(old_memory, c_ulong(new_size)))
}
if memory |old_memory| {
if alignment <= malloc_alignment {
return ptrcast!(u8, c.realloc(old_memory, c_ulong(new_size)))
}
new_memory ?*mut u8 = c_alloc(none, new_size, alignment)
if new_memory |new_bytes| {
copy_size usize = old_size
if new_size < copy_size {
copy_size = new_size
}
i usize = 0
while i < copy_size : i += 1 {
new_bytes[i] = old_memory[i]
}
c.free(old_memory)
}
return new_memory
}
new_memory := c_alloc(null, new_size, alignment)
if new_memory |new_bytes| {
copy_size := old_size
if (new_size < copy_size) copy_size = new_size
memcopy!(new_bytes[..copy_size], old_memory[..copy_size])
c.free(old_memory)
}
return new_memory
}
return c_alloc(none, new_size, alignment)
return c_alloc(null, new_size, alignment)
}
hide c_free func(_ ?@mut anyopaque, memory ?*mut u8, _ usize, _ usize) void {
c.free(memory)
@hide:file
c_free proc(_ ?@mut anyopaque, memory ?*mut u8, _ usize, _ usize) void {
c.free(memory)
}
hide c_vtable AllocatorVTable :: AllocatorVTable {
alloc = c_alloc,
realloc = c_realloc,
free = c_free,
@hide:file
c_vtable AllocatorVTable :: AllocatorVTable {
alloc = c_alloc,
realloc = c_realloc,
free = c_free,
}
c_allocator Allocator :: Allocator {
context = none,
vtable = &c_vtable,
context = null,
vtable = &c_vtable,
}
+48 -28
View File
@@ -1,41 +1,61 @@
Layout :: enum {
auto
c
Layout :: enum { auto, c }
ArrayInfo :: struct {
child type
len usize
}
FieldInfo :: struct {
name []u8
type type
index usize
name []u8
type type
index usize
}
RecordInfo :: struct {
name ?[]u8
fields []FieldInfo
is_tuple bool
layout Layout
name ?[]u8
fields []FieldInfo
is_tuple bool
layout Layout
}
EnumInfo :: struct {
fields []FieldInfo
fields []FieldInfo
}
TypeInfo :: union(enum) {
invalid void
void void
anyopaque void
bool void
integer void
float void
array void
pointer void
slice void
range void
optional void
function void
enum EnumInfo
record RecordInfo
union void
fallible void
distinct void
invalid void
void void
noreturn void
anyopaque void
bool void
integer void
float void
array ArrayInfo
pointer void
slice void
range void
optional void
function void
enum EnumInfo
record RecordInfo
union void
fallible void
distinct type
}
EnumFieldStruct proc($E, $Field type, $default ?Field) type {
match typeinfo!(E) {
.enum |info|: {
names [info.fields.len]mut []u8 := undefined
field_types [info.fields.len]mut type := undefined
defaults [info.fields.len]mut ?Field := undefined
inline for info.fields |field, index| {
names[index] = field.name
field_types[index] = Field
defaults[index] = default
}
return struct_type!(.auto, names, field_types, defaults)
}
else: compile_error!("EnumFieldStruct key must be an enum")
}
}
+65
View File
@@ -0,0 +1,65 @@
import "@std/testing"
TestTokenKind :: enum(u8) {
ident = 3
int = 8
eof = 21
}
TestNames :: alias EnumFieldStruct(TestTokenKind, ?[]u8, some!(null))
TestArrayAlias :: alias [3]u16
TestInner :: distinct u16
TestOuter :: distinct TestInner
TestOuterAlias :: alias TestOuter
@hide
array_info_matches proc($Array, $Child type, $len usize) bool {
return match typeinfo!(Array) {
.array |info|: info.child == Child and info.len == len
else: false
}
}
@hide
distinct_info_matches proc($Distinct, $Backing type) bool {
return match typeinfo!(Distinct) {
.distinct |backing|: backing == Backing
else: false
}
}
array_reflection_exposes_child_and_logical_length test {
try testing.expect($(array_info_matches([4]i32, i32, 4)))
try testing.expect($(array_info_matches([0]bool, bool, 0)))
try testing.expect($(array_info_matches(TestArrayAlias, u16, 3)))
try testing.expect($(array_info_matches([2]mut i64, i64, 2)))
try testing.expect($(array_info_matches([2;0]u8, u8, 2)))
}
distinct_reflection_exposes_immediate_backing test {
try testing.expect($(distinct_info_matches(TestInner, u16)))
try testing.expect($(distinct_info_matches(TestOuter, TestInner)))
try testing.expect($(distinct_info_matches(TestOuterAlias, TestInner)))
}
enum_field_struct_defaults test {
names TestNames := {
ident = "identifier",
int = "integer",
}
if (field!(names, "ident")) |value|
try testing.expect(value.len == 10)
else
try testing.expect(false)
if (field!(names, "int")) |value|
try testing.expect(value.len == 7)
else
try testing.expect(false)
if (field!(names, "eof")) |_| try testing.expect(false)
empty TestNames := {}
if (field!(empty, "ident")) |_| try testing.expect(false)
}
+1 -1
View File
@@ -1,5 +1,5 @@
import "@std/io"
Init :: struct {
io io.Io
io io.Io
}
+6
View File
@@ -1,5 +1,11 @@
import "io"
import "enums/enummap"
import "hashmap"
import "arraylist"
import "strmap"
Io :: alias io.Io
EnumMap :: alias enummap.EnumMap
ArrayList :: alias arraylist.ArrayList
StringHashMap :: alias hashmap.StringHashMap
StringMap :: alias strmap.StringMap
+96
View File
@@ -0,0 +1,96 @@
import "@std/mem"
StringMap proc($V type) type {
return struct {
keys [][]u8
values []V
len_indexes []u32
min_len u32
max_len u32
}
}
@hide:file
Pair proc($V type) type {
return struct { []u8, V }
}
#! initializes a static string map from a list of key-value pairs (constructed at compile-time).
init proc($V type, $N usize, $entries [N]Pair(V)) StringMap(V) {
if N > usize(maxval!(u32)) {
compile_error!("static string map has too many entries")
}
keys [N]mut []u8 := undefined
values [N]mut V := undefined
# assert no duplicate keys
for entries |entry, i| {
if entry.0.len > usize(maxval!(u32)) {
compile_error!("static string map key is too long")
}
for (0..i) |prior| if mem.eql(u8, entry.0, entries[prior].0) {
compile_error!("duplicate static string map key")
}
keys[i] = entry.0
values[i] = entry.1
}
if N == 0 {
len_indexes [0]u32 := undefined
return StringMap(V){
keys = keys[..],
values = values[..],
len_indexes = len_indexes[..],
min_len = 0,
max_len = 0,
}
}
# fixme: insertion sort is compile-time O(N²); replace if large maps affect builds
for 1..N |i| {
key :: keys[i]
value :: values[i]
j := i
while j > 0 and keys[j - 1].len > key.len : j -= 1 {
keys[j] = keys[j - 1]
values[j] = values[j - 1]
}
keys[j] = key
values[j] = value
}
min_len u32 :: u32(keys[0].len)
max_len u32 :: u32(keys[N - 1].len)
len_indexes [usize(max_len) + 1]mut u32 := undefined
entry_index usize := 0
for 0..=usize(max_len) |length| {
while entry_index < N and keys[entry_index].len < length : entry_index += 1 {}
len_indexes[length] = u32(entry_index)
}
return StringMap(V) {
keys = keys[..],
values = values[..],
len_indexes = len_indexes[..],
min_len = min_len,
max_len = max_len,
}
}
get proc($V type, map @StringMap(V), key []u8) ?V {
if (map.keys.len == 0 or key.len > maxval!(u32)) return null
length := u32(key.len)
if (length < map.min_len or length > map.max_len) return null
idx := usize(map.len_indexes[usize(length)])
while idx < map.keys.len : idx += 1 {
candidate :: map.keys[idx]
if (candidate.len != key.len) return null # key not found
if mem.eql(u8, candidate, key) return map.values[idx]
}
return null
}
+1
View File
@@ -0,0 +1 @@
# todo
+68 -25
View File
@@ -1,42 +1,85 @@
import "@std/debug"
import "@std/mem"
Error :: enum {
expectation_failed
expectation_failed
}
SourceLocation :: struct {
file []u8
line usize
column usize
file []u8
line usize
column usize
}
expect func(condition bool, location SourceLocation) void ! Error {
if !condition {
debug.print("{s}:{d}:{d}: expectation failed\n", {location.file, location.line, location.column})
return .expectation_failed
}
expect proc(condition bool, location SourceLocation) void ! Error {
if !condition {
debug.print("{s}:{d}:{d}: expectation failed\n", {
location.file,
location.line,
location.column,
})
return .expectation_failed
}
}
expect_equal func($T type, expected, actual T, location SourceLocation) void ! Error {
if expected != actual {
debug.print("{s}:{d}:{d}: expected {}, found {}\n", {location.file, location.line, location.column, expected, actual})
return .expectation_failed
}
expect_equal proc($T type, expected, actual T, location SourceLocation) void ! Error {
match typeinfo!(T) {
.optional: {
if expected |expected_value| {
if actual |actual_value| {
try expect_equal(expected_value, actual_value, location)
return
}
debug.print("{s}:{d}:{d}: expected an optional value, found null\n", {
location.file,
location.line,
location.column,
})
return .expectation_failed
}
if actual |_| {
debug.print("{s}:{d}:{d}: expected null, found an optional value\n", {
location.file,
location.line,
location.column,
})
return .expectation_failed
}
}
.slice: if !mem.eql(expected, actual) {
debug.print("{s}:{d}:{d}: expected and actual slices differ\n", {
location.file,
location.line,
location.column,
})
return .expectation_failed
}
else: if expected != actual {
debug.print("{s}:{d}:{d}: expected {}, found {}\n", {
location.file,
location.line,
location.column,
expected,
actual,
})
return .expectation_failed
}
}
}
expect_type func($Expected, $Actual type, _ Actual, location SourceLocation) void ! Error {
try expect($(Expected == Actual), location)
expect_type proc($Expected, $Actual type, _ Actual, location SourceLocation) void ! Error {
try expect($(Expected == Actual), location)
}
run func(name []u8, callback *func() void ! Error) bool {
callback() catch |_| {
debug.print("{s} [failed]\n", {name,})
return false
}
debug.print("{s} [ok]\n", {name,})
return true
run proc(name []u8, callback *proc() void ! Error) bool {
callback() catch |_| {
debug.print("{s} ... [\x1b[91mfailed\x1b[0m]\n", {name})
return false
}
debug.print("{s} ... [\x1b[92mok\x1b[0m]\n", {name})
return true
}
summary func(passed, failed i32) void {
debug.print("{d} passed, {d} failed\n", {passed, failed})
summary proc(passed, failed i32) void {
debug.print("{d} passed, {d} failed\n", {passed, failed})
}