feat: execute collection and index iteration

source/compilation/compilation.test.hon

@@ -146,6 +146,18 @@ recompiles_specialized_calls_without_borrowing_previous_ir test {
             `        bias i16 = config.bias + i16(original.counts[1])
             `    }
             `}
+            `Collected proc() type {
+            `    counts [2]mut usize := [1, 1]
+            `    length usize := 0
+            `    for counts |amount, index| {
+            `        counts[1] = 2
+            `        length = length + amount
+            `        if (index == 0) continue
+            `    }
+            `    Element := i8
+            `    for "ok" |byte| { if (byte == u8(107)) Element = i16 }
+            `    return [length]Element
+            `}
             `read_selected proc($choice Choice, value i16) i16 {
             `    match choice {
             `        .payload |detail|: {
@@ -173,6 +185,11 @@ recompiles_specialized_calls_without_borrowing_previous_ir test {
             `    if (optional[0] + optional[1] != 40) return 0
             `    updated Updated() :: { values = optional }
             `    if (updated.bias != 42 or updated.values[1] != 21) return 0
+            `    collected Collected() :: [19, 21, 2]
+            `    same_collected Collected() :: collected
+            `    total i16 := 0
+            `    for same_collected |item| { total = total + item }
+            `    if (total != 42) return 0
             `    payload :: packet.choices[0].payload
             `    if (payload.values[0] != 40 or payload.values[1] != 2 or payload.text.len != 2) return 0
             `    match packet.choices[1] { .empty: {}, .payload: return 0 }

source/diagnostics/diagnostics.hon

@@ -176,7 +176,7 @@ messages :: $enummap.init(Code, []u8, {
     break_outside_loop = "break requires an enclosing loop",
     continue_outside_loop = "continue requires an enclosing loop",
     invalid_range_bound = "range bounds must have compatible integer types",
-    invalid_for_iterable = "for-loop iteration currently requires a direct integer range",
+    invalid_for_iterable = "for-loop iteration requires an array, slice, string, single pointer to an array, or direct integer range",
     range_value_not_supported = "first-class range values are not implemented",
     range_pointer_capture = "range loops do not support pointer captures",
     range_index_capture = "range loops do not support index captures",

source/lower/lower.hon

@@ -482,13 +482,17 @@ predeclare_locals proc(state @mut State, block ast.NodeId) void ! (mem.AllocErro
             }
             .for_stmt: {
                 loop :: ast.for_stmt(state.tree, statement)
+                is_range :: ast.expr_kind(state.tree, loop.iterable) == .range
                 family :: typecheck.type_of(state.checked, loop.iterable).range_element
-                type :: lower_type(state, typecheck.Type{ kind = family })
+                type ir.Type :: if (is_range) lower_type(state, typecheck.Type{ kind = family }) else .usize
                 slot :: try ir.add_local(state.module, ir.FunctionId(state.function), type, ir.no_array)
                 state.storage[usize(statement)] = Storage{ kind = .local, index = u32(slot) }
-                if (loop.capture) |capture| state.storage[usize(capture)] = state.storage[usize(statement)]
+                if (loop.capture) |capture| {
+                    if (is_range) state.storage[usize(capture)] = state.storage[usize(statement)]
+                    else try predeclare_capture(state, capture)
+                }
                 if (loop.index) |index| {
-                    index_slot :: try ir.add_local(state.module, ir.FunctionId(state.function), .int, ir.no_array)
+                    index_slot :: try ir.add_local(state.module, ir.FunctionId(state.function), .usize, ir.no_array)
                     state.storage[usize(index)] = Storage{ kind = .local, index = u32(index_slot) }
                 }
                 try predeclare_locals(state, loop.body)
@@ -705,8 +709,7 @@ lower_while proc(state @mut State, id ast.NodeId) void ! (mem.AllocError | diagn
 lower_for proc(state @mut State, id ast.NodeId) void ! (mem.AllocError | diagnostics.CapacityError) {
     loop :: ast.for_stmt(state.tree, id)
     if (ast.expr_kind(state.tree, loop.iterable) != .range) {
-        _ = try lower_expression(state, loop.iterable)
-        if (!state.terminated) _ = trap_type(state, id)
+        try lower_collection_for(state, id)
         return
     }
     interval :: ast.range_expr(state.tree, loop.iterable)
@@ -765,6 +768,78 @@ lower_for proc(state @mut State, id ast.NodeId) void ! (mem.AllocError | diagnos
     state.terminated = false
 }
 
+@hide:file
+lower_collection_for proc(state @mut State, id ast.NodeId) void ! (mem.AllocError | diagnostics.CapacityError) {
+    loop :: ast.for_stmt(state.tree, id)
+    parts :: try slice_parts(state, loop.iterable)
+    if (state.terminated or trap_type(state, loop.iterable) or trap_type(state, id)) return
+    slot :: state.storage[usize(id)].index
+    zero :: try usize_constant(state, 0, loop.span)
+    _ = try emit(state, ir.Instruction{ op = .store_local, type = .usize, left = zero, slot = slot, span = loop.span })
+    origin :: state.block
+    condition :: try add_block(state)
+    body :: try add_block(state)
+    update :: try add_block(state)
+    exit :: try add_block(state)
+    state.module.functions.items[state.function].blocks.items[usize(origin)].terminator = ir.Terminator{
+        kind = .jump, target = condition, span = loop.span,
+    }
+    state.block = condition
+    current :: try emit(state, ir.Instruction{ op = .load_local, type = .usize, slot = slot, span = loop.span })
+    condition_value :: try emit(state, ir.Instruction{ op = .less, type = .bool, left = current, right = parts.bound, span = loop.span })
+    state.module.functions.items[state.function].blocks.items[usize(state.block)].terminator = ir.Terminator{
+        kind = .branch, value = condition_value, target = body, alternative = exit, span = loop.span,
+    }
+    previous_loop :: state.loop
+    state.loop = LoopTargets{ continue_target = update, exit = exit, cleanup_depth = state.cleanups.items.len }
+    defer state.loop = previous_loop
+    state.block = body
+    if (loop.capture) |capture| {
+        if (!trap_binding(state, capture) and !trap_type(state, capture)) {
+            runtime :: lower_type(state, parts.element)
+            shape :: try array_shape(state, parts.element)
+            pointer :: try emit(state, ir.Instruction{
+                op = .pointer_offset, type = .pointer, left = parts.pointer, right = current,
+                pointee = runtime, array_type = shape, span = loop.span,
+            })
+            value := pointer
+            if (!loop.pointer_capture) {
+                address :: try pointer_address(state, pointer, parts.element, loop.span)
+                value = try emit(state, ir.Instruction{
+                    op = .load_address, type = runtime, array_type = shape, left = address, span = loop.span,
+                })
+            }
+            checked :: typecheck.type_of(state.checked, capture)
+            _ = try emit(state, ir.Instruction{
+                op = .store_local, type = lower_type(state, checked), array_type = try array_shape(state, checked),
+                left = value, slot = state.storage[usize(capture)].index, span = loop.span,
+            })
+        }
+    }
+    if (loop.index) |index| {
+        if (!state.terminated and !trap_binding(state, index) and !trap_type(state, index)) {
+            _ = try emit(state, ir.Instruction{
+                op = .store_local, type = .usize, left = current, slot = state.storage[usize(index)].index, span = loop.span,
+            })
+        }
+    }
+    if (!state.terminated) try lower_block(state, loop.body, true)
+    if (!state.terminated) state.module.functions.items[state.function].blocks.items[usize(state.block)].terminator = ir.Terminator{
+        kind = .jump, target = update, span = loop.span,
+    }
+    state.block = update
+    state.terminated = false
+    last :: try emit(state, ir.Instruction{ op = .load_local, type = .usize, slot = slot, span = loop.span })
+    one :: try usize_constant(state, 1, loop.span)
+    next :: try emit(state, ir.Instruction{ op = .add, type = .usize, left = last, right = one, span = loop.span })
+    _ = try emit(state, ir.Instruction{ op = .store_local, type = .usize, left = next, slot = slot, span = loop.span })
+    state.module.functions.items[state.function].blocks.items[usize(state.block)].terminator = ir.Terminator{
+        kind = .jump, target = condition, span = loop.span,
+    }
+    state.block = exit
+    state.terminated = false
+}
+
 @hide:file
 lower_statement proc(state @mut State, id ast.NodeId) void ! (mem.AllocError | diagnostics.CapacityError) {
     tree :: state.tree

source/sema/check/check.hon

@@ -174,6 +174,7 @@ Cell :: struct {
     literal ast.NodeId = ast.NodeId(maxval!(u32))
     producer ast.NodeId = ast.NodeId(maxval!(u32))
     capture ast.NodeId = ast.NodeId(maxval!(u32))
+    for_loop ast.NodeId = ast.NodeId(maxval!(u32))
     record_checking bool = false
     record_pointer_depth usize = 0
     nominal bool = false
@@ -382,6 +383,7 @@ check_elaborated proc(
             }
         }
         # calls use the signature result before contextual literal inference.
+        try prepare_for_captures(&state, false)
         try prepare_optional_captures(&state, false)
         for (0..count) |i| {
             id :: ast.NodeId(i)
@@ -553,7 +555,7 @@ check_elaborated proc(
         if (cells[i].role != .for_stmt) continue
         iterable :: ast.for_stmt(tree, ast.NodeId(i)).iterable
         actual :: types[usize(iterable)]
-        if (actual.kind != .invalid and actual.kind != .range) {
+        if (actual.kind != .invalid and actual.kind != .range and !collection_type(&state, actual)) {
             try fail(&state, iterable, .invalid_for_iterable, ast.node_span(tree, iterable))
         }
     }
@@ -1073,12 +1075,13 @@ collect_block proc(state @mut State, id, return_annotation ast.NodeId, in_loop b
                 state.cells[usize(loop.iterable)].for_iterable = true
                 try collect_expression(state, loop.iterable)
                 try depend(state, loop.iterable, statement)
-                if (loop.pointer_capture) {
+                direct_range :: ast.expr_kind(tree, loop.iterable) == .range
+                if (direct_range and loop.pointer_capture) {
                     state.cells[usize(statement)].type = invalid_type(try diagnostics.add(
                         state.diagnostics, .range_pointer_capture, loop.span,
                     ))
                 }
-                if (loop.index_capture) {
+                if (direct_range and loop.index_capture) {
                     diagnostic :: try diagnostics.add(state.diagnostics, .range_index_capture, loop.span)
                     if (state.cells[usize(statement)].type.kind != .invalid) {
                         state.cells[usize(statement)].type = invalid_type(diagnostic)
@@ -1086,9 +1089,18 @@ collect_block proc(state @mut State, id, return_annotation ast.NodeId, in_loop b
                 }
                 if (loop.capture) |capture| {
                     try collect_capture(state, capture, loop.body)
+                    if (!direct_range) {
+                        state.cells[usize(capture)].for_loop = statement
+                        if (state.cells[usize(capture)].type.kind != .invalid) state.cells[usize(capture)].type = Type{}
+                    }
                     state.cells[usize(loop.iterable)].capture = capture
                 }
-                if (loop.index) |index| try collect_capture(state, index, loop.body)
+                if (loop.index) |index| {
+                    try collect_capture(state, index, loop.body)
+                    if (!direct_range and state.cells[usize(index)].type.kind != .invalid) {
+                        state.cells[usize(index)].type = Type{ kind = .usize }
+                    }
+                }
                 try collect_block(state, loop.body, return_annotation, true)
                 try depend(state, loop.body, id)
             }
@@ -1763,6 +1775,15 @@ indexable_type proc(state @State, type Type) bool {
     return (pointer.many and pointer.element.kind != .anyopaque) or pointer.element.kind == .array
 }
 
+@hide:file
+collection_type proc(state @State, type Type) bool {
+    if (type.nominal != ast.NodeId(maxval!(u32))) return false
+    if (type.kind == .array or type.kind == .slice or type.kind == .string) return true
+    if (type.kind != .pointer) return false
+    pointer :: state.pointers.items[usize(type.pointer_type)]
+    return !pointer.many and pointer.element.kind == .array and pointer.element.nominal == ast.NodeId(maxval!(u32))
+}
+
 @hide:file
 addressable_target proc(state @mut State, id ast.NodeId) bool {
     kind :: ast.expr_kind(state.tree, id)
@@ -2676,6 +2697,28 @@ infer_shape proc(state @mut State, id ast.NodeId, defaults bool) Type ! (mem.All
     if (state.cells[index].type.kind == .invalid) return state.cells[index].type
     state.cells[index].visiting = true
     defer state.cells[index].visiting = false
+    if (state.cells[usize(id)].role == .loop_capture and state.cells[usize(id)].for_loop != ast.NodeId(maxval!(u32))) {
+        loop :: ast.for_stmt(state.tree, state.cells[usize(id)].for_loop)
+        actual :: try infer_shape(state, loop.iterable, defaults)
+        if (actual.kind == .invalid or state.types[usize(loop.iterable)].kind == .invalid) {
+            type :: if (actual.kind == .invalid) actual else state.types[usize(loop.iterable)]
+            seed(state, id, type)
+            return type
+        }
+        if (!collection_type(state, actual)) {
+            if (actual.kind != .none) {
+                try fail(state, loop.iterable, .invalid_for_iterable, ast.node_span(state.tree, loop.iterable))
+                seed(state, id, state.types[usize(loop.iterable)])
+            }
+            return group_type(state, id)
+        }
+        descriptor :: try sequence_type(state, loop.iterable, defaults)
+        type :: if (loop.pointer_capture)
+            try intern_pointer(state, PointerType{ element = descriptor.element, mutable = descriptor.mutable, many = false })
+        else descriptor.element
+        seed(state, id, type)
+        return type
+    }
     if (state.cells[usize(id)].role == .expression) {
         kind :: ast.expr_kind(state.tree, id)
         if (declaration_reference(state.tree, id) and has_narrowing(state, id)) {
@@ -2685,6 +2728,14 @@ infer_shape proc(state @mut State, id ast.NodeId, defaults bool) Type ! (mem.All
                 return narrowed
             }
         }
+        if (declaration_reference(state.tree, id)) {
+            binding :: sema.binding(state.resolution, id)
+            if (binding.kind == .declaration and state.cells[usize(binding.data.declaration)].role == .loop_capture) {
+                type :: try infer_shape(state, binding.data.declaration, defaults)
+                if (type.kind != .none) seed(state, id, type)
+                return group_type(state, id)
+            }
+        }
         if (kind == .try_expr or kind == .catch_expr) return try infer_fallible_expression(state, id, defaults)
         if (kind == .builtin_call) return try infer_builtin(state, id, defaults)
         if (kind == .slice) return try infer_slice(state, id, defaults)
@@ -2937,6 +2988,15 @@ infer_shape proc(state @mut State, id ast.NodeId, defaults bool) Type ! (mem.All
     return group_type(state, id)
 }
 
+@hide:file
+prepare_for_captures proc(state @mut State, defaults bool) void ! (mem.AllocError | diagnostics.CapacityError) {
+    for 0..(state.cells.len) |i| {
+        if (state.cells[i].role == .loop_capture and state.cells[i].for_loop != ast.NodeId(maxval!(u32))) {
+            _ = try infer_shape(state, ast.NodeId(i), defaults)
+        }
+    }
+}
+
 @hide:file
 prepare_optional_captures proc(state @mut State, defaults bool) void ! (mem.AllocError | diagnostics.CapacityError) {
     for 0..(state.cells.len) |i| {
@@ -2965,6 +3025,7 @@ prepare_optional_captures proc(state @mut State, defaults bool) void ! (mem.Allo
 
 @hide:file
 prepare_shapes proc(state @mut State, assignment_values, defaults bool) void ! (mem.AllocError | diagnostics.CapacityError) {
+    try prepare_for_captures(state, defaults)
     try prepare_optional_captures(state, defaults)
     for 0..(state.cells.len) |i| {
         if (state.cells[i].role != .expression or state.cells[i].assignment_value != assignment_values) continue

source/sema/check/check.test.hon

@@ -1855,7 +1855,7 @@ range_values_nonrange_iterables_and_capture_modes_have_explicit_errors test {
         `    local :: 0..3
         `    for 3 |_| { absent_scalar }
         `    for true |_| {}
-        `    for "text" |_| {}
+        `    for 2.5 |_| {}
         `    for scalar() |_| {}
         `    for stored |_| {}
         `    for 0..3 |@item| { item }

source/sema/specialize/specialize.hon

@@ -1617,10 +1617,153 @@ execute_type_statement proc(state @mut State, execution @mut TypeExecution, stat
     return .next
 }
 
+# retain an array place, but freeze its indices at loop entry. snapshots then read
+# the current owned root after body assignments without re-evaluating the place.
+@hide:file
+type_collection_place proc(state @mut State, execution @mut TypeExecution, id ast.NodeId) ast.NodeId ! Error {
+    resolved :: binding(state, id)
+    if (resolved.kind == .declaration) {
+        for (execution.locals.items) |local| {
+            if (local.declaration == resolved.data.declaration) return id
+        }
+        return no_node
+    }
+    kind :: ast.expr_kind(&state.tree, id)
+    if (kind != .selection and kind != .index) return no_node
+    base :: if (kind == .selection) ast.selection_expr(&state.tree, id).base else ast.index_expr(&state.tree, id).base
+    place :: try type_collection_place(state, execution, base)
+    if (place == no_node) return no_node
+    result := no_node
+    if (kind == .selection) {
+        result = try ast.add_selection(state.builder, ast.node_span(&state.tree, id),
+            place, ast.selection_expr(&state.tree, id).member)
+    } else {
+        index :: try type_snapshot(state, execution, ast.index_expr(&state.tree, id).index)
+        value :: try type_execution_value(state, index, try builtin(state, index, .usize))
+        if (value == no_node) return no_node
+        result = try ast.add_index(state.builder, ast.node_span(&state.tree, id), place, value)
+    }
+    try refresh(state)
+    return result
+}
+
+@hide:file
+type_collection_read proc(state @mut State, execution @mut TypeExecution, id ast.NodeId) ast.NodeId ! Error {
+    if (binding(state, id).kind == .declaration) return try type_snapshot(state, execution, id)
+    kind :: ast.expr_kind(&state.tree, id)
+    base :: if (kind == .selection) ast.selection_expr(&state.tree, id).base else ast.index_expr(&state.tree, id).base
+    value :: try type_collection_read(state, execution, base)
+    result := no_node
+    if (kind == .selection)
+        result = try ast.add_selection(state.builder, ast.node_span(&state.tree, id),
+            value, ast.selection_expr(&state.tree, id).member)
+    else result = try ast.add_index(state.builder, ast.node_span(&state.tree, id),
+        value, ast.index_expr(&state.tree, id).index)
+    try refresh(state)
+    return result
+}
+
+@hide:file
+execute_type_collection proc(state @mut State, execution @mut TypeExecution, statement ast.NodeId) TypeFlow ! Error {
+    loop :: ast.for_stmt(&state.tree, statement)
+    span :: ast.node_span(&state.tree, statement)
+    if (loop.capture) |capture| {
+        if (binding(state, capture).kind == .invalid) {
+            state.evaluation_diagnostic = binding(state, capture).data.diagnostic
+            return .invalid
+        }
+    }
+    if (loop.index) |index| {
+        if (binding(state, index).kind == .invalid) {
+            state.evaluation_diagnostic = binding(state, index).data.diagnostic
+            return .invalid
+        }
+    }
+    snapshot :: try type_snapshot(state, execution, loop.iterable)
+    if (!try expand_type_matches(state, snapshot)) return .invalid
+    try expand(state, snapshot)
+    annotation :: try expression_type(state, snapshot, 0)
+    if (annotation == no_node) return .invalid
+    collection :: try type_execution_value(state, snapshot, annotation)
+    if (collection == no_node) return .invalid
+    # validate the loop contract even for empty collections, without executing its
+    # type-valued body. the synthetic body exits before reading any element.
+    stop :: try ast.add_break_stmt(state.builder, span)
+    stops [1]ast.NodeId :: [stop]
+    body :: try ast.add_block(state.builder, span, stops[..])
+    selector :: try ast.add_for_stmt(state.builder, span, collection, loop.capture, loop.index,
+        loop.pointer_capture, loop.index_capture, body)
+    length := no_node
+    if (ast.expr_kind(&state.tree, annotation) == .array_type) length = ast.array_type(&state.tree, annotation).count?
+    else {
+        value :: try evaluate(state, collection)
+        if (value.kind == .string) {
+            text :: ast.literal_text(&state.tree, value.node)
+            cursor := usize(1)
+            count := u64(0)
+            while (cursor < text.len - 1) {
+                _ = ast.string_next(text, &cursor) catch { return .invalid }
+                count += 1
+            }
+            length = try ast.add_integer(state.builder, span, count)
+        } else length = try ast.add_integer(state.builder, span, 0)
+    }
+    returned :: try ast.add_return_stmt(state.builder, span, length)
+    statements [2]ast.NodeId :: [selector, returned]
+    wrapper :: try ast.add_block(state.builder, span, statements[..])
+    try refresh(state)
+    count :: try evaluate_ir_body(state, statement, wrapper, try builtin(state, statement, .usize))
+    if (count.kind != .integer) return .invalid
+    if (loop.pointer_capture and loop.capture != null) {
+        state.evaluation_diagnostic = try diagnostics.add(state.diagnostics, .comptime_value_escape, span)
+        return .invalid
+    }
+    place := no_node
+    if (ast.expr_kind(&state.tree, annotation) == .array_type)
+        place = try type_collection_place(state, execution, loop.iterable)
+    if (state.evaluation_diagnostic != diagnostics.no_diagnostic) return .invalid
+    saved :: execution.locals.items.len
+    defer execution.locals.items = execution.locals.items[..saved]
+    execution.loop_depth += 1
+    defer execution.loop_depth -= 1
+    for (0..usize(count.magnitude)) |i| {
+        if (!try type_step(state, statement)) return .invalid
+        execution.locals.items = execution.locals.items[..saved]
+        index :: try ast.add_integer(state.builder, span, u64(i))
+        try refresh(state)
+        if (loop.capture) |capture| {
+            current := collection
+            if (place != no_node) current = try type_collection_read(state, execution, place)
+            element :: try ast.add_index(state.builder, span, current, index)
+            try refresh(state)
+            element_type :: try expression_type(state, element, 0)
+            value :: try type_execution_value(state, element, element_type)
+            if (value == no_node) return .invalid
+            try arraylist.append(&execution.locals, TypeLocal{
+                declaration = capture, value = value, annotation = element_type,
+            })
+        }
+        if (loop.index) |capture| {
+            type :: try builtin(state, capture, .usize)
+            value :: try type_execution_value(state, index, type)
+            if (value == no_node) return .invalid
+            try arraylist.append(&execution.locals, TypeLocal{
+                declaration = capture, value = value, annotation = type,
+            })
+        }
+        flow :: try execute_type_block(state, execution, loop.body)
+        if (flow == .returned or flow == .invalid) return flow
+        if (flow == .break) return .next
+    }
+    return .next
+}
+
 @hide:file
 execute_type_for proc(state @mut State, execution @mut TypeExecution, statement ast.NodeId) TypeFlow ! Error {
     loop :: ast.for_stmt(&state.tree, statement)
-    if (loop.pointer_capture or loop.index_capture or ast.expr_kind(&state.tree, loop.iterable) != .range) return .invalid
+    if (ast.expr_kind(&state.tree, loop.iterable) != .range)
+        return try execute_type_collection(state, execution, statement)
+    if (loop.pointer_capture or loop.index_capture) return .invalid
     bounds :: ast.range_expr(&state.tree, loop.iterable)
     start :: try type_snapshot(state, execution, bounds.start)
     end :: try type_snapshot(state, execution, bounds.end)