remora/tests/kind/lib.sh

#!/usr/bin/env bash
set -euo pipefail

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
TESTBED="$ROOT/tests/kind"
RUNTIME="$TESTBED/.runtime"
BIN="$RUNTIME/bin"
KUBECONFIG_PATH="$RUNTIME/kubeconfig"
CONTEXT=kind-remora-testbed
CLUSTER=remora-testbed
NERDCTL_NAMESPACE=remora-wp02
FIXTURE_NAMESPACE=remora-testbed

set -a
source "$TESTBED/versions.env"
set +a

user_kubeconfig_digest() {
  local config_value="${KUBECONFIG:-${HOME:?}/.kube/config}"
  local path
  local paths
  IFS=: read -r -a paths <<< "$config_value"
  {
    for path in "${paths[@]}"; do
      if [[ -f "$path" ]]; then
        sha256sum "$path"
      else
        printf 'missing %s\n' "$path"
      fi
    done
  } | sha256sum | cut -d' ' -f1
}

download_tool() {
  local destination="$1"
  local checksum="$2"
  local url="$3"
  if [[ -x "$destination" ]] && [[ "$(sha256sum "$destination" | cut -d' ' -f1)" = "$checksum" ]]; then
    return
  fi
  curl --fail --location --silent --show-error "$url" --output "$destination.tmp"
  test "$(sha256sum "$destination.tmp" | cut -d' ' -f1)" = "$checksum"
  chmod 0755 "$destination.tmp"
  mv "$destination.tmp" "$destination"
}

prepare_tools() {
  mkdir -p "$BIN"
  download_tool \
    "$BIN/kind" \
    "$KIND_SHA256" \
    "https://github.com/kubernetes-sigs/kind/releases/download/$KIND_VERSION/kind-linux-amd64"
  download_tool \
    "$BIN/kubectl" \
    "$KUBECTL_SHA256" \
    "https://dl.k8s.io/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl"
  test "$("$BIN/kind" version | awk '{print $2}')" = "$KIND_VERSION"
  test "$("$BIN/kubectl" --kubeconfig "$KUBECONFIG_PATH" --context "$CONTEXT" version --client=true -o json | jq -r .clientVersion.gitVersion)" = "$KUBECTL_VERSION"
  test "$(nerdctl --version | awk '{print $3}')" = "$NERDCTL_VERSION"
  test "$(runtime_ctr version | awk '/Client:/{client=1} client && $1=="Version:"{print $2; exit}')" = "v$CONTAINERD_VERSION"
  test "$(runtime_ctr version | awk '/Server:/{server=1} server && $1=="Version:"{print $2; exit}')" = "v$CONTAINERD_VERSION"
}

prepare_provider() {
  configure_provider
  delete_cluster
  remove_runtime_namespace
  "$REMORA_REAL_NERDCTL" namespace create "$NERDCTL_NAMESPACE" >/dev/null
}

acquire_fixture_lock() {
  mkdir -p "$RUNTIME"
  exec 9>"$RUNTIME/fixture.lock"
  if ! flock --nonblock 9; then
    printf 'kind test fixture is already in use\n' >&2
    return 1
  fi
}

configure_provider() {
  if [[ -z "${REMORA_REAL_NERDCTL:-}" ]]; then
    REMORA_REAL_NERDCTL="$(command -v nerdctl)"
  fi
  export REMORA_REAL_NERDCTL
  export CONTAINERD_NAMESPACE="$NERDCTL_NAMESPACE"
  export KIND_EXPERIMENTAL_PROVIDER=nerdctl
  export PATH="$TESTBED/provider:$PATH"
}

remove_runtime_namespace() {
  local status=0
  local digests
  local digest
  local leases
  local lease
  if ! runtime_namespace_exists; then
    return
  fi
  "$REMORA_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" image prune --all --force >/dev/null || status=1
  leases="$(runtime_ctr --namespace "$NERDCTL_NAMESPACE" leases list --quiet)" || status=1
  if [[ -n "$leases" ]]; then
    while read -r lease; do
      runtime_ctr --namespace "$NERDCTL_NAMESPACE" leases delete "$lease" >/dev/null || status=1
    done <<< "$leases"
  fi
  digests="$(runtime_ctr --namespace "$NERDCTL_NAMESPACE" content list --quiet)" || status=1
  if [[ -n "$digests" ]]; then
    while read -r digest; do
      runtime_ctr --namespace "$NERDCTL_NAMESPACE" content remove "$digest" >/dev/null || status=1
    done <<< "$digests"
  fi
  "$REMORA_REAL_NERDCTL" namespace remove "$NERDCTL_NAMESPACE" >/dev/null || status=1
  if runtime_namespace_exists; then
    printf 'containerd namespace %s still exists after teardown\n' "$NERDCTL_NAMESPACE" >&2
    status=1
  fi
  return "$status"
}

runtime_ctr() {
  ctr --address "${CONTAINERD_ADDRESS:-${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/containerd/containerd.sock}" "$@"
}

runtime_namespace_exists() {
  "$REMORA_REAL_NERDCTL" namespace list --quiet | grep -Fx "$NERDCTL_NAMESPACE" >/dev/null
}

cluster_exists() {
  if ! runtime_namespace_exists; then
    return 1
  fi
  "$BIN/kind" get clusters | grep -Fx "$CLUSTER" >/dev/null
}

delete_cluster() {
  if cluster_exists; then
    "$BIN/kind" delete cluster --name "$CLUSTER"
  fi
  if cluster_exists; then
    printf 'kind cluster %s still exists after teardown\n' "$CLUSTER" >&2
    return 1
  fi
}

load_external_image() {
  local name="$1"
  local source="$2"
  local digest="${source##*@}"
  local image="remora-testbed/$name:sha256-${digest#sha256:}"
  local archive="$RUNTIME/image-archives/$digest.tar"
  "$REMORA_REAL_NERDCTL" image pull --platform linux/amd64 --unpack=false --quiet "$source" >&2 || return 1
  "$REMORA_REAL_NERDCTL" image tag "$source" "$image" || return 1
  "$REMORA_REAL_NERDCTL" image save --platform linux/amd64 --output "$archive" "$image" || return 1
  printf 'Loading %s from %s\n' "$image" "$source" >&2
  "$BIN/kind" load image-archive "$archive" --name "$CLUSTER" >&2 || return 1
  printf '%s\n' "$image"
}

load_local_image() {
  local image="$1"
  local namespace="${2:-default}"
  local archive="$RUNTIME/image-archives/package-test.tar"
  rm -f "$archive"
  "$REMORA_REAL_NERDCTL" --namespace "$namespace" image save \
    --platform linux/amd64 \
    --output "$archive" \
    "$image"
  "$BIN/kind" load image-archive "$archive" --name "$CLUSTER"
}

render_manifests() {
  local postgres_image="$1"
  local redis_image="$2"
  local dragonfly_image="$3"
  local vault_image="$4"
  local keycloak_image="$5"
  local zitadel_image="$6"
  local garage_image="$7"
  local versitygw_image="$8"
  local coredns_image="$9"
  local curl_image="${10}"
  local aws_cli_image="${11}"
  local postgres_replicas="${12}"
  local redis_replicas="${13}"
  local dragonfly_replicas="${14}"
  local vault_replicas="${15}"
  local keycloak_replicas="${16}"
  local zitadel_replicas="${17}"
  local garage_replicas="${18}"
  local versitygw_replicas="${19}"
  local coredns_replicas="${20}"
  sed \
    -e "s|@@POSTGRES_IMAGE@@|$postgres_image|g" \
    -e "s|@@REDIS_IMAGE@@|$redis_image|g" \
    -e "s|@@DRAGONFLY_IMAGE@@|$dragonfly_image|g" \
    -e "s|@@VAULT_IMAGE@@|$vault_image|g" \
    -e "s|@@KEYCLOAK_IMAGE@@|$keycloak_image|g" \
    -e "s|@@ZITADEL_IMAGE@@|$zitadel_image|g" \
    -e "s|@@GARAGE_IMAGE@@|$garage_image|g" \
    -e "s|@@VERSITYGW_IMAGE@@|$versitygw_image|g" \
    -e "s|@@COREDNS_IMAGE@@|$coredns_image|g" \
    -e "s|@@CURL_IMAGE@@|$curl_image|g" \
    -e "s|@@AWS_CLI_IMAGE@@|$aws_cli_image|g" \
    -e "s|@@POSTGRES_REPLICAS@@|$postgres_replicas|g" \
    -e "s|@@REDIS_REPLICAS@@|$redis_replicas|g" \
    -e "s|@@DRAGONFLY_REPLICAS@@|$dragonfly_replicas|g" \
    -e "s|@@VAULT_REPLICAS@@|$vault_replicas|g" \
    -e "s|@@KEYCLOAK_REPLICAS@@|$keycloak_replicas|g" \
    -e "s|@@ZITADEL_REPLICAS@@|$zitadel_replicas|g" \
    -e "s|@@GARAGE_REPLICAS@@|$garage_replicas|g" \
    -e "s|@@VERSITYGW_REPLICAS@@|$versitygw_replicas|g" \
    -e "s|@@COREDNS_REPLICAS@@|$coredns_replicas|g" \
    "$TESTBED/dependencies.yaml.in" > "$RUNTIME/dependencies.yaml"
  sed \
    -e "s|@@POSTGRES_IMAGE@@|$postgres_image|g" \
    -e "s|@@REDIS_IMAGE@@|$redis_image|g" \
    -e "s|@@VAULT_IMAGE@@|$vault_image|g" \
    -e "s|@@CURL_IMAGE@@|$curl_image|g" \
    -e "s|@@AWS_CLI_IMAGE@@|$aws_cli_image|g" \
    "$TESTBED/probe.yaml.in" > "$RUNTIME/probe.yaml"
  {
    printf 'POSTGRES_LOCAL_IMAGE=%q\n' "$postgres_image"
    printf 'REDIS_LOCAL_IMAGE=%q\n' "$redis_image"
    printf 'DRAGONFLY_LOCAL_IMAGE=%q\n' "$dragonfly_image"
    printf 'VAULT_LOCAL_IMAGE=%q\n' "$vault_image"
    printf 'KEYCLOAK_LOCAL_IMAGE=%q\n' "$keycloak_image"
    printf 'ZITADEL_LOCAL_IMAGE=%q\n' "$zitadel_image"
    printf 'GARAGE_LOCAL_IMAGE=%q\n' "$garage_image"
    printf 'VERSITYGW_LOCAL_IMAGE=%q\n' "$versitygw_image"
    printf 'COREDNS_LOCAL_IMAGE=%q\n' "$coredns_image"
    printf 'CURL_LOCAL_IMAGE=%q\n' "$curl_image"
    printf 'AWS_CLI_LOCAL_IMAGE=%q\n' "$aws_cli_image"
  } > "$RUNTIME/fixture.env"
}

select_fixture_families() {
  local requested=("$@")
  local family
  local candidate
  local found
  local use_all=false
  FIXTURE_FAMILIES=()
  if [[ "${#requested[@]}" = 0 ]]; then
    requested=(all)
  fi
  for family in "${requested[@]}"; do
    case "$family" in
      all)
        use_all=true
        ;;
      durable | coordination | oidc | target | secret) ;;
      *)
        printf 'unknown fixture family: %s\n' "$family" >&2
        return 1
        ;;
    esac
  done
  if [[ "$use_all" = true ]]; then
    requested=(durable coordination oidc target secret)
  fi
  for candidate in durable coordination oidc target secret; do
    found=false
    for family in "${requested[@]}"; do
      if [[ "$family" = "$candidate" ]]; then
        found=true
      fi
    done
    if [[ "$found" = true ]]; then
      FIXTURE_FAMILIES+=("$candidate")
    fi
  done
}

fixture_has_family() {
  local expected="$1"
  local family
  for family in "${FIXTURE_FAMILIES[@]}"; do
    if [[ "$family" = "$expected" ]]; then
      return
    fi
  done
  return 1
}

load_selected_image() {
  local family="$1"
  local variable="$2"
  local name="$3"
  local source="$4"
  local image
  if fixture_has_family "$family"; then
    image="$(load_external_image "$name" "$source")" || return
    printf -v "$variable" '%s' "$image"
  fi
}

setup_fixture() {
  select_fixture_families "$@" || return
  mkdir -p "$RUNTIME"
  rm -rf "$RUNTIME/image-archives"
  mkdir -p "$RUNTIME/image-archives"
  prepare_tools
  prepare_provider

  "$BIN/kind" create cluster \
    --name "$CLUSTER" \
    --image "$NODE_IMAGE" \
    --config "$TESTBED/cluster.yaml" \
    --kubeconfig "$KUBECONFIG_PATH" \
    --wait 180s

  POSTGRES_LOCAL_IMAGE=remora-testbed/unused:postgres
  REDIS_LOCAL_IMAGE=remora-testbed/unused:redis
  DRAGONFLY_LOCAL_IMAGE=remora-testbed/unused:dragonfly
  VAULT_LOCAL_IMAGE=remora-testbed/unused:vault
  KEYCLOAK_LOCAL_IMAGE=remora-testbed/unused:keycloak
  ZITADEL_LOCAL_IMAGE=remora-testbed/unused:zitadel
  GARAGE_LOCAL_IMAGE=remora-testbed/unused:garage
  VERSITYGW_LOCAL_IMAGE=remora-testbed/unused:versitygw
  COREDNS_LOCAL_IMAGE=remora-testbed/unused:coredns
  CURL_LOCAL_IMAGE=remora-testbed/unused:curl
  AWS_CLI_LOCAL_IMAGE=remora-testbed/unused:aws-cli
  if fixture_has_family durable || fixture_has_family oidc; then
    POSTGRES_LOCAL_IMAGE="$(load_external_image postgres "$POSTGRES_IMAGE")"
  fi
  load_selected_image coordination REDIS_LOCAL_IMAGE redis "$REDIS_IMAGE"
  load_selected_image coordination DRAGONFLY_LOCAL_IMAGE dragonfly "$DRAGONFLY_IMAGE"
  load_selected_image secret VAULT_LOCAL_IMAGE vault "$VAULT_IMAGE"
  load_selected_image oidc KEYCLOAK_LOCAL_IMAGE keycloak "$KEYCLOAK_IMAGE"
  load_selected_image oidc ZITADEL_LOCAL_IMAGE zitadel "$ZITADEL_IMAGE"
  load_selected_image target GARAGE_LOCAL_IMAGE garage "$GARAGE_IMAGE"
  load_selected_image target VERSITYGW_LOCAL_IMAGE versitygw "$VERSITYGW_IMAGE"
  load_selected_image oidc COREDNS_LOCAL_IMAGE coredns "$COREDNS_IMAGE"
  if fixture_has_family oidc || fixture_has_family target; then
    CURL_LOCAL_IMAGE="$(load_external_image curl "$CURL_IMAGE")"
  fi
  load_selected_image target AWS_CLI_LOCAL_IMAGE aws-cli "$AWS_CLI_IMAGE"

  POSTGRES_REPLICAS=0
  REDIS_REPLICAS=0
  DRAGONFLY_REPLICAS=0
  VAULT_REPLICAS=0
  KEYCLOAK_REPLICAS=0
  ZITADEL_REPLICAS=0
  GARAGE_REPLICAS=0
  VERSITYGW_REPLICAS=0
  COREDNS_REPLICAS=0
  if fixture_has_family durable || fixture_has_family oidc; then POSTGRES_REPLICAS=1; fi
  if fixture_has_family coordination; then REDIS_REPLICAS=1; DRAGONFLY_REPLICAS=1; fi
  if fixture_has_family secret; then VAULT_REPLICAS=1; fi
  if fixture_has_family oidc; then KEYCLOAK_REPLICAS=1; ZITADEL_REPLICAS=1; COREDNS_REPLICAS=1; fi
  if fixture_has_family target; then GARAGE_REPLICAS=1; VERSITYGW_REPLICAS=1; fi
  render_manifests \
    "$POSTGRES_LOCAL_IMAGE" \
    "$REDIS_LOCAL_IMAGE" \
    "$DRAGONFLY_LOCAL_IMAGE" \
    "$VAULT_LOCAL_IMAGE" \
    "$KEYCLOAK_LOCAL_IMAGE" \
    "$ZITADEL_LOCAL_IMAGE" \
    "$GARAGE_LOCAL_IMAGE" \
    "$VERSITYGW_LOCAL_IMAGE" \
    "$COREDNS_LOCAL_IMAGE" \
    "$CURL_LOCAL_IMAGE" \
    "$AWS_CLI_LOCAL_IMAGE" \
    "$POSTGRES_REPLICAS" \
    "$REDIS_REPLICAS" \
    "$DRAGONFLY_REPLICAS" \
    "$VAULT_REPLICAS" \
    "$KEYCLOAK_REPLICAS" \
    "$ZITADEL_REPLICAS" \
    "$GARAGE_REPLICAS" \
    "$VERSITYGW_REPLICAS" \
    "$COREDNS_REPLICAS"

  apply_fixture_component common
  if fixture_has_family durable || fixture_has_family oidc; then apply_fixture_component postgres; fi
  if fixture_has_family coordination; then apply_fixture_component redis; apply_fixture_component dragonfly; fi
  if fixture_has_family secret; then apply_fixture_component vault; fi
  if fixture_has_family oidc; then apply_fixture_component keycloak; apply_fixture_component zitadel; apply_fixture_component controlled-dns; fi
  if fixture_has_family target; then apply_fixture_component garage; apply_fixture_component versitygw; fi
  if fixture_has_family durable || fixture_has_family oidc; then wait_for_deployment postgres; fi
  if fixture_has_family coordination; then wait_for_deployment redis; wait_for_deployment dragonfly; fi
  if fixture_has_family secret; then wait_for_deployment vault; fi
  if fixture_has_family oidc; then wait_for_deployment keycloak; wait_for_deployment zitadel; wait_for_deployment controlled-dns; fi
  if fixture_has_family target; then wait_for_deployment garage; wait_for_deployment versitygw; fi
}

apply_fixture_component() {
  kube apply \
    --filename "$RUNTIME/dependencies.yaml" \
    --selector "fixture.remora.dev/component=$1"
}

wait_for_deployment() {
  kube --namespace "$FIXTURE_NAMESPACE" rollout status "deployment/$1" --timeout 240s
}

expand_fixture_matrix() {
  local coordination_providers=(-)
  local oidc_providers=(-)
  local target_providers=(-)
  local coordination
  local oidc
  local target
  local lane=0
  select_fixture_families "$@" || return
  if fixture_has_family coordination; then coordination_providers=(redis dragonfly); fi
  if fixture_has_family oidc; then oidc_providers=(keycloak zitadel); fi
  if fixture_has_family target; then target_providers=(garage versitygw); fi
  for coordination in "${coordination_providers[@]}"; do
    for oidc in "${oidc_providers[@]}"; do
      for target in "${target_providers[@]}"; do
        lane=$((lane + 1))
        printf '%03d\t%s\t%s\t%s\n' "$lane" "$coordination" "$oidc" "$target"
      done
    done
  done
}

reset_lane_durable_store() {
  local lane="$1"
  local database="remora_lane_$lane"
  kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
    env PGPASSWORD=synthetic-postgres psql \
      --username remora \
      --dbname postgres \
      --set ON_ERROR_STOP=1 \
      --command "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$database'" \
      --command "DROP DATABASE IF EXISTS $database" \
      --command "CREATE DATABASE $database" >&2 || return
  printf '%s\n' "$database"
}

reset_lane_target_store() {
  local provider="$1"
  local bucket="$2"
  local deployment
  local endpoint
  if [[ "$provider" = garage ]]; then
    deployment=garage
    endpoint=http://127.0.0.1:3900
  else
    deployment=versitygw
    endpoint=http://127.0.0.1:10000
  fi
  if kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$deployment" --container s3-readiness -- \
    aws --endpoint-url "$endpoint" s3api head-bucket --bucket "$bucket" >/dev/null 2>&1; then
    cleanup_lane_target_store "$provider" "$bucket" || return
  fi
  kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$deployment" --container s3-readiness -- \
    aws --endpoint-url "$endpoint" s3api create-bucket --bucket "$bucket" >&2 || return
}

# Remora storage role for fixture lanes: target-main / target-bucket / models.
remora_fixture_storage_role() {
  local b64url
  b64url() {
    printf '%s' "$1" | openssl base64 -A | tr '+/' '-_' | tr -d '='
  }
  printf 'remora:v1:storage|%s|%s|%s\n' \
    "$(b64url target-main)" \
    "$(b64url target-bucket)" \
    "$(b64url models)"
}

reset_keycloak_lane() {
  local lane="$1"
  local realm="remora-lane-$lane"
  local username="remora-user-$lane"
  local client_id="remora-client-$lane"
  local storage_role
  local client_uuid
  storage_role="$(remora_fixture_storage_role)"
  if kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
    /opt/keycloak/bin/kcadm.sh get "realms/$realm" \
      --no-config \
      --server http://127.0.0.1:8080 \
      --realm master \
      --user remora-admin \
      --password synthetic-keycloak >/dev/null 2>&1; then
    cleanup_keycloak_lane "$lane" || return
  fi
  kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
    /opt/keycloak/bin/kcadm.sh create realms \
      --no-config \
      --server http://127.0.0.1:8080 \
      --realm master \
      --user remora-admin \
      --password synthetic-keycloak \
      --set "realm=$realm" \
      --set enabled=true \
      --set registrationEmailAsUsername=false \
      --set verifyEmail=false \
      --set resetPasswordAllowed=false \
      --set loginWithEmailAllowed=true \
      --set duplicateEmailsAllowed=false \
      --set sslRequired=none >&2 || return
  # Keycloak 26 enables VERIFY_PROFILE by default; that blocks ROPC with
  # "Account is not fully set up".
  kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
    /opt/keycloak/bin/kcadm.sh update authentication/required-actions/VERIFY_PROFILE \
      --no-config \
      --server http://127.0.0.1:8080 \
      --realm master \
      --user remora-admin \
      --password synthetic-keycloak \
      -r "$realm" \
      --set enabled=false \
      --set defaultAction=false >&2 || true
  # Public client with direct access grants so package Jobs can mint ROPC tokens.
  kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
    /opt/keycloak/bin/kcadm.sh create clients \
      --no-config \
      --server http://127.0.0.1:8080 \
      --realm master \
      --user remora-admin \
      --password synthetic-keycloak \
      -r "$realm" \
      --set "clientId=$client_id" \
      --set publicClient=true \
      --set standardFlowEnabled=true \
      --set directAccessGrantsEnabled=true \
      --set 'redirectUris=["http://localhost/callback"]' \
      --set 'attributes={"access.token.lifespan":"3600"}' >&2 || return
  client_uuid="$(kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
    /opt/keycloak/bin/kcadm.sh get clients \
      --no-config \
      --server http://127.0.0.1:8080 \
      --realm master \
      --user remora-admin \
      --password synthetic-keycloak \
      -r "$realm" \
      --query "clientId=$client_id" \
      --fields id \
      --format csv \
      --noquotes)" || return
  client_uuid="$(printf '%s' "$client_uuid" | tr -d '\r' | head -n1)"
  [[ -n "$client_uuid" ]] || return
  # Include this client in access-token `aud` so mandatory audience checks pass.
  kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
    /opt/keycloak/bin/kcadm.sh create "clients/$client_uuid/protocol-mappers/models" \
      --no-config \
      --server http://127.0.0.1:8080 \
      --realm master \
      --user remora-admin \
      --password synthetic-keycloak \
      -r "$realm" \
      --set name=remora-audience \
      --set protocol=openid-connect \
      --set protocolMapper=oidc-audience-mapper \
      --set 'config."included.client.audience"='"$client_id" \
      --set 'config."id.token.claim"=false' \
      --set 'config."access.token.claim"=true' >&2 || return
  kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
    /opt/keycloak/bin/kcadm.sh create roles \
      --no-config \
      --server http://127.0.0.1:8080 \
      --realm master \
      --user remora-admin \
      --password synthetic-keycloak \
      -r "$realm" \
      --set name=test-user >&2 || return
  # Client roles are the only Keycloak claim source Remora trusts (resource_access).
  # Use -b JSON so `|` in storage roles is not percent-encoded by --set.
  kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
    /opt/keycloak/bin/kcadm.sh create "clients/$client_uuid/roles" \
      --no-config \
      --server http://127.0.0.1:8080 \
      --realm master \
      --user remora-admin \
      --password synthetic-keycloak \
      -r "$realm" \
      -b "{\"name\":\"remora:v1:admin\"}" >&2 || return
  kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
    /opt/keycloak/bin/kcadm.sh create "clients/$client_uuid/roles" \
      --no-config \
      --server http://127.0.0.1:8080 \
      --realm master \
      --user remora-admin \
      --password synthetic-keycloak \
      -r "$realm" \
      -b "{\"name\":\"$storage_role\"}" >&2 || return
  kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
    /opt/keycloak/bin/kcadm.sh create users \
      --no-config \
      --server http://127.0.0.1:8080 \
      --realm master \
      --user remora-admin \
      --password synthetic-keycloak \
      -r "$realm" \
      --set "username=$username" \
      --set enabled=true \
      --set emailVerified=true \
      --set "email=$username@remora.test" \
      --set 'requiredActions=[]' >&2 || return
  local user_id
  user_id="$(kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
    /opt/keycloak/bin/kcadm.sh get users \
      --no-config \
      --server http://127.0.0.1:8080 \
      --realm master \
      --user remora-admin \
      --password synthetic-keycloak \
      -r "$realm" \
      --query "username=$username" \
      --fields id \
      --format csv \
      --noquotes)" || return
  user_id="$(printf '%s' "$user_id" | tr -d '\r' | head -n1)"
  [[ -n "$user_id" ]] || return
  # Clear any required actions and mark email verified for ROPC.
  kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
    /opt/keycloak/bin/kcadm.sh update "users/$user_id" \
      --no-config \
      --server http://127.0.0.1:8080 \
      --realm master \
      --user remora-admin \
      --password synthetic-keycloak \
      -r "$realm" \
      --set emailVerified=true \
      --set 'requiredActions=[]' \
      --set enabled=true >&2 || return
  kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
    /opt/keycloak/bin/kcadm.sh set-password \
      --no-config \
      --server http://127.0.0.1:8080 \
      --realm master \
      --user remora-admin \
      --password synthetic-keycloak \
      -r "$realm" \
      --userid "$user_id" \
      --new-password Synthetic-OIDC-1! \
      --temporary=false >&2 || return
  kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
    /opt/keycloak/bin/kcadm.sh add-roles \
      --no-config \
      --server http://127.0.0.1:8080 \
      --realm master \
      --user remora-admin \
      --password synthetic-keycloak \
      -r "$realm" \
      --uusername "$username" \
      --rolename test-user >&2 || return
  kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
    /opt/keycloak/bin/kcadm.sh add-roles \
      --no-config \
      --server http://127.0.0.1:8080 \
      --realm master \
      --user remora-admin \
      --password synthetic-keycloak \
      -r "$realm" \
      --uusername "$username" \
      --cclientid "$client_id" \
      --rolename 'remora:v1:admin' \
      --rolename "$storage_role" >&2 || return
  # issuer \t tenant \t client_id \t username \t storage_role \t project_id \t user_id \t mgmt_token
  # issuer \t tenant \t client_id \t username \t storage_role \t project_id \t user_id \t mgmt_token \t user_key_b64
  printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
    "http://keycloak.remora.test:8080/realms/$realm" \
    "$realm" \
    "$client_id" \
    "$username" \
    "$storage_role" \
    "" \
    "" \
    "" \
    ""
}

zitadel_request() {
  local endpoint="$1"
  local body="$2"
  local organization="${3:-}"
  local method="${4:-POST}"
  kube --namespace "$FIXTURE_NAMESPACE" exec deployment/zitadel --container provisioner -- \
    /bin/sh -ec '
      endpoint="$1"
      body="$2"
      organization="$3"
      method="$4"
      token="$(cat /bootstrap/admin.pat)"
      set -- \
        --fail \
        --silent \
        --show-error \
        --request "$method" \
        --header "Host: zitadel.remora.test:8080" \
        --header "Authorization: Bearer $token" \
        --header "Content-Type: application/json"
      if test -n "$organization"; then
        set -- "$@" --header "x-zitadel-orgid: $organization"
      fi
      curl "$@" --data "$body" "http://127.0.0.1:8080$endpoint"
    ' remora-zitadel-request "$endpoint" "$body" "$organization" "$method"
}

json_string_field() {
  local document="$1"
  local field="$2"
  local value
  value="$(sed -nE 's/.*"'"$field"'"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/p' <<< "$document")"
  if [[ -z "$value" ]]; then
    printf 'ZITADEL response omitted %s\n' "$field" >&2
    return 1
  fi
  printf '%s\n' "$value"
}

reset_zitadel_lane() {
  local lane="$1"
  local tenant="remora-lane-$lane"
  local username="remora-user-$lane"
  local storage_role
  local response
  local organization_id
  local project_id
  local user_id
  local client_id
  local mgmt_token
  local grant_id
  storage_role="$(remora_fixture_storage_role)"
  cleanup_zitadel_lane "$lane" || return
  response="$(zitadel_request /management/v1/orgs "{\"name\":\"$tenant\"}")" || return
  organization_id="$(json_string_field "$response" id)" || return
  response="$(zitadel_request /management/v1/projects "{\"name\":\"$tenant\",\"projectRoleAssertion\":true,\"projectRoleCheck\":true}" "$organization_id")" || return
  project_id="$(json_string_field "$response" id)" || return
  zitadel_request \
    "/management/v1/projects/$project_id/roles" \
    '{"roleKey":"test-user","displayName":"Test User","group":"fixture"}' \
    "$organization_id" >/dev/null || return
  zitadel_request \
    "/management/v1/projects/$project_id/roles" \
    '{"roleKey":"remora:v1:admin","displayName":"Remora Admin","group":"remora"}' \
    "$organization_id" >/dev/null || return
  zitadel_request \
    "/management/v1/projects/$project_id/roles" \
    "{\"roleKey\":\"$storage_role\",\"displayName\":\"Remora Storage\",\"group\":\"remora\"}" \
    "$organization_id" >/dev/null || return
  # Native app: PKCE public client with JWT access tokens and project role assertion.
  response="$(zitadel_request \
    "/management/v1/projects/$project_id/apps/oidc" \
    "{\"name\":\"$tenant\",\"redirectUris\":[\"http://localhost/callback\"],\"postLogoutRedirectUris\":[\"http://localhost/logout\"],\"responseTypes\":[\"OIDC_RESPONSE_TYPE_CODE\"],\"grantTypes\":[\"OIDC_GRANT_TYPE_AUTHORIZATION_CODE\",\"OIDC_GRANT_TYPE_REFRESH_TOKEN\"],\"appType\":\"OIDC_APP_TYPE_NATIVE\",\"authMethodType\":\"OIDC_AUTH_METHOD_TYPE_NONE\",\"version\":\"OIDC_VERSION_1_0\",\"devMode\":true,\"accessTokenType\":\"OIDC_TOKEN_TYPE_JWT\",\"accessTokenRoleAssertion\":true,\"idTokenRoleAssertion\":true,\"idTokenUserinfoAssertion\":true}" \
    "$organization_id")" || return
  client_id="$(json_string_field "$response" clientId)" || return
  response="$(zitadel_request \
    /management/v1/users/human \
    "{\"userName\":\"$username\",\"profile\":{\"firstName\":\"Remora\",\"lastName\":\"Lane $lane\",\"displayName\":\"Remora Lane $lane\",\"preferredLanguage\":\"en\",\"gender\":\"GENDER_UNSPECIFIED\"},\"email\":{\"email\":\"$username@remora.test\",\"isEmailVerified\":true},\"initialPassword\":\"Synthetic-OIDC-1!\"}" \
    "$organization_id")" || return
  user_id="$(json_string_field "$response" userId)" || return
  # Set a permanent password for Session API checks (initialPassword alone is not enough on all versions).
  if ! zitadel_request \
    "/management/v1/users/$user_id/password" \
    '{"password":"Synthetic-OIDC-1!","noChangeRequired":true}' \
    "$organization_id" >/dev/null; then
    # Fallback field name used by older management API payloads.
    zitadel_request \
      "/management/v1/users/$user_id/password" \
      '{"password":"Synthetic-OIDC-1!","changeRequired":false}' \
      "$organization_id" >/dev/null || return
  fi
  response="$(zitadel_request \
    "/management/v1/users/$user_id/grants" \
    "{\"projectId\":\"$project_id\",\"roleKeys\":[\"test-user\",\"remora:v1:admin\",\"$storage_role\"]}" \
    "$organization_id")" || return
  # Machine user + application key for JWT-bearer access tokens with project roles.
  local machine_name="remora-machine-$lane"
  # Bearer defaults to opaque tokens; Remora validates only JWKS-signed JWTs.
  response="$(zitadel_request \
    /management/v1/users/machine \
    "{\"userName\":\"$machine_name\",\"name\":\"Remora Machine $lane\",\"description\":\"JWT bearer minting\",\"accessTokenType\":\"ACCESS_TOKEN_TYPE_JWT\"}" \
    "$organization_id")" || return
  local machine_id
  machine_id="$(json_string_field "$response" userId)" || return
  zitadel_request \
    "/management/v1/users/$machine_id/grants" \
    "{\"projectId\":\"$project_id\",\"roleKeys\":[\"remora:v1:admin\",\"$storage_role\"]}" \
    "$organization_id" >/dev/null || return
  response="$(zitadel_request \
    "/management/v1/users/$machine_id/keys" \
    '{"type":"KEY_TYPE_JSON","expirationDate":"2099-01-01T00:00:00Z"}' \
    "$organization_id")" || return
  local key_details_b64
  key_details_b64="$(json_string_field "$response" keyDetails)" || return
  mgmt_token="$(kube --namespace "$FIXTURE_NAMESPACE" exec deployment/zitadel --container provisioner -- \
    cat /bootstrap/admin.pat)" || return
  mgmt_token="$(printf '%s' "$mgmt_token" | tr -d '\r\n')"
  # issuer \t tenant \t client_id \t username \t storage_role \t project_id \t user_id \t mgmt_token \t key_details_b64
  # user_id is the machine user used for JWT-bearer minting (and grant loss).
  printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
    http://zitadel.remora.test:8080 \
    "$organization_id" \
    "$client_id" \
    "$username" \
    "$storage_role" \
    "$project_id" \
    "$machine_id" \
    "$mgmt_token" \
    "$key_details_b64"
}

reset_lane_oidc_provider() {
  local provider="$1"
  local lane="$2"
  if [[ "$provider" = keycloak ]]; then
    reset_keycloak_lane "$lane"
  else
    reset_zitadel_lane "$lane"
  fi
}

cleanup_lane_durable_store() {
  local database="remora_lane_$1"
  local remaining
  kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
    env PGPASSWORD=synthetic-postgres psql \
      --username remora \
      --dbname postgres \
      --set ON_ERROR_STOP=1 \
      --command "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$database'" \
      --command "DROP DATABASE IF EXISTS $database" >&2 || return
  remaining="$(kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
    env PGPASSWORD=synthetic-postgres psql \
      --username remora \
      --dbname postgres \
      --tuples-only \
      --no-align \
      --command "SELECT count(*) FROM pg_database WHERE datname = '$database'")" || return
  [[ "$remaining" = 0 ]]
}

cleanup_lane_coordination_store() {
  local provider="$1"
  local namespace="$2"
  local keys
  local key
  while true; do
    keys="$(kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" -- \
      redis-cli --scan --pattern "$namespace*")" || return
    [[ -z "$keys" ]] && return
    while IFS= read -r key; do
      kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" -- \
        redis-cli DEL "$key" >/dev/null || return
    done <<< "$keys"
  done
}

cleanup_lane_target_store() {
  local provider="$1"
  local bucket="$2"
  local deployment
  local endpoint
  if [[ "$provider" = garage ]]; then
    deployment=garage
    endpoint=http://127.0.0.1:3900
  else
    deployment=versitygw
    endpoint=http://127.0.0.1:10000
  fi
  kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$deployment" --container s3-readiness -- \
    aws --endpoint-url "$endpoint" s3 rm "s3://$bucket" --recursive >&2 || return
  kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$deployment" --container s3-readiness -- \
    aws --endpoint-url "$endpoint" s3api delete-bucket --bucket "$bucket" >&2 || return
  if kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$deployment" --container s3-readiness -- \
    aws --endpoint-url "$endpoint" s3api head-bucket --bucket "$bucket" >/dev/null 2>&1; then
    return 1
  fi
}

cleanup_keycloak_lane() {
  local realm="remora-lane-$1"
  kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
    /opt/keycloak/bin/kcadm.sh delete "realms/$realm" \
      --no-config \
      --server http://127.0.0.1:8080 \
      --realm master \
      --user remora-admin \
      --password synthetic-keycloak >&2 || return
}

find_zitadel_lane_organizations() {
  local lane="$1"
  local tenant="remora-lane-$lane"
  local response
  response="$(zitadel_request /v2/organizations/_search '{}')" || return
  jq --raw-output --arg tenant "$tenant" \
    '.result[]? | select(.name == $tenant) | .id' <<< "$response"
}

cleanup_zitadel_lane() {
  local lane="$1"
  local organization_ids
  local organization_id
  local status=0
  organization_ids="$(find_zitadel_lane_organizations "$lane")" || return
  while IFS= read -r organization_id; do
    [[ -z "$organization_id" ]] && continue
    zitadel_request "/v2/organizations/$organization_id" '' '' DELETE >/dev/null || status=1
  done <<< "$organization_ids"
  return "$status"
}

cleanup_lane_secret_store() {
  local path="secret/remora/lane-$1"
  kube --namespace "$FIXTURE_NAMESPACE" exec deployment/vault -- \
    env VAULT_ADDR=http://127.0.0.1:8200 VAULT_TOKEN=synthetic-vault-token \
    vault kv metadata delete "$path" >/dev/null || return
}

cleanup_lane_state() {
  local lane="$1"
  local coordination="$2"
  local oidc="$3"
  local target="$4"
  local status=0
  if fixture_has_family durable && ! cleanup_lane_durable_store "$lane"; then
    printf 'durable store cleanup failed for lane %s\n' "$lane" >&2
    status=1
  fi
  if [[ "$coordination" != - ]] && ! cleanup_lane_coordination_store "$coordination" "remora:lane:$lane:"; then
    printf 'coordination store cleanup failed for lane %s\n' "$lane" >&2
    status=1
  fi
  if [[ "$oidc" = keycloak ]] && ! cleanup_keycloak_lane "$lane"; then
    printf 'Keycloak cleanup failed for lane %s\n' "$lane" >&2
    status=1
  fi
  if [[ "$oidc" = zitadel ]] && ! cleanup_zitadel_lane "$lane"; then
    printf 'ZITADEL cleanup failed for lane %s\n' "$lane" >&2
    status=1
  fi
  if [[ "$target" != - ]] && ! cleanup_lane_target_store "$target" "remora-lane-$lane"; then
    printf 'target store cleanup failed for lane %s\n' "$lane" >&2
    status=1
  fi
  if fixture_has_family secret && ! cleanup_lane_secret_store "$lane"; then
    printf 'secret store cleanup failed for lane %s\n' "$lane" >&2
    status=1
  fi
  return "$status"
}

write_lane_profile() {
  local lane="$1"
  local coordination="$2"
  local oidc="$3"
  local target="$4"
  local profile_name="remora-fixture-lane-$lane"
  local database=
  local database_url=
  local coordination_url=
  local coordination_namespace=
  local oidc_issuer=
  local oidc_tenant=
  local oidc_client_id=
  local oidc_username=
  local oidc_storage_role=
  local oidc_project_id=
  local oidc_user_id=
  local oidc_mgmt_token=
  local oidc_user_key_b64=
  local oidc_profile
  local target_endpoint=
  local target_bucket=
  local target_prefix=
  local target_access_key=
  local target_secret_key=
  local target_region=
  local vault_path=
  local profile
  if fixture_has_family durable; then
    database="$(reset_lane_durable_store "$lane")" || return
    database_url="postgres://remora:synthetic-postgres@postgres:5432/$database?sslmode=disable"
  fi
  if [[ "$coordination" != - ]]; then
    coordination_url="redis://$coordination:6379"
    coordination_namespace="remora:lane:$lane:"
    cleanup_lane_coordination_store "$coordination" "$coordination_namespace" || return
  fi
  if [[ "$oidc" != - ]]; then
    oidc_profile="$(reset_lane_oidc_provider "$oidc" "$lane")" || return
    IFS=$'\t' read -r oidc_issuer oidc_tenant oidc_client_id oidc_username \
      oidc_storage_role oidc_project_id oidc_user_id oidc_mgmt_token oidc_user_key_b64 \
      <<< "$oidc_profile"
  fi
  if [[ "$target" != - ]]; then
    target_bucket="remora-lane-$lane"
    target_prefix="lane-$lane/"
    reset_lane_target_store "$target" "$target_bucket" || return
    if [[ "$target" = garage ]]; then
      target_endpoint=http://garage:3900
      target_access_key=GK00000000000000000000000000000000
      target_secret_key=0000000000000000000000000000000000000000000000000000000000000000
      target_region=garage
    else
      target_endpoint=http://versitygw:10000
      target_access_key=synthetic-s3-access
      target_secret_key=synthetic-s3-secret
      target_region=us-east-1
    fi
  fi
  if fixture_has_family secret; then
    vault_path="secret/remora/lane-$lane"
    cleanup_lane_secret_store "$lane" || return
  fi
  printf -v profile '%s\n' \
    "REMORA_FIXTURE_LANE=$lane" \
    "REMORA_COORDINATION_PROVIDER=$coordination" \
    "REMORA_COORDINATION_URL=$coordination_url" \
    "REMORA_COORDINATION_NAMESPACE=$coordination_namespace" \
    "REMORA_OIDC_PROVIDER=$oidc" \
    "REMORA_OIDC_ISSUER=$oidc_issuer" \
    "REMORA_OIDC_TENANT=$oidc_tenant" \
    "REMORA_OIDC_CLIENT_ID=$oidc_client_id" \
    "REMORA_OIDC_USERNAME=$oidc_username" \
    "REMORA_OIDC_PASSWORD=Synthetic-OIDC-1!" \
    "REMORA_OIDC_STORAGE_ROLE=$oidc_storage_role" \
    "REMORA_OIDC_PROJECT_ID=$oidc_project_id" \
    "REMORA_OIDC_USER_ID=$oidc_user_id" \
    "REMORA_OIDC_MGMT_TOKEN=$oidc_mgmt_token" \
    "REMORA_OIDC_USER_KEY_B64=$oidc_user_key_b64" \
    "REMORA_TARGET_PROVIDER=$target" \
    "REMORA_TARGET_ENDPOINT=$target_endpoint" \
    "REMORA_TARGET_BUCKET=$target_bucket" \
    "REMORA_TARGET_PREFIX=$target_prefix" \
    "REMORA_TARGET_ACCESS_KEY=$target_access_key" \
    "REMORA_TARGET_SECRET_KEY=$target_secret_key" \
    "REMORA_TARGET_REGION=$target_region" \
    "REMORA_TARGET_EXCLUSIVE_WRITER=true" \
    "REMORA_VAULT_ADDR=http://vault:8200" \
    "REMORA_VAULT_TOKEN=synthetic-vault-token" \
    "REMORA_VAULT_PATH=$vault_path" \
    "DATABASE_URL=$database_url"
  kube --namespace "$FIXTURE_NAMESPACE" delete \
    "configmap/$profile_name" \
    "configmap/$profile_name-file" \
    --ignore-not-found \
    --wait=true >&2 || return
  kube --namespace "$FIXTURE_NAMESPACE" create configmap "$profile_name" \
    --from-literal="REMORA_FIXTURE_LANE=$lane" \
    --from-literal="REMORA_COORDINATION_PROVIDER=$coordination" \
    --from-literal="REMORA_COORDINATION_URL=$coordination_url" \
    --from-literal="REMORA_COORDINATION_NAMESPACE=$coordination_namespace" \
    --from-literal="REMORA_OIDC_PROVIDER=$oidc" \
    --from-literal="REMORA_OIDC_ISSUER=$oidc_issuer" \
    --from-literal="REMORA_OIDC_TENANT=$oidc_tenant" \
    --from-literal="REMORA_OIDC_CLIENT_ID=$oidc_client_id" \
    --from-literal="REMORA_OIDC_USERNAME=$oidc_username" \
    --from-literal=REMORA_OIDC_PASSWORD=Synthetic-OIDC-1! \
    --from-literal="REMORA_OIDC_STORAGE_ROLE=$oidc_storage_role" \
    --from-literal="REMORA_OIDC_PROJECT_ID=$oidc_project_id" \
    --from-literal="REMORA_OIDC_USER_ID=$oidc_user_id" \
    --from-literal="REMORA_OIDC_MGMT_TOKEN=$oidc_mgmt_token" \
    --from-literal="REMORA_OIDC_USER_KEY_B64=$oidc_user_key_b64" \
    --from-literal="REMORA_TARGET_PROVIDER=$target" \
    --from-literal="REMORA_TARGET_ENDPOINT=$target_endpoint" \
    --from-literal="REMORA_TARGET_BUCKET=$target_bucket" \
    --from-literal="REMORA_TARGET_PREFIX=$target_prefix" \
    --from-literal="REMORA_TARGET_ACCESS_KEY=$target_access_key" \
    --from-literal="REMORA_TARGET_SECRET_KEY=$target_secret_key" \
    --from-literal="REMORA_TARGET_REGION=$target_region" \
    --from-literal=REMORA_TARGET_EXCLUSIVE_WRITER=true \
    --from-literal=REMORA_VAULT_ADDR=http://vault:8200 \
    --from-literal=REMORA_VAULT_TOKEN=synthetic-vault-token \
    --from-literal="REMORA_VAULT_PATH=$vault_path" \
    --from-literal="DATABASE_URL=$database_url" >&2 || return
  kube --namespace "$FIXTURE_NAMESPACE" create configmap "$profile_name-file" \
    --from-literal="profile.env=$profile" >&2 || return
  printf '%s\n' "$profile_name"
}

render_lane_job() {
  local template="$1"
  local output="$2"
  local job="$3"
  local profile="$4"
  local image="$5"
  local postgres_image="${POSTGRES_LOCAL_IMAGE:-remora-testbed/unused:postgres}"
  local redis_image="${REDIS_LOCAL_IMAGE:-remora-testbed/unused:redis}"
  local vault_image="${VAULT_LOCAL_IMAGE:-remora-testbed/unused:vault}"
  local curl_image="${CURL_LOCAL_IMAGE:-remora-testbed/unused:curl}"
  local aws_cli_image="${AWS_CLI_LOCAL_IMAGE:-remora-testbed/unused:aws-cli}"
  sed \
    -e "s|@@JOB_NAME@@|$job|g" \
    -e "s|@@PROFILE_NAME@@|$profile|g" \
    -e "s|@@TEST_IMAGE@@|$image|g" \
    -e "s|@@POSTGRES_IMAGE@@|$postgres_image|g" \
    -e "s|@@REDIS_IMAGE@@|$redis_image|g" \
    -e "s|@@VAULT_IMAGE@@|$vault_image|g" \
    -e "s|@@CURL_IMAGE@@|$curl_image|g" \
    -e "s|@@AWS_CLI_IMAGE@@|$aws_cli_image|g" \
    "$template" > "$output"
}

run_job_matrix() {
  local job_base="$1"
  local template="$2"
  local image="$3"
  shift 3
  local families=("$@")
  local lane
  local coordination
  local oidc
  local target
  local job
  local profile
  local manifest
  select_fixture_families "${families[@]}" || return
  while IFS=$'\t' read -r lane coordination oidc target; do
    job="$job_base-$lane"
    profile="$(write_lane_profile "$lane" "$coordination" "$oidc" "$target")" || return
    manifest="$RUNTIME/$job.yaml"
    render_lane_job "$template" "$manifest" "$job" "$profile" "$image" || return
    printf 'Running fixture lane %s: coordination=%s oidc=%s target=%s\n' "$lane" "$coordination" "$oidc" "$target"
    run_job_manifest "$FIXTURE_NAMESPACE" "$job" "$manifest" || return
    cleanup_lane_state "$lane" "$coordination" "$oidc" "$target" || return
    kube --namespace "$FIXTURE_NAMESPACE" delete \
      "job/$job" \
      "configmap/$profile" \
      "configmap/$profile-file" \
      --wait=true || return
    rm -f "$manifest" || return
  done < <(expand_fixture_matrix "${families[@]}")
}

verify_dependencies() {
  local family
  for family in durable coordination oidc target secret; do
    if ! fixture_has_family "$family"; then
      printf 'released-image dependency probes require the all fixture profile\n' >&2
      return 1
    fi
  done
  kube apply --filename "$RUNTIME/probe.yaml"
  for job in state-probe oidc-probe garage-probe versitygw-probe; do
    wait_for_job "$FIXTURE_NAMESPACE" "$job"
  done
}

destroy_fixture() {
  configure_provider
  delete_cluster
  remove_runtime_namespace
  rm -rf "$RUNTIME/image-archives"
  rm -f \
    "$KUBECONFIG_PATH" \
    "$RUNTIME/dependencies.yaml" \
    "$RUNTIME/fixture.env" \
    "$RUNTIME/probe.yaml" \
    "$RUNTIME/sample-job.yaml" \
    "$RUNTIME/durable-job.yaml"
  find "$RUNTIME" -maxdepth 1 -type f -name '*-[0-9][0-9][0-9].yaml' -delete
}

kube() {
  "$BIN/kubectl" \
    --kubeconfig "$KUBECONFIG_PATH" \
    --context "$CONTEXT" \
    "$@"
}

wait_for_job() {
  local namespace="$1"
  local job="$2"
  local deadline=$((SECONDS + 300))
  local complete
  local failed
  while ((SECONDS < deadline)); do
    complete="$(kube --namespace "$namespace" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Complete")].status}')"
    if [[ "$complete" = True ]]; then
      print_job_logs "$namespace" "$job"
      return
    fi
    failed="$(kube --namespace "$namespace" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Failed")].status}')"
    if [[ "$failed" = True ]]; then
      print_job_logs "$namespace" "$job" >&2 || true
      return 1
    fi
    sleep 1
  done
  print_job_logs "$namespace" "$job" >&2 || true
  printf 'job/%s did not complete within 300 seconds\n' "$job" >&2
  return 1
}

print_job_logs() {
  local namespace="$1"
  local job="$2"
  local pod
  local containers
  local container
  local status=0
  pod="$(kube --namespace "$namespace" get pods \
    --selector "job-name=$job" \
    --output jsonpath='{.items[0].metadata.name}')" || return
  containers="$(kube --namespace "$namespace" get "pod/$pod" \
    --output jsonpath='{range .spec.initContainers[*]}{.name}{"\n"}{end}{range .spec.containers[*]}{.name}{"\n"}{end}')" || return
  while IFS= read -r container; do
    [[ -z "$container" ]] && continue
    kube --namespace "$namespace" logs "pod/$pod" --container "$container" || status=1
  done <<< "$containers"
  return "$status"
}

run_job_manifest() {
  local namespace="$1"
  local job="$2"
  local manifest="$3"
  kube --namespace "$namespace" delete "job/$job" --ignore-not-found --wait=true || return
  kube --namespace "$namespace" apply --filename "$manifest" || return
  wait_for_job "$namespace" "$job" || return
}