nuthatch/tests/kind/lib.sh

#!/usr/bin/env bash
set -euo pipefail

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
TESTBED="$ROOT/tests/kind"
RUNTIME="$TESTBED/.runtime"
BIN="$RUNTIME/bin"
KUBECONFIG_PATH="$RUNTIME/kubeconfig"
FIXTURE_LOCK_DIR="${XDG_RUNTIME_DIR:-/tmp/nuthatch-$(id -u)}"
FIXTURE_LOCK_PATH="$FIXTURE_LOCK_DIR/nuthatch-testbed-kind.lock"
CONTEXT=kind-nuthatch-testbed
CLUSTER=nuthatch-testbed
NERDCTL_NAMESPACE=nuthatch-testbed
FIXTURE_NAMESPACE=nuthatch-testbed

set -a
source "$TESTBED/versions.env"
set +a

user_kubeconfig_digest() {
  local config_value="${KUBECONFIG:-${HOME:?}/.kube/config}"
  local path
  local paths
  IFS=: read -r -a paths <<< "$config_value"
  {
    for path in "${paths[@]}"; do
      if [[ -f "$path" ]]; then
        sha256sum "$path"
      else
        printf 'missing %s\n' "$path"
      fi
    done
  } | sha256sum | cut -d' ' -f1
}

download_tool() {
  local destination="$1"
  local checksum="$2"
  local url="$3"
  if [[ -x "$destination" ]] &&
     [[ "$(sha256sum "$destination" | cut -d' ' -f1)" = "$checksum" ]]; then
    return
  fi
  curl --fail --location --silent --show-error "$url" --output "$destination.tmp"
  test "$(sha256sum "$destination.tmp" | cut -d' ' -f1)" = "$checksum"
  chmod 0755 "$destination.tmp"
  mv "$destination.tmp" "$destination"
}

prepare_tools() {
  mkdir -p "$BIN"
  download_tool \
    "$BIN/kind" \
    "$KIND_SHA256" \
    "https://github.com/kubernetes-sigs/kind/releases/download/$KIND_VERSION/kind-linux-amd64"
  download_tool \
    "$BIN/kubectl" \
    "$KUBECTL_SHA256" \
    "https://dl.k8s.io/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl"
  test "$("$BIN/kind" version | awk '{print $2}')" = "$KIND_VERSION"
  test "$("$BIN/kubectl" --kubeconfig "$KUBECONFIG_PATH" --context "$CONTEXT" version --client=true -o json | jq -r .clientVersion.gitVersion)" = "$KUBECTL_VERSION"
  test "$(nerdctl --version | awk '{print $3}')" = "$NERDCTL_VERSION"
  test "$(runtime_ctr version | awk '/Client:/{client=1} client && $1=="Version:"{print $2; exit}')" = "v$CONTAINERD_VERSION"
  test "$(runtime_ctr version | awk '/Server:/{server=1} server && $1=="Version:"{print $2; exit}')" = "v$CONTAINERD_VERSION"
}

acquire_fixture_lock() {
  mkdir -p "$RUNTIME" "$FIXTURE_LOCK_DIR"
  exec 9>"$FIXTURE_LOCK_PATH"
  if ! flock --nonblock 9; then
    printf 'kind test fixture is already in use\n' >&2
    return 1
  fi
}

configure_provider() {
  if [[ -z "${NUTHATCH_REAL_NERDCTL:-}" ]]; then
    NUTHATCH_REAL_NERDCTL="$(command -v nerdctl)" || return
  fi
  export NUTHATCH_REAL_NERDCTL
  export CONTAINERD_NAMESPACE="$NERDCTL_NAMESPACE"
  export KIND_EXPERIMENTAL_PROVIDER=nerdctl
  export PATH="$TESTBED/provider:$PATH"
}

runtime_ctr() {
  ctr --address "${CONTAINERD_ADDRESS:-${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/containerd/containerd.sock}" "$@"
}

runtime_namespace_exists() {
  "$NUTHATCH_REAL_NERDCTL" namespace list --quiet | grep -Fx "$NERDCTL_NAMESPACE" >/dev/null
}

cluster_exists() {
  runtime_namespace_exists &&
    "$BIN/kind" get clusters | grep -Fx "$CLUSTER" >/dev/null
}

delete_cluster() {
  local container
  local containers
  if cluster_exists; then
    "$BIN/kind" delete cluster --name "$CLUSTER" || true
  fi
  containers="$("$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" ps \
    --all \
    --quiet \
    --filter "label=io.x-k8s.kind.cluster=$CLUSTER")" || return
  while IFS= read -r container; do
    [[ -z "$container" ]] ||
      "$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" rm --force "$container" ||
      return
  done <<< "$containers"
  if cluster_exists; then
    "$BIN/kind" delete cluster --name "$CLUSTER" || true
  fi
  if cluster_exists; then
    printf 'kind cluster %s still exists after teardown\n' "$CLUSTER" >&2
    return 1
  fi
}

remove_runtime_namespace() {
  local status=0
  local digest
  local digests
  local lease
  local leases
  local network
  for network in kind nuthatch-kind; do
    if "$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" network inspect "$network" >/dev/null 2>&1; then
      "$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" network remove "$network" >/dev/null || status=1
    fi
  done
  if ! runtime_namespace_exists; then
    return "$status"
  fi
  "$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" image prune --all --force >/dev/null || status=1
  leases="$(runtime_ctr --namespace "$NERDCTL_NAMESPACE" leases list --quiet)" || status=1
  while IFS= read -r lease; do
    [[ -z "$lease" ]] || runtime_ctr --namespace "$NERDCTL_NAMESPACE" leases delete "$lease" >/dev/null || status=1
  done <<< "$leases"
  digests="$(runtime_ctr --namespace "$NERDCTL_NAMESPACE" content list --quiet)" || status=1
  while IFS= read -r digest; do
    [[ -z "$digest" ]] || runtime_ctr --namespace "$NERDCTL_NAMESPACE" content remove "$digest" >/dev/null || status=1
  done <<< "$digests"
  "$NUTHATCH_REAL_NERDCTL" namespace remove "$NERDCTL_NAMESPACE" >/dev/null || status=1
  runtime_namespace_exists && status=1
  return "$status"
}

prepare_provider() {
  configure_provider
  delete_cluster
  remove_runtime_namespace
  "$NUTHATCH_REAL_NERDCTL" namespace create "$NERDCTL_NAMESPACE" >/dev/null
}

load_external_image() {
  local name="$1"
  local source="$2"
  local digest="${source##*@}"
  local image="nuthatch-testbed/$name:sha256-${digest#sha256:}"
  local archive="$RUNTIME/image-archives/$digest.tar"
  "$NUTHATCH_REAL_NERDCTL" image pull --platform linux/amd64 --unpack=false --quiet "$source" >&2
  "$NUTHATCH_REAL_NERDCTL" image tag "$source" "$image"
  "$NUTHATCH_REAL_NERDCTL" image save --platform linux/amd64 --output "$archive" "$image"
  printf 'Loading %s from %s\n' "$image" "$source" >&2
  "$BIN/kind" load image-archive "$archive" --name "$CLUSTER" >&2
  printf '%s\n' "$image"
}

load_local_image() {
  local image="$1"
  local namespace="${2:-default}"
  local archive="$RUNTIME/image-archives/nuthatch-app.tar"
  rm -f "$archive"
  "$NUTHATCH_REAL_NERDCTL" --namespace "$namespace" image save \
    --platform linux/amd64 \
    --output "$archive" \
    "$image"
  "$BIN/kind" load image-archive "$archive" --name "$CLUSTER"
}

render_manifests() {
  local postgres="$1" redis="$2" dragonfly="$3" keycloak="$4" zitadel="$5"
  local garage="$6" versitygw="$7" coredns="$8" curl="$9" aws_cli="${10}"
  local postgres_replicas="${11}" redis_replicas="${12}" dragonfly_replicas="${13}"
  local keycloak_replicas="${14}" zitadel_replicas="${15}"
  local garage_replicas="${16}" versitygw_replicas="${17}" coredns_replicas="${18}"
  sed \
    -e "s|@@POSTGRES_IMAGE@@|$postgres|g" \
    -e "s|@@REDIS_IMAGE@@|$redis|g" \
    -e "s|@@DRAGONFLY_IMAGE@@|$dragonfly|g" \
    -e "s|@@KEYCLOAK_IMAGE@@|$keycloak|g" \
    -e "s|@@ZITADEL_IMAGE@@|$zitadel|g" \
    -e "s|@@GARAGE_IMAGE@@|$garage|g" \
    -e "s|@@VERSITYGW_IMAGE@@|$versitygw|g" \
    -e "s|@@COREDNS_IMAGE@@|$coredns|g" \
    -e "s|@@AWS_CLI_IMAGE@@|$aws_cli|g" \
    -e "s|@@POSTGRES_REPLICAS@@|$postgres_replicas|g" \
    -e "s|@@REDIS_REPLICAS@@|$redis_replicas|g" \
    -e "s|@@DRAGONFLY_REPLICAS@@|$dragonfly_replicas|g" \
    -e "s|@@KEYCLOAK_REPLICAS@@|$keycloak_replicas|g" \
    -e "s|@@ZITADEL_REPLICAS@@|$zitadel_replicas|g" \
    -e "s|@@GARAGE_REPLICAS@@|$garage_replicas|g" \
    -e "s|@@VERSITYGW_REPLICAS@@|$versitygw_replicas|g" \
    -e "s|@@COREDNS_REPLICAS@@|$coredns_replicas|g" \
    "$TESTBED/dependencies.yaml.in" > "$RUNTIME/dependencies.yaml"
  sed \
    -e "s|@@POSTGRES_IMAGE@@|$postgres|g" \
    -e "s|@@REDIS_IMAGE@@|$redis|g" \
    -e "s|@@CURL_IMAGE@@|$curl|g" \
    -e "s|@@AWS_CLI_IMAGE@@|$aws_cli|g" \
    "$TESTBED/probe.yaml.in" > "$RUNTIME/probe.yaml"
  {
    printf 'POSTGRES_LOCAL_IMAGE=%q\n' "$postgres"
    printf 'REDIS_LOCAL_IMAGE=%q\n' "$redis"
    printf 'DRAGONFLY_LOCAL_IMAGE=%q\n' "$dragonfly"
    printf 'KEYCLOAK_LOCAL_IMAGE=%q\n' "$keycloak"
    printf 'ZITADEL_LOCAL_IMAGE=%q\n' "$zitadel"
    printf 'GARAGE_LOCAL_IMAGE=%q\n' "$garage"
    printf 'VERSITYGW_LOCAL_IMAGE=%q\n' "$versitygw"
    printf 'COREDNS_LOCAL_IMAGE=%q\n' "$coredns"
    printf 'CURL_LOCAL_IMAGE=%q\n' "$curl"
    printf 'AWS_CLI_LOCAL_IMAGE=%q\n' "$aws_cli"
  } > "$RUNTIME/fixture.env"
}

kube() {
  "$BIN/kubectl" \
    --kubeconfig "$KUBECONFIG_PATH" \
    --context "$CONTEXT" \
    "$@"
}

wait_for_job() {
  local job="$1"
  local deadline=$((SECONDS + 300))
  local complete
  local failed
  while ((SECONDS < deadline)); do
    complete="$(kube --namespace "$FIXTURE_NAMESPACE" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Complete")].status}')"
    if [[ "$complete" = True ]]; then
      kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" --all-containers
      return
    fi
    failed="$(kube --namespace "$FIXTURE_NAMESPACE" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Failed")].status}')"
    if [[ "$failed" = True ]]; then
      kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" --all-containers >&2 || true
      return 1
    fi
    sleep 1
  done
  printf 'job/%s did not complete within 300 seconds\n' "$job" >&2
  return 1
}

select_fixture_families() {
  local requested=("$@")
  local family
  local candidate
  local found
  local use_all=false
  FIXTURE_FAMILIES=()
  if [[ "${#requested[@]}" = 0 ]]; then
    requested=(all)
  fi
  for family in "${requested[@]}"; do
    case "$family" in
      all) use_all=true ;;
      base | durable | coordination | oidc | target) ;;
      *)
        printf 'unknown fixture family: %s\n' "$family" >&2
        return 1
        ;;
    esac
  done
  if [[ "$use_all" = true ]]; then
    requested=(durable coordination oidc target)
  fi
  for candidate in durable coordination oidc target; do
    found=false
    for family in "${requested[@]}"; do
      if [[ "$family" = "$candidate" ]]; then
        found=true
      fi
    done
    if [[ "$found" = true ]]; then
      FIXTURE_FAMILIES+=("$candidate")
    fi
  done
}

fixture_has_family() {
  local expected="$1"
  local family
  for family in "${FIXTURE_FAMILIES[@]}"; do
    if [[ "$family" = "$expected" ]]; then
      return
    fi
  done
  return 1
}

expand_fixture_matrix() {
  local coordination_providers=(-)
  local oidc_providers=(-)
  local target_providers=(-)
  local coordination
  local oidc
  local target
  local lane=0
  select_fixture_families "$@" || return
  if fixture_has_family coordination; then
    coordination_providers=(redis dragonfly)
  fi
  if fixture_has_family oidc; then
    oidc_providers=(keycloak zitadel)
  fi
  if fixture_has_family target; then
    target_providers=(garage versitygw)
  fi
  for coordination in "${coordination_providers[@]}"; do
    for oidc in "${oidc_providers[@]}"; do
      for target in "${target_providers[@]}"; do
        lane=$((lane + 1))
        printf '%03d\t%s\t%s\t%s\n' "$lane" "$coordination" "$oidc" "$target"
      done
    done
  done
}

setup_fixture() {
  select_fixture_families "$@" || return
  mkdir -p "$RUNTIME"
  rm -rf "$RUNTIME/image-archives"
  mkdir -p "$RUNTIME/image-archives"
  prepare_tools
  prepare_provider
  "$BIN/kind" create cluster \
    --name "$CLUSTER" \
    --image "$NODE_IMAGE" \
    --config "$TESTBED/cluster.yaml" \
    --kubeconfig "$KUBECONFIG_PATH" \
    --wait 180s
  local postgres=nuthatch-testbed/unused:postgres
  local redis=nuthatch-testbed/unused:redis
  local dragonfly=nuthatch-testbed/unused:dragonfly
  local keycloak=nuthatch-testbed/unused:keycloak
  local zitadel=nuthatch-testbed/unused:zitadel
  local garage=nuthatch-testbed/unused:garage
  local versitygw=nuthatch-testbed/unused:versitygw
  local coredns=nuthatch-testbed/unused:coredns
  local curl=nuthatch-testbed/unused:curl
  local aws_cli=nuthatch-testbed/unused:aws-cli
  local postgres_replicas=0 redis_replicas=0 dragonfly_replicas=0
  local keycloak_replicas=0 zitadel_replicas=0
  local garage_replicas=0 versitygw_replicas=0 coredns_replicas=0
  if fixture_has_family durable || fixture_has_family oidc; then
    postgres="$(load_external_image postgres "$POSTGRES_IMAGE")"
    postgres_replicas=1
  fi
  if fixture_has_family coordination; then
    redis="$(load_external_image redis "$REDIS_IMAGE")"
    dragonfly="$(load_external_image dragonfly "$DRAGONFLY_IMAGE")"
    redis_replicas=1
    dragonfly_replicas=1
  fi
  if fixture_has_family oidc; then
    keycloak="$(load_external_image keycloak "$KEYCLOAK_IMAGE")"
    zitadel="$(load_external_image zitadel "$ZITADEL_IMAGE")"
    coredns="$(load_external_image coredns "$COREDNS_IMAGE")"
    curl="$(load_external_image curl "$CURL_IMAGE")"
    keycloak_replicas=1
    zitadel_replicas=1
    coredns_replicas=1
  fi
  if fixture_has_family target; then
    garage="$(load_external_image garage "$GARAGE_IMAGE")"
    versitygw="$(load_external_image versitygw "$VERSITYGW_IMAGE")"
    aws_cli="$(load_external_image aws-cli "$AWS_CLI_IMAGE")"
    if [[ "$curl" = nuthatch-testbed/unused:curl ]]; then
      curl="$(load_external_image curl "$CURL_IMAGE")"
    fi
    garage_replicas=1
    versitygw_replicas=1
  fi
  render_manifests "$postgres" "$redis" "$dragonfly" "$keycloak" "$zitadel" \
    "$garage" "$versitygw" "$coredns" "$curl" "$aws_cli" \
    "$postgres_replicas" "$redis_replicas" "$dragonfly_replicas" \
    "$keycloak_replicas" "$zitadel_replicas" \
    "$garage_replicas" "$versitygw_replicas" "$coredns_replicas"
  kube apply --filename "$RUNTIME/dependencies.yaml"
  if fixture_has_family durable || fixture_has_family oidc; then
    wait_for_deployment postgres
  fi
  if fixture_has_family coordination; then
    wait_for_deployment redis
    wait_for_deployment dragonfly
  fi
  if fixture_has_family oidc; then
    wait_for_deployment keycloak
    wait_for_deployment zitadel
    wait_for_deployment controlled-dns
  fi
  if fixture_has_family target; then
    wait_for_deployment garage
    wait_for_deployment versitygw
  fi
}

wait_for_deployment() {
  kube --namespace "$FIXTURE_NAMESPACE" rollout status "deployment/$1" --timeout 240s
}

verify_dependencies() {
  kube apply --filename "$RUNTIME/probe.yaml"
  local job
  for job in state-probe oidc-probe garage-probe versitygw-probe; do
    wait_for_job "$job"
  done
}

reset_lane_durable_store() {
  local lane="$1"
  local database="nuthatch_lane_$lane"
  kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
    env PGPASSWORD=synthetic-postgres psql \
      --username nuthatch \
      --dbname postgres \
      --set ON_ERROR_STOP=1 \
      --command "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$database'" \
      --command "DROP DATABASE IF EXISTS $database" \
      --command "CREATE DATABASE $database" >&2 || return
  printf '%s\n' "$database"
}

reset_lane_coordination_store() {
  local provider="$1"
  local lane="$2"
  local database=$((10#$lane))
  kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" -- \
    redis-cli --raw --no-auth-warning -n "$database" FLUSHDB >/dev/null || return
  printf '%s\n' "$database"
}

target_endpoint() {
  case "$1" in
    garage) printf 'http://garage:3900\n' ;;
    versitygw) printf 'http://versitygw:10000\n' ;;
    *) return 1 ;;
  esac
}

target_loopback_endpoint() {
  case "$1" in
    garage) printf 'http://127.0.0.1:3900\n' ;;
    versitygw) printf 'http://127.0.0.1:10000\n' ;;
    *) return 1 ;;
  esac
}

ensure_target_scope_canary() {
  local provider="$1"
  local endpoint
  endpoint="$(target_loopback_endpoint "$provider")" || return
  if ! kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
    aws --endpoint-url "$endpoint" s3api create-bucket --bucket nuthatch-fixture >&2; then
    kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
      aws --endpoint-url "$endpoint" s3api head-bucket --bucket nuthatch-fixture >/dev/null || return
  fi
  kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
    sh -ec 'printf scope-canary >/tmp/scope-canary && aws --endpoint-url "$1" s3api put-object --bucket nuthatch-fixture --key scope-canary --body /tmp/scope-canary >/dev/null' \
    sh "$endpoint"
}

provision_lane_target_principal() {
  local provider="$1"
  local lane="$2"
  local bucket="$3"
  local access_key_var="$4"
  local secret_key_var="$5"
  local endpoint
  local access_key
  local secret_key
  endpoint="$(target_loopback_endpoint "$provider")" || return
  case "$provider" in
    garage)
      printf -v access_key 'GKWP05%028d' "$((10#$lane))"
      printf -v secret_key '%064d' "$((10#$lane))"
      kube --namespace "$FIXTURE_NAMESPACE" exec deployment/garage --container garage -- \
        /garage bucket create "$bucket" >/dev/null || return
      kube --namespace "$FIXTURE_NAMESPACE" exec deployment/garage --container garage -- \
        /garage bucket allow \
        --read \
        --write \
        --owner \
        "$bucket" \
        --key GK00000000000000000000000000000000 >/dev/null || return
      kube --namespace "$FIXTURE_NAMESPACE" exec deployment/garage --container garage -- \
        /garage key import --yes -n "nuthatch-$lane" "$access_key" "$secret_key" >/dev/null || return
      kube --namespace "$FIXTURE_NAMESPACE" exec deployment/garage --container garage -- \
        /garage bucket allow --read --write "$bucket" --key "$access_key" >/dev/null || return
      ;;
    versitygw)
      access_key="nuthatch-wp05-$lane"
      printf -v secret_key 'synthetic-wp05-%049d' "$((10#$lane))"
      kube --namespace "$FIXTURE_NAMESPACE" exec deployment/versitygw --container versitygw -- \
        /usr/local/bin/versitygw admin \
        --endpoint-url http://127.0.0.1:10001 \
        create-user \
        --access "$access_key" \
        --secret "$secret_key" \
        --role user \
        --user-id "$((10#$lane))" \
        --group-id "$((10#$lane))" >/dev/null || return
      kube --namespace "$FIXTURE_NAMESPACE" exec deployment/versitygw --container versitygw -- \
        /usr/local/bin/versitygw admin \
        --endpoint-url http://127.0.0.1:10001 \
        create-bucket \
        --bucket "$bucket" \
        --owner "$access_key" >/dev/null || return
      ;;
    *) return 1 ;;
  esac
  printf -v "$access_key_var" '%s' "$access_key"
  printf -v "$secret_key_var" '%s' "$secret_key"
}

write_lane_profile() {
  local lane="$1"
  local coordination="$2"
  local oidc="$3"
  local target="$4"
  local profile="nuthatch-fixture-lane-$lane"
  local profile_file="$RUNTIME/$profile.env"
  local database=
  local coordination_database=
  local oidc_issuer=
  local target_url=
  local target_bucket=
  local target_region=
  local target_access_key=
  local target_secret_key=
  if fixture_has_family durable; then
    database="$(reset_lane_durable_store "$lane")" || return
  fi
  if fixture_has_family coordination; then
    coordination_database="$(reset_lane_coordination_store "$coordination" "$lane")" || return
  fi
  if fixture_has_family oidc; then
    case "$oidc" in
      keycloak) oidc_issuer=http://keycloak.nuthatch.test:8080/realms/master ;;
      zitadel) oidc_issuer=http://zitadel.nuthatch.test:8080 ;;
      *) return 1 ;;
    esac
  fi
  if fixture_has_family target; then
    target_bucket="nuthatch-lane-$lane"
    target_url="$(target_endpoint "$target")" || return
    ensure_target_scope_canary "$target" || return
    provision_lane_target_principal \
      "$target" \
      "$lane" \
      "$target_bucket" \
      target_access_key \
      target_secret_key || return
    case "$target" in
      garage) target_region=garage ;;
      versitygw) target_region=us-east-1 ;;
      *) return 1 ;;
    esac
  fi
  {
    printf 'NUTHATCH_FIXTURE_PROFILE=%s\n' "$profile"
    printf 'NUTHATCH_FIXTURE_LANE=%s\n' "$lane"
    printf 'NUTHATCH_COORDINATION_PROVIDER=%s\n' "$coordination"
    printf 'NUTHATCH_OIDC_PROVIDER=%s\n' "$oidc"
    printf 'NUTHATCH_TARGET_PROVIDER=%s\n' "$target"
    printf 'NUTHATCH_PG_CONN=host=postgres port=5432 user=nuthatch password=synthetic-postgres dbname=%s\n' "$database"
    printf 'NUTHATCH_REDIS_URL=redis://%s:6379/%s\n' "$coordination" "$coordination_database"
    printf 'NUTHATCH_OIDC_ISSUER=%s\n' "$oidc_issuer"
    printf 'NUTHATCH_S3_ENDPOINT=%s\n' "$target_url"
    printf 'NUTHATCH_S3_BUCKET=%s\n' "$target_bucket"
    printf 'NUTHATCH_S3_REGION=%s\n' "$target_region"
    printf 'NUTHATCH_S3_ACCESS_KEY=%s\n' "$target_access_key"
    printf 'NUTHATCH_S3_SECRET_KEY=%s\n' "$target_secret_key"
    printf 'NUTHATCH_S3_PATH_STYLE=true\n'
  } > "$profile_file" || return
  kube create configmap "$profile" \
    --namespace "$FIXTURE_NAMESPACE" \
    --from-env-file="$profile_file" \
    --dry-run=client \
    --output yaml |
    kube apply --filename - >/dev/null || return
  kube create configmap "$profile-file" \
    --namespace "$FIXTURE_NAMESPACE" \
    --from-file="profile.env=$profile_file" \
    --dry-run=client \
    --output yaml |
    kube apply --filename - >/dev/null || return
  printf '%s\n' "$profile"
}

render_lane_job() {
  local job="$1"
  local manifest="$2"
  local image="$3"
  local profile="$4"
  source "$RUNTIME/fixture.env" || return
  local coordination_image="$REDIS_LOCAL_IMAGE"
  sed \
    -e "s|@@JOB_NAME@@|$job|g" \
    -e "s|@@PROFILE_NAME@@|$profile|g" \
    -e "s|@@TEST_IMAGE@@|$image|g" \
    -e "s|@@POSTGRES_IMAGE@@|$POSTGRES_LOCAL_IMAGE|g" \
    -e "s|@@COORDINATION_IMAGE@@|$coordination_image|g" \
    -e "s|@@CURL_IMAGE@@|$CURL_LOCAL_IMAGE|g" \
    -e "s|@@AWS_CLI_IMAGE@@|$AWS_CLI_LOCAL_IMAGE|g" \
    "$manifest" > "$RUNTIME/$job.yaml"
}

run_job_manifest() {
  local job="$1"
  local manifest="$2"
  kube --namespace "$FIXTURE_NAMESPACE" delete "job/$job" --ignore-not-found >/dev/null || return
  kube apply --filename "$manifest" >/dev/null || return
  wait_for_job "$job"
}

cleanup_lane_durable_store() {
  local lane="$1"
  local database="nuthatch_lane_$lane"
  kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
    env PGPASSWORD=synthetic-postgres psql \
      --username nuthatch \
      --dbname postgres \
      --set ON_ERROR_STOP=1 \
      --command "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$database'" \
      --command "DROP DATABASE IF EXISTS $database" >/dev/null
}

cleanup_lane_coordination_store() {
  local provider="$1"
  local lane="$2"
  local database=$((10#$lane))
  kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" -- \
    redis-cli --raw --no-auth-warning -n "$database" FLUSHDB >/dev/null
}

cleanup_lane_target_store() {
  local provider="$1"
  local bucket="$2"
  local endpoint
  endpoint="$(target_loopback_endpoint "$provider")" || return
  kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
    aws --endpoint-url "$endpoint" s3 rb "s3://$bucket" --force >/dev/null
}

cleanup_lane_state() {
  local lane="$1"
  local coordination="$2"
  local target="$4"
  local status=0
  if fixture_has_family durable; then
    cleanup_lane_durable_store "$lane" || status=1
  fi
  if fixture_has_family coordination; then
    cleanup_lane_coordination_store "$coordination" "$lane" || status=1
  fi
  if fixture_has_family target; then
    cleanup_lane_target_store "$target" "nuthatch-lane-$lane" || status=1
  fi
  return "$status"
}

cleanup_lane_resources() {
  local job="$1"
  local profile="$2"
  kube --namespace "$FIXTURE_NAMESPACE" delete \
    "job/$job" \
    "configmap/$profile" \
    "configmap/$profile-file" \
    --ignore-not-found >/dev/null || return
  rm -f "$RUNTIME/$profile.env" "$RUNTIME/$job.yaml"
}

run_job_matrix() {
  local base_job="$1"
  local manifest="$2"
  local image="$3"
  shift 3
  local lane
  local coordination
  local oidc
  local target
  local profile
  local job
  select_fixture_families "$@" || return
  while IFS=$'\t' read -r lane coordination oidc target; do
    profile="$(write_lane_profile "$lane" "$coordination" "$oidc" "$target")" || return
    job="$base_job-$lane"
    render_lane_job "$job" "$manifest" "$image" "$profile" || return
    printf 'Running lane %s: coordination=%s oidc=%s target=%s\n' \
      "$lane" "$coordination" "$oidc" "$target"
    run_job_manifest "$job" "$RUNTIME/$job.yaml" || return
    cleanup_lane_state "$lane" "$coordination" "$oidc" "$target" || return
    cleanup_lane_resources "$job" "$profile" || return
  done < <(expand_fixture_matrix "$@")
}

destroy_fixture() {
  local status=0
  configure_provider || return
  delete_cluster || status=1
  remove_runtime_namespace || status=1
  rm -rf "$RUNTIME/image-archives" || status=1
  rm -f \
    "$KUBECONFIG_PATH" \
    "$RUNTIME/dependencies.yaml" \
    "$RUNTIME/probe.yaml" \
    "$RUNTIME/fixture.env" \
    "$RUNTIME"/nuthatch-fixture-lane-*.env \
    "$RUNTIME"/wp05-app-*.yaml || status=1
  return "$status"
}