Add a released-image dependency compatibility testbed

.gitignore

@@ -10,6 +10,7 @@ states/
 
 # Rust
 /target/
+/tests/kind/.runtime/
 
 # Web
 /web/node_modules/

Makefile

@@ -1,4 +1,4 @@
-.PHONY: check-fast check-full check-fast-host check-full-host rust-check web-install web-check web-test-fast web-test-full
+.PHONY: check-dependencies check-fast check-full check-fast-host check-full-host rust-check web-install web-check web-test-fast web-test-full
 
 CONTAINER_ENGINE ?= docker
 CONTAINER_RUN_FLAGS ?= --memory=16g --memory-swap=24g --cpus=4
@@ -46,3 +46,6 @@ check-full:
 check-fast-host: rust-check web-check web-test-fast
 
 check-full-host: rust-check web-check web-test-full
+
+check-dependencies:
+	./tests/kind/run.sh

tests/kind/cluster.yaml

@@ -0,0 +1,7 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+networking:
+  apiServerAddress: 127.0.0.1
+  apiServerPort: 35443
+nodes:
+  - role: control-plane

tests/kind/dependencies.yaml.in

@@ -0,0 +1,507 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: remora-testbed
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: postgres-init
+  namespace: remora-testbed
+data:
+  01-keycloak.sql: |
+    CREATE USER keycloak WITH PASSWORD 'synthetic-keycloak-db';
+    CREATE DATABASE keycloak OWNER keycloak;
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: postgres
+  namespace: remora-testbed
+spec:
+  replicas: 1
+  selector:
+    matchLabels: {app: postgres}
+  template:
+    metadata:
+      labels: {app: postgres}
+    spec:
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      containers:
+        - name: postgres
+          image: @@POSTGRES_IMAGE@@
+          imagePullPolicy: Never
+          env:
+            - {name: POSTGRES_DB, value: remora}
+            - {name: POSTGRES_USER, value: remora}
+            - {name: POSTGRES_PASSWORD, value: synthetic-postgres}
+          ports:
+            - {name: postgres, containerPort: 5432}
+          readinessProbe:
+            exec:
+              command: [pg_isready, -U, remora, -d, remora]
+            periodSeconds: 2
+          resources:
+            requests: {cpu: 100m, memory: 256Mi}
+            limits: {cpu: "2", memory: 1Gi}
+          volumeMounts:
+            - {name: data, mountPath: /var/lib/postgresql}
+            - {name: init, mountPath: /docker-entrypoint-initdb.d}
+      volumes:
+        - name: data
+          emptyDir: {}
+        - name: init
+          configMap: {name: postgres-init}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: postgres
+  namespace: remora-testbed
+spec:
+  selector: {app: postgres}
+  ports:
+    - {name: postgres, port: 5432, targetPort: postgres}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: redis
+  namespace: remora-testbed
+spec:
+  replicas: 1
+  selector:
+    matchLabels: {app: redis}
+  template:
+    metadata:
+      labels: {app: redis}
+    spec:
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      containers:
+        - name: redis
+          image: @@REDIS_IMAGE@@
+          imagePullPolicy: Never
+          args: [redis-server, --save, "", --appendonly, "no"]
+          ports:
+            - {name: redis, containerPort: 6379}
+          readinessProbe:
+            exec:
+              command: [redis-cli, ping]
+            periodSeconds: 2
+          resources:
+            requests: {cpu: 25m, memory: 32Mi}
+            limits: {cpu: "1", memory: 256Mi}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: redis
+  namespace: remora-testbed
+spec:
+  selector: {app: redis}
+  ports:
+    - {name: redis, port: 6379, targetPort: redis}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: dragonfly
+  namespace: remora-testbed
+spec:
+  replicas: 1
+  selector:
+    matchLabels: {app: dragonfly}
+  template:
+    metadata:
+      labels: {app: dragonfly}
+    spec:
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      containers:
+        - name: dragonfly
+          image: @@DRAGONFLY_IMAGE@@
+          imagePullPolicy: Never
+          args: [--proactor_threads=1, --maxmemory=256mb]
+          ports:
+            - {name: redis, containerPort: 6379}
+          readinessProbe:
+            tcpSocket: {port: redis}
+            periodSeconds: 2
+          resources:
+            requests: {cpu: 100m, memory: 64Mi}
+            limits: {cpu: "2", memory: 384Mi}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: dragonfly
+  namespace: remora-testbed
+spec:
+  selector: {app: dragonfly}
+  ports:
+    - {name: redis, port: 6379, targetPort: redis}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: vault
+  namespace: remora-testbed
+spec:
+  replicas: 1
+  selector:
+    matchLabels: {app: vault}
+  template:
+    metadata:
+      labels: {app: vault}
+    spec:
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      containers:
+        - name: vault
+          image: @@VAULT_IMAGE@@
+          imagePullPolicy: Never
+          args: [server, -dev]
+          env:
+            - {name: VAULT_DEV_ROOT_TOKEN_ID, value: synthetic-vault-token}
+            - {name: VAULT_DEV_LISTEN_ADDRESS, value: "0.0.0.0:8200"}
+          ports:
+            - {name: http, containerPort: 8200}
+          readinessProbe:
+            httpGet: {path: /v1/sys/health, port: http}
+            periodSeconds: 2
+          resources:
+            requests: {cpu: 50m, memory: 64Mi}
+            limits: {cpu: "1", memory: 256Mi}
+          securityContext:
+            capabilities: {add: [IPC_LOCK]}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: vault
+  namespace: remora-testbed
+spec:
+  selector: {app: vault}
+  ports:
+    - {name: http, port: 8200, targetPort: http}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: keycloak
+  namespace: remora-testbed
+spec:
+  replicas: 1
+  selector:
+    matchLabels: {app: keycloak}
+  template:
+    metadata:
+      labels: {app: keycloak}
+    spec:
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      containers:
+        - name: keycloak
+          image: @@KEYCLOAK_IMAGE@@
+          imagePullPolicy: Never
+          args: [start-dev, --hostname=http://keycloak.remora.test:8080]
+          env:
+            - {name: KC_BOOTSTRAP_ADMIN_USERNAME, value: remora-admin}
+            - {name: KC_BOOTSTRAP_ADMIN_PASSWORD, value: synthetic-keycloak}
+            - {name: KC_DB, value: postgres}
+            - {name: KC_DB_URL, value: jdbc:postgresql://postgres:5432/keycloak}
+            - {name: KC_DB_USERNAME, value: keycloak}
+            - {name: KC_DB_PASSWORD, value: synthetic-keycloak-db}
+            - {name: JAVA_OPTS_KC_HEAP, value: "-XX:MinHeapFreeRatio=10 -XX:MaxHeapFreeRatio=30 -XX:MaxRAMPercentage=65"}
+          ports:
+            - {name: http, containerPort: 8080}
+          readinessProbe:
+            httpGet: {path: /realms/master, port: http}
+            periodSeconds: 3
+            failureThreshold: 60
+          resources:
+            requests: {cpu: 200m, memory: 512Mi}
+            limits: {cpu: "2", memory: 1Gi}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: keycloak
+  namespace: remora-testbed
+spec:
+  clusterIP: 10.96.0.55
+  selector: {app: keycloak}
+  ports:
+    - {name: http, port: 8080, targetPort: http}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: zitadel
+  namespace: remora-testbed
+spec:
+  replicas: 1
+  selector:
+    matchLabels: {app: zitadel}
+  template:
+    metadata:
+      labels: {app: zitadel}
+    spec:
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      containers:
+        - name: zitadel
+          image: @@ZITADEL_IMAGE@@
+          imagePullPolicy: Never
+          args: [start-from-init, --masterkeyFromEnv, --tlsMode, disabled]
+          env:
+            - {name: ZITADEL_MASTERKEY, value: synthetic-masterkey-32-character}
+            - {name: ZITADEL_DATABASE_POSTGRES_DSN, value: "postgresql://remora:synthetic-postgres@postgres:5432/postgres?sslmode=disable"}
+            - {name: ZITADEL_EXTERNALSECURE, value: "false"}
+            - {name: ZITADEL_EXTERNALDOMAIN, value: zitadel.remora.test}
+            - {name: ZITADEL_EXTERNALPORT, value: "8080"}
+            - {name: ZITADEL_FIRSTINSTANCE_ORG_HUMAN_USERNAME, value: remora-admin}
+            - {name: ZITADEL_FIRSTINSTANCE_ORG_HUMAN_PASSWORD, value: Synthetic-Zitadel-1!}
+          ports:
+            - {name: http, containerPort: 8080}
+          readinessProbe:
+            httpGet:
+              path: /.well-known/openid-configuration
+              port: http
+              httpHeaders:
+                - {name: Host, value: zitadel.remora.test}
+            periodSeconds: 3
+            failureThreshold: 80
+          resources:
+            requests: {cpu: 200m, memory: 512Mi}
+            limits: {cpu: "2", memory: 1Gi}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: zitadel
+  namespace: remora-testbed
+spec:
+  clusterIP: 10.96.0.56
+  selector: {app: zitadel}
+  ports:
+    - {name: http, port: 8080, targetPort: http}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: garage-config
+  namespace: remora-testbed
+data:
+  garage.toml: |
+    metadata_dir = "/data/meta"
+    data_dir = "/data/data"
+    db_engine = "sqlite"
+    replication_factor = 1
+    rpc_bind_addr = "[::]:3901"
+    rpc_public_addr = "garage:3901"
+    rpc_secret = "0000000000000000000000000000000000000000000000000000000000000000"
+    [s3_api]
+    s3_region = "garage"
+    api_bind_addr = "[::]:3900"
+    root_domain = ".s3.garage.remora.test"
+    [admin]
+    api_bind_addr = "[::]:3903"
+    admin_token = "synthetic-garage-admin"
+    metrics_token = "synthetic-garage-metrics"
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: garage
+  namespace: remora-testbed
+spec:
+  replicas: 1
+  selector:
+    matchLabels: {app: garage}
+  template:
+    metadata:
+      labels: {app: garage}
+    spec:
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      containers:
+        - name: garage
+          image: @@GARAGE_IMAGE@@
+          imagePullPolicy: Never
+          command: [/garage]
+          args: [server, --single-node, --default-bucket]
+          env:
+            - {name: GARAGE_DEFAULT_ACCESS_KEY, value: GK00000000000000000000000000000000}
+            - {name: GARAGE_DEFAULT_SECRET_KEY, value: "0000000000000000000000000000000000000000000000000000000000000000"}
+            - {name: GARAGE_DEFAULT_BUCKET, value: remora-fixture}
+          ports:
+            - {name: s3, containerPort: 3900}
+          resources:
+            requests: {cpu: 100m, memory: 128Mi}
+            limits: {cpu: "2", memory: 512Mi}
+          volumeMounts:
+            - {name: config, mountPath: /etc/garage.toml, subPath: garage.toml}
+            - {name: data, mountPath: /data}
+        - name: s3-readiness
+          image: @@AWS_CLI_IMAGE@@
+          imagePullPolicy: Never
+          command: [/bin/sh, -ec]
+          args: ["while :; do sleep 3600; done"]
+          env:
+            - {name: AWS_ACCESS_KEY_ID, value: GK00000000000000000000000000000000}
+            - {name: AWS_SECRET_ACCESS_KEY, value: "0000000000000000000000000000000000000000000000000000000000000000"}
+            - {name: AWS_DEFAULT_REGION, value: garage}
+          readinessProbe:
+            exec:
+              command: [aws, --endpoint-url, "http://127.0.0.1:3900", s3api, head-bucket, --bucket, remora-fixture]
+            periodSeconds: 2
+            failureThreshold: 60
+          resources:
+            requests: {cpu: 25m, memory: 64Mi}
+            limits: {cpu: "1", memory: 256Mi}
+      volumes:
+        - name: config
+          configMap: {name: garage-config}
+        - name: data
+          emptyDir: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: garage
+  namespace: remora-testbed
+spec:
+  selector: {app: garage}
+  ports:
+    - {name: s3, port: 3900, targetPort: s3}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: versitygw
+  namespace: remora-testbed
+spec:
+  replicas: 1
+  selector:
+    matchLabels: {app: versitygw}
+  template:
+    metadata:
+      labels: {app: versitygw}
+    spec:
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      containers:
+        - name: versitygw
+          image: @@VERSITYGW_IMAGE@@
+          imagePullPolicy: Never
+          args: [--port, ":10000", --iam-dir, /data, posix, --versioning-dir, /versions, /data]
+          env:
+            - {name: ROOT_ACCESS_KEY, value: synthetic-s3-access}
+            - {name: ROOT_SECRET_KEY, value: synthetic-s3-secret}
+          ports:
+            - {name: s3, containerPort: 10000}
+          resources:
+            requests: {cpu: 100m, memory: 64Mi}
+            limits: {cpu: "2", memory: 512Mi}
+          volumeMounts:
+            - {name: data, mountPath: /data}
+            - {name: versions, mountPath: /versions}
+        - name: s3-readiness
+          image: @@AWS_CLI_IMAGE@@
+          imagePullPolicy: Never
+          command: [/bin/sh, -ec]
+          args: ["while :; do sleep 3600; done"]
+          env:
+            - {name: AWS_ACCESS_KEY_ID, value: synthetic-s3-access}
+            - {name: AWS_SECRET_ACCESS_KEY, value: synthetic-s3-secret}
+            - {name: AWS_DEFAULT_REGION, value: us-east-1}
+          readinessProbe:
+            exec:
+              command: [aws, --endpoint-url, "http://127.0.0.1:10000", s3api, list-buckets]
+            periodSeconds: 2
+            failureThreshold: 60
+          resources:
+            requests: {cpu: 25m, memory: 64Mi}
+            limits: {cpu: "1", memory: 256Mi}
+      volumes:
+        - name: data
+          emptyDir: {}
+        - name: versions
+          emptyDir: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: versitygw
+  namespace: remora-testbed
+spec:
+  selector: {app: versitygw}
+  ports:
+    - {name: s3, port: 10000, targetPort: s3}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: controlled-dns
+  namespace: remora-testbed
+data:
+  Corefile: |
+    .:53 {
+      errors
+      hosts {
+        10.96.0.55 keycloak.remora.test
+        10.96.0.56 zitadel.remora.test
+        fallthrough
+      }
+      forward . 10.96.0.10
+    }
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controlled-dns
+  namespace: remora-testbed
+spec:
+  replicas: 1
+  selector:
+    matchLabels: {app: controlled-dns}
+  template:
+    metadata:
+      labels: {app: controlled-dns}
+    spec:
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      containers:
+        - name: coredns
+          image: @@COREDNS_IMAGE@@
+          imagePullPolicy: Never
+          args: [-conf, /etc/coredns/Corefile]
+          ports:
+            - {name: dns-udp, containerPort: 53, protocol: UDP}
+            - {name: dns-tcp, containerPort: 53, protocol: TCP}
+          readinessProbe:
+            tcpSocket: {port: dns-tcp}
+            periodSeconds: 2
+          resources:
+            requests: {cpu: 25m, memory: 32Mi}
+            limits: {cpu: "1", memory: 128Mi}
+          volumeMounts:
+            - {name: config, mountPath: /etc/coredns}
+      volumes:
+        - name: config
+          configMap: {name: controlled-dns}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: controlled-dns
+  namespace: remora-testbed
+spec:
+  clusterIP: 10.96.0.53
+  selector: {app: controlled-dns}
+  ports:
+    - {name: dns-udp, port: 53, targetPort: dns-udp, protocol: UDP}
+    - {name: dns-tcp, port: 53, targetPort: dns-tcp, protocol: TCP}

tests/kind/destroy.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
+source "$ROOT/tests/kind/lib.sh"
+
+prepare_tools
+configure_provider
+delete_cluster
+remove_runtime_namespace
+rm -rf "$RUNTIME/image-archives"
+rm -f "$KUBECONFIG_PATH" "$RUNTIME/dependencies.yaml" "$RUNTIME/probe.yaml"

tests/kind/lib.sh

@@ -0,0 +1,186 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+TESTBED="$ROOT/tests/kind"
+RUNTIME="$TESTBED/.runtime"
+BIN="$RUNTIME/bin"
+KUBECONFIG_PATH="$RUNTIME/kubeconfig"
+CONTEXT=kind-remora-testbed
+CLUSTER=remora-testbed
+NERDCTL_NAMESPACE=remora-wp02
+
+set -a
+source "$TESTBED/versions.env"
+set +a
+
+user_kubeconfig_digest() {
+  local config_value="${KUBECONFIG:-${HOME:?}/.kube/config}"
+  local path
+  local paths
+  IFS=: read -r -a paths <<< "$config_value"
+  {
+    for path in "${paths[@]}"; do
+      if [[ -f "$path" ]]; then
+        sha256sum "$path"
+      else
+        printf 'missing %s\n' "$path"
+      fi
+    done
+  } | sha256sum | cut -d' ' -f1
+}
+
+download_tool() {
+  local destination="$1"
+  local checksum="$2"
+  local url="$3"
+  if [[ -x "$destination" ]] && [[ "$(sha256sum "$destination" | cut -d' ' -f1)" = "$checksum" ]]; then
+    return
+  fi
+  curl --fail --location --silent --show-error "$url" --output "$destination.tmp"
+  test "$(sha256sum "$destination.tmp" | cut -d' ' -f1)" = "$checksum"
+  chmod 0755 "$destination.tmp"
+  mv "$destination.tmp" "$destination"
+}
+
+prepare_tools() {
+  mkdir -p "$BIN"
+  download_tool \
+    "$BIN/kind" \
+    "$KIND_SHA256" \
+    "https://github.com/kubernetes-sigs/kind/releases/download/$KIND_VERSION/kind-linux-amd64"
+  download_tool \
+    "$BIN/kubectl" \
+    "$KUBECTL_SHA256" \
+    "https://dl.k8s.io/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl"
+  test "$("$BIN/kind" version | awk '{print $2}')" = "$KIND_VERSION"
+  test "$("$BIN/kubectl" --kubeconfig "$KUBECONFIG_PATH" --context "$CONTEXT" version --client=true -o json | jq -r .clientVersion.gitVersion)" = "$KUBECTL_VERSION"
+  test "$(nerdctl --version | awk '{print $3}')" = 2.3.4
+}
+
+prepare_provider() {
+  configure_provider
+  delete_cluster
+  remove_runtime_namespace
+  "$REMORA_REAL_NERDCTL" namespace create "$NERDCTL_NAMESPACE" >/dev/null
+}
+
+configure_provider() {
+  REMORA_REAL_NERDCTL="$(command -v nerdctl)"
+  export REMORA_REAL_NERDCTL
+  export CONTAINERD_NAMESPACE="$NERDCTL_NAMESPACE"
+  export KIND_EXPERIMENTAL_PROVIDER=nerdctl
+  export PATH="$TESTBED/provider:$PATH"
+}
+
+remove_runtime_namespace() {
+  local status=0
+  if ! runtime_namespace_exists; then
+    return
+  fi
+  "$REMORA_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" image prune --all --force >/dev/null || status=1
+  "$REMORA_REAL_NERDCTL" namespace remove "$NERDCTL_NAMESPACE" >/dev/null || status=1
+  if runtime_namespace_exists; then
+    printf 'containerd namespace %s still exists after teardown\n' "$NERDCTL_NAMESPACE" >&2
+    status=1
+  fi
+  return "$status"
+}
+
+runtime_namespace_exists() {
+  "$REMORA_REAL_NERDCTL" namespace list --quiet | grep -Fx "$NERDCTL_NAMESPACE" >/dev/null
+}
+
+cluster_exists() {
+  if ! runtime_namespace_exists; then
+    return 1
+  fi
+  "$BIN/kind" get clusters | grep -Fx "$CLUSTER" >/dev/null
+}
+
+delete_cluster() {
+  if cluster_exists; then
+    "$BIN/kind" delete cluster --name "$CLUSTER"
+  fi
+  if cluster_exists; then
+    printf 'kind cluster %s still exists after teardown\n' "$CLUSTER" >&2
+    return 1
+  fi
+}
+
+load_external_image() {
+  local name="$1"
+  local source="$2"
+  local digest="${source##*@}"
+  local image="remora-testbed/$name:sha256-${digest#sha256:}"
+  local archive="$RUNTIME/image-archives/$digest.tar"
+  "$REMORA_REAL_NERDCTL" image pull --platform linux/amd64 --unpack=false --quiet "$source" >&2 || return 1
+  "$REMORA_REAL_NERDCTL" image tag "$source" "$image" || return 1
+  "$REMORA_REAL_NERDCTL" image save --platform linux/amd64 --output "$archive" "$image" || return 1
+  printf 'Loading %s from %s\n' "$image" "$source" >&2
+  "$BIN/kind" load image-archive "$archive" --name "$CLUSTER" >&2 || return 1
+  printf '%s\n' "$image"
+}
+
+render_manifests() {
+  local postgres_image="$1"
+  local redis_image="$2"
+  local dragonfly_image="$3"
+  local vault_image="$4"
+  local keycloak_image="$5"
+  local zitadel_image="$6"
+  local garage_image="$7"
+  local versitygw_image="$8"
+  local coredns_image="$9"
+  local curl_image="${10}"
+  local aws_cli_image="${11}"
+  sed \
+    -e "s|@@POSTGRES_IMAGE@@|$postgres_image|g" \
+    -e "s|@@REDIS_IMAGE@@|$redis_image|g" \
+    -e "s|@@DRAGONFLY_IMAGE@@|$dragonfly_image|g" \
+    -e "s|@@VAULT_IMAGE@@|$vault_image|g" \
+    -e "s|@@KEYCLOAK_IMAGE@@|$keycloak_image|g" \
+    -e "s|@@ZITADEL_IMAGE@@|$zitadel_image|g" \
+    -e "s|@@GARAGE_IMAGE@@|$garage_image|g" \
+    -e "s|@@VERSITYGW_IMAGE@@|$versitygw_image|g" \
+    -e "s|@@COREDNS_IMAGE@@|$coredns_image|g" \
+    -e "s|@@CURL_IMAGE@@|$curl_image|g" \
+    -e "s|@@AWS_CLI_IMAGE@@|$aws_cli_image|g" \
+    "$TESTBED/dependencies.yaml.in" > "$RUNTIME/dependencies.yaml"
+  sed \
+    -e "s|@@POSTGRES_IMAGE@@|$postgres_image|g" \
+    -e "s|@@REDIS_IMAGE@@|$redis_image|g" \
+    -e "s|@@VAULT_IMAGE@@|$vault_image|g" \
+    -e "s|@@CURL_IMAGE@@|$curl_image|g" \
+    -e "s|@@AWS_CLI_IMAGE@@|$aws_cli_image|g" \
+    "$TESTBED/probe.yaml.in" > "$RUNTIME/probe.yaml"
+}
+
+kube() {
+  "$BIN/kubectl" \
+    --kubeconfig "$KUBECONFIG_PATH" \
+    --context "$CONTEXT" \
+    "$@"
+}
+
+wait_for_job() {
+  local job="$1"
+  local deadline=$((SECONDS + 300))
+  local complete
+  local failed
+  while ((SECONDS < deadline)); do
+    complete="$(kube --namespace remora-testbed get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Complete")].status}')"
+    if [[ "$complete" = True ]]; then
+      kube --namespace remora-testbed logs "job/$job" --all-containers
+      return
+    fi
+    failed="$(kube --namespace remora-testbed get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Failed")].status}')"
+    if [[ "$failed" = True ]]; then
+      kube --namespace remora-testbed logs "job/$job" --all-containers >&2 || true
+      return 1
+    fi
+    sleep 1
+  done
+  printf 'job/%s did not complete within 300 seconds\n' "$job" >&2
+  return 1
+}

tests/kind/probe.yaml.in

@@ -0,0 +1,283 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: probe-payload
+  namespace: remora-testbed
+data:
+  payload: remora-wp02-published-image-transfer
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: state-probe
+  namespace: remora-testbed
+spec:
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 300
+  template:
+    spec:
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      restartPolicy: Never
+      initContainers:
+        - name: postgres
+          image: @@POSTGRES_IMAGE@@
+          imagePullPolicy: Never
+          command: [psql]
+          args:
+            - -h
+            - postgres
+            - -U
+            - remora
+            - -d
+            - remora
+            - -v
+            - ON_ERROR_STOP=1
+            - -c
+            - "CREATE TABLE wp02_probe (value text NOT NULL)"
+            - -c
+            - "INSERT INTO wp02_probe VALUES ('passed')"
+            - -c
+            - "SELECT value = 'passed' FROM wp02_probe"
+          env:
+            - {name: PGPASSWORD, value: synthetic-postgres}
+        - name: redis
+          image: @@REDIS_IMAGE@@
+          imagePullPolicy: Never
+          command: [/bin/sh, -ec]
+          args: ["redis-cli -h redis SET wp02 passed && test \"$(redis-cli -h redis GET wp02)\" = passed"]
+        - name: dragonfly
+          image: @@REDIS_IMAGE@@
+          imagePullPolicy: Never
+          command: [/bin/sh, -ec]
+          args: ["redis-cli -h dragonfly SET wp02 passed && test \"$(redis-cli -h dragonfly GET wp02)\" = passed"]
+        - name: vault-write
+          image: @@VAULT_IMAGE@@
+          imagePullPolicy: Never
+          command: [vault]
+          args: [kv, put, secret/wp02, value=passed]
+          env:
+            - {name: VAULT_ADDR, value: "http://vault:8200"}
+            - {name: VAULT_TOKEN, value: synthetic-vault-token}
+      containers:
+        - name: vault-read
+          image: @@VAULT_IMAGE@@
+          imagePullPolicy: Never
+          command: [/bin/sh, -ec]
+          args: ["test \"$(vault kv get -field=value secret/wp02)\" = passed && printf 'Vault write/read passed\\n'"]
+          env:
+            - {name: VAULT_ADDR, value: "http://vault:8200"}
+            - {name: VAULT_TOKEN, value: synthetic-vault-token}
+          resources:
+            requests: {cpu: 25m, memory: 32Mi}
+            limits: {cpu: "1", memory: 128Mi}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: oidc-probe
+  namespace: remora-testbed
+spec:
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 300
+  template:
+    spec:
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      restartPolicy: Never
+      dnsPolicy: None
+      dnsConfig:
+        nameservers: [10.96.0.53]
+      containers:
+        - name: curl
+          image: @@CURL_IMAGE@@
+          imagePullPolicy: Never
+          command: [/bin/sh, -ec]
+          args:
+            - |
+              probe() {
+                url="$1"
+                pattern="$2"
+                attempt=0
+                until curl --fail --silent --show-error "$url" | grep -q "$pattern"; do
+                  attempt=$((attempt + 1))
+                  test "$attempt" -lt 30
+                  sleep 1
+                done
+              }
+              probe http://keycloak.remora.test:8080/realms/master/.well-known/openid-configuration '"issuer"'
+              probe http://keycloak.remora.test:8080/realms/master/protocol/openid-connect/certs '"keys"'
+              probe http://zitadel.remora.test:8080/.well-known/openid-configuration '"issuer"'
+              probe http://zitadel.remora.test:8080/oauth/v2/keys '"keys"'
+              printf 'Keycloak and ZITADEL discovery passed through controlled DNS\n'
+          resources:
+            requests: {cpu: 25m, memory: 32Mi}
+            limits: {cpu: "1", memory: 128Mi}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: garage-probe
+  namespace: remora-testbed
+spec:
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 300
+  template:
+    spec:
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      restartPolicy: Never
+      initContainers:
+        - name: bucket
+          image: @@AWS_CLI_IMAGE@@
+          imagePullPolicy: Never
+          command: [/bin/sh, -ec]
+          args:
+            - |
+              attempt=0
+              until aws --endpoint-url http://garage:3900 s3api head-bucket --bucket remora-fixture; do
+                attempt=$((attempt + 1))
+                test "$attempt" -lt 30
+                sleep 1
+              done
+          env:
+            - {name: AWS_ACCESS_KEY_ID, value: GK00000000000000000000000000000000}
+            - {name: AWS_SECRET_ACCESS_KEY, value: "0000000000000000000000000000000000000000000000000000000000000000"}
+            - {name: AWS_DEFAULT_REGION, value: garage}
+        - name: upload
+          image: @@AWS_CLI_IMAGE@@
+          imagePullPolicy: Never
+          command: [/bin/sh, -ec]
+          args:
+            - |
+              attempt=0
+              until aws --endpoint-url http://garage:3900 s3api put-object --bucket remora-fixture --key smoke/payload --body /fixture/payload; do
+                attempt=$((attempt + 1))
+                test "$attempt" -lt 30
+                sleep 1
+              done
+          env:
+            - {name: AWS_ACCESS_KEY_ID, value: GK00000000000000000000000000000000}
+            - {name: AWS_SECRET_ACCESS_KEY, value: "0000000000000000000000000000000000000000000000000000000000000000"}
+            - {name: AWS_DEFAULT_REGION, value: garage}
+          volumeMounts:
+            - {name: fixture, mountPath: /fixture}
+        - name: download
+          image: @@AWS_CLI_IMAGE@@
+          imagePullPolicy: Never
+          command: [/bin/sh, -ec]
+          args:
+            - |
+              attempt=0
+              until aws --endpoint-url http://garage:3900 s3api get-object --bucket remora-fixture --key smoke/payload /work/result; do
+                attempt=$((attempt + 1))
+                test "$attempt" -lt 30
+                sleep 1
+              done
+          env:
+            - {name: AWS_ACCESS_KEY_ID, value: GK00000000000000000000000000000000}
+            - {name: AWS_SECRET_ACCESS_KEY, value: "0000000000000000000000000000000000000000000000000000000000000000"}
+            - {name: AWS_DEFAULT_REGION, value: garage}
+          volumeMounts:
+            - {name: work, mountPath: /work}
+      containers:
+        - name: compare
+          image: @@CURL_IMAGE@@
+          imagePullPolicy: Never
+          command: [/bin/sh, -ec]
+          args: ["cmp /fixture/payload /work/result && printf 'Garage transfer passed\\n'"]
+          volumeMounts:
+            - {name: fixture, mountPath: /fixture}
+            - {name: work, mountPath: /work}
+          resources:
+            requests: {cpu: 25m, memory: 32Mi}
+            limits: {cpu: "1", memory: 128Mi}
+      volumes:
+        - name: fixture
+          configMap: {name: probe-payload}
+        - name: work
+          emptyDir: {}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: versitygw-probe
+  namespace: remora-testbed
+spec:
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 300
+  template:
+    spec:
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      restartPolicy: Never
+      initContainers:
+        - name: bucket
+          image: @@AWS_CLI_IMAGE@@
+          imagePullPolicy: Never
+          command: [/bin/sh, -ec]
+          args:
+            - |
+              attempt=0
+              until aws --endpoint-url http://versitygw:10000 s3api create-bucket --bucket remora-fixture || aws --endpoint-url http://versitygw:10000 s3api head-bucket --bucket remora-fixture; do
+                attempt=$((attempt + 1))
+                test "$attempt" -lt 30
+                sleep 1
+              done
+          env:
+            - {name: AWS_ACCESS_KEY_ID, value: synthetic-s3-access}
+            - {name: AWS_SECRET_ACCESS_KEY, value: synthetic-s3-secret}
+            - {name: AWS_DEFAULT_REGION, value: us-east-1}
+        - name: upload
+          image: @@AWS_CLI_IMAGE@@
+          imagePullPolicy: Never
+          command: [/bin/sh, -ec]
+          args:
+            - |
+              attempt=0
+              until aws --endpoint-url http://versitygw:10000 s3api put-object --bucket remora-fixture --key smoke/payload --body /fixture/payload; do
+                attempt=$((attempt + 1))
+                test "$attempt" -lt 30
+                sleep 1
+              done
+          env:
+            - {name: AWS_ACCESS_KEY_ID, value: synthetic-s3-access}
+            - {name: AWS_SECRET_ACCESS_KEY, value: synthetic-s3-secret}
+            - {name: AWS_DEFAULT_REGION, value: us-east-1}
+          volumeMounts:
+            - {name: fixture, mountPath: /fixture}
+        - name: download
+          image: @@AWS_CLI_IMAGE@@
+          imagePullPolicy: Never
+          command: [/bin/sh, -ec]
+          args:
+            - |
+              attempt=0
+              until aws --endpoint-url http://versitygw:10000 s3api get-object --bucket remora-fixture --key smoke/payload /work/result; do
+                attempt=$((attempt + 1))
+                test "$attempt" -lt 30
+                sleep 1
+              done
+          env:
+            - {name: AWS_ACCESS_KEY_ID, value: synthetic-s3-access}
+            - {name: AWS_SECRET_ACCESS_KEY, value: synthetic-s3-secret}
+            - {name: AWS_DEFAULT_REGION, value: us-east-1}
+          volumeMounts:
+            - {name: work, mountPath: /work}
+      containers:
+        - name: compare
+          image: @@CURL_IMAGE@@
+          imagePullPolicy: Never
+          command: [/bin/sh, -ec]
+          args: ["cmp /fixture/payload /work/result && printf 'VersityGW transfer passed\\n'"]
+          volumeMounts:
+            - {name: fixture, mountPath: /fixture}
+            - {name: work, mountPath: /work}
+          resources:
+            requests: {cpu: 25m, memory: 32Mi}
+            limits: {cpu: "1", memory: 128Mi}
+      volumes:
+        - name: fixture
+          configMap: {name: probe-payload}
+        - name: work
+          emptyDir: {}

tests/kind/provider/nerdctl

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ "${1:-}" = run ]]; then
+  shift
+  exec "$REMORA_REAL_NERDCTL" run --memory 24g --memory-swap 32g --cpus 8 "$@"
+fi
+exec "$REMORA_REAL_NERDCTL" "$@"

tests/kind/run.sh

@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
+source "$ROOT/tests/kind/lib.sh"
+
+mkdir -p "$RUNTIME"
+rm -rf "$RUNTIME/image-archives"
+mkdir -p "$RUNTIME/image-archives"
+USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
+SUCCEEDED=false
+
+cleanup() {
+  local status=$?
+  trap - EXIT INT TERM
+  if [[ "${REMORA_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
+    printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
+    printf 'retained context: %s\n' "$CONTEXT"
+    exit "$status"
+  fi
+  if [[ -n "${REMORA_REAL_NERDCTL:-}" ]]; then
+    delete_cluster || status=1
+    remove_runtime_namespace || status=1
+  fi
+  rm -rf "$RUNTIME/image-archives" || status=1
+  rm -f "$KUBECONFIG_PATH" "$RUNTIME/dependencies.yaml" "$RUNTIME/probe.yaml" || status=1
+  if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
+    printf 'user kubeconfig changed unexpectedly\n' >&2
+    status=1
+  fi
+  if [[ "$status" = 0 ]] && [[ "$SUCCEEDED" = true ]]; then
+    printf 'WP-02 hermetic dependency testbed passed\n'
+  fi
+  exit "$status"
+}
+
+trap cleanup EXIT INT TERM
+
+prepare_tools
+prepare_provider
+
+"$BIN/kind" create cluster \
+  --name "$CLUSTER" \
+  --image "$NODE_IMAGE" \
+  --config "$TESTBED/cluster.yaml" \
+  --kubeconfig "$KUBECONFIG_PATH" \
+  --wait 180s
+
+POSTGRES_LOCAL_IMAGE="$(load_external_image postgres "$POSTGRES_IMAGE")"
+REDIS_LOCAL_IMAGE="$(load_external_image redis "$REDIS_IMAGE")"
+DRAGONFLY_LOCAL_IMAGE="$(load_external_image dragonfly "$DRAGONFLY_IMAGE")"
+VAULT_LOCAL_IMAGE="$(load_external_image vault "$VAULT_IMAGE")"
+KEYCLOAK_LOCAL_IMAGE="$(load_external_image keycloak "$KEYCLOAK_IMAGE")"
+ZITADEL_LOCAL_IMAGE="$(load_external_image zitadel "$ZITADEL_IMAGE")"
+GARAGE_LOCAL_IMAGE="$(load_external_image garage "$GARAGE_IMAGE")"
+VERSITYGW_LOCAL_IMAGE="$(load_external_image versitygw "$VERSITYGW_IMAGE")"
+COREDNS_LOCAL_IMAGE="$(load_external_image coredns "$COREDNS_IMAGE")"
+CURL_LOCAL_IMAGE="$(load_external_image curl "$CURL_IMAGE")"
+AWS_CLI_LOCAL_IMAGE="$(load_external_image aws-cli "$AWS_CLI_IMAGE")"
+render_manifests \
+  "$POSTGRES_LOCAL_IMAGE" \
+  "$REDIS_LOCAL_IMAGE" \
+  "$DRAGONFLY_LOCAL_IMAGE" \
+  "$VAULT_LOCAL_IMAGE" \
+  "$KEYCLOAK_LOCAL_IMAGE" \
+  "$ZITADEL_LOCAL_IMAGE" \
+  "$GARAGE_LOCAL_IMAGE" \
+  "$VERSITYGW_LOCAL_IMAGE" \
+  "$COREDNS_LOCAL_IMAGE" \
+  "$CURL_LOCAL_IMAGE" \
+  "$AWS_CLI_LOCAL_IMAGE"
+
+kube apply --filename "$RUNTIME/dependencies.yaml"
+for deployment in postgres redis dragonfly vault keycloak zitadel garage versitygw controlled-dns; do
+  kube --namespace remora-testbed rollout status "deployment/$deployment" --timeout 240s
+done
+
+kube apply --filename "$RUNTIME/probe.yaml"
+for job in state-probe oidc-probe garage-probe versitygw-probe; do
+  wait_for_job "$job"
+done
+
+SUCCEEDED=true

tests/kind/versions.env

@@ -0,0 +1,16 @@
+KIND_VERSION=v0.32.0
+KIND_SHA256=50030de23cf40a18505f20426f6a8506bedf13c6e509244bd1fa9463721b0f54
+KUBECTL_VERSION=v1.36.2
+KUBECTL_SHA256=1e9045ec32bea85da43de85f0065358529ea7c7a152eca78154fba5b58c27d82
+NODE_IMAGE=kindest/node:v1.36.1@sha256:3489c7674813ba5d8b1a9977baea8a6e553784dab7b84759d1014dbd78f7ebd5
+POSTGRES_IMAGE=postgres@sha256:bb9940981eeb86356ec91bda430f8e0ba8729cc0b319615a2c2d7ac932fdf7bb
+REDIS_IMAGE=redis@sha256:cd5f3ac681c77791c6a8eaa62de876ad2be043ee5a428afb7c0095aa08246277
+DRAGONFLY_IMAGE=docker.dragonflydb.io/dragonflydb/dragonfly@sha256:665ee58607d035881384be45b20025490ecf87069418f61f715c19117ed5fb7d
+VAULT_IMAGE=hashicorp/vault@sha256:db6a5f78749204d0a8d002be1cc6e9a2a6de53dffce41a894c4a06d309dca360
+KEYCLOAK_IMAGE=quay.io/keycloak/keycloak@sha256:1362a9d9f13ab325231ea133610cc905e12805804abc7acbef552dd613720aa6
+ZITADEL_IMAGE=ghcr.io/zitadel/zitadel@sha256:d59554f17d20c06c9f6ed114bf9f2c3a9a63698003daea62d27bdae1e9667583
+GARAGE_IMAGE=dxflrs/garage@sha256:dac0c92add4f1a0b41035e94b41036a270ffbe88a37c7ac9c3f19e6dc5bdccf2
+VERSITYGW_IMAGE=versity/versitygw@sha256:4a2c613f6541410f87983c5134cd26a88dfc23f0ad555b7aa51d3d87fd2d347b
+COREDNS_IMAGE=coredns/coredns@sha256:1ba6f47265602e2e50a9c4669e3a955e4298a0d30dc82f39293d4bf1a851e0ff
+CURL_IMAGE=curlimages/curl@sha256:1ab04d023ece37e6ec991bf3306ad04e0ef0084e94a5c6b6563cfcb9563169db
+AWS_CLI_IMAGE=amazon/aws-cli@sha256:10c3a6dff0fdcb3fbc86cd744709548ad041ebb6bbe14fbedc1595a5539d8d27