Group kind tests by owning component

Makefile

@@ -53,49 +53,49 @@ check-fast-host: rust-check web-check web-test-fast
 check-full-host: rust-check web-check web-test-full
 
 check-dependencies:
-	./tests/kind/run.sh
+	./tests/kind/fixture/run.sh
 
 check-fixture-lib:
-	./tests/kind/lib-test.sh
+	./tests/kind/fixture/lib-test.sh
 
 check-kind-sample:
-	./tests/kind/run-sample-job.sh
+	./tests/kind/sample/run.sh
 
 check-durable:
-	./tests/kind/run-durable-job.sh
+	./tests/kind/durable/run.sh
 
 check-coordination:
-	./tests/kind/run-coordination-job.sh
+	./tests/kind/coordination/run.sh
 
 check-identity:
-	./tests/kind/run-identity-job.sh
+	./tests/kind/identity/run.sh
 
 check-credentials:
-	./tests/kind/run-credentials-job.sh
+	./tests/kind/credentials/run.sh
 
 check-admission:
-	./tests/kind/run-admission-job.sh
+	./tests/kind/admission/run.sh
 
 check-attempt:
-	./tests/kind/run-attempt-job.sh
+	./tests/kind/attempt/run.sh
 
 check-attempt-mutations:
-	./tests/kind/run-attempt-mutations-job.sh
+	./tests/kind/attempt/run-mutations.sh
 
 check-recovery:
-	./tests/kind/run-recovery-job.sh
+	./tests/kind/attempt/run-recovery.sh
 
 check-worker-api:
-	./tests/kind/run-worker-api-job.sh
+	./tests/kind/api/run-worker.sh
 
 check-public-api:
-	./tests/kind/run-public-api-job.sh
+	./tests/kind/api/run-public.sh
 
 check-live-events:
-	./tests/kind/run-live-events-job.sh
+	./tests/kind/api/run-live-events.sh
 
 check-worker:
-	./tests/kind/run-worker-job.sh
+	./tests/kind/worker/run.sh
 
 check-stream:
 	cargo fmt --all --check
@@ -104,34 +104,34 @@ check-stream:
 	cargo test --package remora-stream --doc
 
 check-target:
-	./tests/kind/run-target-job.sh
+	./tests/kind/target/run.sh
 
 check-result:
-	./tests/kind/run-result-job.sh
+	./tests/kind/result/run.sh
 
 check-deletion:
-	./tests/kind/run-deletion-job.sh
+	./tests/kind/deletion/run.sh
 
 check-outbound:
-	./tests/kind/run-outbound-job.sh
+	./tests/kind/outbound/run.sh
 
 check-http-source:
-	./tests/kind/run-http-source-job.sh
+	./tests/kind/http-source/run.sh
 
 check-object-source:
-	./tests/kind/run-object-source-job.sh
+	./tests/kind/object-source/run.sh
 
 check-hf-source:
-	./tests/kind/run-hf-source-job.sh
+	./tests/kind/hf-source/run.sh
 
 check-deployment:
-	./tests/kind/run-deployment.sh
+	./tests/kind/deployment/run.sh
 
 check-requirements:
 	./tests/qualification/check-requirements.sh
 
 check-qualification-kind:
-	./tests/kind/run-qualification.sh
+	./tests/kind/qualification/run.sh
-	REMORA_SYSTEM_QUALIFICATION=1 ./tests/kind/run-deployment.sh
+	REMORA_SYSTEM_QUALIFICATION=1 ./tests/kind/deployment/run.sh
 
 check-qualification: check-requirements check-full check-qualification-kind

docs/spec/testing.md

@@ -4,15 +4,15 @@ Backend integration and fault tests execute as Kubernetes `Job` resources inside
 
 ## Fixture lifecycle
 
-`make check-dependencies` remains the dependency compatibility gate: it creates the full fixture, runs the released-image probes, and tears the fixture down. `make check-kind-sample` builds and loads a Remora-owned cargo-nextest runner, then exercises the full provider matrix around [`fixture-consumer-sample`](../../tests/kind/sample-job.yaml.in). Its init containers access the lane database, coordination namespace, OIDC discovery and signing keys, target bucket, and Vault path before cargo-nextest runs. That Job is the reference structure for a `TestComponent` test Job.
+`make check-dependencies` remains the dependency compatibility gate: it creates the full fixture, runs the released-image probes, and tears the fixture down. `make check-kind-sample` builds and loads a Remora-owned cargo-nextest runner, then exercises the full provider matrix around [`fixture-consumer-sample`](../../tests/kind/sample/job.yaml.in). Its init containers access the lane database, coordination namespace, OIDC discovery and signing keys, target bucket, and Vault path before cargo-nextest runs. That Job is the reference structure for a `TestComponent` test Job.
 
-For interactive work, `tests/kind/setup.sh` creates a ready fixture and leaves it running. `tests/kind/destroy.sh` removes it. Setup replaces any fixture left by an earlier run, teardown is idempotent, and a host-runtime lock shared across worktrees serializes lifecycle mutations around the single `remora-testbed` cluster and `remora-dependency-testbed` containerd namespace. `REMORA_TESTBED_RETAIN=1` retains a failed all-in-one run for inspection.
+For interactive work, `tests/kind/fixture/setup.sh` creates a ready fixture and leaves it running. `tests/kind/fixture/destroy.sh` removes it. Setup replaces any fixture left by an earlier run, teardown is idempotent, and a host-runtime lock shared across worktrees serializes lifecycle mutations around the single `remora-testbed` cluster and `remora-dependency-testbed` containerd namespace. `REMORA_TESTBED_RETAIN=1` retains a failed all-in-one run for inspection.
 
-The fixture owns an isolated kubeconfig at `tests/kind/.runtime/kubeconfig` and the `kind-remora-testbed` context. Every Kubernetes operation uses both explicitly. Package runners use the `kube` helper from `tests/kind/lib.sh`; they do not read or change the user's kubeconfig or current context.
+The fixture owns an isolated kubeconfig at `tests/kind/.runtime/kubeconfig` and the `kind-remora-testbed` context. Every Kubernetes operation uses both explicitly. Package runners use the `kube` helper from `tests/kind/fixture/lib.sh`; they do not read or change the user's kubeconfig or current context.
 
 ## Package test Jobs
 
-Each `TestComponent` owns its test code, test-runner image, Job manifest, acceptance command, and required compatibility families. Its all-in-one runner follows [`run-sample-job.sh`](../../tests/kind/run-sample-job.sh): acquire the fixture lock, install a cleanup trap, call `setup_fixture <families...>`, load any component-owned image with `load_local_image <image> [containerd-namespace]`, and call `run_job_matrix <job-name> <manifest> <image> <families...>`. The image namespace defaults to `default`. A retained failure prints the isolated kubeconfig and context.
+Each `TestComponent` owns a directory under `tests/kind/` containing its Job manifests and acceptance runners. It also owns its test code, test-runner image, and required compatibility families. Its all-in-one runner follows [`sample/run.sh`](../../tests/kind/sample/run.sh): acquire the fixture lock, install a cleanup trap, call `setup_fixture <families...>`, load any component-owned image with `load_local_image <image> [containerd-namespace]`, and call `run_job_matrix <job-name> <manifest> <image> <families...>`. The image namespace defaults to `default`. A retained failure prints the isolated kubeconfig and context.
 
 The selectable families are `base`, `durable`, `coordination`, `oidc`, `target`, `secret`, and `all`. `base` provisions only the cluster and common namespace for suites whose controlled fixtures run inside their test pod. Individual implementations are not selectors. A suite that requests `coordination`, `oidc`, and `target` runs the unchanged cargo-nextest archive against the full Redis/DragonflyDB × Keycloak/ZITADEL × Garage/VersityGW Cartesian product. Lanes run sequentially and stop on the first failure.
 
@@ -32,6 +32,6 @@ Package runners may create additional namespaced fixtures and fault injectors ne
 
 ## Network access
 
-Automated backend tests use cluster-local service discovery. Tests of Keycloak or ZITADEL issuer behavior use the `dnsPolicy` and `dnsConfig` pattern in [`fixture-consumer-sample`](../../tests/kind/sample-job.yaml.in) so discovery URLs and token issuers have one identity from every participating pod.
+Automated backend tests use cluster-local service discovery. Tests of Keycloak or ZITADEL issuer behavior use the `dnsPolicy` and `dnsConfig` pattern in [`fixture-consumer-sample`](../../tests/kind/sample/job.yaml.in) so discovery URLs and token issuers have one identity from every participating pod.
 
 Port forwarding is for interactive diagnostics only and is not an acceptance-test transport. Ingress is introduced by deployment and system qualification, where routing is itself under test; substrate test components do not install an ingress controller merely to reach fixture services.

tests/kind/admission/job.yaml.in

@@ -0,0 +1,34 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: @@JOB_NAME@@
+  namespace: remora-testbed
+spec:
+  activeDeadlineSeconds: 600
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 300
+  template:
+    spec:
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      restartPolicy: Never
+      containers:
+        - name: tests
+          image: @@TEST_IMAGE@@
+          imagePullPolicy: Never
+          envFrom:
+            - configMapRef: {name: @@PROFILE_NAME@@}
+          env:
+            - name: REMORA_ADMISSION_REQUIRE
+              value: "1"
+          volumeMounts:
+            - {name: fixture-profile, mountPath: /run/remora-fixture, readOnly: true}
+          resources:
+            requests: {cpu: 250m, memory: 256Mi}
+            limits: {cpu: "2", memory: 1Gi}
+      volumes:
+        - name: fixture-profile
+          configMap:
+            name: @@PROFILE_NAME@@-file
+            items:
+              - {key: profile.env, path: profile.env}

tests/kind/admission/run.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/../../.." && pwd)"
+source "$ROOT/tests/kind/fixture/lib.sh"
+
+USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
+SUCCEEDED=false
+OWNS_FIXTURE=false
+
+cleanup() {
+  local status=$?
+  trap - EXIT INT TERM
+  if [[ "${REMORA_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
+    printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
+    printf 'retained context: %s\n' "$CONTEXT"
+    exit "$status"
+  fi
+  if [[ "$OWNS_FIXTURE" = true ]]; then
+    destroy_fixture || status=1
+  fi
+  if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
+    printf 'user kubeconfig changed unexpectedly\n' >&2
+    status=1
+  fi
+  exit "$status"
+}
+
+trap cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+acquire_fixture_lock
+REMORA_REAL_NERDCTL="$(command -v nerdctl)"
+export REMORA_REAL_NERDCTL
+"$REMORA_REAL_NERDCTL" --namespace default build \
+  --file "$ROOT/Dockerfile.test" \
+  --target kind-admission-runner \
+  --tag remora-kind-admission:local \
+  "$ROOT"
+OWNS_FIXTURE=true
+setup_fixture durable
+load_local_image remora-kind-admission:local
+run_job_matrix \
+  job-admission \
+  "$TESTBED/admission/job.yaml.in" \
+  remora-kind-admission:local \
+  durable
+SUCCEEDED=true
+printf 'job admission package test Job passed\n'

tests/kind/api/live-events-job.yaml.in

@@ -0,0 +1,40 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: @@JOB_NAME@@
+  namespace: remora-testbed
+spec:
+  activeDeadlineSeconds: 900
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 300
+  template:
+    spec:
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      restartPolicy: Never
+      dnsPolicy: None
+      dnsConfig:
+        nameservers: [10.96.0.53]
+        searches: [remora-testbed.svc.cluster.local, svc.cluster.local, cluster.local]
+        options:
+          - {name: ndots, value: "5"}
+      containers:
+        - name: tests
+          image: @@TEST_IMAGE@@
+          imagePullPolicy: Never
+          envFrom:
+            - configMapRef: {name: @@PROFILE_NAME@@}
+          env:
+            - name: REMORA_LIVE_EVENTS_REQUIRE
+              value: "1"
+          volumeMounts:
+            - {name: fixture-profile, mountPath: /run/remora-fixture, readOnly: true}
+          resources:
+            requests: {cpu: 250m, memory: 512Mi}
+            limits: {cpu: "2", memory: 2Gi}
+      volumes:
+        - name: fixture-profile
+          configMap:
+            name: @@PROFILE_NAME@@-file
+            items:
+              - {key: profile.env, path: profile.env}

tests/kind/api/public-job.yaml.in

@@ -0,0 +1,40 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: @@JOB_NAME@@
+  namespace: remora-testbed
+spec:
+  activeDeadlineSeconds: 900
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 300
+  template:
+    spec:
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      restartPolicy: Never
+      dnsPolicy: None
+      dnsConfig:
+        nameservers: [10.96.0.53]
+        searches: [remora-testbed.svc.cluster.local, svc.cluster.local, cluster.local]
+        options:
+          - {name: ndots, value: "5"}
+      containers:
+        - name: tests
+          image: @@TEST_IMAGE@@
+          imagePullPolicy: Never
+          envFrom:
+            - configMapRef: {name: @@PROFILE_NAME@@}
+          env:
+            - name: REMORA_PUBLIC_API_REQUIRE
+              value: "1"
+          volumeMounts:
+            - {name: fixture-profile, mountPath: /run/remora-fixture, readOnly: true}
+          resources:
+            requests: {cpu: 250m, memory: 512Mi}
+            limits: {cpu: "2", memory: 2Gi}
+      volumes:
+        - name: fixture-profile
+          configMap:
+            name: @@PROFILE_NAME@@-file
+            items:
+              - {key: profile.env, path: profile.env}

tests/kind/api/run-live-events.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/../../.." && pwd)"
+source "$ROOT/tests/kind/fixture/lib.sh"
+
+USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
+SUCCEEDED=false
+OWNS_FIXTURE=false
+
+cleanup() {
+  local status=$?
+  trap - EXIT INT TERM
+  if [[ "${REMORA_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
+    printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
+    printf 'retained context: %s\n' "$CONTEXT"
+    exit "$status"
+  fi
+  if [[ "$OWNS_FIXTURE" = true ]]; then
+    destroy_fixture || status=1
+  fi
+  if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
+    printf 'user kubeconfig changed unexpectedly\n' >&2
+    status=1
+  fi
+  exit "$status"
+}
+
+trap cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+acquire_fixture_lock
+REMORA_REAL_NERDCTL="$(command -v nerdctl)"
+export REMORA_REAL_NERDCTL
+"$REMORA_REAL_NERDCTL" --namespace default build \
+  --file "$ROOT/Dockerfile.test" \
+  --target kind-live-events-runner \
+  --tag remora-kind-live-events:local \
+  "$ROOT"
+OWNS_FIXTURE=true
+setup_fixture durable coordination oidc
+load_local_image remora-kind-live-events:local
+run_job_matrix \
+  live-events \
+  "$TESTBED/api/live-events-job.yaml.in" \
+  remora-kind-live-events:local \
+  durable coordination oidc
+SUCCEEDED=true
+printf 'live event package test Jobs passed\n'

tests/kind/api/run-public.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/../../.." && pwd)"
+source "$ROOT/tests/kind/fixture/lib.sh"
+
+USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
+SUCCEEDED=false
+OWNS_FIXTURE=false
+
+cleanup() {
+  local status=$?
+  trap - EXIT INT TERM
+  if [[ "${REMORA_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
+    printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
+    printf 'retained context: %s\n' "$CONTEXT"
+    exit "$status"
+  fi
+  if [[ "$OWNS_FIXTURE" = true ]]; then
+    destroy_fixture || status=1
+  fi
+  if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
+    printf 'user kubeconfig changed unexpectedly\n' >&2
+    status=1
+  fi
+  exit "$status"
+}
+
+trap cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+acquire_fixture_lock
+REMORA_REAL_NERDCTL="$(command -v nerdctl)"
+export REMORA_REAL_NERDCTL
+"$REMORA_REAL_NERDCTL" --namespace default build \
+  --file "$ROOT/Dockerfile.test" \
+  --target kind-public-api-runner \
+  --tag remora-kind-public-api:local \
+  "$ROOT"
+OWNS_FIXTURE=true
+setup_fixture durable coordination oidc target secret
+load_local_image remora-kind-public-api:local
+run_job_matrix \
+  public-api \
+  "$TESTBED/api/public-job.yaml.in" \
+  remora-kind-public-api:local \
+  durable coordination oidc target secret
+SUCCEEDED=true
+printf 'public API package test Jobs passed\n'

tests/kind/api/run-worker.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/../../.." && pwd)"
+source "$ROOT/tests/kind/fixture/lib.sh"
+
+USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
+SUCCEEDED=false
+OWNS_FIXTURE=false
+
+cleanup() {
+  local status=$?
+  trap - EXIT INT TERM
+  if [[ "${REMORA_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
+    printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
+    printf 'retained context: %s\n' "$CONTEXT"
+    exit "$status"
+  fi
+  if [[ "$OWNS_FIXTURE" = true ]]; then
+    destroy_fixture || status=1
+  fi
+  if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
+    printf 'user kubeconfig changed unexpectedly\n' >&2
+    status=1
+  fi
+  exit "$status"
+}
+
+trap cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+acquire_fixture_lock
+REMORA_REAL_NERDCTL="$(command -v nerdctl)"
+export REMORA_REAL_NERDCTL
+"$REMORA_REAL_NERDCTL" --namespace default build \
+  --file "$ROOT/Dockerfile.test" \
+  --target kind-worker-api-runner \
+  --tag remora-kind-worker-api:local \
+  "$ROOT"
+OWNS_FIXTURE=true
+setup_fixture durable coordination
+load_local_image remora-kind-worker-api:local
+run_job_matrix \
+  worker-api \
+  "$TESTBED/api/worker-job.yaml.in" \
+  remora-kind-worker-api:local \
+  durable coordination
+SUCCEEDED=true
+printf 'worker API package test Jobs passed\n'

tests/kind/api/worker-job.yaml.in

@@ -0,0 +1,34 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: @@JOB_NAME@@
+  namespace: remora-testbed
+spec:
+  activeDeadlineSeconds: 600
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 300
+  template:
+    spec:
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      restartPolicy: Never
+      containers:
+        - name: tests
+          image: @@TEST_IMAGE@@
+          imagePullPolicy: Never
+          envFrom:
+            - configMapRef: {name: @@PROFILE_NAME@@}
+          env:
+            - name: REMORA_WORKER_API_REQUIRE
+              value: "1"
+          volumeMounts:
+            - {name: fixture-profile, mountPath: /run/remora-fixture, readOnly: true}
+          resources:
+            requests: {cpu: 250m, memory: 256Mi}
+            limits: {cpu: "2", memory: 1Gi}
+      volumes:
+        - name: fixture-profile
+          configMap:
+            name: @@PROFILE_NAME@@-file
+            items:
+              - {key: profile.env, path: profile.env}

tests/kind/attempt/job.yaml.in

@@ -0,0 +1,34 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: @@JOB_NAME@@
+  namespace: remora-testbed
+spec:
+  activeDeadlineSeconds: 600
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 300
+  template:
+    spec:
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      restartPolicy: Never
+      containers:
+        - name: tests
+          image: @@TEST_IMAGE@@
+          imagePullPolicy: Never
+          envFrom:
+            - configMapRef: {name: @@PROFILE_NAME@@}
+          env:
+            - name: REMORA_ATTEMPT_REQUIRE
+              value: "1"
+          volumeMounts:
+            - {name: fixture-profile, mountPath: /run/remora-fixture, readOnly: true}
+          resources:
+            requests: {cpu: 250m, memory: 256Mi}
+            limits: {cpu: "2", memory: 1Gi}
+      volumes:
+        - name: fixture-profile
+          configMap:
+            name: @@PROFILE_NAME@@-file
+            items:
+              - {key: profile.env, path: profile.env}

tests/kind/attempt/mutations-job.yaml.in

@@ -0,0 +1,34 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: @@JOB_NAME@@
+  namespace: remora-testbed
+spec:
+  activeDeadlineSeconds: 600
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 300
+  template:
+    spec:
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      restartPolicy: Never
+      containers:
+        - name: tests
+          image: @@TEST_IMAGE@@
+          imagePullPolicy: Never
+          envFrom:
+            - configMapRef: {name: @@PROFILE_NAME@@}
+          env:
+            - name: REMORA_ATTEMPT_REQUIRE
+              value: "1"
+          volumeMounts:
+            - {name: fixture-profile, mountPath: /run/remora-fixture, readOnly: true}
+          resources:
+            requests: {cpu: 250m, memory: 256Mi}
+            limits: {cpu: "2", memory: 1Gi}
+      volumes:
+        - name: fixture-profile
+          configMap:
+            name: @@PROFILE_NAME@@-file
+            items:
+              - {key: profile.env, path: profile.env}

tests/kind/attempt/recovery-job.yaml.in

@@ -0,0 +1,34 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: @@JOB_NAME@@
+  namespace: remora-testbed
+spec:
+  activeDeadlineSeconds: 900
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 300
+  template:
+    spec:
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      restartPolicy: Never
+      containers:
+        - name: tests
+          image: @@TEST_IMAGE@@
+          imagePullPolicy: Never
+          envFrom:
+            - configMapRef: {name: @@PROFILE_NAME@@}
+          env:
+            - name: REMORA_RECOVERY_REQUIRE
+              value: "1"
+          volumeMounts:
+            - {name: fixture-profile, mountPath: /run/remora-fixture, readOnly: true}
+          resources:
+            requests: {cpu: 250m, memory: 256Mi}
+            limits: {cpu: "2", memory: 1Gi}
+      volumes:
+        - name: fixture-profile
+          configMap:
+            name: @@PROFILE_NAME@@-file
+            items:
+              - {key: profile.env, path: profile.env}

tests/kind/attempt/run-mutations.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/../../.." && pwd)"
+source "$ROOT/tests/kind/fixture/lib.sh"
+
+USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
+SUCCEEDED=false
+OWNS_FIXTURE=false
+
+cleanup() {
+  local status=$?
+  trap - EXIT INT TERM
+  if [[ "${REMORA_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
+    printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
+    printf 'retained context: %s\n' "$CONTEXT"
+    exit "$status"
+  fi
+  if [[ "$OWNS_FIXTURE" = true ]]; then
+    destroy_fixture || status=1
+  fi
+  if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
+    printf 'user kubeconfig changed unexpectedly\n' >&2
+    status=1
+  fi
+  exit "$status"
+}
+
+trap cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+acquire_fixture_lock
+REMORA_REAL_NERDCTL="$(command -v nerdctl)"
+export REMORA_REAL_NERDCTL
+"$REMORA_REAL_NERDCTL" --namespace default build \
+  --file "$ROOT/Dockerfile.test" \
+  --target kind-attempt-mutations-runner \
+  --tag remora-kind-attempt-mutations:local \
+  "$ROOT"
+OWNS_FIXTURE=true
+setup_fixture durable coordination
+load_local_image remora-kind-attempt-mutations:local
+run_job_matrix \
+  attempt-mutations \
+  "$TESTBED/attempt/mutations-job.yaml.in" \
+  remora-kind-attempt-mutations:local \
+  durable coordination
+SUCCEEDED=true
+printf 'attempt mutation package test Jobs passed\n'

tests/kind/attempt/run-recovery.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/../../.." && pwd)"
+source "$ROOT/tests/kind/fixture/lib.sh"
+
+USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
+SUCCEEDED=false
+OWNS_FIXTURE=false
+
+cleanup() {
+  local status=$?
+  trap - EXIT INT TERM
+  if [[ "${REMORA_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
+    printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
+    printf 'retained context: %s\n' "$CONTEXT"
+    exit "$status"
+  fi
+  if [[ "$OWNS_FIXTURE" = true ]]; then
+    destroy_fixture || status=1
+  fi
+  if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
+    printf 'user kubeconfig changed unexpectedly\n' >&2
+    status=1
+  fi
+  exit "$status"
+}
+
+trap cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+acquire_fixture_lock
+REMORA_REAL_NERDCTL="$(command -v nerdctl)"
+export REMORA_REAL_NERDCTL
+"$REMORA_REAL_NERDCTL" --namespace default build \
+  --file "$ROOT/Dockerfile.test" \
+  --target kind-recovery-runner \
+  --tag remora-kind-recovery:local \
+  "$ROOT"
+OWNS_FIXTURE=true
+setup_fixture durable coordination target
+load_local_image remora-kind-recovery:local
+run_job_matrix \
+  recovery \
+  "$TESTBED/attempt/recovery-job.yaml.in" \
+  remora-kind-recovery:local \
+  durable coordination target
+SUCCEEDED=true
+printf 'recovery package test Jobs passed\n'

tests/kind/attempt/run.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/../../.." && pwd)"
+source "$ROOT/tests/kind/fixture/lib.sh"
+
+USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
+SUCCEEDED=false
+OWNS_FIXTURE=false
+
+cleanup() {
+  local status=$?
+  trap - EXIT INT TERM
+  if [[ "${REMORA_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
+    printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
+    printf 'retained context: %s\n' "$CONTEXT"
+    exit "$status"
+  fi
+  if [[ "$OWNS_FIXTURE" = true ]]; then
+    destroy_fixture || status=1
+  fi
+  if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
+    printf 'user kubeconfig changed unexpectedly\n' >&2
+    status=1
+  fi
+  exit "$status"
+}
+
+trap cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+acquire_fixture_lock
+REMORA_REAL_NERDCTL="$(command -v nerdctl)"
+export REMORA_REAL_NERDCTL
+"$REMORA_REAL_NERDCTL" --namespace default build \
+  --file "$ROOT/Dockerfile.test" \
+  --target kind-attempt-runner \
+  --tag remora-kind-attempt:local \
+  "$ROOT"
+OWNS_FIXTURE=true
+setup_fixture durable coordination
+load_local_image remora-kind-attempt:local
+run_job_matrix \
+  attempt-authority \
+  "$TESTBED/attempt/job.yaml.in" \
+  remora-kind-attempt:local \
+  durable coordination
+SUCCEEDED=true
+printf 'attempt authority package test Jobs passed\n'

tests/kind/coordination/job.yaml.in

@@ -0,0 +1,34 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: @@JOB_NAME@@
+  namespace: remora-testbed
+spec:
+  activeDeadlineSeconds: 600
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 300
+  template:
+    spec:
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      restartPolicy: Never
+      containers:
+        - name: tests
+          image: @@TEST_IMAGE@@
+          imagePullPolicy: Never
+          envFrom:
+            - configMapRef: {name: @@PROFILE_NAME@@}
+          env:
+            - name: REMORA_COORDINATION_REQUIRE
+              value: "1"
+          volumeMounts:
+            - {name: fixture-profile, mountPath: /run/remora-fixture, readOnly: true}
+          resources:
+            requests: {cpu: 250m, memory: 256Mi}
+            limits: {cpu: "2", memory: 1Gi}
+      volumes:
+        - name: fixture-profile
+          configMap:
+            name: @@PROFILE_NAME@@-file
+            items:
+              - {key: profile.env, path: profile.env}

tests/kind/coordination/run.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/../../.." && pwd)"
+source "$ROOT/tests/kind/fixture/lib.sh"
+
+USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
+SUCCEEDED=false
+OWNS_FIXTURE=false
+
+cleanup() {
+  local status=$?
+  trap - EXIT INT TERM
+  if [[ "${REMORA_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
+    printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
+    printf 'retained context: %s\n' "$CONTEXT"
+    exit "$status"
+  fi
+  if [[ "$OWNS_FIXTURE" = true ]]; then
+    destroy_fixture || status=1
+  fi
+  if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
+    printf 'user kubeconfig changed unexpectedly\n' >&2
+    status=1
+  fi
+  exit "$status"
+}
+
+trap cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+acquire_fixture_lock
+REMORA_REAL_NERDCTL="$(command -v nerdctl)"
+export REMORA_REAL_NERDCTL
+"$REMORA_REAL_NERDCTL" --namespace default build \
+  --file "$ROOT/Dockerfile.test" \
+  --target kind-coordination-runner \
+  --tag remora-kind-coordination:local \
+  "$ROOT"
+OWNS_FIXTURE=true
+setup_fixture coordination
+load_local_image remora-kind-coordination:local
+run_job_matrix \
+  coordination-substrate \
+  "$TESTBED/coordination/job.yaml.in" \
+  remora-kind-coordination:local \
+  coordination
+SUCCEEDED=true
+printf 'coordination substrate package test Job passed\n'

tests/kind/credentials/job.yaml.in

@@ -0,0 +1,34 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: @@JOB_NAME@@
+  namespace: remora-testbed
+spec:
+  activeDeadlineSeconds: 600
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 300
+  template:
+    spec:
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      restartPolicy: Never
+      containers:
+        - name: tests
+          image: @@TEST_IMAGE@@
+          imagePullPolicy: Never
+          envFrom:
+            - configMapRef: {name: @@PROFILE_NAME@@}
+          env:
+            - name: REMORA_CREDENTIALS_REQUIRE
+              value: "1"
+          volumeMounts:
+            - {name: fixture-profile, mountPath: /run/remora-fixture, readOnly: true}
+          resources:
+            requests: {cpu: 250m, memory: 256Mi}
+            limits: {cpu: "2", memory: 1Gi}
+      volumes:
+        - name: fixture-profile
+          configMap:
+            name: @@PROFILE_NAME@@-file
+            items:
+              - {key: profile.env, path: profile.env}

tests/kind/credentials/run.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/../../.." && pwd)"
+source "$ROOT/tests/kind/fixture/lib.sh"
+
+USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
+SUCCEEDED=false
+OWNS_FIXTURE=false
+
+cleanup() {
+  local status=$?
+  trap - EXIT INT TERM
+  if [[ "${REMORA_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
+    printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
+    printf 'retained context: %s\n' "$CONTEXT"
+    exit "$status"
+  fi
+  if [[ "$OWNS_FIXTURE" = true ]]; then
+    destroy_fixture || status=1
+  fi
+  if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
+    printf 'user kubeconfig changed unexpectedly\n' >&2
+    status=1
+  fi
+  exit "$status"
+}
+
+trap cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+acquire_fixture_lock
+REMORA_REAL_NERDCTL="$(command -v nerdctl)"
+export REMORA_REAL_NERDCTL
+"$REMORA_REAL_NERDCTL" --namespace default build \
+  --file "$ROOT/Dockerfile.test" \
+  --target kind-credentials-runner \
+  --tag remora-kind-credentials:local \
+  "$ROOT"
+OWNS_FIXTURE=true
+setup_fixture durable secret
+load_local_image remora-kind-credentials:local
+run_job_matrix \
+  credentials-registry \
+  "$TESTBED/credentials/job.yaml.in" \
+  remora-kind-credentials:local \
+  durable secret
+SUCCEEDED=true
+printf 'source credential registry package test Job passed\n'