ms2k/remora
SHA256
Watch
0
Code Releases Activity

Add compatibility matrix fixture execution

Browse source 
Run package test jobs across every selected provider combination with isolated lane state, released-image canaries, controlled OIDC DNS, and deterministic cleanup.
This commit is contained in:
ms2k 2026-07-16 05:39:51 +09:00
commit 20ea9557fd
10 changed files with 989 additions and 53 deletions
Download patch file Download diff file Expand all files Collapse all files

6
Dockerfile.test
View file

@ -40,7 +40,7 @@ WORKDIR /workspace
COPY Cargo.toml ./ COPY Cargo.toml ./
COPY crates crates COPY crates crates
COPY --from=kind-sample-build /tmp/remora-api-tests.tar.zst /tests/remora-api-tests.tar.zst COPY --from=kind-sample-build /tmp/remora-api-tests.tar.zst /tests/remora-api-tests.tar.zst
CMD ["bash", "-ec", "cargo-nextest nextest run --archive-file /tests/remora-api-tests.tar.zst --workspace-remap /workspace && exec 3<>/dev/tcp/postgres/5432"] CMD ["bash", "-ec", "test -s /run/remora-fixture/profile.env && test -n \"$DATABASE_URL\" && test -n \"$REMORA_OIDC_CLIENT_ID\" && test -n \"$REMORA_OIDC_USERNAME\" && case $REMORA_COORDINATION_PROVIDER in redis|dragonfly) ;; *) exit 1 ;; esac && case $REMORA_OIDC_PROVIDER in keycloak|zitadel) ;; *) exit 1 ;; esac && case $REMORA_TARGET_PROVIDER in garage|versitygw) ;; *) exit 1 ;; esac && cargo-nextest nextest run --archive-file /tests/remora-api-tests.tar.zst --workspace-remap /workspace"]
FROM rust-tools AS kind-durable-build FROM rust-tools AS kind-durable-build
WORKDIR /workspace WORKDIR /workspace
@ -60,9 +60,7 @@ WORKDIR /workspace
COPY Cargo.toml ./ COPY Cargo.toml ./
COPY crates crates COPY crates crates
COPY --from=kind-durable-build /tmp/remora-durable-tests.tar.zst /tests/remora-durable-tests.tar.zst COPY --from=kind-durable-build /tmp/remora-durable-tests.tar.zst /tests/remora-durable-tests.tar.zst
ENV DATABASE_URL=postgres://remora:synthetic-postgres@postgres:5432/remora?sslmode=disable CMD ["bash", "-ec", "test -s /run/remora-fixture/profile.env && test -n \"$DATABASE_URL\" && cargo-nextest nextest run --archive-file /tests/remora-durable-tests.tar.zst --workspace-remap /workspace"]
ENV REMORA_DURABLE_REQUIRE_DB=1
CMD ["bash", "-ec", "cargo-nextest nextest run --archive-file /tests/remora-durable-tests.tar.zst --workspace-remap /workspace"]
FROM ${NODE_IMAGE} AS node-runtime FROM ${NODE_IMAGE} AS node-runtime

9
Makefile
View file

@ -1,4 +1,4 @@
.PHONY: check-dependencies check-kind-sample check-durable check-fast check-full check-fast-host check-full-host rust-check web-install web-check web-test-fast web-test-full .PHONY: check-dependencies check-fixture-lib check-kind-sample check-durable check-fast check-full check-fast-host check-full-host rust-check web-install web-check web-test-fast web-test-full
CONTAINER_ENGINE ?= nerdctl CONTAINER_ENGINE ?= nerdctl
CONTAINER_RUN_FLAGS ?= --memory=16g --memory-swap=24g --cpus=4 CONTAINER_RUN_FLAGS ?= --memory=16g --memory-swap=24g --cpus=4
@ -31,13 +31,13 @@ web-test-full: web-install
pnpm --dir web test:component:full pnpm --dir web test:component:full
pnpm --dir web test:e2e:full pnpm --dir web test:e2e:full
check-fast: check-fast: check-fixture-lib
$(CONTAINER_ENGINE) build --file Dockerfile.test --target rust-check --tag remora-rust-check:local . $(CONTAINER_ENGINE) build --file Dockerfile.test --target rust-check --tag remora-rust-check:local .
$(CONTAINER_ENGINE) run --rm $(CONTAINER_RUN_FLAGS) remora-rust-check:local $(CONTAINER_ENGINE) run --rm $(CONTAINER_RUN_FLAGS) remora-rust-check:local
$(CONTAINER_ENGINE) build --file Dockerfile.test --target web-fast --tag remora-web-check:local . $(CONTAINER_ENGINE) build --file Dockerfile.test --target web-fast --tag remora-web-check:local .
$(CONTAINER_ENGINE) run --rm $(CONTAINER_RUN_FLAGS) remora-web-check:local $(CONTAINER_ENGINE) run --rm $(CONTAINER_RUN_FLAGS) remora-web-check:local
check-full: check-full: check-fixture-lib
$(CONTAINER_ENGINE) build --file Dockerfile.test --target rust-check --tag remora-rust-check:local . $(CONTAINER_ENGINE) build --file Dockerfile.test --target rust-check --tag remora-rust-check:local .
$(CONTAINER_ENGINE) run --rm $(CONTAINER_RUN_FLAGS) remora-rust-check:local $(CONTAINER_ENGINE) run --rm $(CONTAINER_RUN_FLAGS) remora-rust-check:local
$(CONTAINER_ENGINE) build --file Dockerfile.test --target web-full --tag remora-web-check:local . $(CONTAINER_ENGINE) build --file Dockerfile.test --target web-full --tag remora-web-check:local .
@ -50,6 +50,9 @@ check-full-host: rust-check web-check web-test-full
check-dependencies: check-dependencies:
./tests/kind/run.sh ./tests/kind/run.sh
check-fixture-lib:
./tests/kind/lib-test.sh
check-kind-sample: check-kind-sample:
./tests/kind/run-sample-job.sh ./tests/kind/run-sample-job.sh

12
docs/spec/testing.md
View file

@ -4,7 +4,7 @@ Backend integration and fault tests execute as Kubernetes `Job` resources inside
## Fixture lifecycle ## Fixture lifecycle
`make check-dependencies` remains the dependency compatibility gate: it creates a fresh fixture, runs the released-image probes, and tears the fixture down. `make check-kind-sample` builds and loads a Remora-owned cargo-nextest runner, then runs the same lifecycle around [`fixture-consumer-sample`](../../tests/kind/sample-job.yaml.in). That Job is the reference structure for a work-package test Job. `make check-dependencies` remains the dependency compatibility gate: it creates the full fixture, runs the released-image probes, and tears the fixture down. `make check-kind-sample` builds and loads a Remora-owned cargo-nextest runner, then exercises the full provider matrix around [`fixture-consumer-sample`](../../tests/kind/sample-job.yaml.in). Its init containers access the lane database, coordination namespace, OIDC discovery and signing keys, target bucket, and Vault path before cargo-nextest runs. That Job is the reference structure for a work-package test Job.
For interactive work, `tests/kind/setup.sh` creates a ready fixture and leaves it running. `tests/kind/destroy.sh` removes it. Setup replaces any fixture left by an earlier run, teardown is idempotent, and the harness serializes lifecycle mutations around the single `remora-testbed` cluster. `REMORA_TESTBED_RETAIN=1` retains a failed all-in-one run for inspection. For interactive work, `tests/kind/setup.sh` creates a ready fixture and leaves it running. `tests/kind/destroy.sh` removes it. Setup replaces any fixture left by an earlier run, teardown is idempotent, and the harness serializes lifecycle mutations around the single `remora-testbed` cluster. `REMORA_TESTBED_RETAIN=1` retains a failed all-in-one run for inspection.
@ -12,7 +12,11 @@ The fixture owns an isolated kubeconfig at `tests/kind/.runtime/kubeconfig` and
## Package test Jobs ## Package test Jobs
Each work package owns its test code, test-runner image, Job manifest, and acceptance command. Its all-in-one runner follows [`run-sample-job.sh`](../../tests/kind/run-sample-job.sh): acquire the fixture lock, install a cleanup trap, call `setup_fixture`, load any package-owned image with `load_local_image <image> [containerd-namespace]`, apply the Job with `run_job_manifest <namespace> <job-name> <manifest>`, and propagate the Job result before teardown. The image namespace defaults to `default`. A retained failure prints the isolated kubeconfig and context. Each work package owns its test code, test-runner image, Job manifest, acceptance command, and required compatibility families. Its all-in-one runner follows [`run-sample-job.sh`](../../tests/kind/run-sample-job.sh): acquire the fixture lock, install a cleanup trap, call `setup_fixture <families...>`, load any package-owned image with `load_local_image <image> [containerd-namespace]`, and call `run_job_matrix <job-name> <manifest> <image> <families...>`. The image namespace defaults to `default`. A retained failure prints the isolated kubeconfig and context.
The selectable families are `durable`, `coordination`, `oidc`, `target`, `secret`, and `all`. Individual implementations are not selectors. A suite that requests `coordination`, `oidc`, and `target` runs the unchanged cargo-nextest archive against the full Redis/DragonflyDB × Keycloak/ZITADEL × Garage/VersityGW Cartesian product. Lanes run sequentially and stop on the first failure.
Before each lane, the runner recreates a lane database, allocates unique coordination keys, provisions an OIDC tenant/client/user/role grant, creates a target bucket and prefix, and allocates a Vault path as required by the selected families. The resulting environment is exposed both through environment variables and a ConfigMap-mounted `/run/remora-fixture/profile.env`. Package manifests use `@@JOB_NAME@@`, `@@PROFILE_NAME@@`, and `@@TEST_IMAGE@@` placeholders so only the profile and lane identity vary between Jobs.
Released third-party dependencies are always the digest-pinned images loaded by the fixture. Work packages may build Remora test-runner images locally; they do not rebuild or wrap dependency images. The Job manifest names every loaded image exactly and sets `imagePullPolicy: Never`. Released third-party dependencies are always the digest-pinned images loaded by the fixture. Work packages may build Remora test-runner images locally; they do not rebuild or wrap dependency images. The Job manifest names every loaded image exactly and sets `imagePullPolicy: Never`.
@ -24,10 +28,10 @@ A test Job:
- exits nonzero for every failed assertion and writes diagnostic output to its container logs; and - exits nonzero for every failed assertion and writes diagnostic output to its container logs; and
- uses ClusterIP service names for dependencies and synthetic fixture credentials only. - uses ClusterIP service names for dependencies and synthetic fixture credentials only.
Package runners may create additional namespaced fixtures and fault injectors needed by their acceptance matrix. They clean those resources or rely on whole-cluster teardown; they never depend on state left by another package. Package runners may create additional namespaced fixtures and fault injectors needed by their acceptance matrix. Process-wide coordination fault tests run exclusively. A failed lane is retained when `REMORA_TESTBED_RETAIN=1`; successful lanes delete their Job and profile before the next lane. Tests never depend on state left by an earlier lane or package.
## Network access ## Network access
Automated backend tests use cluster-local service discovery. Tests of Keycloak or ZITADEL issuer behavior use the `dnsPolicy` and `dnsConfig` pattern in [`oidc-probe`](../../tests/kind/probe.yaml.in) so discovery URLs and token issuers have one identity from every participating pod. Automated backend tests use cluster-local service discovery. Tests of Keycloak or ZITADEL issuer behavior use the `dnsPolicy` and `dnsConfig` pattern in [`fixture-consumer-sample`](../../tests/kind/sample-job.yaml.in) so discovery URLs and token issuers have one identity from every participating pod.
Port forwarding is for interactive diagnostics only and is not an acceptance-test transport. Ingress is introduced by deployment and system qualification, where routing is itself under test; substrate work packages do not install an ingress controller merely to reach fixture services. Port forwarding is for interactive diagnostics only and is not an acceptance-test transport. Ingress is introduced by deployment and system qualification, where routing is itself under test; substrate work packages do not install an ingress controller merely to reach fixture services.

59
tests/kind/dependencies.yaml.in
View file

@ -2,12 +2,14 @@ apiVersion: v1
kind: Namespace kind: Namespace
metadata: metadata:
name: remora-testbed name: remora-testbed
labels: {fixture.remora.dev/component: common}
--- ---
apiVersion: v1 apiVersion: v1
kind: ConfigMap kind: ConfigMap
metadata: metadata:
name: postgres-init name: postgres-init
namespace: remora-testbed namespace: remora-testbed
labels: {fixture.remora.dev/component: postgres}
data: data:
01-keycloak.sql: | 01-keycloak.sql: |
CREATE USER keycloak WITH PASSWORD 'synthetic-keycloak-db'; CREATE USER keycloak WITH PASSWORD 'synthetic-keycloak-db';
@ -18,8 +20,9 @@ kind: Deployment
metadata: metadata:
name: postgres name: postgres
namespace: remora-testbed namespace: remora-testbed
labels: {fixture.remora.dev/component: postgres}
spec: spec:
replicas: 1 replicas: @@POSTGRES_REPLICAS@@
selector: selector:
matchLabels: {app: postgres} matchLabels: {app: postgres}
template: template:
@ -59,6 +62,7 @@ kind: Service
metadata: metadata:
name: postgres name: postgres
namespace: remora-testbed namespace: remora-testbed
labels: {fixture.remora.dev/component: postgres}
spec: spec:
selector: {app: postgres} selector: {app: postgres}
ports: ports:
@ -69,8 +73,9 @@ kind: Deployment
metadata: metadata:
name: redis name: redis
namespace: remora-testbed namespace: remora-testbed
labels: {fixture.remora.dev/component: redis}
spec: spec:
replicas: 1 replicas: @@REDIS_REPLICAS@@
selector: selector:
matchLabels: {app: redis} matchLabels: {app: redis}
template: template:
@ -99,6 +104,7 @@ kind: Service
metadata: metadata:
name: redis name: redis
namespace: remora-testbed namespace: remora-testbed
labels: {fixture.remora.dev/component: redis}
spec: spec:
selector: {app: redis} selector: {app: redis}
ports: ports:
@ -109,8 +115,9 @@ kind: Deployment
metadata: metadata:
name: dragonfly name: dragonfly
namespace: remora-testbed namespace: remora-testbed
labels: {fixture.remora.dev/component: dragonfly}
spec: spec:
replicas: 1 replicas: @@DRAGONFLY_REPLICAS@@
selector: selector:
matchLabels: {app: dragonfly} matchLabels: {app: dragonfly}
template: template:
@ -138,6 +145,7 @@ kind: Service
metadata: metadata:
name: dragonfly name: dragonfly
namespace: remora-testbed namespace: remora-testbed
labels: {fixture.remora.dev/component: dragonfly}
spec: spec:
selector: {app: dragonfly} selector: {app: dragonfly}
ports: ports:
@ -148,8 +156,9 @@ kind: Deployment
metadata: metadata:
name: vault name: vault
namespace: remora-testbed namespace: remora-testbed
labels: {fixture.remora.dev/component: vault}
spec: spec:
replicas: 1 replicas: @@VAULT_REPLICAS@@
selector: selector:
matchLabels: {app: vault} matchLabels: {app: vault}
template: template:
@ -182,6 +191,7 @@ kind: Service
metadata: metadata:
name: vault name: vault
namespace: remora-testbed namespace: remora-testbed
labels: {fixture.remora.dev/component: vault}
spec: spec:
selector: {app: vault} selector: {app: vault}
ports: ports:
@ -192,8 +202,9 @@ kind: Deployment
metadata: metadata:
name: keycloak name: keycloak
namespace: remora-testbed namespace: remora-testbed
labels: {fixture.remora.dev/component: keycloak}
spec: spec:
replicas: 1 replicas: @@KEYCLOAK_REPLICAS@@
selector: selector:
matchLabels: {app: keycloak} matchLabels: {app: keycloak}
template: template:
@ -230,6 +241,7 @@ kind: Service
metadata: metadata:
name: keycloak name: keycloak
namespace: remora-testbed namespace: remora-testbed
labels: {fixture.remora.dev/component: keycloak}
spec: spec:
clusterIP: 10.96.0.55 clusterIP: 10.96.0.55
selector: {app: keycloak} selector: {app: keycloak}
@ -241,8 +253,9 @@ kind: Deployment
metadata: metadata:
name: zitadel name: zitadel
namespace: remora-testbed namespace: remora-testbed
labels: {fixture.remora.dev/component: zitadel}
spec: spec:
replicas: 1 replicas: @@ZITADEL_REPLICAS@@
selector: selector:
matchLabels: {app: zitadel} matchLabels: {app: zitadel}
template: template:
@ -264,6 +277,10 @@ spec:
- {name: ZITADEL_EXTERNALPORT, value: "8080"} - {name: ZITADEL_EXTERNALPORT, value: "8080"}
- {name: ZITADEL_FIRSTINSTANCE_ORG_HUMAN_USERNAME, value: remora-admin} - {name: ZITADEL_FIRSTINSTANCE_ORG_HUMAN_USERNAME, value: remora-admin}
- {name: ZITADEL_FIRSTINSTANCE_ORG_HUMAN_PASSWORD, value: Synthetic-Zitadel-1!} - {name: ZITADEL_FIRSTINSTANCE_ORG_HUMAN_PASSWORD, value: Synthetic-Zitadel-1!}
- {name: ZITADEL_FIRSTINSTANCE_PATPATH, value: /bootstrap/admin.pat}
- {name: ZITADEL_FIRSTINSTANCE_ORG_MACHINE_MACHINE_USERNAME, value: fixture-admin}
- {name: ZITADEL_FIRSTINSTANCE_ORG_MACHINE_MACHINE_NAME, value: Fixture Admin}
- {name: ZITADEL_FIRSTINSTANCE_ORG_MACHINE_PAT_EXPIRATIONDATE, value: "2099-01-01T00:00:00Z"}
ports: ports:
- {name: http, containerPort: 8080} - {name: http, containerPort: 8080}
readinessProbe: readinessProbe:
@ -277,12 +294,28 @@ spec:
resources: resources:
requests: {cpu: 200m, memory: 512Mi} requests: {cpu: 200m, memory: 512Mi}
limits: {cpu: "2", memory: 1Gi} limits: {cpu: "2", memory: 1Gi}
volumeMounts:
- {name: bootstrap, mountPath: /bootstrap}
- name: provisioner
image: @@CURL_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args: ["while :; do sleep 3600; done"]
resources:
requests: {cpu: 10m, memory: 16Mi}
limits: {cpu: "1", memory: 64Mi}
volumeMounts:
- {name: bootstrap, mountPath: /bootstrap, readOnly: true}
volumes:
- name: bootstrap
emptyDir: {}
--- ---
apiVersion: v1 apiVersion: v1
kind: Service kind: Service
metadata: metadata:
name: zitadel name: zitadel
namespace: remora-testbed namespace: remora-testbed
labels: {fixture.remora.dev/component: zitadel}
spec: spec:
clusterIP: 10.96.0.56 clusterIP: 10.96.0.56
selector: {app: zitadel} selector: {app: zitadel}
@ -294,6 +327,7 @@ kind: ConfigMap
metadata: metadata:
name: garage-config name: garage-config
namespace: remora-testbed namespace: remora-testbed
labels: {fixture.remora.dev/component: garage}
data: data:
garage.toml: | garage.toml: |
metadata_dir = "/data/meta" metadata_dir = "/data/meta"
@ -317,8 +351,9 @@ kind: Deployment
metadata: metadata:
name: garage name: garage
namespace: remora-testbed namespace: remora-testbed
labels: {fixture.remora.dev/component: garage}
spec: spec:
replicas: 1 replicas: @@GARAGE_REPLICAS@@
selector: selector:
matchLabels: {app: garage} matchLabels: {app: garage}
template: template:
@ -373,6 +408,7 @@ kind: Service
metadata: metadata:
name: garage name: garage
namespace: remora-testbed namespace: remora-testbed
labels: {fixture.remora.dev/component: garage}
spec: spec:
selector: {app: garage} selector: {app: garage}
ports: ports:
@ -383,8 +419,9 @@ kind: Deployment
metadata: metadata:
name: versitygw name: versitygw
namespace: remora-testbed namespace: remora-testbed
labels: {fixture.remora.dev/component: versitygw}
spec: spec:
replicas: 1 replicas: @@VERSITYGW_REPLICAS@@
selector: selector:
matchLabels: {app: versitygw} matchLabels: {app: versitygw}
template: template:
@ -437,6 +474,7 @@ kind: Service
metadata: metadata:
name: versitygw name: versitygw
namespace: remora-testbed namespace: remora-testbed
labels: {fixture.remora.dev/component: versitygw}
spec: spec:
selector: {app: versitygw} selector: {app: versitygw}
ports: ports:
@ -447,6 +485,7 @@ kind: ConfigMap
metadata: metadata:
name: controlled-dns name: controlled-dns
namespace: remora-testbed namespace: remora-testbed
labels: {fixture.remora.dev/component: controlled-dns}
data: data:
Corefile: | Corefile: |
.:53 { .:53 {
@ -464,8 +503,9 @@ kind: Deployment
metadata: metadata:
name: controlled-dns name: controlled-dns
namespace: remora-testbed namespace: remora-testbed
labels: {fixture.remora.dev/component: controlled-dns}
spec: spec:
replicas: 1 replicas: @@COREDNS_REPLICAS@@
selector: selector:
matchLabels: {app: controlled-dns} matchLabels: {app: controlled-dns}
template: template:
@ -499,6 +539,7 @@ kind: Service
metadata: metadata:
name: controlled-dns name: controlled-dns
namespace: remora-testbed namespace: remora-testbed
labels: {fixture.remora.dev/component: controlled-dns}
spec: spec:
clusterIP: 10.96.0.53 clusterIP: 10.96.0.53
selector: {app: controlled-dns} selector: {app: controlled-dns}

14
tests/kind/durable-job.yaml.in
View file

@ -1,7 +1,7 @@
apiVersion: batch/v1 apiVersion: batch/v1
kind: Job kind: Job
metadata: metadata:
name: durable-substrate name: @@JOB_NAME@@
namespace: remora-testbed namespace: remora-testbed
spec: spec:
activeDeadlineSeconds: 600 activeDeadlineSeconds: 600
@ -16,11 +16,19 @@ spec:
- name: tests - name: tests
image: @@TEST_IMAGE@@ image: @@TEST_IMAGE@@
imagePullPolicy: Never imagePullPolicy: Never
envFrom:
- configMapRef: {name: @@PROFILE_NAME@@}
env: env:
- name: DATABASE_URL
value: postgres://remora:synthetic-postgres@postgres:5432/remora?sslmode=disable
- name: REMORA_DURABLE_REQUIRE_DB - name: REMORA_DURABLE_REQUIRE_DB
value: "1" value: "1"
volumeMounts:
- {name: fixture-profile, mountPath: /run/remora-fixture, readOnly: true}
resources: resources:
requests: {cpu: 250m, memory: 256Mi} requests: {cpu: 250m, memory: 256Mi}
limits: {cpu: "2", memory: 1Gi} limits: {cpu: "2", memory: 1Gi}
volumes:
- name: fixture-profile
configMap:
name: @@PROFILE_NAME@@-file
items:
- {key: profile.env, path: profile.env}

97
tests/kind/lib-test.sh Executable file
View file

@ -0,0 +1,97 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
source "$ROOT/tests/kind/lib.sh"
assert_matrix() {
local expected="$1"
shift
local actual
actual="$(expand_fixture_matrix "$@")"
if [[ "$actual" != "$expected" ]]; then
printf 'unexpected matrix for %s\nexpected:\n%s\nactual:\n%s\n' "$*" "$expected" "$actual" >&2
return 1
fi
}
assert_matrix $'001\t-\t-\t-' durable
assert_matrix $'001\tredis\t-\t-\n002\tdragonfly\t-\t-' coordination
assert_matrix $'001\t-\tkeycloak\tgarage\n002\t-\tkeycloak\tversitygw\n003\t-\tzitadel\tgarage\n004\t-\tzitadel\tversitygw' oidc target
assert_matrix $'001\tredis\tkeycloak\tgarage\n002\tredis\tkeycloak\tversitygw\n003\tredis\tzitadel\tgarage\n004\tredis\tzitadel\tversitygw\n005\tdragonfly\tkeycloak\tgarage\n006\tdragonfly\tkeycloak\tversitygw\n007\tdragonfly\tzitadel\tgarage\n008\tdragonfly\tzitadel\tversitygw' all
if expand_fixture_matrix redis >/dev/null 2>&1; then
printf 'individual provider selector was accepted\n' >&2
exit 1
fi
if expand_fixture_matrix all redis >/dev/null 2>&1; then
printf 'individual provider selector alongside all was accepted\n' >&2
exit 1
fi
select_fixture_families target durable target
if [[ "${FIXTURE_FAMILIES[*]}" != "durable target" ]]; then
printf 'families were not normalized: %s\n' "${FIXTURE_FAMILIES[*]}" >&2
exit 1
fi
kube() {
return 1
}
select_fixture_families durable
if write_lane_profile 001 - - - >/dev/null 2>&1; then
printf 'lane provisioning failure was ignored\n' >&2
exit 1
fi
write_lane_profile() {
printf 'remora-fixture-lane-001\n'
}
render_lane_job() {
return 0
}
run_job_manifest() {
return 42
}
cleanup_called=false
cleanup_lane_state() {
cleanup_called=true
}
if run_job_matrix fixture-consumer /dev/null remora-test:local durable; then
printf 'Job failure was ignored\n' >&2
exit 1
fi
if [[ "$cleanup_called" = true ]]; then
printf 'lane cleanup ran after a failed Job\n' >&2
exit 1
fi
source "$ROOT/tests/kind/lib.sh"
select_fixture_families all
cleanup_calls=()
cleanup_lane_durable_store() {
cleanup_calls+=(durable)
return 1
}
cleanup_lane_coordination_store() {
cleanup_calls+=(coordination)
}
cleanup_keycloak_lane() {
cleanup_calls+=(oidc)
}
cleanup_lane_target_store() {
cleanup_calls+=(target)
}
cleanup_lane_secret_store() {
cleanup_calls+=(secret)
}
if cleanup_lane_state 001 redis keycloak garage >/dev/null 2>&1; then
printf 'partial lane cleanup failure was ignored\n' >&2
exit 1
fi
if [[ "${cleanup_calls[*]}" != "durable coordination oidc target secret" ]]; then
printf 'lane cleanup stopped early: %s\n' "${cleanup_calls[*]}" >&2
exit 1
fi
printf 'fixture family matrix tests passed\n'

727
tests/kind/lib.sh
View file

@ -180,6 +180,15 @@ render_manifests() {
local coredns_image="$9" local coredns_image="$9"
local curl_image="${10}" local curl_image="${10}"
local aws_cli_image="${11}" local aws_cli_image="${11}"
local postgres_replicas="${12}"
local redis_replicas="${13}"
local dragonfly_replicas="${14}"
local vault_replicas="${15}"
local keycloak_replicas="${16}"
local zitadel_replicas="${17}"
local garage_replicas="${18}"
local versitygw_replicas="${19}"
local coredns_replicas="${20}"
sed \ sed \
-e "s|@@POSTGRES_IMAGE@@|$postgres_image|g" \ -e "s|@@POSTGRES_IMAGE@@|$postgres_image|g" \
-e "s|@@REDIS_IMAGE@@|$redis_image|g" \ -e "s|@@REDIS_IMAGE@@|$redis_image|g" \
@ -192,6 +201,15 @@ render_manifests() {
-e "s|@@COREDNS_IMAGE@@|$coredns_image|g" \ -e "s|@@COREDNS_IMAGE@@|$coredns_image|g" \
-e "s|@@CURL_IMAGE@@|$curl_image|g" \ -e "s|@@CURL_IMAGE@@|$curl_image|g" \
-e "s|@@AWS_CLI_IMAGE@@|$aws_cli_image|g" \ -e "s|@@AWS_CLI_IMAGE@@|$aws_cli_image|g" \
-e "s|@@POSTGRES_REPLICAS@@|$postgres_replicas|g" \
-e "s|@@REDIS_REPLICAS@@|$redis_replicas|g" \
-e "s|@@DRAGONFLY_REPLICAS@@|$dragonfly_replicas|g" \
-e "s|@@VAULT_REPLICAS@@|$vault_replicas|g" \
-e "s|@@KEYCLOAK_REPLICAS@@|$keycloak_replicas|g" \
-e "s|@@ZITADEL_REPLICAS@@|$zitadel_replicas|g" \
-e "s|@@GARAGE_REPLICAS@@|$garage_replicas|g" \
-e "s|@@VERSITYGW_REPLICAS@@|$versitygw_replicas|g" \
-e "s|@@COREDNS_REPLICAS@@|$coredns_replicas|g" \
"$TESTBED/dependencies.yaml.in" > "$RUNTIME/dependencies.yaml" "$TESTBED/dependencies.yaml.in" > "$RUNTIME/dependencies.yaml"
sed \ sed \
-e "s|@@POSTGRES_IMAGE@@|$postgres_image|g" \ -e "s|@@POSTGRES_IMAGE@@|$postgres_image|g" \
@ -215,7 +233,69 @@ render_manifests() {
} > "$RUNTIME/fixture.env" } > "$RUNTIME/fixture.env"
} }
select_fixture_families() {
local requested=("$@")
local family
local candidate
local found
local use_all=false
FIXTURE_FAMILIES=()
if [[ "${#requested[@]}" = 0 ]]; then
requested=(all)
fi
for family in "${requested[@]}"; do
case "$family" in
all)
use_all=true
;;
durable | coordination | oidc | target | secret) ;;
*)
printf 'unknown fixture family: %s\n' "$family" >&2
return 1
;;
esac
done
if [[ "$use_all" = true ]]; then
requested=(durable coordination oidc target secret)
fi
for candidate in durable coordination oidc target secret; do
found=false
for family in "${requested[@]}"; do
if [[ "$family" = "$candidate" ]]; then
found=true
fi
done
if [[ "$found" = true ]]; then
FIXTURE_FAMILIES+=("$candidate")
fi
done
}
fixture_has_family() {
local expected="$1"
local family
for family in "${FIXTURE_FAMILIES[@]}"; do
if [[ "$family" = "$expected" ]]; then
return
fi
done
return 1
}
load_selected_image() {
local family="$1"
local variable="$2"
local name="$3"
local source="$4"
local image
if fixture_has_family "$family"; then
image="$(load_external_image "$name" "$source")" || return
printf -v "$variable" '%s' "$image"
fi
}
setup_fixture() { setup_fixture() {
select_fixture_families "$@" || return
mkdir -p "$RUNTIME" mkdir -p "$RUNTIME"
rm -rf "$RUNTIME/image-archives" rm -rf "$RUNTIME/image-archives"
mkdir -p "$RUNTIME/image-archives" mkdir -p "$RUNTIME/image-archives"
@ -229,17 +309,47 @@ setup_fixture() {
--kubeconfig "$KUBECONFIG_PATH" \ --kubeconfig "$KUBECONFIG_PATH" \
--wait 180s --wait 180s
POSTGRES_LOCAL_IMAGE="$(load_external_image postgres "$POSTGRES_IMAGE")" POSTGRES_LOCAL_IMAGE=remora-testbed/unused:postgres
REDIS_LOCAL_IMAGE="$(load_external_image redis "$REDIS_IMAGE")" REDIS_LOCAL_IMAGE=remora-testbed/unused:redis
DRAGONFLY_LOCAL_IMAGE="$(load_external_image dragonfly "$DRAGONFLY_IMAGE")" DRAGONFLY_LOCAL_IMAGE=remora-testbed/unused:dragonfly
VAULT_LOCAL_IMAGE="$(load_external_image vault "$VAULT_IMAGE")" VAULT_LOCAL_IMAGE=remora-testbed/unused:vault
KEYCLOAK_LOCAL_IMAGE="$(load_external_image keycloak "$KEYCLOAK_IMAGE")" KEYCLOAK_LOCAL_IMAGE=remora-testbed/unused:keycloak
ZITADEL_LOCAL_IMAGE="$(load_external_image zitadel "$ZITADEL_IMAGE")" ZITADEL_LOCAL_IMAGE=remora-testbed/unused:zitadel
GARAGE_LOCAL_IMAGE="$(load_external_image garage "$GARAGE_IMAGE")" GARAGE_LOCAL_IMAGE=remora-testbed/unused:garage
VERSITYGW_LOCAL_IMAGE="$(load_external_image versitygw "$VERSITYGW_IMAGE")" VERSITYGW_LOCAL_IMAGE=remora-testbed/unused:versitygw
COREDNS_LOCAL_IMAGE="$(load_external_image coredns "$COREDNS_IMAGE")" COREDNS_LOCAL_IMAGE=remora-testbed/unused:coredns
CURL_LOCAL_IMAGE="$(load_external_image curl "$CURL_IMAGE")" CURL_LOCAL_IMAGE=remora-testbed/unused:curl
AWS_CLI_LOCAL_IMAGE="$(load_external_image aws-cli "$AWS_CLI_IMAGE")" AWS_CLI_LOCAL_IMAGE=remora-testbed/unused:aws-cli
if fixture_has_family durable || fixture_has_family oidc; then
POSTGRES_LOCAL_IMAGE="$(load_external_image postgres "$POSTGRES_IMAGE")"
fi
load_selected_image coordination REDIS_LOCAL_IMAGE redis "$REDIS_IMAGE"
load_selected_image coordination DRAGONFLY_LOCAL_IMAGE dragonfly "$DRAGONFLY_IMAGE"
load_selected_image secret VAULT_LOCAL_IMAGE vault "$VAULT_IMAGE"
load_selected_image oidc KEYCLOAK_LOCAL_IMAGE keycloak "$KEYCLOAK_IMAGE"
load_selected_image oidc ZITADEL_LOCAL_IMAGE zitadel "$ZITADEL_IMAGE"
load_selected_image target GARAGE_LOCAL_IMAGE garage "$GARAGE_IMAGE"
load_selected_image target VERSITYGW_LOCAL_IMAGE versitygw "$VERSITYGW_IMAGE"
load_selected_image oidc COREDNS_LOCAL_IMAGE coredns "$COREDNS_IMAGE"
if fixture_has_family oidc || fixture_has_family target; then
CURL_LOCAL_IMAGE="$(load_external_image curl "$CURL_IMAGE")"
fi
load_selected_image target AWS_CLI_LOCAL_IMAGE aws-cli "$AWS_CLI_IMAGE"
POSTGRES_REPLICAS=0
REDIS_REPLICAS=0
DRAGONFLY_REPLICAS=0
VAULT_REPLICAS=0
KEYCLOAK_REPLICAS=0
ZITADEL_REPLICAS=0
GARAGE_REPLICAS=0
VERSITYGW_REPLICAS=0
COREDNS_REPLICAS=0
if fixture_has_family durable || fixture_has_family oidc; then POSTGRES_REPLICAS=1; fi
if fixture_has_family coordination; then REDIS_REPLICAS=1; DRAGONFLY_REPLICAS=1; fi
if fixture_has_family secret; then VAULT_REPLICAS=1; fi
if fixture_has_family oidc; then KEYCLOAK_REPLICAS=1; ZITADEL_REPLICAS=1; COREDNS_REPLICAS=1; fi
if fixture_has_family target; then GARAGE_REPLICAS=1; VERSITYGW_REPLICAS=1; fi
render_manifests \ render_manifests \
"$POSTGRES_LOCAL_IMAGE" \ "$POSTGRES_LOCAL_IMAGE" \
"$REDIS_LOCAL_IMAGE" \ "$REDIS_LOCAL_IMAGE" \
@ -251,15 +361,570 @@ setup_fixture() {
"$VERSITYGW_LOCAL_IMAGE" \ "$VERSITYGW_LOCAL_IMAGE" \
"$COREDNS_LOCAL_IMAGE" \ "$COREDNS_LOCAL_IMAGE" \
"$CURL_LOCAL_IMAGE" \ "$CURL_LOCAL_IMAGE" \
"$AWS_CLI_LOCAL_IMAGE" "$AWS_CLI_LOCAL_IMAGE" \
"$POSTGRES_REPLICAS" \
"$REDIS_REPLICAS" \
"$DRAGONFLY_REPLICAS" \
"$VAULT_REPLICAS" \
"$KEYCLOAK_REPLICAS" \
"$ZITADEL_REPLICAS" \
"$GARAGE_REPLICAS" \
"$VERSITYGW_REPLICAS" \
"$COREDNS_REPLICAS"
kube apply --filename "$RUNTIME/dependencies.yaml" apply_fixture_component common
for deployment in postgres redis dragonfly vault keycloak zitadel garage versitygw controlled-dns; do if fixture_has_family durable || fixture_has_family oidc; then apply_fixture_component postgres; fi
kube --namespace "$FIXTURE_NAMESPACE" rollout status "deployment/$deployment" --timeout 240s if fixture_has_family coordination; then apply_fixture_component redis; apply_fixture_component dragonfly; fi
if fixture_has_family secret; then apply_fixture_component vault; fi
if fixture_has_family oidc; then apply_fixture_component keycloak; apply_fixture_component zitadel; apply_fixture_component controlled-dns; fi
if fixture_has_family target; then apply_fixture_component garage; apply_fixture_component versitygw; fi
if fixture_has_family durable || fixture_has_family oidc; then wait_for_deployment postgres; fi
if fixture_has_family coordination; then wait_for_deployment redis; wait_for_deployment dragonfly; fi
if fixture_has_family secret; then wait_for_deployment vault; fi
if fixture_has_family oidc; then wait_for_deployment keycloak; wait_for_deployment zitadel; wait_for_deployment controlled-dns; fi
if fixture_has_family target; then wait_for_deployment garage; wait_for_deployment versitygw; fi
}
apply_fixture_component() {
kube apply \
--filename "$RUNTIME/dependencies.yaml" \
--selector "fixture.remora.dev/component=$1"
}
wait_for_deployment() {
kube --namespace "$FIXTURE_NAMESPACE" rollout status "deployment/$1" --timeout 240s
}
expand_fixture_matrix() {
local coordination_providers=(-)
local oidc_providers=(-)
local target_providers=(-)
local coordination
local oidc
local target
local lane=0
select_fixture_families "$@" || return
if fixture_has_family coordination; then coordination_providers=(redis dragonfly); fi
if fixture_has_family oidc; then oidc_providers=(keycloak zitadel); fi
if fixture_has_family target; then target_providers=(garage versitygw); fi
for coordination in "${coordination_providers[@]}"; do
for oidc in "${oidc_providers[@]}"; do
for target in "${target_providers[@]}"; do
lane=$((lane + 1))
printf '%03d\t%s\t%s\t%s\n' "$lane" "$coordination" "$oidc" "$target"
done
done
done done
} }
reset_lane_durable_store() {
local lane="$1"
local database="remora_lane_$lane"
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
env PGPASSWORD=synthetic-postgres psql \
--username remora \
--dbname postgres \
--set ON_ERROR_STOP=1 \
--command "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$database'" \
--command "DROP DATABASE IF EXISTS $database" \
--command "CREATE DATABASE $database" >&2 || return
printf '%s\n' "$database"
}
reset_lane_target_store() {
local provider="$1"
local bucket="$2"
local deployment
local endpoint
if [[ "$provider" = garage ]]; then
deployment=garage
endpoint=http://127.0.0.1:3900
else
deployment=versitygw
endpoint=http://127.0.0.1:10000
fi
if kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$deployment" --container s3-readiness -- \
aws --endpoint-url "$endpoint" s3api head-bucket --bucket "$bucket" >/dev/null 2>&1; then
cleanup_lane_target_store "$provider" "$bucket" || return
fi
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$deployment" --container s3-readiness -- \
aws --endpoint-url "$endpoint" s3api create-bucket --bucket "$bucket" >&2 || return
}
reset_keycloak_lane() {
local lane="$1"
local realm="remora-lane-$lane"
local username="remora-user-$lane"
local client_id="remora-client-$lane"
if kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
/opt/keycloak/bin/kcadm.sh get "realms/$realm" \
--no-config \
--server http://127.0.0.1:8080 \
--realm master \
--user remora-admin \
--password synthetic-keycloak >/dev/null 2>&1; then
cleanup_keycloak_lane "$lane" || return
fi
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
/opt/keycloak/bin/kcadm.sh create realms \
--no-config \
--server http://127.0.0.1:8080 \
--realm master \
--user remora-admin \
--password synthetic-keycloak \
--set "realm=$realm" \
--set enabled=true >&2 || return
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
/opt/keycloak/bin/kcadm.sh create clients \
--no-config \
--server http://127.0.0.1:8080 \
--realm master \
--user remora-admin \
--password synthetic-keycloak \
-r "$realm" \
--set "clientId=$client_id" \
--set publicClient=true \
--set standardFlowEnabled=true \
--set 'redirectUris=["http://localhost/callback"]' >&2 || return
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
/opt/keycloak/bin/kcadm.sh create roles \
--no-config \
--server http://127.0.0.1:8080 \
--realm master \
--user remora-admin \
--password synthetic-keycloak \
-r "$realm" \
--set name=test-user >&2 || return
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
/opt/keycloak/bin/kcadm.sh create users \
--no-config \
--server http://127.0.0.1:8080 \
--realm master \
--user remora-admin \
--password synthetic-keycloak \
-r "$realm" \
--set "username=$username" \
--set enabled=true \
--set emailVerified=true \
--set "email=$username@remora.test" >&2 || return
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
/opt/keycloak/bin/kcadm.sh set-password \
--no-config \
--server http://127.0.0.1:8080 \
--realm master \
--user remora-admin \
--password synthetic-keycloak \
-r "$realm" \
--username "$username" \
--new-password Synthetic-OIDC-1! >&2 || return
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
/opt/keycloak/bin/kcadm.sh add-roles \
--no-config \
--server http://127.0.0.1:8080 \
--realm master \
--user remora-admin \
--password synthetic-keycloak \
-r "$realm" \
--uusername "$username" \
--rolename test-user >&2 || return
printf '%s\t%s\t%s\t%s\n' \
"http://keycloak.remora.test:8080/realms/$realm" \
"$realm" \
"$client_id" \
"$username"
}
zitadel_request() {
local endpoint="$1"
local body="$2"
local organization="${3:-}"
local method="${4:-POST}"
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/zitadel --container provisioner -- \
/bin/sh -ec '
endpoint="$1"
body="$2"
organization="$3"
method="$4"
token="$(cat /bootstrap/admin.pat)"
set -- \
--fail \
--silent \
--show-error \
--request "$method" \
--header "Host: zitadel.remora.test" \
--header "Authorization: Bearer $token" \
--header "Content-Type: application/json"
if test -n "$organization"; then
set -- "$@" --header "x-zitadel-orgid: $organization"
fi
curl "$@" --data "$body" "http://127.0.0.1:8080$endpoint"
' remora-zitadel-request "$endpoint" "$body" "$organization" "$method"
}
json_string_field() {
local document="$1"
local field="$2"
local value
value="$(sed -nE 's/.*"'"$field"'"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/p' <<< "$document")"
if [[ -z "$value" ]]; then
printf 'ZITADEL response omitted %s\n' "$field" >&2
return 1
fi
printf '%s\n' "$value"
}
reset_zitadel_lane() {
local lane="$1"
local tenant="remora-lane-$lane"
local username="remora-user-$lane"
local response
local organization_id
local project_id
local user_id
local client_id
cleanup_zitadel_lane "$lane" || return
response="$(zitadel_request /management/v1/orgs "{\"name\":\"$tenant\"}")" || return
organization_id="$(json_string_field "$response" id)" || return
response="$(zitadel_request /management/v1/projects "{\"name\":\"$tenant\",\"projectRoleAssertion\":true,\"projectRoleCheck\":true}" "$organization_id")" || return
project_id="$(json_string_field "$response" id)" || return
zitadel_request \
"/management/v1/projects/$project_id/roles" \
'{"roleKey":"test-user","displayName":"Test User","group":"fixture"}' \
"$organization_id" >/dev/null || return
response="$(zitadel_request \
"/management/v1/projects/$project_id/apps/oidc" \
"{\"name\":\"$tenant\",\"redirectUris\":[\"http://localhost/callback\"],\"postLogoutRedirectUris\":[\"http://localhost/logout\"],\"responseTypes\":[\"OIDC_RESPONSE_TYPE_CODE\"],\"grantTypes\":[\"OIDC_GRANT_TYPE_AUTHORIZATION_CODE\",\"OIDC_GRANT_TYPE_REFRESH_TOKEN\"],\"appType\":\"OIDC_APP_TYPE_USER_AGENT\",\"authMethodType\":\"OIDC_AUTH_METHOD_TYPE_NONE\",\"version\":\"OIDC_VERSION_1_0\",\"devMode\":true,\"accessTokenType\":\"OIDC_TOKEN_TYPE_JWT\",\"accessTokenRoleAssertion\":true}" \
"$organization_id")" || return
client_id="$(json_string_field "$response" clientId)" || return
response="$(zitadel_request \
/management/v1/users/human \
"{\"userName\":\"$username\",\"profile\":{\"firstName\":\"Remora\",\"lastName\":\"Lane $lane\",\"displayName\":\"Remora Lane $lane\",\"preferredLanguage\":\"en\",\"gender\":\"GENDER_UNSPECIFIED\"},\"email\":{\"email\":\"$username@remora.test\",\"isEmailVerified\":true},\"password\":\"Synthetic-OIDC-1!\",\"passwordChangeRequired\":false}" \
"$organization_id")" || return
user_id="$(json_string_field "$response" userId)" || return
zitadel_request \
"/management/v1/users/$user_id/grants" \
"{\"projectId\":\"$project_id\",\"roleKeys\":[\"test-user\"]}" \
"$organization_id" >/dev/null || return
printf '%s\t%s\t%s\t%s\n' \
http://zitadel.remora.test:8080 \
"$organization_id" \
"$client_id" \
"$username"
}
reset_lane_oidc_provider() {
local provider="$1"
local lane="$2"
if [[ "$provider" = keycloak ]]; then
reset_keycloak_lane "$lane"
else
reset_zitadel_lane "$lane"
fi
}
cleanup_lane_durable_store() {
local database="remora_lane_$1"
local remaining
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
env PGPASSWORD=synthetic-postgres psql \
--username remora \
--dbname postgres \
--set ON_ERROR_STOP=1 \
--command "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$database'" \
--command "DROP DATABASE IF EXISTS $database" >&2 || return
remaining="$(kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
env PGPASSWORD=synthetic-postgres psql \
--username remora \
--dbname postgres \
--tuples-only \
--no-align \
--command "SELECT count(*) FROM pg_database WHERE datname = '$database'")" || return
[[ "$remaining" = 0 ]]
}
cleanup_lane_coordination_store() {
local provider="$1"
local namespace="$2"
local keys
local key
while true; do
keys="$(kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" -- \
redis-cli --scan --pattern "$namespace*")" || return
[[ -z "$keys" ]] && return
while IFS= read -r key; do
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" -- \
redis-cli DEL "$key" >/dev/null || return
done <<< "$keys"
done
}
cleanup_lane_target_store() {
local provider="$1"
local bucket="$2"
local deployment
local endpoint
if [[ "$provider" = garage ]]; then
deployment=garage
endpoint=http://127.0.0.1:3900
else
deployment=versitygw
endpoint=http://127.0.0.1:10000
fi
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$deployment" --container s3-readiness -- \
aws --endpoint-url "$endpoint" s3 rm "s3://$bucket" --recursive >&2 || return
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$deployment" --container s3-readiness -- \
aws --endpoint-url "$endpoint" s3api delete-bucket --bucket "$bucket" >&2 || return
if kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$deployment" --container s3-readiness -- \
aws --endpoint-url "$endpoint" s3api head-bucket --bucket "$bucket" >/dev/null 2>&1; then
return 1
fi
}
cleanup_keycloak_lane() {
local realm="remora-lane-$1"
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
/opt/keycloak/bin/kcadm.sh delete "realms/$realm" \
--no-config \
--server http://127.0.0.1:8080 \
--realm master \
--user remora-admin \
--password synthetic-keycloak >&2 || return
}
find_zitadel_lane_organizations() {
local lane="$1"
local tenant="remora-lane-$lane"
local response
response="$(zitadel_request /v2/organizations/_search '{}')" || return
jq --raw-output --arg tenant "$tenant" \
'.result[]? | select(.name == $tenant) | .id' <<< "$response"
}
cleanup_zitadel_lane() {
local lane="$1"
local organization_ids
local organization_id
local status=0
organization_ids="$(find_zitadel_lane_organizations "$lane")" || return
while IFS= read -r organization_id; do
[[ -z "$organization_id" ]] && continue
zitadel_request "/v2/organizations/$organization_id" '' '' DELETE >/dev/null || status=1
done <<< "$organization_ids"
return "$status"
}
cleanup_lane_secret_store() {
local path="secret/remora/lane-$1"
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/vault -- \
env VAULT_ADDR=http://127.0.0.1:8200 VAULT_TOKEN=synthetic-vault-token \
vault kv metadata delete "$path" >/dev/null || return
}
cleanup_lane_state() {
local lane="$1"
local coordination="$2"
local oidc="$3"
local target="$4"
local status=0
if fixture_has_family durable && ! cleanup_lane_durable_store "$lane"; then
printf 'durable store cleanup failed for lane %s\n' "$lane" >&2
status=1
fi
if [[ "$coordination" != - ]] && ! cleanup_lane_coordination_store "$coordination" "remora:lane:$lane:"; then
printf 'coordination store cleanup failed for lane %s\n' "$lane" >&2
status=1
fi
if [[ "$oidc" = keycloak ]] && ! cleanup_keycloak_lane "$lane"; then
printf 'Keycloak cleanup failed for lane %s\n' "$lane" >&2
status=1
fi
if [[ "$oidc" = zitadel ]] && ! cleanup_zitadel_lane "$lane"; then
printf 'ZITADEL cleanup failed for lane %s\n' "$lane" >&2
status=1
fi
if [[ "$target" != - ]] && ! cleanup_lane_target_store "$target" "remora-lane-$lane"; then
printf 'target store cleanup failed for lane %s\n' "$lane" >&2
status=1
fi
if fixture_has_family secret && ! cleanup_lane_secret_store "$lane"; then
printf 'secret store cleanup failed for lane %s\n' "$lane" >&2
status=1
fi
return "$status"
}
write_lane_profile() {
local lane="$1"
local coordination="$2"
local oidc="$3"
local target="$4"
local profile_name="remora-fixture-lane-$lane"
local database=
local database_url=
local coordination_url=
local coordination_namespace=
local oidc_issuer=
local oidc_tenant=
local oidc_client_id=
local oidc_username=
local oidc_profile
local target_endpoint=
local target_bucket=
local target_prefix=
local target_access_key=
local target_secret_key=
local target_region=
local vault_path=
local profile
if fixture_has_family durable; then
database="$(reset_lane_durable_store "$lane")" || return
database_url="postgres://remora:synthetic-postgres@postgres:5432/$database?sslmode=disable"
fi
if [[ "$coordination" != - ]]; then
coordination_url="redis://$coordination:6379"
coordination_namespace="remora:lane:$lane:"
cleanup_lane_coordination_store "$coordination" "$coordination_namespace" || return
fi
if [[ "$oidc" != - ]]; then
oidc_profile="$(reset_lane_oidc_provider "$oidc" "$lane")" || return
IFS=$'\t' read -r oidc_issuer oidc_tenant oidc_client_id oidc_username <<< "$oidc_profile"
fi
if [[ "$target" != - ]]; then
target_bucket="remora-lane-$lane"
target_prefix="lane-$lane/"
reset_lane_target_store "$target" "$target_bucket" || return
if [[ "$target" = garage ]]; then
target_endpoint=http://garage:3900
target_access_key=GK00000000000000000000000000000000
target_secret_key=0000000000000000000000000000000000000000000000000000000000000000
target_region=garage
else
target_endpoint=http://versitygw:10000
target_access_key=synthetic-s3-access
target_secret_key=synthetic-s3-secret
target_region=us-east-1
fi
fi
if fixture_has_family secret; then
vault_path="secret/remora/lane-$lane"
cleanup_lane_secret_store "$lane" || return
fi
printf -v profile '%s\n' \
"REMORA_FIXTURE_LANE=$lane" \
"REMORA_COORDINATION_PROVIDER=$coordination" \
"REMORA_COORDINATION_URL=$coordination_url" \
"REMORA_COORDINATION_NAMESPACE=$coordination_namespace" \
"REMORA_OIDC_PROVIDER=$oidc" \
"REMORA_OIDC_ISSUER=$oidc_issuer" \
"REMORA_OIDC_TENANT=$oidc_tenant" \
"REMORA_OIDC_CLIENT_ID=$oidc_client_id" \
"REMORA_OIDC_USERNAME=$oidc_username" \
"REMORA_OIDC_PASSWORD=Synthetic-OIDC-1!" \
"REMORA_TARGET_PROVIDER=$target" \
"REMORA_TARGET_ENDPOINT=$target_endpoint" \
"REMORA_TARGET_BUCKET=$target_bucket" \
"REMORA_TARGET_PREFIX=$target_prefix" \
"REMORA_TARGET_ACCESS_KEY=$target_access_key" \
"REMORA_TARGET_SECRET_KEY=$target_secret_key" \
"REMORA_TARGET_REGION=$target_region" \
"REMORA_VAULT_ADDR=http://vault:8200" \
"REMORA_VAULT_TOKEN=synthetic-vault-token" \
"REMORA_VAULT_PATH=$vault_path" \
"DATABASE_URL=$database_url"
kube --namespace "$FIXTURE_NAMESPACE" delete \
"configmap/$profile_name" \
"configmap/$profile_name-file" \
--ignore-not-found \
--wait=true >&2 || return
kube --namespace "$FIXTURE_NAMESPACE" create configmap "$profile_name" \
--from-literal="REMORA_FIXTURE_LANE=$lane" \
--from-literal="REMORA_COORDINATION_PROVIDER=$coordination" \
--from-literal="REMORA_COORDINATION_URL=$coordination_url" \
--from-literal="REMORA_COORDINATION_NAMESPACE=$coordination_namespace" \
--from-literal="REMORA_OIDC_PROVIDER=$oidc" \
--from-literal="REMORA_OIDC_ISSUER=$oidc_issuer" \
--from-literal="REMORA_OIDC_TENANT=$oidc_tenant" \
--from-literal="REMORA_OIDC_CLIENT_ID=$oidc_client_id" \
--from-literal="REMORA_OIDC_USERNAME=$oidc_username" \
--from-literal=REMORA_OIDC_PASSWORD=Synthetic-OIDC-1! \
--from-literal="REMORA_TARGET_PROVIDER=$target" \
--from-literal="REMORA_TARGET_ENDPOINT=$target_endpoint" \
--from-literal="REMORA_TARGET_BUCKET=$target_bucket" \
--from-literal="REMORA_TARGET_PREFIX=$target_prefix" \
--from-literal="REMORA_TARGET_ACCESS_KEY=$target_access_key" \
--from-literal="REMORA_TARGET_SECRET_KEY=$target_secret_key" \
--from-literal="REMORA_TARGET_REGION=$target_region" \
--from-literal=REMORA_VAULT_ADDR=http://vault:8200 \
--from-literal=REMORA_VAULT_TOKEN=synthetic-vault-token \
--from-literal="REMORA_VAULT_PATH=$vault_path" \
--from-literal="DATABASE_URL=$database_url" >&2 || return
kube --namespace "$FIXTURE_NAMESPACE" create configmap "$profile_name-file" \
--from-literal="profile.env=$profile" >&2 || return
printf '%s\n' "$profile_name"
}
render_lane_job() {
local template="$1"
local output="$2"
local job="$3"
local profile="$4"
local image="$5"
local postgres_image="${POSTGRES_LOCAL_IMAGE:-remora-testbed/unused:postgres}"
local redis_image="${REDIS_LOCAL_IMAGE:-remora-testbed/unused:redis}"
local vault_image="${VAULT_LOCAL_IMAGE:-remora-testbed/unused:vault}"
local curl_image="${CURL_LOCAL_IMAGE:-remora-testbed/unused:curl}"
local aws_cli_image="${AWS_CLI_LOCAL_IMAGE:-remora-testbed/unused:aws-cli}"
sed \
-e "s|@@JOB_NAME@@|$job|g" \
-e "s|@@PROFILE_NAME@@|$profile|g" \
-e "s|@@TEST_IMAGE@@|$image|g" \
-e "s|@@POSTGRES_IMAGE@@|$postgres_image|g" \
-e "s|@@REDIS_IMAGE@@|$redis_image|g" \
-e "s|@@VAULT_IMAGE@@|$vault_image|g" \
-e "s|@@CURL_IMAGE@@|$curl_image|g" \
-e "s|@@AWS_CLI_IMAGE@@|$aws_cli_image|g" \
"$template" > "$output"
}
run_job_matrix() {
local job_base="$1"
local template="$2"
local image="$3"
shift 3
local families=("$@")
local lane
local coordination
local oidc
local target
local job
local profile
local manifest
select_fixture_families "${families[@]}" || return
while IFS=$'\t' read -r lane coordination oidc target; do
job="$job_base-$lane"
profile="$(write_lane_profile "$lane" "$coordination" "$oidc" "$target")" || return
manifest="$RUNTIME/$job.yaml"
render_lane_job "$template" "$manifest" "$job" "$profile" "$image" || return
printf 'Running fixture lane %s: coordination=%s oidc=%s target=%s\n' "$lane" "$coordination" "$oidc" "$target"
run_job_manifest "$FIXTURE_NAMESPACE" "$job" "$manifest" || return
cleanup_lane_state "$lane" "$coordination" "$oidc" "$target" || return
kube --namespace "$FIXTURE_NAMESPACE" delete \
"job/$job" \
"configmap/$profile" \
"configmap/$profile-file" \
--wait=true || return
rm -f "$manifest" || return
done < <(expand_fixture_matrix "${families[@]}")
}
verify_dependencies() { verify_dependencies() {
local family
for family in durable coordination oidc target secret; do
if ! fixture_has_family "$family"; then
printf 'released-image dependency probes require the all fixture profile\n' >&2
return 1
fi
done
kube apply --filename "$RUNTIME/probe.yaml" kube apply --filename "$RUNTIME/probe.yaml"
for job in state-probe oidc-probe garage-probe versitygw-probe; do for job in state-probe oidc-probe garage-probe versitygw-probe; do
wait_for_job "$FIXTURE_NAMESPACE" "$job" wait_for_job "$FIXTURE_NAMESPACE" "$job"
@ -278,6 +943,7 @@ destroy_fixture() {
"$RUNTIME/probe.yaml" \ "$RUNTIME/probe.yaml" \
"$RUNTIME/sample-job.yaml" \ "$RUNTIME/sample-job.yaml" \
"$RUNTIME/durable-job.yaml" "$RUNTIME/durable-job.yaml"
find "$RUNTIME" -maxdepth 1 -type f -name '*-[0-9][0-9][0-9].yaml' -delete
} }
kube() { kube() {
@ -296,26 +962,45 @@ wait_for_job() {
while ((SECONDS < deadline)); do while ((SECONDS < deadline)); do
complete="$(kube --namespace "$namespace" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Complete")].status}')" complete="$(kube --namespace "$namespace" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Complete")].status}')"
if [[ "$complete" = True ]]; then if [[ "$complete" = True ]]; then
kube --namespace "$namespace" logs "job/$job" --all-containers print_job_logs "$namespace" "$job"
return return
fi fi
failed="$(kube --namespace "$namespace" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Failed")].status}')" failed="$(kube --namespace "$namespace" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Failed")].status}')"
if [[ "$failed" = True ]]; then if [[ "$failed" = True ]]; then
kube --namespace "$namespace" logs "job/$job" --all-containers >&2 || true print_job_logs "$namespace" "$job" >&2 || true
return 1 return 1
fi fi
sleep 1 sleep 1
done done
kube --namespace "$namespace" logs "job/$job" --all-containers >&2 || true print_job_logs "$namespace" "$job" >&2 || true
printf 'job/%s did not complete within 300 seconds\n' "$job" >&2 printf 'job/%s did not complete within 300 seconds\n' "$job" >&2
return 1 return 1
} }
print_job_logs() {
local namespace="$1"
local job="$2"
local pod
local containers
local container
local status=0
pod="$(kube --namespace "$namespace" get pods \
--selector "job-name=$job" \
--output jsonpath='{.items[0].metadata.name}')" || return
containers="$(kube --namespace "$namespace" get "pod/$pod" \
--output jsonpath='{range .spec.initContainers[*]}{.name}{"\n"}{end}{range .spec.containers[*]}{.name}{"\n"}{end}')" || return
while IFS= read -r container; do
[[ -z "$container" ]] && continue
kube --namespace "$namespace" logs "pod/$pod" --container "$container" || status=1
done <<< "$containers"
return "$status"
}
run_job_manifest() { run_job_manifest() {
local namespace="$1" local namespace="$1"
local job="$2" local job="$2"
local manifest="$3" local manifest="$3"
kube --namespace "$namespace" delete "job/$job" --ignore-not-found --wait=true kube --namespace "$namespace" delete "job/$job" --ignore-not-found --wait=true || return
kube --namespace "$namespace" apply --filename "$manifest" kube --namespace "$namespace" apply --filename "$manifest" || return
wait_for_job "$namespace" "$job" wait_for_job "$namespace" "$job" || return
} }

10
tests/kind/run-durable-job.sh
View file

@ -39,10 +39,12 @@ export REMORA_REAL_NERDCTL
--tag remora-kind-durable:local \ --tag remora-kind-durable:local \
"$ROOT" "$ROOT"
OWNS_FIXTURE=true OWNS_FIXTURE=true
setup_fixture setup_fixture durable
load_local_image remora-kind-durable:local load_local_image remora-kind-durable:local
sed "s|@@TEST_IMAGE@@|remora-kind-durable:local|g" \ run_job_matrix \
"$TESTBED/durable-job.yaml.in" > "$RUNTIME/durable-job.yaml" durable-substrate \
run_job_manifest "$FIXTURE_NAMESPACE" durable-substrate "$RUNTIME/durable-job.yaml" "$TESTBED/durable-job.yaml.in" \
remora-kind-durable:local \
durable
SUCCEEDED=true SUCCEEDED=true
printf 'durable substrate package test Job passed\n' printf 'durable substrate package test Job passed\n'

10
tests/kind/run-sample-job.sh
View file

@ -39,10 +39,12 @@ export REMORA_REAL_NERDCTL
--tag remora-kind-sample:local \ --tag remora-kind-sample:local \
"$ROOT" "$ROOT"
OWNS_FIXTURE=true OWNS_FIXTURE=true
setup_fixture setup_fixture all
load_local_image remora-kind-sample:local load_local_image remora-kind-sample:local
sed "s|@@TEST_IMAGE@@|remora-kind-sample:local|g" \ run_job_matrix \
"$TESTBED/sample-job.yaml.in" > "$RUNTIME/sample-job.yaml" fixture-consumer-sample \
run_job_manifest "$FIXTURE_NAMESPACE" fixture-consumer-sample "$RUNTIME/sample-job.yaml" "$TESTBED/sample-job.yaml.in" \
remora-kind-sample:local \
all
SUCCEEDED=true SUCCEEDED=true
printf 'sample package test Job passed\n' printf 'sample package test Job passed\n'

98
tests/kind/sample-job.yaml.in
View file

@ -1,7 +1,7 @@
apiVersion: batch/v1 apiVersion: batch/v1
kind: Job kind: Job
metadata: metadata:
name: fixture-consumer-sample name: @@JOB_NAME@@
namespace: remora-testbed namespace: remora-testbed
spec: spec:
activeDeadlineSeconds: 300 activeDeadlineSeconds: 300
@ -12,10 +12,106 @@ spec:
automountServiceAccountToken: false automountServiceAccountToken: false
enableServiceLinks: false enableServiceLinks: false
restartPolicy: Never restartPolicy: Never
dnsPolicy: None
dnsConfig:
nameservers: [10.96.0.53]
searches: [remora-testbed.svc.cluster.local, svc.cluster.local, cluster.local]
options:
- {name: ndots, value: "5"}
initContainers:
- name: durable-canary
image: @@POSTGRES_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args: [psql "$DATABASE_URL" --set ON_ERROR_STOP=1 --command "SELECT 1"]
envFrom:
- configMapRef: {name: @@PROFILE_NAME@@}
resources:
requests: {cpu: 25m, memory: 32Mi}
limits: {cpu: "1", memory: 128Mi}
- name: coordination-canary
image: @@REDIS_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args:
- |
key="${REMORA_COORDINATION_NAMESPACE}sample-canary"
redis-cli -u "$REMORA_COORDINATION_URL" SET "$key" passed >/dev/null
test "$(redis-cli -u "$REMORA_COORDINATION_URL" GET "$key")" = passed
envFrom:
- configMapRef: {name: @@PROFILE_NAME@@}
resources:
requests: {cpu: 25m, memory: 32Mi}
limits: {cpu: "1", memory: 128Mi}
- name: oidc-canary
image: @@CURL_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args:
- |
discovery="$(curl --fail --silent --show-error "$REMORA_OIDC_ISSUER/.well-known/openid-configuration")"
printf '%s' "$discovery" | grep -F '"issuer"' >/dev/null
printf '%s' "$discovery" | grep -F "$REMORA_OIDC_ISSUER" >/dev/null
case "$REMORA_OIDC_PROVIDER" in
keycloak) keys="$REMORA_OIDC_ISSUER/protocol/openid-connect/certs" ;;
zitadel) keys="$REMORA_OIDC_ISSUER/oauth/v2/keys" ;;
*) exit 1 ;;
esac
curl --fail --silent --show-error "$keys" | grep -F '"keys"' >/dev/null
envFrom:
- configMapRef: {name: @@PROFILE_NAME@@}
resources:
requests: {cpu: 25m, memory: 32Mi}
limits: {cpu: "1", memory: 128Mi}
- name: target-canary
image: @@AWS_CLI_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args:
- |
export AWS_ACCESS_KEY_ID="$REMORA_TARGET_ACCESS_KEY"
export AWS_SECRET_ACCESS_KEY="$REMORA_TARGET_SECRET_KEY"
export AWS_DEFAULT_REGION="$REMORA_TARGET_REGION"
key="${REMORA_TARGET_PREFIX}sample-canary"
printf 'passed\n' >/tmp/input
aws --endpoint-url "$REMORA_TARGET_ENDPOINT" s3api put-object --bucket "$REMORA_TARGET_BUCKET" --key "$key" --body /tmp/input >/dev/null
aws --endpoint-url "$REMORA_TARGET_ENDPOINT" s3api get-object --bucket "$REMORA_TARGET_BUCKET" --key "$key" /tmp/output >/dev/null
IFS= read -r output </tmp/output
test "$output" = passed
envFrom:
- configMapRef: {name: @@PROFILE_NAME@@}
resources:
requests: {cpu: 25m, memory: 64Mi}
limits: {cpu: "1", memory: 256Mi}
- name: secret-canary
image: @@VAULT_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args:
- |
export VAULT_ADDR="$REMORA_VAULT_ADDR"
export VAULT_TOKEN="$REMORA_VAULT_TOKEN"
vault kv put "$REMORA_VAULT_PATH" value=passed >/dev/null
test "$(vault kv get -field=value "$REMORA_VAULT_PATH")" = passed
envFrom:
- configMapRef: {name: @@PROFILE_NAME@@}
resources:
requests: {cpu: 25m, memory: 64Mi}
limits: {cpu: "1", memory: 256Mi}
containers: containers:
- name: tests - name: tests
image: @@TEST_IMAGE@@ image: @@TEST_IMAGE@@
imagePullPolicy: Never imagePullPolicy: Never
envFrom:
- configMapRef: {name: @@PROFILE_NAME@@}
volumeMounts:
- {name: fixture-profile, mountPath: /run/remora-fixture, readOnly: true}
resources: resources:
requests: {cpu: 100m, memory: 64Mi} requests: {cpu: 100m, memory: 64Mi}
limits: {cpu: "2", memory: 512Mi} limits: {cpu: "2", memory: 512Mi}
volumes:
- name: fixture-profile
configMap:
name: @@PROFILE_NAME@@-file
items:
- {key: profile.env, path: profile.env}