Add reusable kind test job execution
This commit is contained in:
parent
3ed528b288
commit
af8bb1a03a
10 changed files with 310 additions and 64 deletions
|
|
@ -21,6 +21,27 @@ COPY crates crates
|
||||||
COPY docs/api docs/api
|
COPY docs/api docs/api
|
||||||
CMD ["sh", "-c", "cargo fmt --all --check && cargo clippy --workspace --all-targets --all-features -- -D warnings && cargo nextest run --workspace --all-features && cargo test --workspace --doc"]
|
CMD ["sh", "-c", "cargo fmt --all --check && cargo clippy --workspace --all-targets --all-features -- -D warnings && cargo nextest run --workspace --all-features && cargo test --workspace --doc"]
|
||||||
|
|
||||||
|
FROM rust-tools AS kind-sample-build
|
||||||
|
WORKDIR /workspace
|
||||||
|
COPY Cargo.toml Cargo.lock rust-toolchain.toml ./
|
||||||
|
COPY .config .config
|
||||||
|
COPY contracts contracts
|
||||||
|
COPY crates crates
|
||||||
|
COPY docs/api docs/api
|
||||||
|
RUN cargo nextest archive \
|
||||||
|
--package remora-api \
|
||||||
|
--lib \
|
||||||
|
--build-jobs 4 \
|
||||||
|
--locked \
|
||||||
|
--archive-file /tmp/remora-api-tests.tar.zst
|
||||||
|
|
||||||
|
FROM rust-tools AS kind-sample-runner
|
||||||
|
WORKDIR /workspace
|
||||||
|
COPY Cargo.toml ./
|
||||||
|
COPY crates crates
|
||||||
|
COPY --from=kind-sample-build /tmp/remora-api-tests.tar.zst /tests/remora-api-tests.tar.zst
|
||||||
|
CMD ["bash", "-ec", "cargo-nextest nextest run --archive-file /tests/remora-api-tests.tar.zst --workspace-remap /workspace && exec 3<>/dev/tcp/postgres/5432"]
|
||||||
|
|
||||||
FROM ${NODE_IMAGE} AS node-runtime
|
FROM ${NODE_IMAGE} AS node-runtime
|
||||||
|
|
||||||
FROM ${PLAYWRIGHT_IMAGE} AS web-dependencies
|
FROM ${PLAYWRIGHT_IMAGE} AS web-dependencies
|
||||||
|
|
|
||||||
5
Makefile
5
Makefile
|
|
@ -1,4 +1,4 @@
|
||||||
.PHONY: check-dependencies check-fast check-full check-fast-host check-full-host rust-check web-install web-check web-test-fast web-test-full
|
.PHONY: check-dependencies check-kind-sample check-fast check-full check-fast-host check-full-host rust-check web-install web-check web-test-fast web-test-full
|
||||||
|
|
||||||
CONTAINER_ENGINE ?= nerdctl
|
CONTAINER_ENGINE ?= nerdctl
|
||||||
CONTAINER_RUN_FLAGS ?= --memory=16g --memory-swap=24g --cpus=4
|
CONTAINER_RUN_FLAGS ?= --memory=16g --memory-swap=24g --cpus=4
|
||||||
|
|
@ -49,3 +49,6 @@ check-full-host: rust-check web-check web-test-full
|
||||||
|
|
||||||
check-dependencies:
|
check-dependencies:
|
||||||
./tests/kind/run.sh
|
./tests/kind/run.sh
|
||||||
|
|
||||||
|
check-kind-sample:
|
||||||
|
./tests/kind/run-sample-job.sh
|
||||||
|
|
|
||||||
33
docs/spec/testing.md
Normal file
33
docs/spec/testing.md
Normal file
|
|
@ -0,0 +1,33 @@
|
||||||
|
# Implementation Testing
|
||||||
|
|
||||||
|
Backend integration and fault tests execute as Kubernetes `Job` resources inside the kind dependency fixture. Cluster-local execution gives test code the same DNS, service routing, and failure boundaries as the deployed services without exposing fixture dependencies on the host.
|
||||||
|
|
||||||
|
## Fixture lifecycle
|
||||||
|
|
||||||
|
`make check-dependencies` remains the dependency compatibility gate: it creates a fresh fixture, runs the released-image probes, and tears the fixture down. `make check-kind-sample` builds and loads a Remora-owned cargo-nextest runner, then runs the same lifecycle around [`fixture-consumer-sample`](../../tests/kind/sample-job.yaml.in). That Job is the reference structure for a work-package test Job.
|
||||||
|
|
||||||
|
For interactive work, `tests/kind/setup.sh` creates a ready fixture and leaves it running. `tests/kind/destroy.sh` removes it. Setup replaces any fixture left by an earlier run, teardown is idempotent, and the harness serializes lifecycle mutations around the single `remora-testbed` cluster. `REMORA_TESTBED_RETAIN=1` retains a failed all-in-one run for inspection.
|
||||||
|
|
||||||
|
The fixture owns an isolated kubeconfig at `tests/kind/.runtime/kubeconfig` and the `kind-remora-testbed` context. Every Kubernetes operation uses both explicitly. Package runners use the `kube` helper from `tests/kind/lib.sh`; they do not read or change the user's kubeconfig or current context.
|
||||||
|
|
||||||
|
## Package test Jobs
|
||||||
|
|
||||||
|
Each work package owns its test code, test-runner image, Job manifest, and acceptance command. Its all-in-one runner follows [`run-sample-job.sh`](../../tests/kind/run-sample-job.sh): acquire the fixture lock, install a cleanup trap, call `setup_fixture`, load any package-owned image with `load_local_image <image> [containerd-namespace]`, apply the Job with `run_job_manifest <namespace> <job-name> <manifest>`, and propagate the Job result before teardown. The image namespace defaults to `default`. A retained failure prints the isolated kubeconfig and context.
|
||||||
|
|
||||||
|
Released third-party dependencies are always the digest-pinned images loaded by the fixture. Work packages may build Remora test-runner images locally; they do not rebuild or wrap dependency images. The Job manifest names every loaded image exactly and sets `imagePullPolicy: Never`.
|
||||||
|
|
||||||
|
A test Job:
|
||||||
|
|
||||||
|
- runs in `remora-testbed` with `restartPolicy: Never`, `backoffLimit: 0`, and a finite `activeDeadlineSeconds`;
|
||||||
|
- disables service-account token mounting and Kubernetes service-link environment variables unless the test explicitly requires them;
|
||||||
|
- declares CPU and memory requests and limits;
|
||||||
|
- exits nonzero for every failed assertion and writes diagnostic output to its container logs; and
|
||||||
|
- uses ClusterIP service names for dependencies and synthetic fixture credentials only.
|
||||||
|
|
||||||
|
Package runners may create additional namespaced fixtures and fault injectors needed by their acceptance matrix. They clean those resources or rely on whole-cluster teardown; they never depend on state left by another package.
|
||||||
|
|
||||||
|
## Network access
|
||||||
|
|
||||||
|
Automated backend tests use cluster-local service discovery. Tests of Keycloak or ZITADEL issuer behavior use the `dnsPolicy` and `dnsConfig` pattern in [`oidc-probe`](../../tests/kind/probe.yaml.in) so discovery URLs and token issuers have one identity from every participating pod.
|
||||||
|
|
||||||
|
Port forwarding is for interactive diagnostics only and is not an acceptance-test transport. Ingress is introduced by deployment and system qualification, where routing is itself under test; substrate work packages do not install an ingress controller merely to reach fixture services.
|
||||||
|
|
@ -4,9 +4,6 @@ set -euo pipefail
|
||||||
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||||
source "$ROOT/tests/kind/lib.sh"
|
source "$ROOT/tests/kind/lib.sh"
|
||||||
|
|
||||||
|
acquire_fixture_lock
|
||||||
prepare_tools
|
prepare_tools
|
||||||
configure_provider
|
destroy_fixture
|
||||||
delete_cluster
|
|
||||||
remove_runtime_namespace
|
|
||||||
rm -rf "$RUNTIME/image-archives"
|
|
||||||
rm -f "$KUBECONFIG_PATH" "$RUNTIME/dependencies.yaml" "$RUNTIME/probe.yaml"
|
|
||||||
|
|
|
||||||
|
|
@ -9,6 +9,7 @@ KUBECONFIG_PATH="$RUNTIME/kubeconfig"
|
||||||
CONTEXT=kind-remora-testbed
|
CONTEXT=kind-remora-testbed
|
||||||
CLUSTER=remora-testbed
|
CLUSTER=remora-testbed
|
||||||
NERDCTL_NAMESPACE=remora-wp02
|
NERDCTL_NAMESPACE=remora-wp02
|
||||||
|
FIXTURE_NAMESPACE=remora-testbed
|
||||||
|
|
||||||
set -a
|
set -a
|
||||||
source "$TESTBED/versions.env"
|
source "$TESTBED/versions.env"
|
||||||
|
|
@ -55,7 +56,9 @@ prepare_tools() {
|
||||||
"https://dl.k8s.io/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl"
|
"https://dl.k8s.io/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl"
|
||||||
test "$("$BIN/kind" version | awk '{print $2}')" = "$KIND_VERSION"
|
test "$("$BIN/kind" version | awk '{print $2}')" = "$KIND_VERSION"
|
||||||
test "$("$BIN/kubectl" --kubeconfig "$KUBECONFIG_PATH" --context "$CONTEXT" version --client=true -o json | jq -r .clientVersion.gitVersion)" = "$KUBECTL_VERSION"
|
test "$("$BIN/kubectl" --kubeconfig "$KUBECONFIG_PATH" --context "$CONTEXT" version --client=true -o json | jq -r .clientVersion.gitVersion)" = "$KUBECTL_VERSION"
|
||||||
test "$(nerdctl --version | awk '{print $3}')" = 2.3.4
|
test "$(nerdctl --version | awk '{print $3}')" = "$NERDCTL_VERSION"
|
||||||
|
test "$(runtime_ctr version | awk '/Client:/{client=1} client && $1=="Version:"{print $2; exit}')" = "v$CONTAINERD_VERSION"
|
||||||
|
test "$(runtime_ctr version | awk '/Server:/{server=1} server && $1=="Version:"{print $2; exit}')" = "v$CONTAINERD_VERSION"
|
||||||
}
|
}
|
||||||
|
|
||||||
prepare_provider() {
|
prepare_provider() {
|
||||||
|
|
@ -65,8 +68,19 @@ prepare_provider() {
|
||||||
"$REMORA_REAL_NERDCTL" namespace create "$NERDCTL_NAMESPACE" >/dev/null
|
"$REMORA_REAL_NERDCTL" namespace create "$NERDCTL_NAMESPACE" >/dev/null
|
||||||
}
|
}
|
||||||
|
|
||||||
|
acquire_fixture_lock() {
|
||||||
|
mkdir -p "$RUNTIME"
|
||||||
|
exec 9>"$RUNTIME/fixture.lock"
|
||||||
|
if ! flock --nonblock 9; then
|
||||||
|
printf 'kind test fixture is already in use\n' >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
configure_provider() {
|
configure_provider() {
|
||||||
|
if [[ -z "${REMORA_REAL_NERDCTL:-}" ]]; then
|
||||||
REMORA_REAL_NERDCTL="$(command -v nerdctl)"
|
REMORA_REAL_NERDCTL="$(command -v nerdctl)"
|
||||||
|
fi
|
||||||
export REMORA_REAL_NERDCTL
|
export REMORA_REAL_NERDCTL
|
||||||
export CONTAINERD_NAMESPACE="$NERDCTL_NAMESPACE"
|
export CONTAINERD_NAMESPACE="$NERDCTL_NAMESPACE"
|
||||||
export KIND_EXPERIMENTAL_PROVIDER=nerdctl
|
export KIND_EXPERIMENTAL_PROVIDER=nerdctl
|
||||||
|
|
@ -75,10 +89,26 @@ configure_provider() {
|
||||||
|
|
||||||
remove_runtime_namespace() {
|
remove_runtime_namespace() {
|
||||||
local status=0
|
local status=0
|
||||||
|
local digests
|
||||||
|
local digest
|
||||||
|
local leases
|
||||||
|
local lease
|
||||||
if ! runtime_namespace_exists; then
|
if ! runtime_namespace_exists; then
|
||||||
return
|
return
|
||||||
fi
|
fi
|
||||||
"$REMORA_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" image prune --all --force >/dev/null || status=1
|
"$REMORA_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" image prune --all --force >/dev/null || status=1
|
||||||
|
leases="$(runtime_ctr --namespace "$NERDCTL_NAMESPACE" leases list --quiet)" || status=1
|
||||||
|
if [[ -n "$leases" ]]; then
|
||||||
|
while read -r lease; do
|
||||||
|
runtime_ctr --namespace "$NERDCTL_NAMESPACE" leases delete "$lease" >/dev/null || status=1
|
||||||
|
done <<< "$leases"
|
||||||
|
fi
|
||||||
|
digests="$(runtime_ctr --namespace "$NERDCTL_NAMESPACE" content list --quiet)" || status=1
|
||||||
|
if [[ -n "$digests" ]]; then
|
||||||
|
while read -r digest; do
|
||||||
|
runtime_ctr --namespace "$NERDCTL_NAMESPACE" content remove "$digest" >/dev/null || status=1
|
||||||
|
done <<< "$digests"
|
||||||
|
fi
|
||||||
"$REMORA_REAL_NERDCTL" namespace remove "$NERDCTL_NAMESPACE" >/dev/null || status=1
|
"$REMORA_REAL_NERDCTL" namespace remove "$NERDCTL_NAMESPACE" >/dev/null || status=1
|
||||||
if runtime_namespace_exists; then
|
if runtime_namespace_exists; then
|
||||||
printf 'containerd namespace %s still exists after teardown\n' "$NERDCTL_NAMESPACE" >&2
|
printf 'containerd namespace %s still exists after teardown\n' "$NERDCTL_NAMESPACE" >&2
|
||||||
|
|
@ -87,6 +117,10 @@ remove_runtime_namespace() {
|
||||||
return "$status"
|
return "$status"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
runtime_ctr() {
|
||||||
|
ctr --address "${CONTAINERD_ADDRESS:-${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/containerd/containerd.sock}" "$@"
|
||||||
|
}
|
||||||
|
|
||||||
runtime_namespace_exists() {
|
runtime_namespace_exists() {
|
||||||
"$REMORA_REAL_NERDCTL" namespace list --quiet | grep -Fx "$NERDCTL_NAMESPACE" >/dev/null
|
"$REMORA_REAL_NERDCTL" namespace list --quiet | grep -Fx "$NERDCTL_NAMESPACE" >/dev/null
|
||||||
}
|
}
|
||||||
|
|
@ -122,6 +156,18 @@ load_external_image() {
|
||||||
printf '%s\n' "$image"
|
printf '%s\n' "$image"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
load_local_image() {
|
||||||
|
local image="$1"
|
||||||
|
local namespace="${2:-default}"
|
||||||
|
local archive="$RUNTIME/image-archives/package-test.tar"
|
||||||
|
rm -f "$archive"
|
||||||
|
"$REMORA_REAL_NERDCTL" --namespace "$namespace" image save \
|
||||||
|
--platform linux/amd64 \
|
||||||
|
--output "$archive" \
|
||||||
|
"$image"
|
||||||
|
"$BIN/kind" load image-archive "$archive" --name "$CLUSTER"
|
||||||
|
}
|
||||||
|
|
||||||
render_manifests() {
|
render_manifests() {
|
||||||
local postgres_image="$1"
|
local postgres_image="$1"
|
||||||
local redis_image="$2"
|
local redis_image="$2"
|
||||||
|
|
@ -154,6 +200,83 @@ render_manifests() {
|
||||||
-e "s|@@CURL_IMAGE@@|$curl_image|g" \
|
-e "s|@@CURL_IMAGE@@|$curl_image|g" \
|
||||||
-e "s|@@AWS_CLI_IMAGE@@|$aws_cli_image|g" \
|
-e "s|@@AWS_CLI_IMAGE@@|$aws_cli_image|g" \
|
||||||
"$TESTBED/probe.yaml.in" > "$RUNTIME/probe.yaml"
|
"$TESTBED/probe.yaml.in" > "$RUNTIME/probe.yaml"
|
||||||
|
{
|
||||||
|
printf 'POSTGRES_LOCAL_IMAGE=%q\n' "$postgres_image"
|
||||||
|
printf 'REDIS_LOCAL_IMAGE=%q\n' "$redis_image"
|
||||||
|
printf 'DRAGONFLY_LOCAL_IMAGE=%q\n' "$dragonfly_image"
|
||||||
|
printf 'VAULT_LOCAL_IMAGE=%q\n' "$vault_image"
|
||||||
|
printf 'KEYCLOAK_LOCAL_IMAGE=%q\n' "$keycloak_image"
|
||||||
|
printf 'ZITADEL_LOCAL_IMAGE=%q\n' "$zitadel_image"
|
||||||
|
printf 'GARAGE_LOCAL_IMAGE=%q\n' "$garage_image"
|
||||||
|
printf 'VERSITYGW_LOCAL_IMAGE=%q\n' "$versitygw_image"
|
||||||
|
printf 'COREDNS_LOCAL_IMAGE=%q\n' "$coredns_image"
|
||||||
|
printf 'CURL_LOCAL_IMAGE=%q\n' "$curl_image"
|
||||||
|
printf 'AWS_CLI_LOCAL_IMAGE=%q\n' "$aws_cli_image"
|
||||||
|
} > "$RUNTIME/fixture.env"
|
||||||
|
}
|
||||||
|
|
||||||
|
setup_fixture() {
|
||||||
|
mkdir -p "$RUNTIME"
|
||||||
|
rm -rf "$RUNTIME/image-archives"
|
||||||
|
mkdir -p "$RUNTIME/image-archives"
|
||||||
|
prepare_tools
|
||||||
|
prepare_provider
|
||||||
|
|
||||||
|
"$BIN/kind" create cluster \
|
||||||
|
--name "$CLUSTER" \
|
||||||
|
--image "$NODE_IMAGE" \
|
||||||
|
--config "$TESTBED/cluster.yaml" \
|
||||||
|
--kubeconfig "$KUBECONFIG_PATH" \
|
||||||
|
--wait 180s
|
||||||
|
|
||||||
|
POSTGRES_LOCAL_IMAGE="$(load_external_image postgres "$POSTGRES_IMAGE")"
|
||||||
|
REDIS_LOCAL_IMAGE="$(load_external_image redis "$REDIS_IMAGE")"
|
||||||
|
DRAGONFLY_LOCAL_IMAGE="$(load_external_image dragonfly "$DRAGONFLY_IMAGE")"
|
||||||
|
VAULT_LOCAL_IMAGE="$(load_external_image vault "$VAULT_IMAGE")"
|
||||||
|
KEYCLOAK_LOCAL_IMAGE="$(load_external_image keycloak "$KEYCLOAK_IMAGE")"
|
||||||
|
ZITADEL_LOCAL_IMAGE="$(load_external_image zitadel "$ZITADEL_IMAGE")"
|
||||||
|
GARAGE_LOCAL_IMAGE="$(load_external_image garage "$GARAGE_IMAGE")"
|
||||||
|
VERSITYGW_LOCAL_IMAGE="$(load_external_image versitygw "$VERSITYGW_IMAGE")"
|
||||||
|
COREDNS_LOCAL_IMAGE="$(load_external_image coredns "$COREDNS_IMAGE")"
|
||||||
|
CURL_LOCAL_IMAGE="$(load_external_image curl "$CURL_IMAGE")"
|
||||||
|
AWS_CLI_LOCAL_IMAGE="$(load_external_image aws-cli "$AWS_CLI_IMAGE")"
|
||||||
|
render_manifests \
|
||||||
|
"$POSTGRES_LOCAL_IMAGE" \
|
||||||
|
"$REDIS_LOCAL_IMAGE" \
|
||||||
|
"$DRAGONFLY_LOCAL_IMAGE" \
|
||||||
|
"$VAULT_LOCAL_IMAGE" \
|
||||||
|
"$KEYCLOAK_LOCAL_IMAGE" \
|
||||||
|
"$ZITADEL_LOCAL_IMAGE" \
|
||||||
|
"$GARAGE_LOCAL_IMAGE" \
|
||||||
|
"$VERSITYGW_LOCAL_IMAGE" \
|
||||||
|
"$COREDNS_LOCAL_IMAGE" \
|
||||||
|
"$CURL_LOCAL_IMAGE" \
|
||||||
|
"$AWS_CLI_LOCAL_IMAGE"
|
||||||
|
|
||||||
|
kube apply --filename "$RUNTIME/dependencies.yaml"
|
||||||
|
for deployment in postgres redis dragonfly vault keycloak zitadel garage versitygw controlled-dns; do
|
||||||
|
kube --namespace "$FIXTURE_NAMESPACE" rollout status "deployment/$deployment" --timeout 240s
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
verify_dependencies() {
|
||||||
|
kube apply --filename "$RUNTIME/probe.yaml"
|
||||||
|
for job in state-probe oidc-probe garage-probe versitygw-probe; do
|
||||||
|
wait_for_job "$FIXTURE_NAMESPACE" "$job"
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
destroy_fixture() {
|
||||||
|
configure_provider
|
||||||
|
delete_cluster
|
||||||
|
remove_runtime_namespace
|
||||||
|
rm -rf "$RUNTIME/image-archives"
|
||||||
|
rm -f \
|
||||||
|
"$KUBECONFIG_PATH" \
|
||||||
|
"$RUNTIME/dependencies.yaml" \
|
||||||
|
"$RUNTIME/fixture.env" \
|
||||||
|
"$RUNTIME/probe.yaml" \
|
||||||
|
"$RUNTIME/sample-job.yaml"
|
||||||
}
|
}
|
||||||
|
|
||||||
kube() {
|
kube() {
|
||||||
|
|
@ -164,23 +287,34 @@ kube() {
|
||||||
}
|
}
|
||||||
|
|
||||||
wait_for_job() {
|
wait_for_job() {
|
||||||
local job="$1"
|
local namespace="$1"
|
||||||
|
local job="$2"
|
||||||
local deadline=$((SECONDS + 300))
|
local deadline=$((SECONDS + 300))
|
||||||
local complete
|
local complete
|
||||||
local failed
|
local failed
|
||||||
while ((SECONDS < deadline)); do
|
while ((SECONDS < deadline)); do
|
||||||
complete="$(kube --namespace remora-testbed get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Complete")].status}')"
|
complete="$(kube --namespace "$namespace" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Complete")].status}')"
|
||||||
if [[ "$complete" = True ]]; then
|
if [[ "$complete" = True ]]; then
|
||||||
kube --namespace remora-testbed logs "job/$job" --all-containers
|
kube --namespace "$namespace" logs "job/$job" --all-containers
|
||||||
return
|
return
|
||||||
fi
|
fi
|
||||||
failed="$(kube --namespace remora-testbed get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Failed")].status}')"
|
failed="$(kube --namespace "$namespace" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Failed")].status}')"
|
||||||
if [[ "$failed" = True ]]; then
|
if [[ "$failed" = True ]]; then
|
||||||
kube --namespace remora-testbed logs "job/$job" --all-containers >&2 || true
|
kube --namespace "$namespace" logs "job/$job" --all-containers >&2 || true
|
||||||
return 1
|
return 1
|
||||||
fi
|
fi
|
||||||
sleep 1
|
sleep 1
|
||||||
done
|
done
|
||||||
|
kube --namespace "$namespace" logs "job/$job" --all-containers >&2 || true
|
||||||
printf 'job/%s did not complete within 300 seconds\n' "$job" >&2
|
printf 'job/%s did not complete within 300 seconds\n' "$job" >&2
|
||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
run_job_manifest() {
|
||||||
|
local namespace="$1"
|
||||||
|
local job="$2"
|
||||||
|
local manifest="$3"
|
||||||
|
kube --namespace "$namespace" delete "job/$job" --ignore-not-found --wait=true
|
||||||
|
kube --namespace "$namespace" apply --filename "$manifest"
|
||||||
|
wait_for_job "$namespace" "$job"
|
||||||
|
}
|
||||||
|
|
|
||||||
48
tests/kind/run-sample-job.sh
Executable file
48
tests/kind/run-sample-job.sh
Executable file
|
|
@ -0,0 +1,48 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||||
|
source "$ROOT/tests/kind/lib.sh"
|
||||||
|
|
||||||
|
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
|
||||||
|
SUCCEEDED=false
|
||||||
|
OWNS_FIXTURE=false
|
||||||
|
|
||||||
|
cleanup() {
|
||||||
|
local status=$?
|
||||||
|
trap - EXIT INT TERM
|
||||||
|
if [[ "${REMORA_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
|
||||||
|
printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
|
||||||
|
printf 'retained context: %s\n' "$CONTEXT"
|
||||||
|
exit "$status"
|
||||||
|
fi
|
||||||
|
if [[ "$OWNS_FIXTURE" = true ]]; then
|
||||||
|
destroy_fixture || status=1
|
||||||
|
fi
|
||||||
|
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
|
||||||
|
printf 'user kubeconfig changed unexpectedly\n' >&2
|
||||||
|
status=1
|
||||||
|
fi
|
||||||
|
exit "$status"
|
||||||
|
}
|
||||||
|
|
||||||
|
trap cleanup EXIT
|
||||||
|
trap 'exit 130' INT
|
||||||
|
trap 'exit 143' TERM
|
||||||
|
|
||||||
|
acquire_fixture_lock
|
||||||
|
REMORA_REAL_NERDCTL="$(command -v nerdctl)"
|
||||||
|
export REMORA_REAL_NERDCTL
|
||||||
|
"$REMORA_REAL_NERDCTL" --namespace default build \
|
||||||
|
--file "$ROOT/Dockerfile.test" \
|
||||||
|
--target kind-sample-runner \
|
||||||
|
--tag remora-kind-sample:local \
|
||||||
|
"$ROOT"
|
||||||
|
OWNS_FIXTURE=true
|
||||||
|
setup_fixture
|
||||||
|
load_local_image remora-kind-sample:local
|
||||||
|
sed "s|@@TEST_IMAGE@@|remora-kind-sample:local|g" \
|
||||||
|
"$TESTBED/sample-job.yaml.in" > "$RUNTIME/sample-job.yaml"
|
||||||
|
run_job_manifest "$FIXTURE_NAMESPACE" fixture-consumer-sample "$RUNTIME/sample-job.yaml"
|
||||||
|
SUCCEEDED=true
|
||||||
|
printf 'sample package test Job passed\n'
|
||||||
|
|
@ -4,9 +4,6 @@ set -euo pipefail
|
||||||
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||||
source "$ROOT/tests/kind/lib.sh"
|
source "$ROOT/tests/kind/lib.sh"
|
||||||
|
|
||||||
mkdir -p "$RUNTIME"
|
|
||||||
rm -rf "$RUNTIME/image-archives"
|
|
||||||
mkdir -p "$RUNTIME/image-archives"
|
|
||||||
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
|
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
|
||||||
SUCCEEDED=false
|
SUCCEEDED=false
|
||||||
|
|
||||||
|
|
@ -19,11 +16,8 @@ cleanup() {
|
||||||
exit "$status"
|
exit "$status"
|
||||||
fi
|
fi
|
||||||
if [[ -n "${REMORA_REAL_NERDCTL:-}" ]]; then
|
if [[ -n "${REMORA_REAL_NERDCTL:-}" ]]; then
|
||||||
delete_cluster || status=1
|
destroy_fixture || status=1
|
||||||
remove_runtime_namespace || status=1
|
|
||||||
fi
|
fi
|
||||||
rm -rf "$RUNTIME/image-archives" || status=1
|
|
||||||
rm -f "$KUBECONFIG_PATH" "$RUNTIME/dependencies.yaml" "$RUNTIME/probe.yaml" || status=1
|
|
||||||
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
|
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
|
||||||
printf 'user kubeconfig changed unexpectedly\n' >&2
|
printf 'user kubeconfig changed unexpectedly\n' >&2
|
||||||
status=1
|
status=1
|
||||||
|
|
@ -34,50 +28,12 @@ cleanup() {
|
||||||
exit "$status"
|
exit "$status"
|
||||||
}
|
}
|
||||||
|
|
||||||
trap cleanup EXIT INT TERM
|
trap cleanup EXIT
|
||||||
|
trap 'exit 130' INT
|
||||||
|
trap 'exit 143' TERM
|
||||||
|
|
||||||
prepare_tools
|
acquire_fixture_lock
|
||||||
prepare_provider
|
setup_fixture
|
||||||
|
verify_dependencies
|
||||||
"$BIN/kind" create cluster \
|
|
||||||
--name "$CLUSTER" \
|
|
||||||
--image "$NODE_IMAGE" \
|
|
||||||
--config "$TESTBED/cluster.yaml" \
|
|
||||||
--kubeconfig "$KUBECONFIG_PATH" \
|
|
||||||
--wait 180s
|
|
||||||
|
|
||||||
POSTGRES_LOCAL_IMAGE="$(load_external_image postgres "$POSTGRES_IMAGE")"
|
|
||||||
REDIS_LOCAL_IMAGE="$(load_external_image redis "$REDIS_IMAGE")"
|
|
||||||
DRAGONFLY_LOCAL_IMAGE="$(load_external_image dragonfly "$DRAGONFLY_IMAGE")"
|
|
||||||
VAULT_LOCAL_IMAGE="$(load_external_image vault "$VAULT_IMAGE")"
|
|
||||||
KEYCLOAK_LOCAL_IMAGE="$(load_external_image keycloak "$KEYCLOAK_IMAGE")"
|
|
||||||
ZITADEL_LOCAL_IMAGE="$(load_external_image zitadel "$ZITADEL_IMAGE")"
|
|
||||||
GARAGE_LOCAL_IMAGE="$(load_external_image garage "$GARAGE_IMAGE")"
|
|
||||||
VERSITYGW_LOCAL_IMAGE="$(load_external_image versitygw "$VERSITYGW_IMAGE")"
|
|
||||||
COREDNS_LOCAL_IMAGE="$(load_external_image coredns "$COREDNS_IMAGE")"
|
|
||||||
CURL_LOCAL_IMAGE="$(load_external_image curl "$CURL_IMAGE")"
|
|
||||||
AWS_CLI_LOCAL_IMAGE="$(load_external_image aws-cli "$AWS_CLI_IMAGE")"
|
|
||||||
render_manifests \
|
|
||||||
"$POSTGRES_LOCAL_IMAGE" \
|
|
||||||
"$REDIS_LOCAL_IMAGE" \
|
|
||||||
"$DRAGONFLY_LOCAL_IMAGE" \
|
|
||||||
"$VAULT_LOCAL_IMAGE" \
|
|
||||||
"$KEYCLOAK_LOCAL_IMAGE" \
|
|
||||||
"$ZITADEL_LOCAL_IMAGE" \
|
|
||||||
"$GARAGE_LOCAL_IMAGE" \
|
|
||||||
"$VERSITYGW_LOCAL_IMAGE" \
|
|
||||||
"$COREDNS_LOCAL_IMAGE" \
|
|
||||||
"$CURL_LOCAL_IMAGE" \
|
|
||||||
"$AWS_CLI_LOCAL_IMAGE"
|
|
||||||
|
|
||||||
kube apply --filename "$RUNTIME/dependencies.yaml"
|
|
||||||
for deployment in postgres redis dragonfly vault keycloak zitadel garage versitygw controlled-dns; do
|
|
||||||
kube --namespace remora-testbed rollout status "deployment/$deployment" --timeout 240s
|
|
||||||
done
|
|
||||||
|
|
||||||
kube apply --filename "$RUNTIME/probe.yaml"
|
|
||||||
for job in state-probe oidc-probe garage-probe versitygw-probe; do
|
|
||||||
wait_for_job "$job"
|
|
||||||
done
|
|
||||||
|
|
||||||
SUCCEEDED=true
|
SUCCEEDED=true
|
||||||
|
|
|
||||||
21
tests/kind/sample-job.yaml.in
Normal file
21
tests/kind/sample-job.yaml.in
Normal file
|
|
@ -0,0 +1,21 @@
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: Job
|
||||||
|
metadata:
|
||||||
|
name: fixture-consumer-sample
|
||||||
|
namespace: remora-testbed
|
||||||
|
spec:
|
||||||
|
activeDeadlineSeconds: 300
|
||||||
|
backoffLimit: 0
|
||||||
|
ttlSecondsAfterFinished: 300
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
automountServiceAccountToken: false
|
||||||
|
enableServiceLinks: false
|
||||||
|
restartPolicy: Never
|
||||||
|
containers:
|
||||||
|
- name: tests
|
||||||
|
image: @@TEST_IMAGE@@
|
||||||
|
imagePullPolicy: Never
|
||||||
|
resources:
|
||||||
|
requests: {cpu: 100m, memory: 64Mi}
|
||||||
|
limits: {cpu: "2", memory: 512Mi}
|
||||||
31
tests/kind/setup.sh
Executable file
31
tests/kind/setup.sh
Executable file
|
|
@ -0,0 +1,31 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||||
|
source "$ROOT/tests/kind/lib.sh"
|
||||||
|
|
||||||
|
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
|
||||||
|
SUCCEEDED=false
|
||||||
|
|
||||||
|
cleanup() {
|
||||||
|
local status=$?
|
||||||
|
trap - EXIT INT TERM
|
||||||
|
if [[ "$SUCCEEDED" != true ]] && [[ "${REMORA_TESTBED_RETAIN:-0}" != 1 ]]; then
|
||||||
|
destroy_fixture || status=1
|
||||||
|
fi
|
||||||
|
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
|
||||||
|
printf 'user kubeconfig changed unexpectedly\n' >&2
|
||||||
|
status=1
|
||||||
|
fi
|
||||||
|
exit "$status"
|
||||||
|
}
|
||||||
|
|
||||||
|
trap cleanup EXIT
|
||||||
|
trap 'exit 130' INT
|
||||||
|
trap 'exit 143' TERM
|
||||||
|
|
||||||
|
acquire_fixture_lock
|
||||||
|
setup_fixture
|
||||||
|
SUCCEEDED=true
|
||||||
|
printf 'fixture kubeconfig: %s\n' "$KUBECONFIG_PATH"
|
||||||
|
printf 'fixture context: %s\n' "$CONTEXT"
|
||||||
|
|
@ -2,6 +2,8 @@ KIND_VERSION=v0.32.0
|
||||||
KIND_SHA256=50030de23cf40a18505f20426f6a8506bedf13c6e509244bd1fa9463721b0f54
|
KIND_SHA256=50030de23cf40a18505f20426f6a8506bedf13c6e509244bd1fa9463721b0f54
|
||||||
KUBECTL_VERSION=v1.36.2
|
KUBECTL_VERSION=v1.36.2
|
||||||
KUBECTL_SHA256=1e9045ec32bea85da43de85f0065358529ea7c7a152eca78154fba5b58c27d82
|
KUBECTL_SHA256=1e9045ec32bea85da43de85f0065358529ea7c7a152eca78154fba5b58c27d82
|
||||||
|
CONTAINERD_VERSION=2.3.3
|
||||||
|
NERDCTL_VERSION=2.3.4
|
||||||
NODE_IMAGE=kindest/node:v1.36.1@sha256:3489c7674813ba5d8b1a9977baea8a6e553784dab7b84759d1014dbd78f7ebd5
|
NODE_IMAGE=kindest/node:v1.36.1@sha256:3489c7674813ba5d8b1a9977baea8a6e553784dab7b84759d1014dbd78f7ebd5
|
||||||
POSTGRES_IMAGE=postgres@sha256:bb9940981eeb86356ec91bda430f8e0ba8729cc0b319615a2c2d7ac932fdf7bb
|
POSTGRES_IMAGE=postgres@sha256:bb9940981eeb86356ec91bda430f8e0ba8729cc0b319615a2c2d7ac932fdf7bb
|
||||||
REDIS_IMAGE=redis@sha256:cd5f3ac681c77791c6a8eaa62de876ad2be043ee5a428afb7c0095aa08246277
|
REDIS_IMAGE=redis@sha256:cd5f3ac681c77791c6a8eaa62de876ad2be043ee5a428afb7c0095aa08246277
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue