Add reusable kind test job execution

This commit is contained in:
ms2k 2026-07-16 02:36:58 +09:00
commit af8bb1a03a
7 changed files with 310 additions and 64 deletions

View file

@ -4,9 +4,6 @@ set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
source "$ROOT/tests/kind/lib.sh"
acquire_fixture_lock
prepare_tools
configure_provider
delete_cluster
remove_runtime_namespace
rm -rf "$RUNTIME/image-archives"
rm -f "$KUBECONFIG_PATH" "$RUNTIME/dependencies.yaml" "$RUNTIME/probe.yaml"
destroy_fixture

View file

@ -9,6 +9,7 @@ KUBECONFIG_PATH="$RUNTIME/kubeconfig"
CONTEXT=kind-remora-testbed
CLUSTER=remora-testbed
NERDCTL_NAMESPACE=remora-wp02
FIXTURE_NAMESPACE=remora-testbed
set -a
source "$TESTBED/versions.env"
@ -55,7 +56,9 @@ prepare_tools() {
"https://dl.k8s.io/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl"
test "$("$BIN/kind" version | awk '{print $2}')" = "$KIND_VERSION"
test "$("$BIN/kubectl" --kubeconfig "$KUBECONFIG_PATH" --context "$CONTEXT" version --client=true -o json | jq -r .clientVersion.gitVersion)" = "$KUBECTL_VERSION"
test "$(nerdctl --version | awk '{print $3}')" = 2.3.4
test "$(nerdctl --version | awk '{print $3}')" = "$NERDCTL_VERSION"
test "$(runtime_ctr version | awk '/Client:/{client=1} client && $1=="Version:"{print $2; exit}')" = "v$CONTAINERD_VERSION"
test "$(runtime_ctr version | awk '/Server:/{server=1} server && $1=="Version:"{print $2; exit}')" = "v$CONTAINERD_VERSION"
}
prepare_provider() {
@ -65,8 +68,19 @@ prepare_provider() {
"$REMORA_REAL_NERDCTL" namespace create "$NERDCTL_NAMESPACE" >/dev/null
}
acquire_fixture_lock() {
mkdir -p "$RUNTIME"
exec 9>"$RUNTIME/fixture.lock"
if ! flock --nonblock 9; then
printf 'kind test fixture is already in use\n' >&2
return 1
fi
}
configure_provider() {
REMORA_REAL_NERDCTL="$(command -v nerdctl)"
if [[ -z "${REMORA_REAL_NERDCTL:-}" ]]; then
REMORA_REAL_NERDCTL="$(command -v nerdctl)"
fi
export REMORA_REAL_NERDCTL
export CONTAINERD_NAMESPACE="$NERDCTL_NAMESPACE"
export KIND_EXPERIMENTAL_PROVIDER=nerdctl
@ -75,10 +89,26 @@ configure_provider() {
remove_runtime_namespace() {
local status=0
local digests
local digest
local leases
local lease
if ! runtime_namespace_exists; then
return
fi
"$REMORA_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" image prune --all --force >/dev/null || status=1
leases="$(runtime_ctr --namespace "$NERDCTL_NAMESPACE" leases list --quiet)" || status=1
if [[ -n "$leases" ]]; then
while read -r lease; do
runtime_ctr --namespace "$NERDCTL_NAMESPACE" leases delete "$lease" >/dev/null || status=1
done <<< "$leases"
fi
digests="$(runtime_ctr --namespace "$NERDCTL_NAMESPACE" content list --quiet)" || status=1
if [[ -n "$digests" ]]; then
while read -r digest; do
runtime_ctr --namespace "$NERDCTL_NAMESPACE" content remove "$digest" >/dev/null || status=1
done <<< "$digests"
fi
"$REMORA_REAL_NERDCTL" namespace remove "$NERDCTL_NAMESPACE" >/dev/null || status=1
if runtime_namespace_exists; then
printf 'containerd namespace %s still exists after teardown\n' "$NERDCTL_NAMESPACE" >&2
@ -87,6 +117,10 @@ remove_runtime_namespace() {
return "$status"
}
runtime_ctr() {
ctr --address "${CONTAINERD_ADDRESS:-${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/containerd/containerd.sock}" "$@"
}
runtime_namespace_exists() {
"$REMORA_REAL_NERDCTL" namespace list --quiet | grep -Fx "$NERDCTL_NAMESPACE" >/dev/null
}
@ -122,6 +156,18 @@ load_external_image() {
printf '%s\n' "$image"
}
load_local_image() {
local image="$1"
local namespace="${2:-default}"
local archive="$RUNTIME/image-archives/package-test.tar"
rm -f "$archive"
"$REMORA_REAL_NERDCTL" --namespace "$namespace" image save \
--platform linux/amd64 \
--output "$archive" \
"$image"
"$BIN/kind" load image-archive "$archive" --name "$CLUSTER"
}
render_manifests() {
local postgres_image="$1"
local redis_image="$2"
@ -154,6 +200,83 @@ render_manifests() {
-e "s|@@CURL_IMAGE@@|$curl_image|g" \
-e "s|@@AWS_CLI_IMAGE@@|$aws_cli_image|g" \
"$TESTBED/probe.yaml.in" > "$RUNTIME/probe.yaml"
{
printf 'POSTGRES_LOCAL_IMAGE=%q\n' "$postgres_image"
printf 'REDIS_LOCAL_IMAGE=%q\n' "$redis_image"
printf 'DRAGONFLY_LOCAL_IMAGE=%q\n' "$dragonfly_image"
printf 'VAULT_LOCAL_IMAGE=%q\n' "$vault_image"
printf 'KEYCLOAK_LOCAL_IMAGE=%q\n' "$keycloak_image"
printf 'ZITADEL_LOCAL_IMAGE=%q\n' "$zitadel_image"
printf 'GARAGE_LOCAL_IMAGE=%q\n' "$garage_image"
printf 'VERSITYGW_LOCAL_IMAGE=%q\n' "$versitygw_image"
printf 'COREDNS_LOCAL_IMAGE=%q\n' "$coredns_image"
printf 'CURL_LOCAL_IMAGE=%q\n' "$curl_image"
printf 'AWS_CLI_LOCAL_IMAGE=%q\n' "$aws_cli_image"
} > "$RUNTIME/fixture.env"
}
setup_fixture() {
mkdir -p "$RUNTIME"
rm -rf "$RUNTIME/image-archives"
mkdir -p "$RUNTIME/image-archives"
prepare_tools
prepare_provider
"$BIN/kind" create cluster \
--name "$CLUSTER" \
--image "$NODE_IMAGE" \
--config "$TESTBED/cluster.yaml" \
--kubeconfig "$KUBECONFIG_PATH" \
--wait 180s
POSTGRES_LOCAL_IMAGE="$(load_external_image postgres "$POSTGRES_IMAGE")"
REDIS_LOCAL_IMAGE="$(load_external_image redis "$REDIS_IMAGE")"
DRAGONFLY_LOCAL_IMAGE="$(load_external_image dragonfly "$DRAGONFLY_IMAGE")"
VAULT_LOCAL_IMAGE="$(load_external_image vault "$VAULT_IMAGE")"
KEYCLOAK_LOCAL_IMAGE="$(load_external_image keycloak "$KEYCLOAK_IMAGE")"
ZITADEL_LOCAL_IMAGE="$(load_external_image zitadel "$ZITADEL_IMAGE")"
GARAGE_LOCAL_IMAGE="$(load_external_image garage "$GARAGE_IMAGE")"
VERSITYGW_LOCAL_IMAGE="$(load_external_image versitygw "$VERSITYGW_IMAGE")"
COREDNS_LOCAL_IMAGE="$(load_external_image coredns "$COREDNS_IMAGE")"
CURL_LOCAL_IMAGE="$(load_external_image curl "$CURL_IMAGE")"
AWS_CLI_LOCAL_IMAGE="$(load_external_image aws-cli "$AWS_CLI_IMAGE")"
render_manifests \
"$POSTGRES_LOCAL_IMAGE" \
"$REDIS_LOCAL_IMAGE" \
"$DRAGONFLY_LOCAL_IMAGE" \
"$VAULT_LOCAL_IMAGE" \
"$KEYCLOAK_LOCAL_IMAGE" \
"$ZITADEL_LOCAL_IMAGE" \
"$GARAGE_LOCAL_IMAGE" \
"$VERSITYGW_LOCAL_IMAGE" \
"$COREDNS_LOCAL_IMAGE" \
"$CURL_LOCAL_IMAGE" \
"$AWS_CLI_LOCAL_IMAGE"
kube apply --filename "$RUNTIME/dependencies.yaml"
for deployment in postgres redis dragonfly vault keycloak zitadel garage versitygw controlled-dns; do
kube --namespace "$FIXTURE_NAMESPACE" rollout status "deployment/$deployment" --timeout 240s
done
}
verify_dependencies() {
kube apply --filename "$RUNTIME/probe.yaml"
for job in state-probe oidc-probe garage-probe versitygw-probe; do
wait_for_job "$FIXTURE_NAMESPACE" "$job"
done
}
destroy_fixture() {
configure_provider
delete_cluster
remove_runtime_namespace
rm -rf "$RUNTIME/image-archives"
rm -f \
"$KUBECONFIG_PATH" \
"$RUNTIME/dependencies.yaml" \
"$RUNTIME/fixture.env" \
"$RUNTIME/probe.yaml" \
"$RUNTIME/sample-job.yaml"
}
kube() {
@ -164,23 +287,34 @@ kube() {
}
wait_for_job() {
local job="$1"
local namespace="$1"
local job="$2"
local deadline=$((SECONDS + 300))
local complete
local failed
while ((SECONDS < deadline)); do
complete="$(kube --namespace remora-testbed get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Complete")].status}')"
complete="$(kube --namespace "$namespace" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Complete")].status}')"
if [[ "$complete" = True ]]; then
kube --namespace remora-testbed logs "job/$job" --all-containers
kube --namespace "$namespace" logs "job/$job" --all-containers
return
fi
failed="$(kube --namespace remora-testbed get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Failed")].status}')"
failed="$(kube --namespace "$namespace" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Failed")].status}')"
if [[ "$failed" = True ]]; then
kube --namespace remora-testbed logs "job/$job" --all-containers >&2 || true
kube --namespace "$namespace" logs "job/$job" --all-containers >&2 || true
return 1
fi
sleep 1
done
kube --namespace "$namespace" logs "job/$job" --all-containers >&2 || true
printf 'job/%s did not complete within 300 seconds\n' "$job" >&2
return 1
}
run_job_manifest() {
local namespace="$1"
local job="$2"
local manifest="$3"
kube --namespace "$namespace" delete "job/$job" --ignore-not-found --wait=true
kube --namespace "$namespace" apply --filename "$manifest"
wait_for_job "$namespace" "$job"
}

48
tests/kind/run-sample-job.sh Executable file
View file

@ -0,0 +1,48 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
source "$ROOT/tests/kind/lib.sh"
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
SUCCEEDED=false
OWNS_FIXTURE=false
cleanup() {
local status=$?
trap - EXIT INT TERM
if [[ "${REMORA_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
printf 'retained context: %s\n' "$CONTEXT"
exit "$status"
fi
if [[ "$OWNS_FIXTURE" = true ]]; then
destroy_fixture || status=1
fi
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
printf 'user kubeconfig changed unexpectedly\n' >&2
status=1
fi
exit "$status"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
acquire_fixture_lock
REMORA_REAL_NERDCTL="$(command -v nerdctl)"
export REMORA_REAL_NERDCTL
"$REMORA_REAL_NERDCTL" --namespace default build \
--file "$ROOT/Dockerfile.test" \
--target kind-sample-runner \
--tag remora-kind-sample:local \
"$ROOT"
OWNS_FIXTURE=true
setup_fixture
load_local_image remora-kind-sample:local
sed "s|@@TEST_IMAGE@@|remora-kind-sample:local|g" \
"$TESTBED/sample-job.yaml.in" > "$RUNTIME/sample-job.yaml"
run_job_manifest "$FIXTURE_NAMESPACE" fixture-consumer-sample "$RUNTIME/sample-job.yaml"
SUCCEEDED=true
printf 'sample package test Job passed\n'

View file

@ -4,9 +4,6 @@ set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
source "$ROOT/tests/kind/lib.sh"
mkdir -p "$RUNTIME"
rm -rf "$RUNTIME/image-archives"
mkdir -p "$RUNTIME/image-archives"
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
SUCCEEDED=false
@ -19,11 +16,8 @@ cleanup() {
exit "$status"
fi
if [[ -n "${REMORA_REAL_NERDCTL:-}" ]]; then
delete_cluster || status=1
remove_runtime_namespace || status=1
destroy_fixture || status=1
fi
rm -rf "$RUNTIME/image-archives" || status=1
rm -f "$KUBECONFIG_PATH" "$RUNTIME/dependencies.yaml" "$RUNTIME/probe.yaml" || status=1
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
printf 'user kubeconfig changed unexpectedly\n' >&2
status=1
@ -34,50 +28,12 @@ cleanup() {
exit "$status"
}
trap cleanup EXIT INT TERM
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
prepare_tools
prepare_provider
"$BIN/kind" create cluster \
--name "$CLUSTER" \
--image "$NODE_IMAGE" \
--config "$TESTBED/cluster.yaml" \
--kubeconfig "$KUBECONFIG_PATH" \
--wait 180s
POSTGRES_LOCAL_IMAGE="$(load_external_image postgres "$POSTGRES_IMAGE")"
REDIS_LOCAL_IMAGE="$(load_external_image redis "$REDIS_IMAGE")"
DRAGONFLY_LOCAL_IMAGE="$(load_external_image dragonfly "$DRAGONFLY_IMAGE")"
VAULT_LOCAL_IMAGE="$(load_external_image vault "$VAULT_IMAGE")"
KEYCLOAK_LOCAL_IMAGE="$(load_external_image keycloak "$KEYCLOAK_IMAGE")"
ZITADEL_LOCAL_IMAGE="$(load_external_image zitadel "$ZITADEL_IMAGE")"
GARAGE_LOCAL_IMAGE="$(load_external_image garage "$GARAGE_IMAGE")"
VERSITYGW_LOCAL_IMAGE="$(load_external_image versitygw "$VERSITYGW_IMAGE")"
COREDNS_LOCAL_IMAGE="$(load_external_image coredns "$COREDNS_IMAGE")"
CURL_LOCAL_IMAGE="$(load_external_image curl "$CURL_IMAGE")"
AWS_CLI_LOCAL_IMAGE="$(load_external_image aws-cli "$AWS_CLI_IMAGE")"
render_manifests \
"$POSTGRES_LOCAL_IMAGE" \
"$REDIS_LOCAL_IMAGE" \
"$DRAGONFLY_LOCAL_IMAGE" \
"$VAULT_LOCAL_IMAGE" \
"$KEYCLOAK_LOCAL_IMAGE" \
"$ZITADEL_LOCAL_IMAGE" \
"$GARAGE_LOCAL_IMAGE" \
"$VERSITYGW_LOCAL_IMAGE" \
"$COREDNS_LOCAL_IMAGE" \
"$CURL_LOCAL_IMAGE" \
"$AWS_CLI_LOCAL_IMAGE"
kube apply --filename "$RUNTIME/dependencies.yaml"
for deployment in postgres redis dragonfly vault keycloak zitadel garage versitygw controlled-dns; do
kube --namespace remora-testbed rollout status "deployment/$deployment" --timeout 240s
done
kube apply --filename "$RUNTIME/probe.yaml"
for job in state-probe oidc-probe garage-probe versitygw-probe; do
wait_for_job "$job"
done
acquire_fixture_lock
setup_fixture
verify_dependencies
SUCCEEDED=true

View file

@ -0,0 +1,21 @@
apiVersion: batch/v1
kind: Job
metadata:
name: fixture-consumer-sample
namespace: remora-testbed
spec:
activeDeadlineSeconds: 300
backoffLimit: 0
ttlSecondsAfterFinished: 300
template:
spec:
automountServiceAccountToken: false
enableServiceLinks: false
restartPolicy: Never
containers:
- name: tests
image: @@TEST_IMAGE@@
imagePullPolicy: Never
resources:
requests: {cpu: 100m, memory: 64Mi}
limits: {cpu: "2", memory: 512Mi}

31
tests/kind/setup.sh Executable file
View file

@ -0,0 +1,31 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
source "$ROOT/tests/kind/lib.sh"
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
SUCCEEDED=false
cleanup() {
local status=$?
trap - EXIT INT TERM
if [[ "$SUCCEEDED" != true ]] && [[ "${REMORA_TESTBED_RETAIN:-0}" != 1 ]]; then
destroy_fixture || status=1
fi
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
printf 'user kubeconfig changed unexpectedly\n' >&2
status=1
fi
exit "$status"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
acquire_fixture_lock
setup_fixture
SUCCEEDED=true
printf 'fixture kubeconfig: %s\n' "$KUBECONFIG_PATH"
printf 'fixture context: %s\n' "$CONTEXT"

View file

@ -2,6 +2,8 @@ KIND_VERSION=v0.32.0
KIND_SHA256=50030de23cf40a18505f20426f6a8506bedf13c6e509244bd1fa9463721b0f54
KUBECTL_VERSION=v1.36.2
KUBECTL_SHA256=1e9045ec32bea85da43de85f0065358529ea7c7a152eca78154fba5b58c27d82
CONTAINERD_VERSION=2.3.3
NERDCTL_VERSION=2.3.4
NODE_IMAGE=kindest/node:v1.36.1@sha256:3489c7674813ba5d8b1a9977baea8a6e553784dab7b84759d1014dbd78f7ebd5
POSTGRES_IMAGE=postgres@sha256:bb9940981eeb86356ec91bda430f8e0ba8729cc0b319615a2c2d7ac932fdf7bb
REDIS_IMAGE=redis@sha256:cd5f3ac681c77791c6a8eaa62de876ad2be043ee5a428afb7c0095aa08246277