Add reusable kind test job execution
This commit is contained in:
parent
3ed528b288
commit
af8bb1a03a
1 changed files with 310 additions and 64 deletions
|
|
@ -9,6 +9,7 @@ KUBECONFIG_PATH="$RUNTIME/kubeconfig"
|
|||
CONTEXT=kind-remora-testbed
|
||||
CLUSTER=remora-testbed
|
||||
NERDCTL_NAMESPACE=remora-wp02
|
||||
FIXTURE_NAMESPACE=remora-testbed
|
||||
|
||||
set -a
|
||||
source "$TESTBED/versions.env"
|
||||
|
|
@ -55,7 +56,9 @@ prepare_tools() {
|
|||
"https://dl.k8s.io/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl"
|
||||
test "$("$BIN/kind" version | awk '{print $2}')" = "$KIND_VERSION"
|
||||
test "$("$BIN/kubectl" --kubeconfig "$KUBECONFIG_PATH" --context "$CONTEXT" version --client=true -o json | jq -r .clientVersion.gitVersion)" = "$KUBECTL_VERSION"
|
||||
test "$(nerdctl --version | awk '{print $3}')" = 2.3.4
|
||||
test "$(nerdctl --version | awk '{print $3}')" = "$NERDCTL_VERSION"
|
||||
test "$(runtime_ctr version | awk '/Client:/{client=1} client && $1=="Version:"{print $2; exit}')" = "v$CONTAINERD_VERSION"
|
||||
test "$(runtime_ctr version | awk '/Server:/{server=1} server && $1=="Version:"{print $2; exit}')" = "v$CONTAINERD_VERSION"
|
||||
}
|
||||
|
||||
prepare_provider() {
|
||||
|
|
@ -65,8 +68,19 @@ prepare_provider() {
|
|||
"$REMORA_REAL_NERDCTL" namespace create "$NERDCTL_NAMESPACE" >/dev/null
|
||||
}
|
||||
|
||||
acquire_fixture_lock() {
|
||||
mkdir -p "$RUNTIME"
|
||||
exec 9>"$RUNTIME/fixture.lock"
|
||||
if ! flock --nonblock 9; then
|
||||
printf 'kind test fixture is already in use\n' >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
configure_provider() {
|
||||
REMORA_REAL_NERDCTL="$(command -v nerdctl)"
|
||||
if [[ -z "${REMORA_REAL_NERDCTL:-}" ]]; then
|
||||
REMORA_REAL_NERDCTL="$(command -v nerdctl)"
|
||||
fi
|
||||
export REMORA_REAL_NERDCTL
|
||||
export CONTAINERD_NAMESPACE="$NERDCTL_NAMESPACE"
|
||||
export KIND_EXPERIMENTAL_PROVIDER=nerdctl
|
||||
|
|
@ -75,10 +89,26 @@ configure_provider() {
|
|||
|
||||
remove_runtime_namespace() {
|
||||
local status=0
|
||||
local digests
|
||||
local digest
|
||||
local leases
|
||||
local lease
|
||||
if ! runtime_namespace_exists; then
|
||||
return
|
||||
fi
|
||||
"$REMORA_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" image prune --all --force >/dev/null || status=1
|
||||
leases="$(runtime_ctr --namespace "$NERDCTL_NAMESPACE" leases list --quiet)" || status=1
|
||||
if [[ -n "$leases" ]]; then
|
||||
while read -r lease; do
|
||||
runtime_ctr --namespace "$NERDCTL_NAMESPACE" leases delete "$lease" >/dev/null || status=1
|
||||
done <<< "$leases"
|
||||
fi
|
||||
digests="$(runtime_ctr --namespace "$NERDCTL_NAMESPACE" content list --quiet)" || status=1
|
||||
if [[ -n "$digests" ]]; then
|
||||
while read -r digest; do
|
||||
runtime_ctr --namespace "$NERDCTL_NAMESPACE" content remove "$digest" >/dev/null || status=1
|
||||
done <<< "$digests"
|
||||
fi
|
||||
"$REMORA_REAL_NERDCTL" namespace remove "$NERDCTL_NAMESPACE" >/dev/null || status=1
|
||||
if runtime_namespace_exists; then
|
||||
printf 'containerd namespace %s still exists after teardown\n' "$NERDCTL_NAMESPACE" >&2
|
||||
|
|
@ -87,6 +117,10 @@ remove_runtime_namespace() {
|
|||
return "$status"
|
||||
}
|
||||
|
||||
runtime_ctr() {
|
||||
ctr --address "${CONTAINERD_ADDRESS:-${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/containerd/containerd.sock}" "$@"
|
||||
}
|
||||
|
||||
runtime_namespace_exists() {
|
||||
"$REMORA_REAL_NERDCTL" namespace list --quiet | grep -Fx "$NERDCTL_NAMESPACE" >/dev/null
|
||||
}
|
||||
|
|
@ -122,6 +156,18 @@ load_external_image() {
|
|||
printf '%s\n' "$image"
|
||||
}
|
||||
|
||||
load_local_image() {
|
||||
local image="$1"
|
||||
local namespace="${2:-default}"
|
||||
local archive="$RUNTIME/image-archives/package-test.tar"
|
||||
rm -f "$archive"
|
||||
"$REMORA_REAL_NERDCTL" --namespace "$namespace" image save \
|
||||
--platform linux/amd64 \
|
||||
--output "$archive" \
|
||||
"$image"
|
||||
"$BIN/kind" load image-archive "$archive" --name "$CLUSTER"
|
||||
}
|
||||
|
||||
render_manifests() {
|
||||
local postgres_image="$1"
|
||||
local redis_image="$2"
|
||||
|
|
@ -154,6 +200,83 @@ render_manifests() {
|
|||
-e "s|@@CURL_IMAGE@@|$curl_image|g" \
|
||||
-e "s|@@AWS_CLI_IMAGE@@|$aws_cli_image|g" \
|
||||
"$TESTBED/probe.yaml.in" > "$RUNTIME/probe.yaml"
|
||||
{
|
||||
printf 'POSTGRES_LOCAL_IMAGE=%q\n' "$postgres_image"
|
||||
printf 'REDIS_LOCAL_IMAGE=%q\n' "$redis_image"
|
||||
printf 'DRAGONFLY_LOCAL_IMAGE=%q\n' "$dragonfly_image"
|
||||
printf 'VAULT_LOCAL_IMAGE=%q\n' "$vault_image"
|
||||
printf 'KEYCLOAK_LOCAL_IMAGE=%q\n' "$keycloak_image"
|
||||
printf 'ZITADEL_LOCAL_IMAGE=%q\n' "$zitadel_image"
|
||||
printf 'GARAGE_LOCAL_IMAGE=%q\n' "$garage_image"
|
||||
printf 'VERSITYGW_LOCAL_IMAGE=%q\n' "$versitygw_image"
|
||||
printf 'COREDNS_LOCAL_IMAGE=%q\n' "$coredns_image"
|
||||
printf 'CURL_LOCAL_IMAGE=%q\n' "$curl_image"
|
||||
printf 'AWS_CLI_LOCAL_IMAGE=%q\n' "$aws_cli_image"
|
||||
} > "$RUNTIME/fixture.env"
|
||||
}
|
||||
|
||||
setup_fixture() {
|
||||
mkdir -p "$RUNTIME"
|
||||
rm -rf "$RUNTIME/image-archives"
|
||||
mkdir -p "$RUNTIME/image-archives"
|
||||
prepare_tools
|
||||
prepare_provider
|
||||
|
||||
"$BIN/kind" create cluster \
|
||||
--name "$CLUSTER" \
|
||||
--image "$NODE_IMAGE" \
|
||||
--config "$TESTBED/cluster.yaml" \
|
||||
--kubeconfig "$KUBECONFIG_PATH" \
|
||||
--wait 180s
|
||||
|
||||
POSTGRES_LOCAL_IMAGE="$(load_external_image postgres "$POSTGRES_IMAGE")"
|
||||
REDIS_LOCAL_IMAGE="$(load_external_image redis "$REDIS_IMAGE")"
|
||||
DRAGONFLY_LOCAL_IMAGE="$(load_external_image dragonfly "$DRAGONFLY_IMAGE")"
|
||||
VAULT_LOCAL_IMAGE="$(load_external_image vault "$VAULT_IMAGE")"
|
||||
KEYCLOAK_LOCAL_IMAGE="$(load_external_image keycloak "$KEYCLOAK_IMAGE")"
|
||||
ZITADEL_LOCAL_IMAGE="$(load_external_image zitadel "$ZITADEL_IMAGE")"
|
||||
GARAGE_LOCAL_IMAGE="$(load_external_image garage "$GARAGE_IMAGE")"
|
||||
VERSITYGW_LOCAL_IMAGE="$(load_external_image versitygw "$VERSITYGW_IMAGE")"
|
||||
COREDNS_LOCAL_IMAGE="$(load_external_image coredns "$COREDNS_IMAGE")"
|
||||
CURL_LOCAL_IMAGE="$(load_external_image curl "$CURL_IMAGE")"
|
||||
AWS_CLI_LOCAL_IMAGE="$(load_external_image aws-cli "$AWS_CLI_IMAGE")"
|
||||
render_manifests \
|
||||
"$POSTGRES_LOCAL_IMAGE" \
|
||||
"$REDIS_LOCAL_IMAGE" \
|
||||
"$DRAGONFLY_LOCAL_IMAGE" \
|
||||
"$VAULT_LOCAL_IMAGE" \
|
||||
"$KEYCLOAK_LOCAL_IMAGE" \
|
||||
"$ZITADEL_LOCAL_IMAGE" \
|
||||
"$GARAGE_LOCAL_IMAGE" \
|
||||
"$VERSITYGW_LOCAL_IMAGE" \
|
||||
"$COREDNS_LOCAL_IMAGE" \
|
||||
"$CURL_LOCAL_IMAGE" \
|
||||
"$AWS_CLI_LOCAL_IMAGE"
|
||||
|
||||
kube apply --filename "$RUNTIME/dependencies.yaml"
|
||||
for deployment in postgres redis dragonfly vault keycloak zitadel garage versitygw controlled-dns; do
|
||||
kube --namespace "$FIXTURE_NAMESPACE" rollout status "deployment/$deployment" --timeout 240s
|
||||
done
|
||||
}
|
||||
|
||||
verify_dependencies() {
|
||||
kube apply --filename "$RUNTIME/probe.yaml"
|
||||
for job in state-probe oidc-probe garage-probe versitygw-probe; do
|
||||
wait_for_job "$FIXTURE_NAMESPACE" "$job"
|
||||
done
|
||||
}
|
||||
|
||||
destroy_fixture() {
|
||||
configure_provider
|
||||
delete_cluster
|
||||
remove_runtime_namespace
|
||||
rm -rf "$RUNTIME/image-archives"
|
||||
rm -f \
|
||||
"$KUBECONFIG_PATH" \
|
||||
"$RUNTIME/dependencies.yaml" \
|
||||
"$RUNTIME/fixture.env" \
|
||||
"$RUNTIME/probe.yaml" \
|
||||
"$RUNTIME/sample-job.yaml"
|
||||
}
|
||||
|
||||
kube() {
|
||||
|
|
@ -164,23 +287,34 @@ kube() {
|
|||
}
|
||||
|
||||
wait_for_job() {
|
||||
local job="$1"
|
||||
local namespace="$1"
|
||||
local job="$2"
|
||||
local deadline=$((SECONDS + 300))
|
||||
local complete
|
||||
local failed
|
||||
while ((SECONDS < deadline)); do
|
||||
complete="$(kube --namespace remora-testbed get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Complete")].status}')"
|
||||
complete="$(kube --namespace "$namespace" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Complete")].status}')"
|
||||
if [[ "$complete" = True ]]; then
|
||||
kube --namespace remora-testbed logs "job/$job" --all-containers
|
||||
kube --namespace "$namespace" logs "job/$job" --all-containers
|
||||
return
|
||||
fi
|
||||
failed="$(kube --namespace remora-testbed get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Failed")].status}')"
|
||||
failed="$(kube --namespace "$namespace" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Failed")].status}')"
|
||||
if [[ "$failed" = True ]]; then
|
||||
kube --namespace remora-testbed logs "job/$job" --all-containers >&2 || true
|
||||
kube --namespace "$namespace" logs "job/$job" --all-containers >&2 || true
|
||||
return 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
kube --namespace "$namespace" logs "job/$job" --all-containers >&2 || true
|
||||
printf 'job/%s did not complete within 300 seconds\n' "$job" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
run_job_manifest() {
|
||||
local namespace="$1"
|
||||
local job="$2"
|
||||
local manifest="$3"
|
||||
kube --namespace "$namespace" delete "job/$job" --ignore-not-found --wait=true
|
||||
kube --namespace "$namespace" apply --filename "$manifest"
|
||||
wait_for_job "$namespace" "$job"
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue