#!/usr/bin/env bash set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" TESTBED="$ROOT/tests/kind" RUNTIME="$TESTBED/.runtime" BIN="$RUNTIME/bin" KUBECONFIG_PATH="$RUNTIME/kubeconfig" CONTEXT=kind-remora-testbed CLUSTER=remora-testbed NERDCTL_NAMESPACE=remora-wp02 FIXTURE_NAMESPACE=remora-testbed set -a source "$TESTBED/versions.env" set +a user_kubeconfig_digest() { local config_value="${KUBECONFIG:-${HOME:?}/.kube/config}" local path local paths IFS=: read -r -a paths <<< "$config_value" { for path in "${paths[@]}"; do if [[ -f "$path" ]]; then sha256sum "$path" else printf 'missing %s\n' "$path" fi done } | sha256sum | cut -d' ' -f1 } download_tool() { local destination="$1" local checksum="$2" local url="$3" if [[ -x "$destination" ]] && [[ "$(sha256sum "$destination" | cut -d' ' -f1)" = "$checksum" ]]; then return fi curl --fail --location --silent --show-error "$url" --output "$destination.tmp" test "$(sha256sum "$destination.tmp" | cut -d' ' -f1)" = "$checksum" chmod 0755 "$destination.tmp" mv "$destination.tmp" "$destination" } prepare_tools() { mkdir -p "$BIN" download_tool \ "$BIN/kind" \ "$KIND_SHA256" \ "https://github.com/kubernetes-sigs/kind/releases/download/$KIND_VERSION/kind-linux-amd64" download_tool \ "$BIN/kubectl" \ "$KUBECTL_SHA256" \ "https://dl.k8s.io/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl" test "$("$BIN/kind" version | awk '{print $2}')" = "$KIND_VERSION" test "$("$BIN/kubectl" --kubeconfig "$KUBECONFIG_PATH" --context "$CONTEXT" version --client=true -o json | jq -r .clientVersion.gitVersion)" = "$KUBECTL_VERSION" test "$(nerdctl --version | awk '{print $3}')" = "$NERDCTL_VERSION" test "$(runtime_ctr version | awk '/Client:/{client=1} client && $1=="Version:"{print $2; exit}')" = "v$CONTAINERD_VERSION" test "$(runtime_ctr version | awk '/Server:/{server=1} server && $1=="Version:"{print $2; exit}')" = "v$CONTAINERD_VERSION" } prepare_provider() { configure_provider delete_cluster remove_runtime_namespace "$REMORA_REAL_NERDCTL" namespace create "$NERDCTL_NAMESPACE" >/dev/null } acquire_fixture_lock() { mkdir -p "$RUNTIME" exec 9>"$RUNTIME/fixture.lock" if ! flock --nonblock 9; then printf 'kind test fixture is already in use\n' >&2 return 1 fi } configure_provider() { if [[ -z "${REMORA_REAL_NERDCTL:-}" ]]; then REMORA_REAL_NERDCTL="$(command -v nerdctl)" fi export REMORA_REAL_NERDCTL export CONTAINERD_NAMESPACE="$NERDCTL_NAMESPACE" export KIND_EXPERIMENTAL_PROVIDER=nerdctl export PATH="$TESTBED/provider:$PATH" } remove_runtime_namespace() { local status=0 local digests local digest local leases local lease if ! runtime_namespace_exists; then return fi "$REMORA_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" image prune --all --force >/dev/null || status=1 leases="$(runtime_ctr --namespace "$NERDCTL_NAMESPACE" leases list --quiet)" || status=1 if [[ -n "$leases" ]]; then while read -r lease; do runtime_ctr --namespace "$NERDCTL_NAMESPACE" leases delete "$lease" >/dev/null || status=1 done <<< "$leases" fi digests="$(runtime_ctr --namespace "$NERDCTL_NAMESPACE" content list --quiet)" || status=1 if [[ -n "$digests" ]]; then while read -r digest; do runtime_ctr --namespace "$NERDCTL_NAMESPACE" content remove "$digest" >/dev/null || status=1 done <<< "$digests" fi "$REMORA_REAL_NERDCTL" namespace remove "$NERDCTL_NAMESPACE" >/dev/null || status=1 if runtime_namespace_exists; then printf 'containerd namespace %s still exists after teardown\n' "$NERDCTL_NAMESPACE" >&2 status=1 fi return "$status" } runtime_ctr() { ctr --address "${CONTAINERD_ADDRESS:-${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/containerd/containerd.sock}" "$@" } runtime_namespace_exists() { "$REMORA_REAL_NERDCTL" namespace list --quiet | grep -Fx "$NERDCTL_NAMESPACE" >/dev/null } cluster_exists() { if ! runtime_namespace_exists; then return 1 fi "$BIN/kind" get clusters | grep -Fx "$CLUSTER" >/dev/null } delete_cluster() { if cluster_exists; then "$BIN/kind" delete cluster --name "$CLUSTER" fi if cluster_exists; then printf 'kind cluster %s still exists after teardown\n' "$CLUSTER" >&2 return 1 fi } load_external_image() { local name="$1" local source="$2" local digest="${source##*@}" local image="remora-testbed/$name:sha256-${digest#sha256:}" local archive="$RUNTIME/image-archives/$digest.tar" "$REMORA_REAL_NERDCTL" image pull --platform linux/amd64 --unpack=false --quiet "$source" >&2 || return 1 "$REMORA_REAL_NERDCTL" image tag "$source" "$image" || return 1 "$REMORA_REAL_NERDCTL" image save --platform linux/amd64 --output "$archive" "$image" || return 1 printf 'Loading %s from %s\n' "$image" "$source" >&2 "$BIN/kind" load image-archive "$archive" --name "$CLUSTER" >&2 || return 1 printf '%s\n' "$image" } load_local_image() { local image="$1" local namespace="${2:-default}" local archive="$RUNTIME/image-archives/package-test.tar" rm -f "$archive" "$REMORA_REAL_NERDCTL" --namespace "$namespace" image save \ --platform linux/amd64 \ --output "$archive" \ "$image" "$BIN/kind" load image-archive "$archive" --name "$CLUSTER" } render_manifests() { local postgres_image="$1" local redis_image="$2" local dragonfly_image="$3" local vault_image="$4" local keycloak_image="$5" local zitadel_image="$6" local garage_image="$7" local versitygw_image="$8" local coredns_image="$9" local curl_image="${10}" local aws_cli_image="${11}" sed \ -e "s|@@POSTGRES_IMAGE@@|$postgres_image|g" \ -e "s|@@REDIS_IMAGE@@|$redis_image|g" \ -e "s|@@DRAGONFLY_IMAGE@@|$dragonfly_image|g" \ -e "s|@@VAULT_IMAGE@@|$vault_image|g" \ -e "s|@@KEYCLOAK_IMAGE@@|$keycloak_image|g" \ -e "s|@@ZITADEL_IMAGE@@|$zitadel_image|g" \ -e "s|@@GARAGE_IMAGE@@|$garage_image|g" \ -e "s|@@VERSITYGW_IMAGE@@|$versitygw_image|g" \ -e "s|@@COREDNS_IMAGE@@|$coredns_image|g" \ -e "s|@@CURL_IMAGE@@|$curl_image|g" \ -e "s|@@AWS_CLI_IMAGE@@|$aws_cli_image|g" \ "$TESTBED/dependencies.yaml.in" > "$RUNTIME/dependencies.yaml" sed \ -e "s|@@POSTGRES_IMAGE@@|$postgres_image|g" \ -e "s|@@REDIS_IMAGE@@|$redis_image|g" \ -e "s|@@VAULT_IMAGE@@|$vault_image|g" \ -e "s|@@CURL_IMAGE@@|$curl_image|g" \ -e "s|@@AWS_CLI_IMAGE@@|$aws_cli_image|g" \ "$TESTBED/probe.yaml.in" > "$RUNTIME/probe.yaml" { printf 'POSTGRES_LOCAL_IMAGE=%q\n' "$postgres_image" printf 'REDIS_LOCAL_IMAGE=%q\n' "$redis_image" printf 'DRAGONFLY_LOCAL_IMAGE=%q\n' "$dragonfly_image" printf 'VAULT_LOCAL_IMAGE=%q\n' "$vault_image" printf 'KEYCLOAK_LOCAL_IMAGE=%q\n' "$keycloak_image" printf 'ZITADEL_LOCAL_IMAGE=%q\n' "$zitadel_image" printf 'GARAGE_LOCAL_IMAGE=%q\n' "$garage_image" printf 'VERSITYGW_LOCAL_IMAGE=%q\n' "$versitygw_image" printf 'COREDNS_LOCAL_IMAGE=%q\n' "$coredns_image" printf 'CURL_LOCAL_IMAGE=%q\n' "$curl_image" printf 'AWS_CLI_LOCAL_IMAGE=%q\n' "$aws_cli_image" } > "$RUNTIME/fixture.env" } setup_fixture() { mkdir -p "$RUNTIME" rm -rf "$RUNTIME/image-archives" mkdir -p "$RUNTIME/image-archives" prepare_tools prepare_provider "$BIN/kind" create cluster \ --name "$CLUSTER" \ --image "$NODE_IMAGE" \ --config "$TESTBED/cluster.yaml" \ --kubeconfig "$KUBECONFIG_PATH" \ --wait 180s POSTGRES_LOCAL_IMAGE="$(load_external_image postgres "$POSTGRES_IMAGE")" REDIS_LOCAL_IMAGE="$(load_external_image redis "$REDIS_IMAGE")" DRAGONFLY_LOCAL_IMAGE="$(load_external_image dragonfly "$DRAGONFLY_IMAGE")" VAULT_LOCAL_IMAGE="$(load_external_image vault "$VAULT_IMAGE")" KEYCLOAK_LOCAL_IMAGE="$(load_external_image keycloak "$KEYCLOAK_IMAGE")" ZITADEL_LOCAL_IMAGE="$(load_external_image zitadel "$ZITADEL_IMAGE")" GARAGE_LOCAL_IMAGE="$(load_external_image garage "$GARAGE_IMAGE")" VERSITYGW_LOCAL_IMAGE="$(load_external_image versitygw "$VERSITYGW_IMAGE")" COREDNS_LOCAL_IMAGE="$(load_external_image coredns "$COREDNS_IMAGE")" CURL_LOCAL_IMAGE="$(load_external_image curl "$CURL_IMAGE")" AWS_CLI_LOCAL_IMAGE="$(load_external_image aws-cli "$AWS_CLI_IMAGE")" render_manifests \ "$POSTGRES_LOCAL_IMAGE" \ "$REDIS_LOCAL_IMAGE" \ "$DRAGONFLY_LOCAL_IMAGE" \ "$VAULT_LOCAL_IMAGE" \ "$KEYCLOAK_LOCAL_IMAGE" \ "$ZITADEL_LOCAL_IMAGE" \ "$GARAGE_LOCAL_IMAGE" \ "$VERSITYGW_LOCAL_IMAGE" \ "$COREDNS_LOCAL_IMAGE" \ "$CURL_LOCAL_IMAGE" \ "$AWS_CLI_LOCAL_IMAGE" kube apply --filename "$RUNTIME/dependencies.yaml" for deployment in postgres redis dragonfly vault keycloak zitadel garage versitygw controlled-dns; do kube --namespace "$FIXTURE_NAMESPACE" rollout status "deployment/$deployment" --timeout 240s done } verify_dependencies() { kube apply --filename "$RUNTIME/probe.yaml" for job in state-probe oidc-probe garage-probe versitygw-probe; do wait_for_job "$FIXTURE_NAMESPACE" "$job" done } destroy_fixture() { configure_provider delete_cluster remove_runtime_namespace rm -rf "$RUNTIME/image-archives" rm -f \ "$KUBECONFIG_PATH" \ "$RUNTIME/dependencies.yaml" \ "$RUNTIME/fixture.env" \ "$RUNTIME/probe.yaml" \ "$RUNTIME/sample-job.yaml" \ "$RUNTIME/durable-job.yaml" } kube() { "$BIN/kubectl" \ --kubeconfig "$KUBECONFIG_PATH" \ --context "$CONTEXT" \ "$@" } wait_for_job() { local namespace="$1" local job="$2" local deadline=$((SECONDS + 300)) local complete local failed while ((SECONDS < deadline)); do complete="$(kube --namespace "$namespace" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Complete")].status}')" if [[ "$complete" = True ]]; then kube --namespace "$namespace" logs "job/$job" --all-containers return fi failed="$(kube --namespace "$namespace" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Failed")].status}')" if [[ "$failed" = True ]]; then kube --namespace "$namespace" logs "job/$job" --all-containers >&2 || true return 1 fi sleep 1 done kube --namespace "$namespace" logs "job/$job" --all-containers >&2 || true printf 'job/%s did not complete within 300 seconds\n' "$job" >&2 return 1 } run_job_manifest() { local namespace="$1" local job="$2" local manifest="$3" kube --namespace "$namespace" delete "job/$job" --ignore-not-found --wait=true kube --namespace "$namespace" apply --filename "$manifest" wait_for_job "$namespace" "$job" }