#!/usr/bin/env bash set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" TESTBED="$ROOT/tests/kind" RUNTIME="$TESTBED/.runtime" BIN="$RUNTIME/bin" KUBECONFIG_PATH="$RUNTIME/kubeconfig" FIXTURE_LOCK_DIR="${XDG_RUNTIME_DIR:-/tmp/nuthatch-$(id -u)}" FIXTURE_LOCK_PATH="$FIXTURE_LOCK_DIR/nuthatch-testbed-kind.lock" CONTEXT=kind-nuthatch-testbed CLUSTER=nuthatch-testbed NERDCTL_NAMESPACE=nuthatch-testbed FIXTURE_NAMESPACE=nuthatch-testbed set -a source "$TESTBED/versions.env" set +a user_kubeconfig_digest() { local config_value="${KUBECONFIG:-${HOME:?}/.kube/config}" local path local paths IFS=: read -r -a paths <<< "$config_value" { for path in "${paths[@]}"; do if [[ -f "$path" ]]; then sha256sum "$path" else printf 'missing %s\n' "$path" fi done } | sha256sum | cut -d' ' -f1 } download_tool() { local destination="$1" local checksum="$2" local url="$3" if [[ -x "$destination" ]] && [[ "$(sha256sum "$destination" | cut -d' ' -f1)" = "$checksum" ]]; then return fi curl --fail --location --silent --show-error "$url" --output "$destination.tmp" test "$(sha256sum "$destination.tmp" | cut -d' ' -f1)" = "$checksum" chmod 0755 "$destination.tmp" mv "$destination.tmp" "$destination" } prepare_tools() { mkdir -p "$BIN" download_tool \ "$BIN/kind" \ "$KIND_SHA256" \ "https://github.com/kubernetes-sigs/kind/releases/download/$KIND_VERSION/kind-linux-amd64" download_tool \ "$BIN/kubectl" \ "$KUBECTL_SHA256" \ "https://dl.k8s.io/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl" test "$("$BIN/kind" version | awk '{print $2}')" = "$KIND_VERSION" test "$("$BIN/kubectl" --kubeconfig "$KUBECONFIG_PATH" --context "$CONTEXT" version --client=true -o json | jq -r .clientVersion.gitVersion)" = "$KUBECTL_VERSION" test "$(nerdctl --version | awk '{print $3}')" = "$NERDCTL_VERSION" test "$(runtime_ctr version | awk '/Client:/{client=1} client && $1=="Version:"{print $2; exit}')" = "v$CONTAINERD_VERSION" test "$(runtime_ctr version | awk '/Server:/{server=1} server && $1=="Version:"{print $2; exit}')" = "v$CONTAINERD_VERSION" } acquire_fixture_lock() { mkdir -p "$RUNTIME" "$FIXTURE_LOCK_DIR" exec 9>"$FIXTURE_LOCK_PATH" if ! flock --nonblock 9; then printf 'kind test fixture is already in use\n' >&2 return 1 fi } configure_provider() { if [[ -z "${NUTHATCH_REAL_NERDCTL:-}" ]]; then NUTHATCH_REAL_NERDCTL="$(command -v nerdctl)" fi export NUTHATCH_REAL_NERDCTL export CONTAINERD_NAMESPACE="$NERDCTL_NAMESPACE" export KIND_EXPERIMENTAL_PROVIDER=nerdctl export PATH="$TESTBED/provider:$PATH" } runtime_ctr() { ctr --address "${CONTAINERD_ADDRESS:-${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/containerd/containerd.sock}" "$@" } runtime_namespace_exists() { "$NUTHATCH_REAL_NERDCTL" namespace list --quiet | grep -Fx "$NERDCTL_NAMESPACE" >/dev/null } cluster_exists() { runtime_namespace_exists && "$BIN/kind" get clusters | grep -Fx "$CLUSTER" >/dev/null } delete_cluster() { local container local containers if cluster_exists; then "$BIN/kind" delete cluster --name "$CLUSTER" || true fi containers="$("$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" ps \ --all \ --quiet \ --filter "label=io.x-k8s.kind.cluster=$CLUSTER")" while IFS= read -r container; do [[ -z "$container" ]] || "$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" rm --force "$container" done <<< "$containers" if cluster_exists; then "$BIN/kind" delete cluster --name "$CLUSTER" || true fi if cluster_exists; then printf 'kind cluster %s still exists after teardown\n' "$CLUSTER" >&2 return 1 fi } remove_runtime_namespace() { local status=0 local digest local digests local lease local leases local network for network in kind nuthatch-kind; do if "$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" network inspect "$network" >/dev/null 2>&1; then "$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" network remove "$network" >/dev/null || status=1 fi done if ! runtime_namespace_exists; then return "$status" fi "$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" image prune --all --force >/dev/null || status=1 leases="$(runtime_ctr --namespace "$NERDCTL_NAMESPACE" leases list --quiet)" || status=1 while IFS= read -r lease; do [[ -z "$lease" ]] || runtime_ctr --namespace "$NERDCTL_NAMESPACE" leases delete "$lease" >/dev/null || status=1 done <<< "$leases" digests="$(runtime_ctr --namespace "$NERDCTL_NAMESPACE" content list --quiet)" || status=1 while IFS= read -r digest; do [[ -z "$digest" ]] || runtime_ctr --namespace "$NERDCTL_NAMESPACE" content remove "$digest" >/dev/null || status=1 done <<< "$digests" "$NUTHATCH_REAL_NERDCTL" namespace remove "$NERDCTL_NAMESPACE" >/dev/null || status=1 runtime_namespace_exists && status=1 return "$status" } prepare_provider() { configure_provider delete_cluster remove_runtime_namespace "$NUTHATCH_REAL_NERDCTL" namespace create "$NERDCTL_NAMESPACE" >/dev/null } load_external_image() { local name="$1" local source="$2" local digest="${source##*@}" local image="nuthatch-testbed/$name:sha256-${digest#sha256:}" local archive="$RUNTIME/image-archives/$digest.tar" "$NUTHATCH_REAL_NERDCTL" image pull --platform linux/amd64 --unpack=false --quiet "$source" >&2 "$NUTHATCH_REAL_NERDCTL" image tag "$source" "$image" "$NUTHATCH_REAL_NERDCTL" image save --platform linux/amd64 --output "$archive" "$image" printf 'Loading %s from %s\n' "$image" "$source" >&2 "$BIN/kind" load image-archive "$archive" --name "$CLUSTER" >&2 printf '%s\n' "$image" } render_manifests() { local postgres="$1" redis="$2" dragonfly="$3" keycloak="$4" zitadel="$5" local garage="$6" versitygw="$7" coredns="$8" curl="$9" aws_cli="${10}" sed \ -e "s|@@POSTGRES_IMAGE@@|$postgres|g" \ -e "s|@@REDIS_IMAGE@@|$redis|g" \ -e "s|@@DRAGONFLY_IMAGE@@|$dragonfly|g" \ -e "s|@@KEYCLOAK_IMAGE@@|$keycloak|g" \ -e "s|@@ZITADEL_IMAGE@@|$zitadel|g" \ -e "s|@@GARAGE_IMAGE@@|$garage|g" \ -e "s|@@VERSITYGW_IMAGE@@|$versitygw|g" \ -e "s|@@COREDNS_IMAGE@@|$coredns|g" \ -e "s|@@AWS_CLI_IMAGE@@|$aws_cli|g" \ "$TESTBED/dependencies.yaml.in" > "$RUNTIME/dependencies.yaml" sed \ -e "s|@@POSTGRES_IMAGE@@|$postgres|g" \ -e "s|@@REDIS_IMAGE@@|$redis|g" \ -e "s|@@CURL_IMAGE@@|$curl|g" \ -e "s|@@AWS_CLI_IMAGE@@|$aws_cli|g" \ "$TESTBED/probe.yaml.in" > "$RUNTIME/probe.yaml" } kube() { "$BIN/kubectl" \ --kubeconfig "$KUBECONFIG_PATH" \ --context "$CONTEXT" \ "$@" } wait_for_job() { local job="$1" local deadline=$((SECONDS + 300)) local complete local failed while ((SECONDS < deadline)); do complete="$(kube --namespace "$FIXTURE_NAMESPACE" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Complete")].status}')" if [[ "$complete" = True ]]; then kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" --all-containers return fi failed="$(kube --namespace "$FIXTURE_NAMESPACE" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Failed")].status}')" if [[ "$failed" = True ]]; then kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" --all-containers >&2 || true return 1 fi sleep 1 done printf 'job/%s did not complete within 300 seconds\n' "$job" >&2 return 1 } setup_fixture() { mkdir -p "$RUNTIME" rm -rf "$RUNTIME/image-archives" mkdir -p "$RUNTIME/image-archives" prepare_tools prepare_provider "$BIN/kind" create cluster \ --name "$CLUSTER" \ --image "$NODE_IMAGE" \ --config "$TESTBED/cluster.yaml" \ --kubeconfig "$KUBECONFIG_PATH" \ --wait 180s local postgres redis dragonfly keycloak zitadel garage versitygw coredns curl aws_cli postgres="$(load_external_image postgres "$POSTGRES_IMAGE")" redis="$(load_external_image redis "$REDIS_IMAGE")" dragonfly="$(load_external_image dragonfly "$DRAGONFLY_IMAGE")" keycloak="$(load_external_image keycloak "$KEYCLOAK_IMAGE")" zitadel="$(load_external_image zitadel "$ZITADEL_IMAGE")" garage="$(load_external_image garage "$GARAGE_IMAGE")" versitygw="$(load_external_image versitygw "$VERSITYGW_IMAGE")" coredns="$(load_external_image coredns "$COREDNS_IMAGE")" curl="$(load_external_image curl "$CURL_IMAGE")" aws_cli="$(load_external_image aws-cli "$AWS_CLI_IMAGE")" render_manifests "$postgres" "$redis" "$dragonfly" "$keycloak" "$zitadel" \ "$garage" "$versitygw" "$coredns" "$curl" "$aws_cli" kube apply --filename "$RUNTIME/dependencies.yaml" local deployment for deployment in postgres redis dragonfly keycloak zitadel garage versitygw controlled-dns; do kube --namespace "$FIXTURE_NAMESPACE" rollout status "deployment/$deployment" --timeout 240s done } verify_dependencies() { kube apply --filename "$RUNTIME/probe.yaml" local job for job in state-probe oidc-probe garage-probe versitygw-probe; do wait_for_job "$job" done } destroy_fixture() { configure_provider delete_cluster remove_runtime_namespace rm -rf "$RUNTIME/image-archives" rm -f "$KUBECONFIG_PATH" "$RUNTIME/dependencies.yaml" "$RUNTIME/probe.yaml" }