Add isolated kind dependency testbed
This commit is contained in:
parent
74ac822d43
commit
5759938f3f
1 changed files with 1155 additions and 0 deletions
269
tests/kind/lib.sh
Executable file
269
tests/kind/lib.sh
Executable file
|
|
@ -0,0 +1,269 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
TESTBED="$ROOT/tests/kind"
|
||||
RUNTIME="$TESTBED/.runtime"
|
||||
BIN="$RUNTIME/bin"
|
||||
KUBECONFIG_PATH="$RUNTIME/kubeconfig"
|
||||
FIXTURE_LOCK_DIR="${XDG_RUNTIME_DIR:-/tmp/nuthatch-$(id -u)}"
|
||||
FIXTURE_LOCK_PATH="$FIXTURE_LOCK_DIR/nuthatch-testbed-kind.lock"
|
||||
CONTEXT=kind-nuthatch-testbed
|
||||
CLUSTER=nuthatch-testbed
|
||||
NERDCTL_NAMESPACE=nuthatch-testbed
|
||||
FIXTURE_NAMESPACE=nuthatch-testbed
|
||||
|
||||
set -a
|
||||
source "$TESTBED/versions.env"
|
||||
set +a
|
||||
|
||||
user_kubeconfig_digest() {
|
||||
local config_value="${KUBECONFIG:-${HOME:?}/.kube/config}"
|
||||
local path
|
||||
local paths
|
||||
IFS=: read -r -a paths <<< "$config_value"
|
||||
{
|
||||
for path in "${paths[@]}"; do
|
||||
if [[ -f "$path" ]]; then
|
||||
sha256sum "$path"
|
||||
else
|
||||
printf 'missing %s\n' "$path"
|
||||
fi
|
||||
done
|
||||
} | sha256sum | cut -d' ' -f1
|
||||
}
|
||||
|
||||
download_tool() {
|
||||
local destination="$1"
|
||||
local checksum="$2"
|
||||
local url="$3"
|
||||
if [[ -x "$destination" ]] &&
|
||||
[[ "$(sha256sum "$destination" | cut -d' ' -f1)" = "$checksum" ]]; then
|
||||
return
|
||||
fi
|
||||
curl --fail --location --silent --show-error "$url" --output "$destination.tmp"
|
||||
test "$(sha256sum "$destination.tmp" | cut -d' ' -f1)" = "$checksum"
|
||||
chmod 0755 "$destination.tmp"
|
||||
mv "$destination.tmp" "$destination"
|
||||
}
|
||||
|
||||
prepare_tools() {
|
||||
mkdir -p "$BIN"
|
||||
download_tool \
|
||||
"$BIN/kind" \
|
||||
"$KIND_SHA256" \
|
||||
"https://github.com/kubernetes-sigs/kind/releases/download/$KIND_VERSION/kind-linux-amd64"
|
||||
download_tool \
|
||||
"$BIN/kubectl" \
|
||||
"$KUBECTL_SHA256" \
|
||||
"https://dl.k8s.io/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl"
|
||||
test "$("$BIN/kind" version | awk '{print $2}')" = "$KIND_VERSION"
|
||||
test "$("$BIN/kubectl" --kubeconfig "$KUBECONFIG_PATH" --context "$CONTEXT" version --client=true -o json | jq -r .clientVersion.gitVersion)" = "$KUBECTL_VERSION"
|
||||
test "$(nerdctl --version | awk '{print $3}')" = "$NERDCTL_VERSION"
|
||||
test "$(runtime_ctr version | awk '/Client:/{client=1} client && $1=="Version:"{print $2; exit}')" = "v$CONTAINERD_VERSION"
|
||||
test "$(runtime_ctr version | awk '/Server:/{server=1} server && $1=="Version:"{print $2; exit}')" = "v$CONTAINERD_VERSION"
|
||||
}
|
||||
|
||||
acquire_fixture_lock() {
|
||||
mkdir -p "$RUNTIME" "$FIXTURE_LOCK_DIR"
|
||||
exec 9>"$FIXTURE_LOCK_PATH"
|
||||
if ! flock --nonblock 9; then
|
||||
printf 'kind test fixture is already in use\n' >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
configure_provider() {
|
||||
if [[ -z "${NUTHATCH_REAL_NERDCTL:-}" ]]; then
|
||||
NUTHATCH_REAL_NERDCTL="$(command -v nerdctl)"
|
||||
fi
|
||||
export NUTHATCH_REAL_NERDCTL
|
||||
export CONTAINERD_NAMESPACE="$NERDCTL_NAMESPACE"
|
||||
export KIND_EXPERIMENTAL_PROVIDER=nerdctl
|
||||
export PATH="$TESTBED/provider:$PATH"
|
||||
}
|
||||
|
||||
runtime_ctr() {
|
||||
ctr --address "${CONTAINERD_ADDRESS:-${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/containerd/containerd.sock}" "$@"
|
||||
}
|
||||
|
||||
runtime_namespace_exists() {
|
||||
"$NUTHATCH_REAL_NERDCTL" namespace list --quiet | grep -Fx "$NERDCTL_NAMESPACE" >/dev/null
|
||||
}
|
||||
|
||||
cluster_exists() {
|
||||
runtime_namespace_exists &&
|
||||
"$BIN/kind" get clusters | grep -Fx "$CLUSTER" >/dev/null
|
||||
}
|
||||
|
||||
delete_cluster() {
|
||||
local container
|
||||
local containers
|
||||
if cluster_exists; then
|
||||
"$BIN/kind" delete cluster --name "$CLUSTER" || true
|
||||
fi
|
||||
containers="$("$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" ps \
|
||||
--all \
|
||||
--quiet \
|
||||
--filter "label=io.x-k8s.kind.cluster=$CLUSTER")"
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] ||
|
||||
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" rm --force "$container"
|
||||
done <<< "$containers"
|
||||
if cluster_exists; then
|
||||
"$BIN/kind" delete cluster --name "$CLUSTER" || true
|
||||
fi
|
||||
if cluster_exists; then
|
||||
printf 'kind cluster %s still exists after teardown\n' "$CLUSTER" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
remove_runtime_namespace() {
|
||||
local status=0
|
||||
local digest
|
||||
local digests
|
||||
local lease
|
||||
local leases
|
||||
local network
|
||||
for network in kind nuthatch-kind; do
|
||||
if "$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" network inspect "$network" >/dev/null 2>&1; then
|
||||
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" network remove "$network" >/dev/null || status=1
|
||||
fi
|
||||
done
|
||||
if ! runtime_namespace_exists; then
|
||||
return "$status"
|
||||
fi
|
||||
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" image prune --all --force >/dev/null || status=1
|
||||
leases="$(runtime_ctr --namespace "$NERDCTL_NAMESPACE" leases list --quiet)" || status=1
|
||||
while IFS= read -r lease; do
|
||||
[[ -z "$lease" ]] || runtime_ctr --namespace "$NERDCTL_NAMESPACE" leases delete "$lease" >/dev/null || status=1
|
||||
done <<< "$leases"
|
||||
digests="$(runtime_ctr --namespace "$NERDCTL_NAMESPACE" content list --quiet)" || status=1
|
||||
while IFS= read -r digest; do
|
||||
[[ -z "$digest" ]] || runtime_ctr --namespace "$NERDCTL_NAMESPACE" content remove "$digest" >/dev/null || status=1
|
||||
done <<< "$digests"
|
||||
"$NUTHATCH_REAL_NERDCTL" namespace remove "$NERDCTL_NAMESPACE" >/dev/null || status=1
|
||||
runtime_namespace_exists && status=1
|
||||
return "$status"
|
||||
}
|
||||
|
||||
prepare_provider() {
|
||||
configure_provider
|
||||
delete_cluster
|
||||
remove_runtime_namespace
|
||||
"$NUTHATCH_REAL_NERDCTL" namespace create "$NERDCTL_NAMESPACE" >/dev/null
|
||||
}
|
||||
|
||||
load_external_image() {
|
||||
local name="$1"
|
||||
local source="$2"
|
||||
local digest="${source##*@}"
|
||||
local image="nuthatch-testbed/$name:sha256-${digest#sha256:}"
|
||||
local archive="$RUNTIME/image-archives/$digest.tar"
|
||||
"$NUTHATCH_REAL_NERDCTL" image pull --platform linux/amd64 --unpack=false --quiet "$source" >&2
|
||||
"$NUTHATCH_REAL_NERDCTL" image tag "$source" "$image"
|
||||
"$NUTHATCH_REAL_NERDCTL" image save --platform linux/amd64 --output "$archive" "$image"
|
||||
printf 'Loading %s from %s\n' "$image" "$source" >&2
|
||||
"$BIN/kind" load image-archive "$archive" --name "$CLUSTER" >&2
|
||||
printf '%s\n' "$image"
|
||||
}
|
||||
|
||||
render_manifests() {
|
||||
local postgres="$1" redis="$2" dragonfly="$3" keycloak="$4" zitadel="$5"
|
||||
local garage="$6" versitygw="$7" coredns="$8" curl="$9" aws_cli="${10}"
|
||||
sed \
|
||||
-e "s|@@POSTGRES_IMAGE@@|$postgres|g" \
|
||||
-e "s|@@REDIS_IMAGE@@|$redis|g" \
|
||||
-e "s|@@DRAGONFLY_IMAGE@@|$dragonfly|g" \
|
||||
-e "s|@@KEYCLOAK_IMAGE@@|$keycloak|g" \
|
||||
-e "s|@@ZITADEL_IMAGE@@|$zitadel|g" \
|
||||
-e "s|@@GARAGE_IMAGE@@|$garage|g" \
|
||||
-e "s|@@VERSITYGW_IMAGE@@|$versitygw|g" \
|
||||
-e "s|@@COREDNS_IMAGE@@|$coredns|g" \
|
||||
-e "s|@@AWS_CLI_IMAGE@@|$aws_cli|g" \
|
||||
"$TESTBED/dependencies.yaml.in" > "$RUNTIME/dependencies.yaml"
|
||||
sed \
|
||||
-e "s|@@POSTGRES_IMAGE@@|$postgres|g" \
|
||||
-e "s|@@REDIS_IMAGE@@|$redis|g" \
|
||||
-e "s|@@CURL_IMAGE@@|$curl|g" \
|
||||
-e "s|@@AWS_CLI_IMAGE@@|$aws_cli|g" \
|
||||
"$TESTBED/probe.yaml.in" > "$RUNTIME/probe.yaml"
|
||||
}
|
||||
|
||||
kube() {
|
||||
"$BIN/kubectl" \
|
||||
--kubeconfig "$KUBECONFIG_PATH" \
|
||||
--context "$CONTEXT" \
|
||||
"$@"
|
||||
}
|
||||
|
||||
wait_for_job() {
|
||||
local job="$1"
|
||||
local deadline=$((SECONDS + 300))
|
||||
local complete
|
||||
local failed
|
||||
while ((SECONDS < deadline)); do
|
||||
complete="$(kube --namespace "$FIXTURE_NAMESPACE" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Complete")].status}')"
|
||||
if [[ "$complete" = True ]]; then
|
||||
kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" --all-containers
|
||||
return
|
||||
fi
|
||||
failed="$(kube --namespace "$FIXTURE_NAMESPACE" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Failed")].status}')"
|
||||
if [[ "$failed" = True ]]; then
|
||||
kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" --all-containers >&2 || true
|
||||
return 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
printf 'job/%s did not complete within 300 seconds\n' "$job" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
setup_fixture() {
|
||||
mkdir -p "$RUNTIME"
|
||||
rm -rf "$RUNTIME/image-archives"
|
||||
mkdir -p "$RUNTIME/image-archives"
|
||||
prepare_tools
|
||||
prepare_provider
|
||||
"$BIN/kind" create cluster \
|
||||
--name "$CLUSTER" \
|
||||
--image "$NODE_IMAGE" \
|
||||
--config "$TESTBED/cluster.yaml" \
|
||||
--kubeconfig "$KUBECONFIG_PATH" \
|
||||
--wait 180s
|
||||
local postgres redis dragonfly keycloak zitadel garage versitygw coredns curl aws_cli
|
||||
postgres="$(load_external_image postgres "$POSTGRES_IMAGE")"
|
||||
redis="$(load_external_image redis "$REDIS_IMAGE")"
|
||||
dragonfly="$(load_external_image dragonfly "$DRAGONFLY_IMAGE")"
|
||||
keycloak="$(load_external_image keycloak "$KEYCLOAK_IMAGE")"
|
||||
zitadel="$(load_external_image zitadel "$ZITADEL_IMAGE")"
|
||||
garage="$(load_external_image garage "$GARAGE_IMAGE")"
|
||||
versitygw="$(load_external_image versitygw "$VERSITYGW_IMAGE")"
|
||||
coredns="$(load_external_image coredns "$COREDNS_IMAGE")"
|
||||
curl="$(load_external_image curl "$CURL_IMAGE")"
|
||||
aws_cli="$(load_external_image aws-cli "$AWS_CLI_IMAGE")"
|
||||
render_manifests "$postgres" "$redis" "$dragonfly" "$keycloak" "$zitadel" \
|
||||
"$garage" "$versitygw" "$coredns" "$curl" "$aws_cli"
|
||||
kube apply --filename "$RUNTIME/dependencies.yaml"
|
||||
local deployment
|
||||
for deployment in postgres redis dragonfly keycloak zitadel garage versitygw controlled-dns; do
|
||||
kube --namespace "$FIXTURE_NAMESPACE" rollout status "deployment/$deployment" --timeout 240s
|
||||
done
|
||||
}
|
||||
|
||||
verify_dependencies() {
|
||||
kube apply --filename "$RUNTIME/probe.yaml"
|
||||
local job
|
||||
for job in state-probe oidc-probe garage-probe versitygw-probe; do
|
||||
wait_for_job "$job"
|
||||
done
|
||||
}
|
||||
|
||||
destroy_fixture() {
|
||||
configure_provider
|
||||
delete_cluster
|
||||
remove_runtime_namespace
|
||||
rm -rf "$RUNTIME/image-archives"
|
||||
rm -f "$KUBECONFIG_PATH" "$RUNTIME/dependencies.yaml" "$RUNTIME/probe.yaml"
|
||||
}
|
||||
Loading…
Reference in a new issue