674 lines
22 KiB
Shell
Executable file
674 lines
22 KiB
Shell
Executable file
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
TESTBED="$ROOT/tests/kind"
|
|
RUNTIME="$TESTBED/.runtime"
|
|
BIN="$RUNTIME/bin"
|
|
KUBECONFIG_PATH="$RUNTIME/kubeconfig"
|
|
FIXTURE_LOCK_DIR="${XDG_RUNTIME_DIR:-/tmp/nuthatch-$(id -u)}"
|
|
FIXTURE_LOCK_PATH="$FIXTURE_LOCK_DIR/nuthatch-testbed-kind.lock"
|
|
CONTEXT=kind-nuthatch-testbed
|
|
CLUSTER=nuthatch-testbed
|
|
NERDCTL_NAMESPACE=nuthatch-testbed
|
|
FIXTURE_NAMESPACE=nuthatch-testbed
|
|
|
|
set -a
|
|
source "$TESTBED/versions.env"
|
|
set +a
|
|
|
|
user_kubeconfig_digest() {
|
|
local config_value="${KUBECONFIG:-${HOME:?}/.kube/config}"
|
|
local path
|
|
local paths
|
|
IFS=: read -r -a paths <<< "$config_value"
|
|
{
|
|
for path in "${paths[@]}"; do
|
|
if [[ -f "$path" ]]; then
|
|
sha256sum "$path"
|
|
else
|
|
printf 'missing %s\n' "$path"
|
|
fi
|
|
done
|
|
} | sha256sum | cut -d' ' -f1
|
|
}
|
|
|
|
download_tool() {
|
|
local destination="$1"
|
|
local checksum="$2"
|
|
local url="$3"
|
|
if [[ -x "$destination" ]] &&
|
|
[[ "$(sha256sum "$destination" | cut -d' ' -f1)" = "$checksum" ]]; then
|
|
return
|
|
fi
|
|
curl --fail --location --silent --show-error "$url" --output "$destination.tmp"
|
|
test "$(sha256sum "$destination.tmp" | cut -d' ' -f1)" = "$checksum"
|
|
chmod 0755 "$destination.tmp"
|
|
mv "$destination.tmp" "$destination"
|
|
}
|
|
|
|
prepare_tools() {
|
|
mkdir -p "$BIN"
|
|
download_tool \
|
|
"$BIN/kind" \
|
|
"$KIND_SHA256" \
|
|
"https://github.com/kubernetes-sigs/kind/releases/download/$KIND_VERSION/kind-linux-amd64"
|
|
download_tool \
|
|
"$BIN/kubectl" \
|
|
"$KUBECTL_SHA256" \
|
|
"https://dl.k8s.io/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl"
|
|
test "$("$BIN/kind" version | awk '{print $2}')" = "$KIND_VERSION"
|
|
test "$("$BIN/kubectl" --kubeconfig "$KUBECONFIG_PATH" --context "$CONTEXT" version --client=true -o json | jq -r .clientVersion.gitVersion)" = "$KUBECTL_VERSION"
|
|
test "$(nerdctl --version | awk '{print $3}')" = "$NERDCTL_VERSION"
|
|
test "$(runtime_ctr version | awk '/Client:/{client=1} client && $1=="Version:"{print $2; exit}')" = "v$CONTAINERD_VERSION"
|
|
test "$(runtime_ctr version | awk '/Server:/{server=1} server && $1=="Version:"{print $2; exit}')" = "v$CONTAINERD_VERSION"
|
|
}
|
|
|
|
acquire_fixture_lock() {
|
|
mkdir -p "$RUNTIME" "$FIXTURE_LOCK_DIR"
|
|
exec 9>"$FIXTURE_LOCK_PATH"
|
|
if ! flock --nonblock 9; then
|
|
printf 'kind test fixture is already in use\n' >&2
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
configure_provider() {
|
|
if [[ -z "${NUTHATCH_REAL_NERDCTL:-}" ]]; then
|
|
NUTHATCH_REAL_NERDCTL="$(command -v nerdctl)" || return
|
|
fi
|
|
export NUTHATCH_REAL_NERDCTL
|
|
export CONTAINERD_NAMESPACE="$NERDCTL_NAMESPACE"
|
|
export KIND_EXPERIMENTAL_PROVIDER=nerdctl
|
|
export PATH="$TESTBED/provider:$PATH"
|
|
}
|
|
|
|
runtime_ctr() {
|
|
ctr --address "${CONTAINERD_ADDRESS:-${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/containerd/containerd.sock}" "$@"
|
|
}
|
|
|
|
runtime_namespace_exists() {
|
|
"$NUTHATCH_REAL_NERDCTL" namespace list --quiet | grep -Fx "$NERDCTL_NAMESPACE" >/dev/null
|
|
}
|
|
|
|
cluster_exists() {
|
|
runtime_namespace_exists &&
|
|
"$BIN/kind" get clusters | grep -Fx "$CLUSTER" >/dev/null
|
|
}
|
|
|
|
delete_cluster() {
|
|
local container
|
|
local containers
|
|
if cluster_exists; then
|
|
"$BIN/kind" delete cluster --name "$CLUSTER" || true
|
|
fi
|
|
containers="$("$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" ps \
|
|
--all \
|
|
--quiet \
|
|
--filter "label=io.x-k8s.kind.cluster=$CLUSTER")" || return
|
|
while IFS= read -r container; do
|
|
[[ -z "$container" ]] ||
|
|
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" rm --force "$container" ||
|
|
return
|
|
done <<< "$containers"
|
|
if cluster_exists; then
|
|
"$BIN/kind" delete cluster --name "$CLUSTER" || true
|
|
fi
|
|
if cluster_exists; then
|
|
printf 'kind cluster %s still exists after teardown\n' "$CLUSTER" >&2
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
remove_runtime_namespace() {
|
|
local status=0
|
|
local digest
|
|
local digests
|
|
local lease
|
|
local leases
|
|
local network
|
|
for network in kind nuthatch-kind; do
|
|
if "$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" network inspect "$network" >/dev/null 2>&1; then
|
|
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" network remove "$network" >/dev/null || status=1
|
|
fi
|
|
done
|
|
if ! runtime_namespace_exists; then
|
|
return "$status"
|
|
fi
|
|
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" image prune --all --force >/dev/null || status=1
|
|
leases="$(runtime_ctr --namespace "$NERDCTL_NAMESPACE" leases list --quiet)" || status=1
|
|
while IFS= read -r lease; do
|
|
[[ -z "$lease" ]] || runtime_ctr --namespace "$NERDCTL_NAMESPACE" leases delete "$lease" >/dev/null || status=1
|
|
done <<< "$leases"
|
|
digests="$(runtime_ctr --namespace "$NERDCTL_NAMESPACE" content list --quiet)" || status=1
|
|
while IFS= read -r digest; do
|
|
[[ -z "$digest" ]] || runtime_ctr --namespace "$NERDCTL_NAMESPACE" content remove "$digest" >/dev/null || status=1
|
|
done <<< "$digests"
|
|
"$NUTHATCH_REAL_NERDCTL" namespace remove "$NERDCTL_NAMESPACE" >/dev/null || status=1
|
|
runtime_namespace_exists && status=1
|
|
return "$status"
|
|
}
|
|
|
|
prepare_provider() {
|
|
configure_provider
|
|
delete_cluster
|
|
remove_runtime_namespace
|
|
"$NUTHATCH_REAL_NERDCTL" namespace create "$NERDCTL_NAMESPACE" >/dev/null
|
|
}
|
|
|
|
load_external_image() {
|
|
local name="$1"
|
|
local source="$2"
|
|
local digest="${source##*@}"
|
|
local image="nuthatch-testbed/$name:sha256-${digest#sha256:}"
|
|
local archive="$RUNTIME/image-archives/$digest.tar"
|
|
"$NUTHATCH_REAL_NERDCTL" image pull --platform linux/amd64 --unpack=false --quiet "$source" >&2
|
|
"$NUTHATCH_REAL_NERDCTL" image tag "$source" "$image"
|
|
"$NUTHATCH_REAL_NERDCTL" image save --platform linux/amd64 --output "$archive" "$image"
|
|
printf 'Loading %s from %s\n' "$image" "$source" >&2
|
|
"$BIN/kind" load image-archive "$archive" --name "$CLUSTER" >&2
|
|
printf '%s\n' "$image"
|
|
}
|
|
|
|
load_local_image() {
|
|
local image="$1"
|
|
local namespace="${2:-default}"
|
|
local archive="$RUNTIME/image-archives/nuthatch-app.tar"
|
|
rm -f "$archive"
|
|
"$NUTHATCH_REAL_NERDCTL" --namespace "$namespace" image save \
|
|
--platform linux/amd64 \
|
|
--output "$archive" \
|
|
"$image"
|
|
"$BIN/kind" load image-archive "$archive" --name "$CLUSTER"
|
|
}
|
|
|
|
render_manifests() {
|
|
local postgres="$1" redis="$2" dragonfly="$3" keycloak="$4" zitadel="$5"
|
|
local garage="$6" versitygw="$7" coredns="$8" curl="$9" aws_cli="${10}"
|
|
local postgres_replicas="${11}" redis_replicas="${12}" dragonfly_replicas="${13}"
|
|
local keycloak_replicas="${14}" zitadel_replicas="${15}"
|
|
local garage_replicas="${16}" versitygw_replicas="${17}" coredns_replicas="${18}"
|
|
sed \
|
|
-e "s|@@POSTGRES_IMAGE@@|$postgres|g" \
|
|
-e "s|@@REDIS_IMAGE@@|$redis|g" \
|
|
-e "s|@@DRAGONFLY_IMAGE@@|$dragonfly|g" \
|
|
-e "s|@@KEYCLOAK_IMAGE@@|$keycloak|g" \
|
|
-e "s|@@ZITADEL_IMAGE@@|$zitadel|g" \
|
|
-e "s|@@GARAGE_IMAGE@@|$garage|g" \
|
|
-e "s|@@VERSITYGW_IMAGE@@|$versitygw|g" \
|
|
-e "s|@@COREDNS_IMAGE@@|$coredns|g" \
|
|
-e "s|@@AWS_CLI_IMAGE@@|$aws_cli|g" \
|
|
-e "s|@@POSTGRES_REPLICAS@@|$postgres_replicas|g" \
|
|
-e "s|@@REDIS_REPLICAS@@|$redis_replicas|g" \
|
|
-e "s|@@DRAGONFLY_REPLICAS@@|$dragonfly_replicas|g" \
|
|
-e "s|@@KEYCLOAK_REPLICAS@@|$keycloak_replicas|g" \
|
|
-e "s|@@ZITADEL_REPLICAS@@|$zitadel_replicas|g" \
|
|
-e "s|@@GARAGE_REPLICAS@@|$garage_replicas|g" \
|
|
-e "s|@@VERSITYGW_REPLICAS@@|$versitygw_replicas|g" \
|
|
-e "s|@@COREDNS_REPLICAS@@|$coredns_replicas|g" \
|
|
"$TESTBED/dependencies.yaml.in" > "$RUNTIME/dependencies.yaml"
|
|
sed \
|
|
-e "s|@@POSTGRES_IMAGE@@|$postgres|g" \
|
|
-e "s|@@REDIS_IMAGE@@|$redis|g" \
|
|
-e "s|@@CURL_IMAGE@@|$curl|g" \
|
|
-e "s|@@AWS_CLI_IMAGE@@|$aws_cli|g" \
|
|
"$TESTBED/probe.yaml.in" > "$RUNTIME/probe.yaml"
|
|
{
|
|
printf 'POSTGRES_LOCAL_IMAGE=%q\n' "$postgres"
|
|
printf 'REDIS_LOCAL_IMAGE=%q\n' "$redis"
|
|
printf 'DRAGONFLY_LOCAL_IMAGE=%q\n' "$dragonfly"
|
|
printf 'KEYCLOAK_LOCAL_IMAGE=%q\n' "$keycloak"
|
|
printf 'ZITADEL_LOCAL_IMAGE=%q\n' "$zitadel"
|
|
printf 'GARAGE_LOCAL_IMAGE=%q\n' "$garage"
|
|
printf 'VERSITYGW_LOCAL_IMAGE=%q\n' "$versitygw"
|
|
printf 'COREDNS_LOCAL_IMAGE=%q\n' "$coredns"
|
|
printf 'CURL_LOCAL_IMAGE=%q\n' "$curl"
|
|
printf 'AWS_CLI_LOCAL_IMAGE=%q\n' "$aws_cli"
|
|
} > "$RUNTIME/fixture.env"
|
|
}
|
|
|
|
kube() {
|
|
"$BIN/kubectl" \
|
|
--kubeconfig "$KUBECONFIG_PATH" \
|
|
--context "$CONTEXT" \
|
|
"$@"
|
|
}
|
|
|
|
wait_for_job() {
|
|
local job="$1"
|
|
local deadline=$((SECONDS + 300))
|
|
local complete
|
|
local failed
|
|
while ((SECONDS < deadline)); do
|
|
complete="$(kube --namespace "$FIXTURE_NAMESPACE" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Complete")].status}')"
|
|
if [[ "$complete" = True ]]; then
|
|
kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" --all-containers
|
|
return
|
|
fi
|
|
failed="$(kube --namespace "$FIXTURE_NAMESPACE" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Failed")].status}')"
|
|
if [[ "$failed" = True ]]; then
|
|
kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" --all-containers >&2 || true
|
|
return 1
|
|
fi
|
|
sleep 1
|
|
done
|
|
printf 'job/%s did not complete within 300 seconds\n' "$job" >&2
|
|
return 1
|
|
}
|
|
|
|
select_fixture_families() {
|
|
local requested=("$@")
|
|
local family
|
|
local candidate
|
|
local found
|
|
local use_all=false
|
|
FIXTURE_FAMILIES=()
|
|
if [[ "${#requested[@]}" = 0 ]]; then
|
|
requested=(all)
|
|
fi
|
|
for family in "${requested[@]}"; do
|
|
case "$family" in
|
|
all) use_all=true ;;
|
|
base | durable | coordination | oidc | target) ;;
|
|
*)
|
|
printf 'unknown fixture family: %s\n' "$family" >&2
|
|
return 1
|
|
;;
|
|
esac
|
|
done
|
|
if [[ "$use_all" = true ]]; then
|
|
requested=(durable coordination oidc target)
|
|
fi
|
|
for candidate in durable coordination oidc target; do
|
|
found=false
|
|
for family in "${requested[@]}"; do
|
|
if [[ "$family" = "$candidate" ]]; then
|
|
found=true
|
|
fi
|
|
done
|
|
if [[ "$found" = true ]]; then
|
|
FIXTURE_FAMILIES+=("$candidate")
|
|
fi
|
|
done
|
|
}
|
|
|
|
fixture_has_family() {
|
|
local expected="$1"
|
|
local family
|
|
for family in "${FIXTURE_FAMILIES[@]}"; do
|
|
if [[ "$family" = "$expected" ]]; then
|
|
return
|
|
fi
|
|
done
|
|
return 1
|
|
}
|
|
|
|
expand_fixture_matrix() {
|
|
local coordination_providers=(-)
|
|
local oidc_providers=(-)
|
|
local target_providers=(-)
|
|
local coordination
|
|
local oidc
|
|
local target
|
|
local lane=0
|
|
select_fixture_families "$@" || return
|
|
if fixture_has_family coordination; then
|
|
coordination_providers=(redis dragonfly)
|
|
fi
|
|
if fixture_has_family oidc; then
|
|
oidc_providers=(keycloak zitadel)
|
|
fi
|
|
if fixture_has_family target; then
|
|
target_providers=(garage versitygw)
|
|
fi
|
|
for coordination in "${coordination_providers[@]}"; do
|
|
for oidc in "${oidc_providers[@]}"; do
|
|
for target in "${target_providers[@]}"; do
|
|
lane=$((lane + 1))
|
|
printf '%03d\t%s\t%s\t%s\n' "$lane" "$coordination" "$oidc" "$target"
|
|
done
|
|
done
|
|
done
|
|
}
|
|
|
|
setup_fixture() {
|
|
select_fixture_families "$@" || return
|
|
mkdir -p "$RUNTIME"
|
|
rm -rf "$RUNTIME/image-archives"
|
|
mkdir -p "$RUNTIME/image-archives"
|
|
prepare_tools
|
|
prepare_provider
|
|
"$BIN/kind" create cluster \
|
|
--name "$CLUSTER" \
|
|
--image "$NODE_IMAGE" \
|
|
--config "$TESTBED/cluster.yaml" \
|
|
--kubeconfig "$KUBECONFIG_PATH" \
|
|
--wait 180s
|
|
local postgres=nuthatch-testbed/unused:postgres
|
|
local redis=nuthatch-testbed/unused:redis
|
|
local dragonfly=nuthatch-testbed/unused:dragonfly
|
|
local keycloak=nuthatch-testbed/unused:keycloak
|
|
local zitadel=nuthatch-testbed/unused:zitadel
|
|
local garage=nuthatch-testbed/unused:garage
|
|
local versitygw=nuthatch-testbed/unused:versitygw
|
|
local coredns=nuthatch-testbed/unused:coredns
|
|
local curl=nuthatch-testbed/unused:curl
|
|
local aws_cli=nuthatch-testbed/unused:aws-cli
|
|
local postgres_replicas=0 redis_replicas=0 dragonfly_replicas=0
|
|
local keycloak_replicas=0 zitadel_replicas=0
|
|
local garage_replicas=0 versitygw_replicas=0 coredns_replicas=0
|
|
if fixture_has_family durable || fixture_has_family oidc; then
|
|
postgres="$(load_external_image postgres "$POSTGRES_IMAGE")"
|
|
postgres_replicas=1
|
|
fi
|
|
if fixture_has_family coordination; then
|
|
redis="$(load_external_image redis "$REDIS_IMAGE")"
|
|
dragonfly="$(load_external_image dragonfly "$DRAGONFLY_IMAGE")"
|
|
redis_replicas=1
|
|
dragonfly_replicas=1
|
|
fi
|
|
if fixture_has_family oidc; then
|
|
keycloak="$(load_external_image keycloak "$KEYCLOAK_IMAGE")"
|
|
zitadel="$(load_external_image zitadel "$ZITADEL_IMAGE")"
|
|
coredns="$(load_external_image coredns "$COREDNS_IMAGE")"
|
|
curl="$(load_external_image curl "$CURL_IMAGE")"
|
|
keycloak_replicas=1
|
|
zitadel_replicas=1
|
|
coredns_replicas=1
|
|
fi
|
|
if fixture_has_family target; then
|
|
garage="$(load_external_image garage "$GARAGE_IMAGE")"
|
|
versitygw="$(load_external_image versitygw "$VERSITYGW_IMAGE")"
|
|
aws_cli="$(load_external_image aws-cli "$AWS_CLI_IMAGE")"
|
|
if [[ "$curl" = nuthatch-testbed/unused:curl ]]; then
|
|
curl="$(load_external_image curl "$CURL_IMAGE")"
|
|
fi
|
|
garage_replicas=1
|
|
versitygw_replicas=1
|
|
fi
|
|
render_manifests "$postgres" "$redis" "$dragonfly" "$keycloak" "$zitadel" \
|
|
"$garage" "$versitygw" "$coredns" "$curl" "$aws_cli" \
|
|
"$postgres_replicas" "$redis_replicas" "$dragonfly_replicas" \
|
|
"$keycloak_replicas" "$zitadel_replicas" \
|
|
"$garage_replicas" "$versitygw_replicas" "$coredns_replicas"
|
|
kube apply --filename "$RUNTIME/dependencies.yaml"
|
|
if fixture_has_family durable || fixture_has_family oidc; then
|
|
wait_for_deployment postgres
|
|
fi
|
|
if fixture_has_family coordination; then
|
|
wait_for_deployment redis
|
|
wait_for_deployment dragonfly
|
|
fi
|
|
if fixture_has_family oidc; then
|
|
wait_for_deployment keycloak
|
|
wait_for_deployment zitadel
|
|
wait_for_deployment controlled-dns
|
|
fi
|
|
if fixture_has_family target; then
|
|
wait_for_deployment garage
|
|
wait_for_deployment versitygw
|
|
fi
|
|
}
|
|
|
|
wait_for_deployment() {
|
|
kube --namespace "$FIXTURE_NAMESPACE" rollout status "deployment/$1" --timeout 240s
|
|
}
|
|
|
|
verify_dependencies() {
|
|
kube apply --filename "$RUNTIME/probe.yaml"
|
|
local job
|
|
for job in state-probe oidc-probe garage-probe versitygw-probe; do
|
|
wait_for_job "$job"
|
|
done
|
|
}
|
|
|
|
reset_lane_durable_store() {
|
|
local lane="$1"
|
|
local database="nuthatch_lane_$lane"
|
|
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
|
|
env PGPASSWORD=synthetic-postgres psql \
|
|
--username nuthatch \
|
|
--dbname postgres \
|
|
--set ON_ERROR_STOP=1 \
|
|
--command "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$database'" \
|
|
--command "DROP DATABASE IF EXISTS $database" \
|
|
--command "CREATE DATABASE $database" >&2 || return
|
|
printf '%s\n' "$database"
|
|
}
|
|
|
|
reset_lane_coordination_store() {
|
|
local provider="$1"
|
|
local lane="$2"
|
|
local database=$((10#$lane))
|
|
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" -- \
|
|
redis-cli --raw --no-auth-warning -n "$database" FLUSHDB >/dev/null || return
|
|
printf '%s\n' "$database"
|
|
}
|
|
|
|
target_endpoint() {
|
|
case "$1" in
|
|
garage) printf 'http://garage:3900\n' ;;
|
|
versitygw) printf 'http://versitygw:10000\n' ;;
|
|
*) return 1 ;;
|
|
esac
|
|
}
|
|
|
|
target_loopback_endpoint() {
|
|
case "$1" in
|
|
garage) printf 'http://127.0.0.1:3900\n' ;;
|
|
versitygw) printf 'http://127.0.0.1:10000\n' ;;
|
|
*) return 1 ;;
|
|
esac
|
|
}
|
|
|
|
reset_lane_target_store() {
|
|
local provider="$1"
|
|
local bucket="$2"
|
|
local endpoint
|
|
endpoint="$(target_loopback_endpoint "$provider")" || return
|
|
if ! kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
|
|
aws --endpoint-url "$endpoint" s3api create-bucket --bucket "$bucket" >&2; then
|
|
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
|
|
aws --endpoint-url "$endpoint" s3api head-bucket --bucket "$bucket" >/dev/null || return
|
|
fi
|
|
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
|
|
aws --endpoint-url "$endpoint" s3 rm "s3://$bucket" --recursive >/dev/null || return
|
|
}
|
|
|
|
write_lane_profile() {
|
|
local lane="$1"
|
|
local coordination="$2"
|
|
local oidc="$3"
|
|
local target="$4"
|
|
local profile="nuthatch-fixture-lane-$lane"
|
|
local profile_file="$RUNTIME/$profile.env"
|
|
local database=
|
|
local coordination_database=
|
|
local oidc_issuer=
|
|
local target_url=
|
|
local target_bucket=
|
|
local target_region=
|
|
local target_access_key=
|
|
local target_secret_key=
|
|
if fixture_has_family durable; then
|
|
database="$(reset_lane_durable_store "$lane")" || return
|
|
fi
|
|
if fixture_has_family coordination; then
|
|
coordination_database="$(reset_lane_coordination_store "$coordination" "$lane")" || return
|
|
fi
|
|
if fixture_has_family oidc; then
|
|
case "$oidc" in
|
|
keycloak) oidc_issuer=http://keycloak.nuthatch.test:8080/realms/master ;;
|
|
zitadel) oidc_issuer=http://zitadel.nuthatch.test:8080 ;;
|
|
*) return 1 ;;
|
|
esac
|
|
fi
|
|
if fixture_has_family target; then
|
|
target_bucket="nuthatch-lane-$lane"
|
|
reset_lane_target_store "$target" "$target_bucket" || return
|
|
target_url="$(target_endpoint "$target")" || return
|
|
case "$target" in
|
|
garage)
|
|
target_region=garage
|
|
target_access_key=GK00000000000000000000000000000000
|
|
target_secret_key=0000000000000000000000000000000000000000000000000000000000000000
|
|
;;
|
|
versitygw)
|
|
target_region=us-east-1
|
|
target_access_key=synthetic-s3-access
|
|
target_secret_key=synthetic-s3-secret
|
|
;;
|
|
*) return 1 ;;
|
|
esac
|
|
fi
|
|
{
|
|
printf 'NUTHATCH_FIXTURE_PROFILE=%s\n' "$profile"
|
|
printf 'NUTHATCH_FIXTURE_LANE=%s\n' "$lane"
|
|
printf 'NUTHATCH_COORDINATION_PROVIDER=%s\n' "$coordination"
|
|
printf 'NUTHATCH_OIDC_PROVIDER=%s\n' "$oidc"
|
|
printf 'NUTHATCH_TARGET_PROVIDER=%s\n' "$target"
|
|
printf 'NUTHATCH_PG_CONN=host=postgres port=5432 user=nuthatch password=synthetic-postgres dbname=%s\n' "$database"
|
|
printf 'NUTHATCH_REDIS_URL=redis://%s:6379/%s\n' "$coordination" "$coordination_database"
|
|
printf 'NUTHATCH_OIDC_ISSUER=%s\n' "$oidc_issuer"
|
|
printf 'NUTHATCH_S3_ENDPOINT=%s\n' "$target_url"
|
|
printf 'NUTHATCH_S3_BUCKET=%s\n' "$target_bucket"
|
|
printf 'NUTHATCH_S3_REGION=%s\n' "$target_region"
|
|
printf 'NUTHATCH_S3_ACCESS_KEY=%s\n' "$target_access_key"
|
|
printf 'NUTHATCH_S3_SECRET_KEY=%s\n' "$target_secret_key"
|
|
printf 'NUTHATCH_S3_PATH_STYLE=true\n'
|
|
} > "$profile_file" || return
|
|
kube create configmap "$profile" \
|
|
--namespace "$FIXTURE_NAMESPACE" \
|
|
--from-env-file="$profile_file" \
|
|
--dry-run=client \
|
|
--output yaml |
|
|
kube apply --filename - >/dev/null || return
|
|
kube create configmap "$profile-file" \
|
|
--namespace "$FIXTURE_NAMESPACE" \
|
|
--from-file="profile.env=$profile_file" \
|
|
--dry-run=client \
|
|
--output yaml |
|
|
kube apply --filename - >/dev/null || return
|
|
printf '%s\n' "$profile"
|
|
}
|
|
|
|
render_lane_job() {
|
|
local job="$1"
|
|
local manifest="$2"
|
|
local image="$3"
|
|
local profile="$4"
|
|
source "$RUNTIME/fixture.env" || return
|
|
local coordination_image="$REDIS_LOCAL_IMAGE"
|
|
sed \
|
|
-e "s|@@JOB_NAME@@|$job|g" \
|
|
-e "s|@@PROFILE_NAME@@|$profile|g" \
|
|
-e "s|@@TEST_IMAGE@@|$image|g" \
|
|
-e "s|@@POSTGRES_IMAGE@@|$POSTGRES_LOCAL_IMAGE|g" \
|
|
-e "s|@@COORDINATION_IMAGE@@|$coordination_image|g" \
|
|
-e "s|@@CURL_IMAGE@@|$CURL_LOCAL_IMAGE|g" \
|
|
-e "s|@@AWS_CLI_IMAGE@@|$AWS_CLI_LOCAL_IMAGE|g" \
|
|
"$manifest" > "$RUNTIME/$job.yaml"
|
|
}
|
|
|
|
run_job_manifest() {
|
|
local job="$1"
|
|
local manifest="$2"
|
|
kube --namespace "$FIXTURE_NAMESPACE" delete "job/$job" --ignore-not-found >/dev/null || return
|
|
kube apply --filename "$manifest" >/dev/null || return
|
|
wait_for_job "$job"
|
|
}
|
|
|
|
cleanup_lane_durable_store() {
|
|
local lane="$1"
|
|
local database="nuthatch_lane_$lane"
|
|
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
|
|
env PGPASSWORD=synthetic-postgres psql \
|
|
--username nuthatch \
|
|
--dbname postgres \
|
|
--set ON_ERROR_STOP=1 \
|
|
--command "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$database'" \
|
|
--command "DROP DATABASE IF EXISTS $database" >/dev/null
|
|
}
|
|
|
|
cleanup_lane_coordination_store() {
|
|
local provider="$1"
|
|
local lane="$2"
|
|
local database=$((10#$lane))
|
|
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" -- \
|
|
redis-cli --raw --no-auth-warning -n "$database" FLUSHDB >/dev/null
|
|
}
|
|
|
|
cleanup_lane_target_store() {
|
|
local provider="$1"
|
|
local bucket="$2"
|
|
local endpoint
|
|
endpoint="$(target_loopback_endpoint "$provider")" || return
|
|
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
|
|
aws --endpoint-url "$endpoint" s3 rb "s3://$bucket" --force >/dev/null
|
|
}
|
|
|
|
cleanup_lane_state() {
|
|
local lane="$1"
|
|
local coordination="$2"
|
|
local target="$4"
|
|
local status=0
|
|
if fixture_has_family durable; then
|
|
cleanup_lane_durable_store "$lane" || status=1
|
|
fi
|
|
if fixture_has_family coordination; then
|
|
cleanup_lane_coordination_store "$coordination" "$lane" || status=1
|
|
fi
|
|
if fixture_has_family target; then
|
|
cleanup_lane_target_store "$target" "nuthatch-lane-$lane" || status=1
|
|
fi
|
|
return "$status"
|
|
}
|
|
|
|
cleanup_lane_resources() {
|
|
local job="$1"
|
|
local profile="$2"
|
|
kube --namespace "$FIXTURE_NAMESPACE" delete \
|
|
"job/$job" \
|
|
"configmap/$profile" \
|
|
"configmap/$profile-file" \
|
|
--ignore-not-found >/dev/null || return
|
|
rm -f "$RUNTIME/$profile.env" "$RUNTIME/$job.yaml"
|
|
}
|
|
|
|
run_job_matrix() {
|
|
local base_job="$1"
|
|
local manifest="$2"
|
|
local image="$3"
|
|
shift 3
|
|
local lane
|
|
local coordination
|
|
local oidc
|
|
local target
|
|
local profile
|
|
local job
|
|
select_fixture_families "$@" || return
|
|
while IFS=$'\t' read -r lane coordination oidc target; do
|
|
profile="$(write_lane_profile "$lane" "$coordination" "$oidc" "$target")" || return
|
|
job="$base_job-$lane"
|
|
render_lane_job "$job" "$manifest" "$image" "$profile" || return
|
|
printf 'Running lane %s: coordination=%s oidc=%s target=%s\n' \
|
|
"$lane" "$coordination" "$oidc" "$target"
|
|
run_job_manifest "$job" "$RUNTIME/$job.yaml" || return
|
|
cleanup_lane_state "$lane" "$coordination" "$oidc" "$target" || return
|
|
cleanup_lane_resources "$job" "$profile" || return
|
|
done < <(expand_fixture_matrix "$@")
|
|
}
|
|
|
|
destroy_fixture() {
|
|
local status=0
|
|
configure_provider || return
|
|
delete_cluster || status=1
|
|
remove_runtime_namespace || status=1
|
|
rm -rf "$RUNTIME/image-archives" || status=1
|
|
rm -f \
|
|
"$KUBECONFIG_PATH" \
|
|
"$RUNTIME/dependencies.yaml" \
|
|
"$RUNTIME/probe.yaml" \
|
|
"$RUNTIME/fixture.env" || status=1
|
|
return "$status"
|
|
}
|