Split kind tests into component-owned probes
This commit is contained in:
parent
b3c0f6d58a
commit
691ad92e30
1 changed files with 4269 additions and 3108 deletions
|
|
@ -1,921 +1,4 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
TESTBED="$ROOT/tests/kind"
|
||||
RUNTIME="$TESTBED/.runtime"
|
||||
BIN="$RUNTIME/bin"
|
||||
KUBECONFIG_PATH="$RUNTIME/kubeconfig"
|
||||
FIXTURE_LOCK_DIR="${XDG_RUNTIME_DIR:-/tmp/nuthatch-$(id -u)}"
|
||||
FIXTURE_LOCK_PATH="$FIXTURE_LOCK_DIR/nuthatch-testbed-kind.lock"
|
||||
CONTEXT=kind-nuthatch-testbed
|
||||
CLUSTER=nuthatch-testbed
|
||||
NERDCTL_NAMESPACE=nuthatch-testbed
|
||||
FIXTURE_NAMESPACE=nuthatch-testbed
|
||||
|
||||
set -a
|
||||
source "$TESTBED/versions.env"
|
||||
set +a
|
||||
|
||||
user_kubeconfig_digest() {
|
||||
local config_value="${KUBECONFIG:-${HOME:?}/.kube/config}"
|
||||
local path
|
||||
local paths
|
||||
IFS=: read -r -a paths <<< "$config_value"
|
||||
{
|
||||
for path in "${paths[@]}"; do
|
||||
if [[ -f "$path" ]]; then
|
||||
sha256sum "$path"
|
||||
else
|
||||
printf 'missing %s\n' "$path"
|
||||
fi
|
||||
done
|
||||
} | sha256sum | cut -d' ' -f1
|
||||
}
|
||||
|
||||
download_tool() {
|
||||
local destination="$1"
|
||||
local checksum="$2"
|
||||
local url="$3"
|
||||
if [[ -x "$destination" ]] &&
|
||||
[[ "$(sha256sum "$destination" | cut -d' ' -f1)" = "$checksum" ]]; then
|
||||
return
|
||||
fi
|
||||
curl --fail --location --silent --show-error "$url" --output "$destination.tmp"
|
||||
test "$(sha256sum "$destination.tmp" | cut -d' ' -f1)" = "$checksum"
|
||||
chmod 0755 "$destination.tmp"
|
||||
mv "$destination.tmp" "$destination"
|
||||
}
|
||||
|
||||
prepare_tools() {
|
||||
mkdir -p "$BIN"
|
||||
download_tool \
|
||||
"$BIN/kind" \
|
||||
"$KIND_SHA256" \
|
||||
"https://github.com/kubernetes-sigs/kind/releases/download/$KIND_VERSION/kind-linux-amd64"
|
||||
download_tool \
|
||||
"$BIN/kubectl" \
|
||||
"$KUBECTL_SHA256" \
|
||||
"https://dl.k8s.io/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl"
|
||||
test "$("$BIN/kind" version | awk '{print $2}')" = "$KIND_VERSION"
|
||||
test "$("$BIN/kubectl" --kubeconfig "$KUBECONFIG_PATH" --context "$CONTEXT" version --client=true -o json | jq -r .clientVersion.gitVersion)" = "$KUBECTL_VERSION"
|
||||
test "$(nerdctl --version | awk '{print $3}')" = "$NERDCTL_VERSION"
|
||||
test "$(runtime_ctr version | awk '/Client:/{client=1} client && $1=="Version:"{print $2; exit}')" = "v$CONTAINERD_VERSION"
|
||||
test "$(runtime_ctr version | awk '/Server:/{server=1} server && $1=="Version:"{print $2; exit}')" = "v$CONTAINERD_VERSION"
|
||||
}
|
||||
|
||||
acquire_fixture_lock() {
|
||||
mkdir -p "$RUNTIME" "$FIXTURE_LOCK_DIR"
|
||||
exec 9>"$FIXTURE_LOCK_PATH"
|
||||
if ! flock --nonblock 9; then
|
||||
printf 'kind test fixture is already in use\n' >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
configure_provider() {
|
||||
if [[ -z "${NUTHATCH_REAL_NERDCTL:-}" ]]; then
|
||||
NUTHATCH_REAL_NERDCTL="$(command -v nerdctl)" || return
|
||||
fi
|
||||
export NUTHATCH_REAL_NERDCTL
|
||||
export CONTAINERD_NAMESPACE="$NERDCTL_NAMESPACE"
|
||||
export KIND_EXPERIMENTAL_PROVIDER=nerdctl
|
||||
export PATH="$TESTBED/provider:$PATH"
|
||||
}
|
||||
|
||||
runtime_ctr() {
|
||||
ctr --address "${CONTAINERD_ADDRESS:-${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/containerd/containerd.sock}" "$@"
|
||||
}
|
||||
|
||||
runtime_namespace_exists() {
|
||||
"$NUTHATCH_REAL_NERDCTL" namespace list --quiet | grep -Fx "$NERDCTL_NAMESPACE" >/dev/null
|
||||
}
|
||||
|
||||
cluster_exists() {
|
||||
runtime_namespace_exists &&
|
||||
"$BIN/kind" get clusters | grep -Fx "$CLUSTER" >/dev/null
|
||||
}
|
||||
|
||||
delete_cluster() {
|
||||
local container
|
||||
local containers
|
||||
if cluster_exists; then
|
||||
"$BIN/kind" delete cluster --name "$CLUSTER" || true
|
||||
fi
|
||||
containers="$("$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" ps \
|
||||
--all \
|
||||
--quiet \
|
||||
--filter "label=io.x-k8s.kind.cluster=$CLUSTER")" || return
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] ||
|
||||
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" rm --force "$container" ||
|
||||
return
|
||||
done <<< "$containers"
|
||||
if cluster_exists; then
|
||||
"$BIN/kind" delete cluster --name "$CLUSTER" || true
|
||||
fi
|
||||
if cluster_exists; then
|
||||
printf 'kind cluster %s still exists after teardown\n' "$CLUSTER" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
remove_runtime_namespace() {
|
||||
local status=0
|
||||
local digest
|
||||
local digests
|
||||
local lease
|
||||
local leases
|
||||
local network
|
||||
for network in kind nuthatch-kind; do
|
||||
if "$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" network inspect "$network" >/dev/null 2>&1; then
|
||||
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" network remove "$network" >/dev/null || status=1
|
||||
fi
|
||||
done
|
||||
if ! runtime_namespace_exists; then
|
||||
return "$status"
|
||||
fi
|
||||
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" image prune --all --force >/dev/null || status=1
|
||||
leases="$(runtime_ctr --namespace "$NERDCTL_NAMESPACE" leases list --quiet)" || status=1
|
||||
while IFS= read -r lease; do
|
||||
[[ -z "$lease" ]] || runtime_ctr --namespace "$NERDCTL_NAMESPACE" leases delete "$lease" >/dev/null || status=1
|
||||
done <<< "$leases"
|
||||
digests="$(runtime_ctr --namespace "$NERDCTL_NAMESPACE" content list --quiet)" || status=1
|
||||
while IFS= read -r digest; do
|
||||
[[ -z "$digest" ]] || runtime_ctr --namespace "$NERDCTL_NAMESPACE" content remove "$digest" >/dev/null || status=1
|
||||
done <<< "$digests"
|
||||
"$NUTHATCH_REAL_NERDCTL" namespace remove "$NERDCTL_NAMESPACE" >/dev/null || status=1
|
||||
runtime_namespace_exists && status=1
|
||||
return "$status"
|
||||
}
|
||||
|
||||
prepare_provider() {
|
||||
configure_provider
|
||||
delete_cluster
|
||||
remove_runtime_namespace
|
||||
"$NUTHATCH_REAL_NERDCTL" namespace create "$NERDCTL_NAMESPACE" >/dev/null
|
||||
}
|
||||
|
||||
load_external_image() {
|
||||
local name="$1"
|
||||
local source="$2"
|
||||
local digest="${source##*@}"
|
||||
local image="nuthatch-testbed/$name:sha256-${digest#sha256:}"
|
||||
local archive="$RUNTIME/image-archives/$digest.tar"
|
||||
"$NUTHATCH_REAL_NERDCTL" image pull --platform linux/amd64 --unpack=false --quiet "$source" >&2
|
||||
"$NUTHATCH_REAL_NERDCTL" image tag "$source" "$image"
|
||||
"$NUTHATCH_REAL_NERDCTL" image save --platform linux/amd64 --output "$archive" "$image"
|
||||
printf 'Loading %s from %s\n' "$image" "$source" >&2
|
||||
"$BIN/kind" load image-archive "$archive" --name "$CLUSTER" >&2
|
||||
printf '%s\n' "$image"
|
||||
}
|
||||
|
||||
load_local_image() {
|
||||
local image="$1"
|
||||
local namespace="${2:-default}"
|
||||
local archive="$RUNTIME/image-archives/nuthatch-app.tar"
|
||||
rm -f "$archive"
|
||||
"$NUTHATCH_REAL_NERDCTL" --namespace "$namespace" image save \
|
||||
--platform linux/amd64 \
|
||||
--output "$archive" \
|
||||
"$image"
|
||||
"$BIN/kind" load image-archive "$archive" --name "$CLUSTER"
|
||||
}
|
||||
|
||||
render_manifests() {
|
||||
local postgres="$1" redis="$2" dragonfly="$3" keycloak="$4" zitadel="$5"
|
||||
local garage="$6" versitygw="$7" coredns="$8" curl="$9" aws_cli="${10}"
|
||||
local postgres_replicas="${11}" redis_replicas="${12}" dragonfly_replicas="${13}"
|
||||
local keycloak_replicas="${14}" zitadel_replicas="${15}"
|
||||
local garage_replicas="${16}" versitygw_replicas="${17}" coredns_replicas="${18}"
|
||||
sed \
|
||||
-e "s|@@POSTGRES_IMAGE@@|$postgres|g" \
|
||||
-e "s|@@REDIS_IMAGE@@|$redis|g" \
|
||||
-e "s|@@DRAGONFLY_IMAGE@@|$dragonfly|g" \
|
||||
-e "s|@@KEYCLOAK_IMAGE@@|$keycloak|g" \
|
||||
-e "s|@@ZITADEL_IMAGE@@|$zitadel|g" \
|
||||
-e "s|@@GARAGE_IMAGE@@|$garage|g" \
|
||||
-e "s|@@VERSITYGW_IMAGE@@|$versitygw|g" \
|
||||
-e "s|@@COREDNS_IMAGE@@|$coredns|g" \
|
||||
-e "s|@@CURL_IMAGE@@|$curl|g" \
|
||||
-e "s|@@AWS_CLI_IMAGE@@|$aws_cli|g" \
|
||||
-e "s|@@POSTGRES_REPLICAS@@|$postgres_replicas|g" \
|
||||
-e "s|@@REDIS_REPLICAS@@|$redis_replicas|g" \
|
||||
-e "s|@@DRAGONFLY_REPLICAS@@|$dragonfly_replicas|g" \
|
||||
-e "s|@@KEYCLOAK_REPLICAS@@|$keycloak_replicas|g" \
|
||||
-e "s|@@ZITADEL_REPLICAS@@|$zitadel_replicas|g" \
|
||||
-e "s|@@GARAGE_REPLICAS@@|$garage_replicas|g" \
|
||||
-e "s|@@VERSITYGW_REPLICAS@@|$versitygw_replicas|g" \
|
||||
-e "s|@@COREDNS_REPLICAS@@|$coredns_replicas|g" \
|
||||
"$TESTBED/dependencies.yaml.in" > "$RUNTIME/dependencies.yaml"
|
||||
sed \
|
||||
-e "s|@@POSTGRES_IMAGE@@|$postgres|g" \
|
||||
-e "s|@@REDIS_IMAGE@@|$redis|g" \
|
||||
-e "s|@@CURL_IMAGE@@|$curl|g" \
|
||||
-e "s|@@AWS_CLI_IMAGE@@|$aws_cli|g" \
|
||||
"$TESTBED/probe.yaml.in" > "$RUNTIME/probe.yaml"
|
||||
{
|
||||
printf 'POSTGRES_LOCAL_IMAGE=%q\n' "$postgres"
|
||||
printf 'REDIS_LOCAL_IMAGE=%q\n' "$redis"
|
||||
printf 'DRAGONFLY_LOCAL_IMAGE=%q\n' "$dragonfly"
|
||||
printf 'KEYCLOAK_LOCAL_IMAGE=%q\n' "$keycloak"
|
||||
printf 'ZITADEL_LOCAL_IMAGE=%q\n' "$zitadel"
|
||||
printf 'GARAGE_LOCAL_IMAGE=%q\n' "$garage"
|
||||
printf 'VERSITYGW_LOCAL_IMAGE=%q\n' "$versitygw"
|
||||
printf 'COREDNS_LOCAL_IMAGE=%q\n' "$coredns"
|
||||
printf 'CURL_LOCAL_IMAGE=%q\n' "$curl"
|
||||
printf 'AWS_CLI_LOCAL_IMAGE=%q\n' "$aws_cli"
|
||||
} > "$RUNTIME/fixture.env"
|
||||
}
|
||||
|
||||
kube() {
|
||||
"$BIN/kubectl" \
|
||||
--kubeconfig "$KUBECONFIG_PATH" \
|
||||
--context "$CONTEXT" \
|
||||
"$@"
|
||||
}
|
||||
|
||||
wait_for_job() {
|
||||
local job="$1"
|
||||
local deadline=$((SECONDS + 300))
|
||||
local complete
|
||||
local failed
|
||||
while ((SECONDS < deadline)); do
|
||||
complete="$(kube --namespace "$FIXTURE_NAMESPACE" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Complete")].status}')"
|
||||
if [[ "$complete" = True ]]; then
|
||||
kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" --all-containers
|
||||
return
|
||||
fi
|
||||
failed="$(kube --namespace "$FIXTURE_NAMESPACE" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Failed")].status}')"
|
||||
if [[ "$failed" = True ]]; then
|
||||
kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" --all-containers >&2 || true
|
||||
return 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
printf 'job/%s did not complete within 300 seconds\n' "$job" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
select_fixture_families() {
|
||||
local requested=("$@")
|
||||
local family
|
||||
local candidate
|
||||
local found
|
||||
local use_all=false
|
||||
FIXTURE_FAMILIES=()
|
||||
if [[ "${#requested[@]}" = 0 ]]; then
|
||||
requested=(all)
|
||||
fi
|
||||
for family in "${requested[@]}"; do
|
||||
case "$family" in
|
||||
all) use_all=true ;;
|
||||
base | durable | coordination | oidc | target) ;;
|
||||
*)
|
||||
printf 'unknown fixture family: %s\n' "$family" >&2
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
if [[ "$use_all" = true ]]; then
|
||||
requested=(durable coordination oidc target)
|
||||
fi
|
||||
for candidate in durable coordination oidc target; do
|
||||
found=false
|
||||
for family in "${requested[@]}"; do
|
||||
if [[ "$family" = "$candidate" ]]; then
|
||||
found=true
|
||||
fi
|
||||
done
|
||||
if [[ "$found" = true ]]; then
|
||||
FIXTURE_FAMILIES+=("$candidate")
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
fixture_has_family() {
|
||||
local expected="$1"
|
||||
local family
|
||||
for family in "${FIXTURE_FAMILIES[@]}"; do
|
||||
if [[ "$family" = "$expected" ]]; then
|
||||
return
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
expand_fixture_matrix() {
|
||||
local coordination_providers=(-)
|
||||
local oidc_providers=(-)
|
||||
local target_providers=(-)
|
||||
local coordination
|
||||
local oidc
|
||||
local target
|
||||
local lane=0
|
||||
select_fixture_families "$@" || return
|
||||
if fixture_has_family coordination; then
|
||||
coordination_providers=(redis dragonfly)
|
||||
fi
|
||||
if fixture_has_family oidc; then
|
||||
oidc_providers=(keycloak zitadel)
|
||||
fi
|
||||
if fixture_has_family target; then
|
||||
target_providers=(garage versitygw)
|
||||
fi
|
||||
for coordination in "${coordination_providers[@]}"; do
|
||||
for oidc in "${oidc_providers[@]}"; do
|
||||
for target in "${target_providers[@]}"; do
|
||||
lane=$((lane + 1))
|
||||
printf '%03d\t%s\t%s\t%s\n' "$lane" "$coordination" "$oidc" "$target"
|
||||
done
|
||||
done
|
||||
done
|
||||
}
|
||||
|
||||
setup_fixture() {
|
||||
select_fixture_families "$@" || return
|
||||
mkdir -p "$RUNTIME"
|
||||
rm -rf "$RUNTIME/image-archives"
|
||||
mkdir -p "$RUNTIME/image-archives"
|
||||
prepare_tools
|
||||
prepare_provider
|
||||
"$BIN/kind" create cluster \
|
||||
--name "$CLUSTER" \
|
||||
--image "$NODE_IMAGE" \
|
||||
--config "$TESTBED/cluster.yaml" \
|
||||
--kubeconfig "$KUBECONFIG_PATH" \
|
||||
--wait 180s
|
||||
local postgres=nuthatch-testbed/unused:postgres
|
||||
local redis=nuthatch-testbed/unused:redis
|
||||
local dragonfly=nuthatch-testbed/unused:dragonfly
|
||||
local keycloak=nuthatch-testbed/unused:keycloak
|
||||
local zitadel=nuthatch-testbed/unused:zitadel
|
||||
local garage=nuthatch-testbed/unused:garage
|
||||
local versitygw=nuthatch-testbed/unused:versitygw
|
||||
local coredns=nuthatch-testbed/unused:coredns
|
||||
local curl=nuthatch-testbed/unused:curl
|
||||
local aws_cli=nuthatch-testbed/unused:aws-cli
|
||||
local postgres_replicas=0 redis_replicas=0 dragonfly_replicas=0
|
||||
local keycloak_replicas=0 zitadel_replicas=0
|
||||
local garage_replicas=0 versitygw_replicas=0 coredns_replicas=0
|
||||
if fixture_has_family durable || fixture_has_family oidc; then
|
||||
postgres="$(load_external_image postgres "$POSTGRES_IMAGE")"
|
||||
postgres_replicas=1
|
||||
fi
|
||||
if fixture_has_family coordination; then
|
||||
redis="$(load_external_image redis "$REDIS_IMAGE")"
|
||||
dragonfly="$(load_external_image dragonfly "$DRAGONFLY_IMAGE")"
|
||||
redis_replicas=1
|
||||
dragonfly_replicas=1
|
||||
fi
|
||||
if fixture_has_family oidc; then
|
||||
keycloak="$(load_external_image keycloak "$KEYCLOAK_IMAGE")"
|
||||
zitadel="$(load_external_image zitadel "$ZITADEL_IMAGE")"
|
||||
coredns="$(load_external_image coredns "$COREDNS_IMAGE")"
|
||||
curl="$(load_external_image curl "$CURL_IMAGE")"
|
||||
keycloak_replicas=1
|
||||
zitadel_replicas=1
|
||||
coredns_replicas=1
|
||||
fi
|
||||
if fixture_has_family target; then
|
||||
garage="$(load_external_image garage "$GARAGE_IMAGE")"
|
||||
versitygw="$(load_external_image versitygw "$VERSITYGW_IMAGE")"
|
||||
aws_cli="$(load_external_image aws-cli "$AWS_CLI_IMAGE")"
|
||||
if [[ "$curl" = nuthatch-testbed/unused:curl ]]; then
|
||||
curl="$(load_external_image curl "$CURL_IMAGE")"
|
||||
fi
|
||||
garage_replicas=1
|
||||
versitygw_replicas=1
|
||||
fi
|
||||
render_manifests "$postgres" "$redis" "$dragonfly" "$keycloak" "$zitadel" \
|
||||
"$garage" "$versitygw" "$coredns" "$curl" "$aws_cli" \
|
||||
"$postgres_replicas" "$redis_replicas" "$dragonfly_replicas" \
|
||||
"$keycloak_replicas" "$zitadel_replicas" \
|
||||
"$garage_replicas" "$versitygw_replicas" "$coredns_replicas"
|
||||
kube apply --filename "$RUNTIME/dependencies.yaml"
|
||||
if fixture_has_family durable || fixture_has_family oidc; then
|
||||
wait_for_deployment postgres
|
||||
fi
|
||||
if fixture_has_family coordination; then
|
||||
wait_for_deployment redis
|
||||
wait_for_deployment dragonfly
|
||||
fi
|
||||
if fixture_has_family oidc; then
|
||||
wait_for_deployment keycloak
|
||||
wait_for_deployment zitadel
|
||||
wait_for_deployment controlled-dns
|
||||
fi
|
||||
if fixture_has_family target; then
|
||||
wait_for_deployment garage
|
||||
wait_for_deployment versitygw
|
||||
fi
|
||||
}
|
||||
|
||||
wait_for_deployment() {
|
||||
kube --namespace "$FIXTURE_NAMESPACE" rollout status "deployment/$1" --timeout 240s
|
||||
}
|
||||
|
||||
verify_dependencies() {
|
||||
kube apply --filename "$RUNTIME/probe.yaml"
|
||||
local job
|
||||
for job in state-probe oidc-probe garage-probe versitygw-probe; do
|
||||
wait_for_job "$job"
|
||||
done
|
||||
}
|
||||
|
||||
reset_lane_durable_store() {
|
||||
local lane="$1"
|
||||
local database="nuthatch_lane_$lane"
|
||||
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
|
||||
env PGPASSWORD=synthetic-postgres psql \
|
||||
--username nuthatch \
|
||||
--dbname postgres \
|
||||
--set ON_ERROR_STOP=1 \
|
||||
--command "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$database'" \
|
||||
--command "DROP DATABASE IF EXISTS $database" \
|
||||
--command "CREATE DATABASE $database" >&2 || return
|
||||
printf '%s\n' "$database"
|
||||
}
|
||||
|
||||
reset_lane_coordination_store() {
|
||||
local provider="$1"
|
||||
local lane="$2"
|
||||
local database=$((10#$lane))
|
||||
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" -- \
|
||||
redis-cli --raw --no-auth-warning -n "$database" FLUSHDB >/dev/null || return
|
||||
printf '%s\n' "$database"
|
||||
}
|
||||
|
||||
target_endpoint() {
|
||||
case "$1" in
|
||||
garage) printf 'http://garage:3900\n' ;;
|
||||
versitygw) printf 'http://versitygw:10000\n' ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
target_loopback_endpoint() {
|
||||
case "$1" in
|
||||
garage) printf 'http://127.0.0.1:3900\n' ;;
|
||||
versitygw) printf 'http://127.0.0.1:10000\n' ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
ensure_target_scope_canary() {
|
||||
local provider="$1"
|
||||
local endpoint
|
||||
endpoint="$(target_loopback_endpoint "$provider")" || return
|
||||
if ! kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
|
||||
aws --endpoint-url "$endpoint" s3api create-bucket --bucket nuthatch-fixture >&2; then
|
||||
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
|
||||
aws --endpoint-url "$endpoint" s3api head-bucket --bucket nuthatch-fixture >/dev/null || return
|
||||
fi
|
||||
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
|
||||
sh -ec 'printf scope-canary >/tmp/scope-canary && aws --endpoint-url "$1" s3api put-object --bucket nuthatch-fixture --key scope-canary --body /tmp/scope-canary >/dev/null' \
|
||||
sh "$endpoint"
|
||||
}
|
||||
|
||||
provision_lane_target_principal() {
|
||||
local provider="$1"
|
||||
local lane="$2"
|
||||
local bucket="$3"
|
||||
local access_key_var="$4"
|
||||
local secret_key_var="$5"
|
||||
local endpoint
|
||||
local access_key
|
||||
local secret_key
|
||||
endpoint="$(target_loopback_endpoint "$provider")" || return
|
||||
case "$provider" in
|
||||
garage)
|
||||
printf -v access_key 'GKNUTHATCH%024d' "$((10#$lane))"
|
||||
printf -v secret_key '%064d' "$((10#$lane))"
|
||||
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/garage --container garage -- \
|
||||
/garage bucket create "$bucket" >/dev/null || return
|
||||
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/garage --container garage -- \
|
||||
/garage bucket allow \
|
||||
--read \
|
||||
--write \
|
||||
--owner \
|
||||
"$bucket" \
|
||||
--key GK00000000000000000000000000000000 >/dev/null || return
|
||||
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/garage --container garage -- \
|
||||
/garage key import --yes -n "nuthatch-$lane" "$access_key" "$secret_key" >/dev/null || return
|
||||
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/garage --container garage -- \
|
||||
/garage bucket allow --read --write "$bucket" --key "$access_key" >/dev/null || return
|
||||
;;
|
||||
versitygw)
|
||||
access_key="nuthatch-lane-$lane"
|
||||
printf -v secret_key 'synthetic-lane-%049d' "$((10#$lane))"
|
||||
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/versitygw --container versitygw -- \
|
||||
/usr/local/bin/versitygw admin \
|
||||
--endpoint-url http://127.0.0.1:10001 \
|
||||
create-user \
|
||||
--access "$access_key" \
|
||||
--secret "$secret_key" \
|
||||
--role user \
|
||||
--user-id "$((10#$lane))" \
|
||||
--group-id "$((10#$lane))" >/dev/null || return
|
||||
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/versitygw --container versitygw -- \
|
||||
/usr/local/bin/versitygw admin \
|
||||
--endpoint-url http://127.0.0.1:10001 \
|
||||
create-bucket \
|
||||
--bucket "$bucket" \
|
||||
--owner "$access_key" >/dev/null || return
|
||||
;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
printf -v "$access_key_var" '%s' "$access_key"
|
||||
printf -v "$secret_key_var" '%s' "$secret_key"
|
||||
}
|
||||
|
||||
keycloak_admin_token() {
|
||||
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
|
||||
sh -ec '
|
||||
/opt/keycloak/bin/kcadm.sh config credentials \
|
||||
--server http://127.0.0.1:8080 \
|
||||
--realm master \
|
||||
--user nuthatch-admin \
|
||||
--password synthetic-keycloak >/dev/null
|
||||
/opt/keycloak/bin/kcadm.sh get serverinfo >/dev/null
|
||||
' >/dev/null
|
||||
}
|
||||
|
||||
provision_lane_keycloak_registration() {
|
||||
local lane="$1"
|
||||
local redirect_uri="$2"
|
||||
local client_id_var="$3"
|
||||
local client_secret_var="$4"
|
||||
local additional_audiences_var="$5"
|
||||
local user_a_var="$6"
|
||||
local user_b_var="$7"
|
||||
local client_id="nuthatch-$lane"
|
||||
local client_secret="synthetic-keycloak-client-secret-$lane"
|
||||
local username
|
||||
keycloak_admin_token || return
|
||||
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
|
||||
/opt/keycloak/bin/kcadm.sh create clients \
|
||||
--realm master \
|
||||
-s "clientId=$client_id" \
|
||||
-s enabled=true \
|
||||
-s protocol=openid-connect \
|
||||
-s publicClient=false \
|
||||
-s clientAuthenticatorType=client-secret \
|
||||
-s standardFlowEnabled=true \
|
||||
-s directAccessGrantsEnabled=false \
|
||||
-s "secret=$client_secret" \
|
||||
-s "redirectUris=[\"$redirect_uri\"]" \
|
||||
-s 'attributes={"pkce.code.challenge.method":"S256"}' >/dev/null || return
|
||||
for username in "nuthatch-$lane-a" "nuthatch-$lane-b"; do
|
||||
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
|
||||
/opt/keycloak/bin/kcadm.sh create users \
|
||||
--realm master \
|
||||
-s "username=$username" \
|
||||
-s enabled=true \
|
||||
-s "firstName=$username" >/dev/null || return
|
||||
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
|
||||
/opt/keycloak/bin/kcadm.sh set-password \
|
||||
--realm master \
|
||||
--username "$username" \
|
||||
--new-password Synthetic-OIDC-1! >/dev/null || return
|
||||
done
|
||||
printf -v "$client_id_var" '%s' "$client_id"
|
||||
printf -v "$client_secret_var" '%s' "$client_secret"
|
||||
printf -v "$additional_audiences_var" ''
|
||||
printf -v "$user_a_var" '%s' "nuthatch-$lane-a"
|
||||
printf -v "$user_b_var" '%s' "nuthatch-$lane-b"
|
||||
}
|
||||
|
||||
zitadel_api() {
|
||||
local token_path="$1"
|
||||
local method="$2"
|
||||
local path="$3"
|
||||
local body="$4"
|
||||
kube --namespace "$FIXTURE_NAMESPACE" exec \
|
||||
deployment/zitadel \
|
||||
--container oidc-provisioner \
|
||||
-- sh -ec '
|
||||
token="$(tr -d "\r\n" < "$1")"
|
||||
curl --fail --silent --show-error \
|
||||
--request "$2" \
|
||||
--header "Host: zitadel.nuthatch.test" \
|
||||
--header "Authorization: Bearer $token" \
|
||||
--header "Content-Type: application/json" \
|
||||
--data "$4" \
|
||||
"http://127.0.0.1:8080$3"
|
||||
' sh "$token_path" "$method" "$path" "$body"
|
||||
}
|
||||
|
||||
provision_lane_zitadel_registration() {
|
||||
local lane="$1"
|
||||
local redirect_uri="$2"
|
||||
local client_id_var="$3"
|
||||
local client_secret_var="$4"
|
||||
local additional_audiences_var="$5"
|
||||
local user_a_var="$6"
|
||||
local user_b_var="$7"
|
||||
local project_response
|
||||
local project_id
|
||||
local app_response
|
||||
local client_id
|
||||
local client_secret
|
||||
local username
|
||||
local user_response
|
||||
local user_ids=()
|
||||
project_response="$(
|
||||
zitadel_api \
|
||||
/bootstrap/admin.pat \
|
||||
POST \
|
||||
/management/v1/projects \
|
||||
"{\"name\":\"Nuthatch lane $lane\"}"
|
||||
)" || return
|
||||
project_id="$(jq -er .id <<< "$project_response")" || return
|
||||
app_response="$(
|
||||
zitadel_api \
|
||||
/bootstrap/admin.pat \
|
||||
POST \
|
||||
"/management/v1/projects/$project_id/apps/oidc" \
|
||||
"{\"projectId\":\"$project_id\",\"name\":\"Nuthatch lane $lane\",\"redirectUris\":[\"$redirect_uri\"],\"responseTypes\":[\"OIDC_RESPONSE_TYPE_CODE\"],\"grantTypes\":[\"OIDC_GRANT_TYPE_AUTHORIZATION_CODE\"],\"appType\":\"OIDC_APP_TYPE_WEB\",\"authMethodType\":\"OIDC_AUTH_METHOD_TYPE_BASIC\",\"version\":\"OIDC_VERSION_1_0\",\"devMode\":false}"
|
||||
)" || return
|
||||
client_id="$(jq -er .clientId <<< "$app_response")" || return
|
||||
client_secret="$(jq -er .clientSecret <<< "$app_response")" || return
|
||||
for username in "nuthatch-$lane-a" "nuthatch-$lane-b"; do
|
||||
user_response="$(
|
||||
zitadel_api \
|
||||
/bootstrap/admin.pat \
|
||||
POST \
|
||||
/management/v1/users/human \
|
||||
"{\"userName\":\"$username\",\"profile\":{\"firstName\":\"Nuthatch\",\"lastName\":\"$username\",\"displayName\":\"$username\",\"preferredLanguage\":\"en\"},\"email\":{\"email\":\"$username@example.test\",\"isEmailVerified\":true},\"initialPassword\":\"Synthetic-OIDC-1!\"}"
|
||||
)" || return
|
||||
user_ids+=("$(jq -er .userId <<< "$user_response")") || return
|
||||
done
|
||||
printf -v "$client_id_var" '%s' "$client_id"
|
||||
printf -v "$client_secret_var" '%s' "$client_secret"
|
||||
printf -v "$additional_audiences_var" '%s' "$project_id"
|
||||
printf -v "$user_a_var" '%s' "${user_ids[0]}"
|
||||
printf -v "$user_b_var" '%s' "${user_ids[1]}"
|
||||
}
|
||||
|
||||
create_oidc_automation_secret() {
|
||||
local lane="$1"
|
||||
local provider="$2"
|
||||
local login_pat=unused
|
||||
if [[ "$provider" = zitadel ]]; then
|
||||
login_pat="$(
|
||||
kube --namespace "$FIXTURE_NAMESPACE" exec \
|
||||
deployment/zitadel \
|
||||
--container oidc-provisioner \
|
||||
-- sh -ec 'tr -d "\r\n" < /bootstrap/login-client.pat'
|
||||
)" || return
|
||||
fi
|
||||
kube create secret generic "nuthatch-oidc-automation-$lane" \
|
||||
--namespace "$FIXTURE_NAMESPACE" \
|
||||
--from-literal="NUTHATCH_OIDC_LOGIN_PAT=$login_pat" \
|
||||
--dry-run=client \
|
||||
--output yaml |
|
||||
kube apply --filename - >/dev/null
|
||||
}
|
||||
|
||||
write_lane_profile() {
|
||||
local lane="$1"
|
||||
local coordination="$2"
|
||||
local oidc="$3"
|
||||
local target="$4"
|
||||
local profile="nuthatch-fixture-lane-$lane"
|
||||
local profile_file="$RUNTIME/$profile.env"
|
||||
local database=
|
||||
local coordination_database=
|
||||
local oidc_issuer=
|
||||
local oidc_client_id=
|
||||
local oidc_client_secret=
|
||||
local oidc_additional_audiences=
|
||||
local oidc_redirect_uri=
|
||||
local oidc_user_a=
|
||||
local oidc_user_b=
|
||||
local target_url=
|
||||
local target_bucket=
|
||||
local target_region=
|
||||
local target_access_key=
|
||||
local target_secret_key=
|
||||
if fixture_has_family durable; then
|
||||
database="$(reset_lane_durable_store "$lane")" || return
|
||||
fi
|
||||
if fixture_has_family coordination; then
|
||||
coordination_database="$(reset_lane_coordination_store "$coordination" "$lane")" || return
|
||||
fi
|
||||
if fixture_has_family oidc; then
|
||||
oidc_redirect_uri="https://nuthatch-$lane:8080/oidc/callback"
|
||||
case "$oidc" in
|
||||
keycloak) oidc_issuer=http://keycloak.nuthatch.test:8080/realms/master ;;
|
||||
zitadel) oidc_issuer=http://zitadel.nuthatch.test:8080 ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
case "$oidc" in
|
||||
keycloak)
|
||||
provision_lane_keycloak_registration \
|
||||
"$lane" \
|
||||
"$oidc_redirect_uri" \
|
||||
oidc_client_id \
|
||||
oidc_client_secret \
|
||||
oidc_additional_audiences \
|
||||
oidc_user_a \
|
||||
oidc_user_b || return
|
||||
;;
|
||||
zitadel)
|
||||
provision_lane_zitadel_registration \
|
||||
"$lane" \
|
||||
"$oidc_redirect_uri" \
|
||||
oidc_client_id \
|
||||
oidc_client_secret \
|
||||
oidc_additional_audiences \
|
||||
oidc_user_a \
|
||||
oidc_user_b || return
|
||||
;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
create_oidc_automation_secret "$lane" "$oidc" || return
|
||||
fi
|
||||
if fixture_has_family target; then
|
||||
target_bucket="nuthatch-lane-$lane"
|
||||
target_url="$(target_endpoint "$target")" || return
|
||||
ensure_target_scope_canary "$target" || return
|
||||
provision_lane_target_principal \
|
||||
"$target" \
|
||||
"$lane" \
|
||||
"$target_bucket" \
|
||||
target_access_key \
|
||||
target_secret_key || return
|
||||
case "$target" in
|
||||
garage) target_region=garage ;;
|
||||
versitygw) target_region=us-east-1 ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
fi
|
||||
{
|
||||
printf 'NUTHATCH_FIXTURE_PROFILE=%s\n' "$profile"
|
||||
printf 'NUTHATCH_FIXTURE_LANE=%s\n' "$lane"
|
||||
printf 'NUTHATCH_COORDINATION_PROVIDER=%s\n' "$coordination"
|
||||
printf 'NUTHATCH_OIDC_PROVIDER=%s\n' "$oidc"
|
||||
printf 'NUTHATCH_TARGET_PROVIDER=%s\n' "$target"
|
||||
printf 'NUTHATCH_PG_CONN=host=postgres port=5432 user=nuthatch password=synthetic-postgres dbname=%s\n' "$database"
|
||||
printf 'NUTHATCH_REDIS_URL=redis://%s:6379/%s\n' "$coordination" "$coordination_database"
|
||||
printf 'NUTHATCH_OIDC_ISSUER=%s\n' "$oidc_issuer"
|
||||
printf 'NUTHATCH_OIDC_CLIENT_ID=%s\n' "$oidc_client_id"
|
||||
printf 'NUTHATCH_OIDC_ADDITIONAL_AUDIENCES=%s\n' "$oidc_additional_audiences"
|
||||
printf 'NUTHATCH_OIDC_REDIRECT_URI=%s\n' "$oidc_redirect_uri"
|
||||
printf 'NUTHATCH_OIDC_USER_A=%s\n' "$oidc_user_a"
|
||||
printf 'NUTHATCH_OIDC_USER_B=%s\n' "$oidc_user_b"
|
||||
printf 'NUTHATCH_OIDC_USER_PASSWORD=Synthetic-OIDC-1!\n'
|
||||
printf 'NUTHATCH_S3_ENDPOINT=%s\n' "$target_url"
|
||||
printf 'NUTHATCH_S3_BUCKET=%s\n' "$target_bucket"
|
||||
printf 'NUTHATCH_S3_REGION=%s\n' "$target_region"
|
||||
printf 'NUTHATCH_S3_ACCESS_KEY=%s\n' "$target_access_key"
|
||||
printf 'NUTHATCH_S3_SECRET_KEY=%s\n' "$target_secret_key"
|
||||
printf 'NUTHATCH_S3_PATH_STYLE=true\n'
|
||||
} > "$profile_file" || return
|
||||
if fixture_has_family oidc; then
|
||||
printf '%s' "$oidc_client_secret" > "$profile_file.oidc-secret" || return
|
||||
chmod 0600 "$profile_file.oidc-secret" || return
|
||||
fi
|
||||
kube create configmap "$profile" \
|
||||
--namespace "$FIXTURE_NAMESPACE" \
|
||||
--from-env-file="$profile_file" \
|
||||
--dry-run=client \
|
||||
--output yaml |
|
||||
kube apply --filename - >/dev/null || return
|
||||
kube create configmap "$profile-file" \
|
||||
--namespace "$FIXTURE_NAMESPACE" \
|
||||
--from-file="profile.env=$profile_file" \
|
||||
--dry-run=client \
|
||||
--output yaml |
|
||||
kube apply --filename - >/dev/null || return
|
||||
printf '%s\n' "$profile"
|
||||
}
|
||||
|
||||
render_lane_job() {
|
||||
local job="$1"
|
||||
local manifest="$2"
|
||||
local image="$3"
|
||||
local profile="$4"
|
||||
source "$RUNTIME/fixture.env" || return
|
||||
local coordination_image="$REDIS_LOCAL_IMAGE"
|
||||
sed \
|
||||
-e "s|@@JOB_NAME@@|$job|g" \
|
||||
-e "s|@@PROFILE_NAME@@|$profile|g" \
|
||||
-e "s|@@TEST_IMAGE@@|$image|g" \
|
||||
-e "s|@@POSTGRES_IMAGE@@|$POSTGRES_LOCAL_IMAGE|g" \
|
||||
-e "s|@@COORDINATION_IMAGE@@|$coordination_image|g" \
|
||||
-e "s|@@CURL_IMAGE@@|$CURL_LOCAL_IMAGE|g" \
|
||||
-e "s|@@AWS_CLI_IMAGE@@|$AWS_CLI_LOCAL_IMAGE|g" \
|
||||
"$manifest" > "$RUNTIME/$job.yaml"
|
||||
}
|
||||
|
||||
run_job_manifest() {
|
||||
local job="$1"
|
||||
local manifest="$2"
|
||||
kube --namespace "$FIXTURE_NAMESPACE" delete "job/$job" --ignore-not-found >/dev/null || return
|
||||
kube apply --filename "$manifest" >/dev/null || return
|
||||
wait_for_job "$job"
|
||||
}
|
||||
|
||||
cleanup_lane_durable_store() {
|
||||
local lane="$1"
|
||||
local database="nuthatch_lane_$lane"
|
||||
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
|
||||
env PGPASSWORD=synthetic-postgres psql \
|
||||
--username nuthatch \
|
||||
--dbname postgres \
|
||||
--set ON_ERROR_STOP=1 \
|
||||
--command "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$database'" \
|
||||
--command "DROP DATABASE IF EXISTS $database" >/dev/null
|
||||
}
|
||||
|
||||
cleanup_lane_coordination_store() {
|
||||
local provider="$1"
|
||||
local lane="$2"
|
||||
local database=$((10#$lane))
|
||||
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" -- \
|
||||
redis-cli --raw --no-auth-warning -n "$database" FLUSHDB >/dev/null
|
||||
}
|
||||
|
||||
cleanup_lane_target_store() {
|
||||
local provider="$1"
|
||||
local bucket="$2"
|
||||
local endpoint
|
||||
endpoint="$(target_loopback_endpoint "$provider")" || return
|
||||
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
|
||||
aws --endpoint-url "$endpoint" s3 rb "s3://$bucket" --force >/dev/null
|
||||
}
|
||||
|
||||
cleanup_lane_state() {
|
||||
local lane="$1"
|
||||
local coordination="$2"
|
||||
local target="$4"
|
||||
local status=0
|
||||
if fixture_has_family durable; then
|
||||
cleanup_lane_durable_store "$lane" || status=1
|
||||
fi
|
||||
if fixture_has_family coordination; then
|
||||
cleanup_lane_coordination_store "$coordination" "$lane" || status=1
|
||||
fi
|
||||
if fixture_has_family target; then
|
||||
cleanup_lane_target_store "$target" "nuthatch-lane-$lane" || status=1
|
||||
fi
|
||||
return "$status"
|
||||
}
|
||||
|
||||
cleanup_lane_resources() {
|
||||
local job="$1"
|
||||
local profile="$2"
|
||||
kube --namespace "$FIXTURE_NAMESPACE" delete \
|
||||
"job/$job" \
|
||||
"configmap/$profile" \
|
||||
"configmap/$profile-file" \
|
||||
"secret/nuthatch-oidc-automation-${profile##*-}" \
|
||||
--ignore-not-found >/dev/null || return
|
||||
rm -f \
|
||||
"$RUNTIME/$profile.env" \
|
||||
"$RUNTIME/$profile.env.oidc-secret" \
|
||||
"$RUNTIME/$job.yaml"
|
||||
}
|
||||
|
||||
run_job_matrix() {
|
||||
local base_job="$1"
|
||||
local manifest="$2"
|
||||
local image="$3"
|
||||
shift 3
|
||||
local lane
|
||||
local coordination
|
||||
local oidc
|
||||
local target
|
||||
local profile
|
||||
local job
|
||||
select_fixture_families "$@" || return
|
||||
while IFS=$'\t' read -r lane coordination oidc target; do
|
||||
profile="$(write_lane_profile "$lane" "$coordination" "$oidc" "$target")" || return
|
||||
job="$base_job-$lane"
|
||||
render_lane_job "$job" "$manifest" "$image" "$profile" || return
|
||||
printf 'Running lane %s: coordination=%s oidc=%s target=%s\n' \
|
||||
"$lane" "$coordination" "$oidc" "$target"
|
||||
run_job_manifest "$job" "$RUNTIME/$job.yaml" || return
|
||||
cleanup_lane_state "$lane" "$coordination" "$oidc" "$target" || return
|
||||
cleanup_lane_resources "$job" "$profile" || return
|
||||
done < <(expand_fixture_matrix "$@")
|
||||
}
|
||||
|
||||
destroy_fixture() {
|
||||
local status=0
|
||||
configure_provider || return
|
||||
delete_cluster || status=1
|
||||
remove_runtime_namespace || status=1
|
||||
rm -rf "$RUNTIME/image-archives" || status=1
|
||||
rm -f \
|
||||
"$KUBECONFIG_PATH" \
|
||||
"$RUNTIME/dependencies.yaml" \
|
||||
"$RUNTIME/probe.yaml" \
|
||||
"$RUNTIME/fixture.env" \
|
||||
"$RUNTIME"/nuthatch-fixture-lane-*.env \
|
||||
"$RUNTIME"/nuthatch-fixture-lane-*.env.oidc-secret \
|
||||
"$RUNTIME"/product-app-*.yaml || status=1
|
||||
return "$status"
|
||||
}
|
||||
source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/fixture/core.sh"
|
||||
|
|
|
|||
Loading…
Reference in a new issue