ms2k/nuthatch
SHA256
Watch
0
Code Releases Activity
nuthatch/tests/kind/lib.sh
ms2k 15eb0038d2 Replace planning-era names and squash initial schema 
Use descriptive test and executable names so implementation surfaces do not depend on journal organization. Consolidate the unreleased migration history into the authoritative initial schema because no deployed database requires an upgrade path.
2026-07-19 15:21:24 +09:00

921 lines
31 KiB
Shell
Executable file
Raw Blame History

#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
TESTBED="$ROOT/tests/kind"
RUNTIME="$TESTBED/.runtime"
BIN="$RUNTIME/bin"
KUBECONFIG_PATH="$RUNTIME/kubeconfig"
FIXTURE_LOCK_DIR="${XDG_RUNTIME_DIR:-/tmp/nuthatch-$(id -u)}"
FIXTURE_LOCK_PATH="$FIXTURE_LOCK_DIR/nuthatch-testbed-kind.lock"
CONTEXT=kind-nuthatch-testbed
CLUSTER=nuthatch-testbed
NERDCTL_NAMESPACE=nuthatch-testbed
FIXTURE_NAMESPACE=nuthatch-testbed
set -a
source "$TESTBED/versions.env"
set +a
user_kubeconfig_digest() {
local config_value="${KUBECONFIG:-${HOME:?}/.kube/config}"
local path
local paths
IFS=: read -r -a paths <<< "$config_value"
{
for path in "${paths[@]}"; do
if [[ -f "$path" ]]; then
sha256sum "$path"
else
printf 'missing %s\n' "$path"
fi
done
} | sha256sum | cut -d' ' -f1
}
download_tool() {
local destination="$1"
local checksum="$2"
local url="$3"
if [[ -x "$destination" ]] &&
[[ "$(sha256sum "$destination" | cut -d' ' -f1)" = "$checksum" ]]; then
return
fi
curl --fail --location --silent --show-error "$url" --output "$destination.tmp"
test "$(sha256sum "$destination.tmp" | cut -d' ' -f1)" = "$checksum"
chmod 0755 "$destination.tmp"
mv "$destination.tmp" "$destination"
}
prepare_tools() {
mkdir -p "$BIN"
download_tool \
"$BIN/kind" \
"$KIND_SHA256" \
"https://github.com/kubernetes-sigs/kind/releases/download/$KIND_VERSION/kind-linux-amd64"
download_tool \
"$BIN/kubectl" \
"$KUBECTL_SHA256" \
"https://dl.k8s.io/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl"
test "$("$BIN/kind" version | awk '{print $2}')" = "$KIND_VERSION"
test "$("$BIN/kubectl" --kubeconfig "$KUBECONFIG_PATH" --context "$CONTEXT" version --client=true -o json | jq -r .clientVersion.gitVersion)" = "$KUBECTL_VERSION"
test "$(nerdctl --version | awk '{print $3}')" = "$NERDCTL_VERSION"
test "$(runtime_ctr version | awk '/Client:/{client=1} client && $1=="Version:"{print $2; exit}')" = "v$CONTAINERD_VERSION"
test "$(runtime_ctr version | awk '/Server:/{server=1} server && $1=="Version:"{print $2; exit}')" = "v$CONTAINERD_VERSION"
}
acquire_fixture_lock() {
mkdir -p "$RUNTIME" "$FIXTURE_LOCK_DIR"
exec 9>"$FIXTURE_LOCK_PATH"
if ! flock --nonblock 9; then
printf 'kind test fixture is already in use\n' >&2
return 1
fi
}
configure_provider() {
if [[ -z "${NUTHATCH_REAL_NERDCTL:-}" ]]; then
NUTHATCH_REAL_NERDCTL="$(command -v nerdctl)" || return
fi
export NUTHATCH_REAL_NERDCTL
export CONTAINERD_NAMESPACE="$NERDCTL_NAMESPACE"
export KIND_EXPERIMENTAL_PROVIDER=nerdctl
export PATH="$TESTBED/provider:$PATH"
}
runtime_ctr() {
ctr --address "${CONTAINERD_ADDRESS:-${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/containerd/containerd.sock}" "$@"
}
runtime_namespace_exists() {
"$NUTHATCH_REAL_NERDCTL" namespace list --quiet | grep -Fx "$NERDCTL_NAMESPACE" >/dev/null
}
cluster_exists() {
runtime_namespace_exists &&
"$BIN/kind" get clusters | grep -Fx "$CLUSTER" >/dev/null
}
delete_cluster() {
local container
local containers
if cluster_exists; then
"$BIN/kind" delete cluster --name "$CLUSTER" || true
fi
containers="$("$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" ps \
--all \
--quiet \
--filter "label=io.x-k8s.kind.cluster=$CLUSTER")" || return
while IFS= read -r container; do
[[ -z "$container" ]] ||
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" rm --force "$container" ||
return
done <<< "$containers"
if cluster_exists; then
"$BIN/kind" delete cluster --name "$CLUSTER" || true
fi
if cluster_exists; then
printf 'kind cluster %s still exists after teardown\n' "$CLUSTER" >&2
return 1
fi
}
remove_runtime_namespace() {
local status=0
local digest
local digests
local lease
local leases
local network
for network in kind nuthatch-kind; do
if "$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" network inspect "$network" >/dev/null 2>&1; then
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" network remove "$network" >/dev/null || status=1
fi
done
if ! runtime_namespace_exists; then
return "$status"
fi
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" image prune --all --force >/dev/null || status=1
leases="$(runtime_ctr --namespace "$NERDCTL_NAMESPACE" leases list --quiet)" || status=1
while IFS= read -r lease; do
[[ -z "$lease" ]] || runtime_ctr --namespace "$NERDCTL_NAMESPACE" leases delete "$lease" >/dev/null || status=1
done <<< "$leases"
digests="$(runtime_ctr --namespace "$NERDCTL_NAMESPACE" content list --quiet)" || status=1
while IFS= read -r digest; do
[[ -z "$digest" ]] || runtime_ctr --namespace "$NERDCTL_NAMESPACE" content remove "$digest" >/dev/null || status=1
done <<< "$digests"
"$NUTHATCH_REAL_NERDCTL" namespace remove "$NERDCTL_NAMESPACE" >/dev/null || status=1
runtime_namespace_exists && status=1
return "$status"
}
prepare_provider() {
configure_provider
delete_cluster
remove_runtime_namespace
"$NUTHATCH_REAL_NERDCTL" namespace create "$NERDCTL_NAMESPACE" >/dev/null
}
load_external_image() {
local name="$1"
local source="$2"
local digest="${source##*@}"
local image="nuthatch-testbed/$name:sha256-${digest#sha256:}"
local archive="$RUNTIME/image-archives/$digest.tar"
"$NUTHATCH_REAL_NERDCTL" image pull --platform linux/amd64 --unpack=false --quiet "$source" >&2
"$NUTHATCH_REAL_NERDCTL" image tag "$source" "$image"
"$NUTHATCH_REAL_NERDCTL" image save --platform linux/amd64 --output "$archive" "$image"
printf 'Loading %s from %s\n' "$image" "$source" >&2
"$BIN/kind" load image-archive "$archive" --name "$CLUSTER" >&2
printf '%s\n' "$image"
}
load_local_image() {
local image="$1"
local namespace="${2:-default}"
local archive="$RUNTIME/image-archives/nuthatch-app.tar"
rm -f "$archive"
"$NUTHATCH_REAL_NERDCTL" --namespace "$namespace" image save \
--platform linux/amd64 \
--output "$archive" \
"$image"
"$BIN/kind" load image-archive "$archive" --name "$CLUSTER"
}
render_manifests() {
local postgres="$1" redis="$2" dragonfly="$3" keycloak="$4" zitadel="$5"
local garage="$6" versitygw="$7" coredns="$8" curl="$9" aws_cli="${10}"
local postgres_replicas="${11}" redis_replicas="${12}" dragonfly_replicas="${13}"
local keycloak_replicas="${14}" zitadel_replicas="${15}"
local garage_replicas="${16}" versitygw_replicas="${17}" coredns_replicas="${18}"
sed \
-e "s|@@POSTGRES_IMAGE@@|$postgres|g" \
-e "s|@@REDIS_IMAGE@@|$redis|g" \
-e "s|@@DRAGONFLY_IMAGE@@|$dragonfly|g" \
-e "s|@@KEYCLOAK_IMAGE@@|$keycloak|g" \
-e "s|@@ZITADEL_IMAGE@@|$zitadel|g" \
-e "s|@@GARAGE_IMAGE@@|$garage|g" \
-e "s|@@VERSITYGW_IMAGE@@|$versitygw|g" \
-e "s|@@COREDNS_IMAGE@@|$coredns|g" \
-e "s|@@CURL_IMAGE@@|$curl|g" \
-e "s|@@AWS_CLI_IMAGE@@|$aws_cli|g" \
-e "s|@@POSTGRES_REPLICAS@@|$postgres_replicas|g" \
-e "s|@@REDIS_REPLICAS@@|$redis_replicas|g" \
-e "s|@@DRAGONFLY_REPLICAS@@|$dragonfly_replicas|g" \
-e "s|@@KEYCLOAK_REPLICAS@@|$keycloak_replicas|g" \
-e "s|@@ZITADEL_REPLICAS@@|$zitadel_replicas|g" \
-e "s|@@GARAGE_REPLICAS@@|$garage_replicas|g" \
-e "s|@@VERSITYGW_REPLICAS@@|$versitygw_replicas|g" \
-e "s|@@COREDNS_REPLICAS@@|$coredns_replicas|g" \
"$TESTBED/dependencies.yaml.in" > "$RUNTIME/dependencies.yaml"
sed \
-e "s|@@POSTGRES_IMAGE@@|$postgres|g" \
-e "s|@@REDIS_IMAGE@@|$redis|g" \
-e "s|@@CURL_IMAGE@@|$curl|g" \
-e "s|@@AWS_CLI_IMAGE@@|$aws_cli|g" \
"$TESTBED/probe.yaml.in" > "$RUNTIME/probe.yaml"
{
printf 'POSTGRES_LOCAL_IMAGE=%q\n' "$postgres"
printf 'REDIS_LOCAL_IMAGE=%q\n' "$redis"
printf 'DRAGONFLY_LOCAL_IMAGE=%q\n' "$dragonfly"
printf 'KEYCLOAK_LOCAL_IMAGE=%q\n' "$keycloak"
printf 'ZITADEL_LOCAL_IMAGE=%q\n' "$zitadel"
printf 'GARAGE_LOCAL_IMAGE=%q\n' "$garage"
printf 'VERSITYGW_LOCAL_IMAGE=%q\n' "$versitygw"
printf 'COREDNS_LOCAL_IMAGE=%q\n' "$coredns"
printf 'CURL_LOCAL_IMAGE=%q\n' "$curl"
printf 'AWS_CLI_LOCAL_IMAGE=%q\n' "$aws_cli"
} > "$RUNTIME/fixture.env"
}
kube() {
"$BIN/kubectl" \
--kubeconfig "$KUBECONFIG_PATH" \
--context "$CONTEXT" \
"$@"
}
wait_for_job() {
local job="$1"
local deadline=$((SECONDS + 300))
local complete
local failed
while ((SECONDS < deadline)); do
complete="$(kube --namespace "$FIXTURE_NAMESPACE" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Complete")].status}')"
if [[ "$complete" = True ]]; then
kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" --all-containers
return
fi
failed="$(kube --namespace "$FIXTURE_NAMESPACE" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Failed")].status}')"
if [[ "$failed" = True ]]; then
kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" --all-containers >&2 || true
return 1
fi
sleep 1
done
printf 'job/%s did not complete within 300 seconds\n' "$job" >&2
return 1
}
select_fixture_families() {
local requested=("$@")
local family
local candidate
local found
local use_all=false
FIXTURE_FAMILIES=()
if [[ "${#requested[@]}" = 0 ]]; then
requested=(all)
fi
for family in "${requested[@]}"; do
case "$family" in
all) use_all=true ;;
base | durable | coordination | oidc | target) ;;
*)
printf 'unknown fixture family: %s\n' "$family" >&2
return 1
;;
esac
done
if [[ "$use_all" = true ]]; then
requested=(durable coordination oidc target)
fi
for candidate in durable coordination oidc target; do
found=false
for family in "${requested[@]}"; do
if [[ "$family" = "$candidate" ]]; then
found=true
fi
done
if [[ "$found" = true ]]; then
FIXTURE_FAMILIES+=("$candidate")
fi
done
}
fixture_has_family() {
local expected="$1"
local family
for family in "${FIXTURE_FAMILIES[@]}"; do
if [[ "$family" = "$expected" ]]; then
return
fi
done
return 1
}
expand_fixture_matrix() {
local coordination_providers=(-)
local oidc_providers=(-)
local target_providers=(-)
local coordination
local oidc
local target
local lane=0
select_fixture_families "$@" || return
if fixture_has_family coordination; then
coordination_providers=(redis dragonfly)
fi
if fixture_has_family oidc; then
oidc_providers=(keycloak zitadel)
fi
if fixture_has_family target; then
target_providers=(garage versitygw)
fi
for coordination in "${coordination_providers[@]}"; do
for oidc in "${oidc_providers[@]}"; do
for target in "${target_providers[@]}"; do
lane=$((lane + 1))
printf '%03d\t%s\t%s\t%s\n' "$lane" "$coordination" "$oidc" "$target"
done
done
done
}
setup_fixture() {
select_fixture_families "$@" || return
mkdir -p "$RUNTIME"
rm -rf "$RUNTIME/image-archives"
mkdir -p "$RUNTIME/image-archives"
prepare_tools
prepare_provider
"$BIN/kind" create cluster \
--name "$CLUSTER" \
--image "$NODE_IMAGE" \
--config "$TESTBED/cluster.yaml" \
--kubeconfig "$KUBECONFIG_PATH" \
--wait 180s
local postgres=nuthatch-testbed/unused:postgres
local redis=nuthatch-testbed/unused:redis
local dragonfly=nuthatch-testbed/unused:dragonfly
local keycloak=nuthatch-testbed/unused:keycloak
local zitadel=nuthatch-testbed/unused:zitadel
local garage=nuthatch-testbed/unused:garage
local versitygw=nuthatch-testbed/unused:versitygw
local coredns=nuthatch-testbed/unused:coredns
local curl=nuthatch-testbed/unused:curl
local aws_cli=nuthatch-testbed/unused:aws-cli
local postgres_replicas=0 redis_replicas=0 dragonfly_replicas=0
local keycloak_replicas=0 zitadel_replicas=0
local garage_replicas=0 versitygw_replicas=0 coredns_replicas=0
if fixture_has_family durable || fixture_has_family oidc; then
postgres="$(load_external_image postgres "$POSTGRES_IMAGE")"
postgres_replicas=1
fi
if fixture_has_family coordination; then
redis="$(load_external_image redis "$REDIS_IMAGE")"
dragonfly="$(load_external_image dragonfly "$DRAGONFLY_IMAGE")"
redis_replicas=1
dragonfly_replicas=1
fi
if fixture_has_family oidc; then
keycloak="$(load_external_image keycloak "$KEYCLOAK_IMAGE")"
zitadel="$(load_external_image zitadel "$ZITADEL_IMAGE")"
coredns="$(load_external_image coredns "$COREDNS_IMAGE")"
curl="$(load_external_image curl "$CURL_IMAGE")"
keycloak_replicas=1
zitadel_replicas=1
coredns_replicas=1
fi
if fixture_has_family target; then
garage="$(load_external_image garage "$GARAGE_IMAGE")"
versitygw="$(load_external_image versitygw "$VERSITYGW_IMAGE")"
aws_cli="$(load_external_image aws-cli "$AWS_CLI_IMAGE")"
if [[ "$curl" = nuthatch-testbed/unused:curl ]]; then
curl="$(load_external_image curl "$CURL_IMAGE")"
fi
garage_replicas=1
versitygw_replicas=1
fi
render_manifests "$postgres" "$redis" "$dragonfly" "$keycloak" "$zitadel" \
"$garage" "$versitygw" "$coredns" "$curl" "$aws_cli" \
"$postgres_replicas" "$redis_replicas" "$dragonfly_replicas" \
"$keycloak_replicas" "$zitadel_replicas" \
"$garage_replicas" "$versitygw_replicas" "$coredns_replicas"
kube apply --filename "$RUNTIME/dependencies.yaml"
if fixture_has_family durable || fixture_has_family oidc; then
wait_for_deployment postgres
fi
if fixture_has_family coordination; then
wait_for_deployment redis
wait_for_deployment dragonfly
fi
if fixture_has_family oidc; then
wait_for_deployment keycloak
wait_for_deployment zitadel
wait_for_deployment controlled-dns
fi
if fixture_has_family target; then
wait_for_deployment garage
wait_for_deployment versitygw
fi
}
wait_for_deployment() {
kube --namespace "$FIXTURE_NAMESPACE" rollout status "deployment/$1" --timeout 240s
}
verify_dependencies() {
kube apply --filename "$RUNTIME/probe.yaml"
local job
for job in state-probe oidc-probe garage-probe versitygw-probe; do
wait_for_job "$job"
done
}
reset_lane_durable_store() {
local lane="$1"
local database="nuthatch_lane_$lane"
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
env PGPASSWORD=synthetic-postgres psql \
--username nuthatch \
--dbname postgres \
--set ON_ERROR_STOP=1 \
--command "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$database'" \
--command "DROP DATABASE IF EXISTS $database" \
--command "CREATE DATABASE $database" >&2 || return
printf '%s\n' "$database"
}
reset_lane_coordination_store() {
local provider="$1"
local lane="$2"
local database=$((10#$lane))
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" -- \
redis-cli --raw --no-auth-warning -n "$database" FLUSHDB >/dev/null || return
printf '%s\n' "$database"
}
target_endpoint() {
case "$1" in
garage) printf 'http://garage:3900\n' ;;
versitygw) printf 'http://versitygw:10000\n' ;;
*) return 1 ;;
esac
}
target_loopback_endpoint() {
case "$1" in
garage) printf 'http://127.0.0.1:3900\n' ;;
versitygw) printf 'http://127.0.0.1:10000\n' ;;
*) return 1 ;;
esac
}
ensure_target_scope_canary() {
local provider="$1"
local endpoint
endpoint="$(target_loopback_endpoint "$provider")" || return
if ! kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
aws --endpoint-url "$endpoint" s3api create-bucket --bucket nuthatch-fixture >&2; then
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
aws --endpoint-url "$endpoint" s3api head-bucket --bucket nuthatch-fixture >/dev/null || return
fi
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
sh -ec 'printf scope-canary >/tmp/scope-canary && aws --endpoint-url "$1" s3api put-object --bucket nuthatch-fixture --key scope-canary --body /tmp/scope-canary >/dev/null' \
sh "$endpoint"
}
provision_lane_target_principal() {
local provider="$1"
local lane="$2"
local bucket="$3"
local access_key_var="$4"
local secret_key_var="$5"
local endpoint
local access_key
local secret_key
endpoint="$(target_loopback_endpoint "$provider")" || return
case "$provider" in
garage)
printf -v access_key 'GKNUTHATCH%024d' "$((10#$lane))"
printf -v secret_key '%064d' "$((10#$lane))"
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/garage --container garage -- \
/garage bucket create "$bucket" >/dev/null || return
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/garage --container garage -- \
/garage bucket allow \
--read \
--write \
--owner \
"$bucket" \
--key GK00000000000000000000000000000000 >/dev/null || return
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/garage --container garage -- \
/garage key import --yes -n "nuthatch-$lane" "$access_key" "$secret_key" >/dev/null || return
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/garage --container garage -- \
/garage bucket allow --read --write "$bucket" --key "$access_key" >/dev/null || return
;;
versitygw)
access_key="nuthatch-lane-$lane"
printf -v secret_key 'synthetic-lane-%049d' "$((10#$lane))"
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/versitygw --container versitygw -- \
/usr/local/bin/versitygw admin \
--endpoint-url http://127.0.0.1:10001 \
create-user \
--access "$access_key" \
--secret "$secret_key" \
--role user \
--user-id "$((10#$lane))" \
--group-id "$((10#$lane))" >/dev/null || return
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/versitygw --container versitygw -- \
/usr/local/bin/versitygw admin \
--endpoint-url http://127.0.0.1:10001 \
create-bucket \
--bucket "$bucket" \
--owner "$access_key" >/dev/null || return
;;
*) return 1 ;;
esac
printf -v "$access_key_var" '%s' "$access_key"
printf -v "$secret_key_var" '%s' "$secret_key"
}
keycloak_admin_token() {
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
sh -ec '
/opt/keycloak/bin/kcadm.sh config credentials \
--server http://127.0.0.1:8080 \
--realm master \
--user nuthatch-admin \
--password synthetic-keycloak >/dev/null
/opt/keycloak/bin/kcadm.sh get serverinfo >/dev/null
' >/dev/null
}
provision_lane_keycloak_registration() {
local lane="$1"
local redirect_uri="$2"
local client_id_var="$3"
local client_secret_var="$4"
local additional_audiences_var="$5"
local user_a_var="$6"
local user_b_var="$7"
local client_id="nuthatch-$lane"
local client_secret="synthetic-keycloak-client-secret-$lane"
local username
keycloak_admin_token || return
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
/opt/keycloak/bin/kcadm.sh create clients \
--realm master \
-s "clientId=$client_id" \
-s enabled=true \
-s protocol=openid-connect \
-s publicClient=false \
-s clientAuthenticatorType=client-secret \
-s standardFlowEnabled=true \
-s directAccessGrantsEnabled=false \
-s "secret=$client_secret" \
-s "redirectUris=[\"$redirect_uri\"]" \
-s 'attributes={"pkce.code.challenge.method":"S256"}' >/dev/null || return
for username in "nuthatch-$lane-a" "nuthatch-$lane-b"; do
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
/opt/keycloak/bin/kcadm.sh create users \
--realm master \
-s "username=$username" \
-s enabled=true \
-s "firstName=$username" >/dev/null || return
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
/opt/keycloak/bin/kcadm.sh set-password \
--realm master \
--username "$username" \
--new-password Synthetic-OIDC-1! >/dev/null || return
done
printf -v "$client_id_var" '%s' "$client_id"
printf -v "$client_secret_var" '%s' "$client_secret"
printf -v "$additional_audiences_var" ''
printf -v "$user_a_var" '%s' "nuthatch-$lane-a"
printf -v "$user_b_var" '%s' "nuthatch-$lane-b"
}
zitadel_api() {
local token_path="$1"
local method="$2"
local path="$3"
local body="$4"
kube --namespace "$FIXTURE_NAMESPACE" exec \
deployment/zitadel \
--container oidc-provisioner \
-- sh -ec '
token="$(tr -d "\r\n" < "$1")"
curl --fail --silent --show-error \
--request "$2" \
--header "Host: zitadel.nuthatch.test" \
--header "Authorization: Bearer $token" \
--header "Content-Type: application/json" \
--data "$4" \
"http://127.0.0.1:8080$3"
' sh "$token_path" "$method" "$path" "$body"
}
provision_lane_zitadel_registration() {
local lane="$1"
local redirect_uri="$2"
local client_id_var="$3"
local client_secret_var="$4"
local additional_audiences_var="$5"
local user_a_var="$6"
local user_b_var="$7"
local project_response
local project_id
local app_response
local client_id
local client_secret
local username
local user_response
local user_ids=()
project_response="$(
zitadel_api \
/bootstrap/admin.pat \
POST \
/management/v1/projects \
"{\"name\":\"Nuthatch lane $lane\"}"
)" || return
project_id="$(jq -er .id <<< "$project_response")" || return
app_response="$(
zitadel_api \
/bootstrap/admin.pat \
POST \
"/management/v1/projects/$project_id/apps/oidc" \
"{\"projectId\":\"$project_id\",\"name\":\"Nuthatch lane $lane\",\"redirectUris\":[\"$redirect_uri\"],\"responseTypes\":[\"OIDC_RESPONSE_TYPE_CODE\"],\"grantTypes\":[\"OIDC_GRANT_TYPE_AUTHORIZATION_CODE\"],\"appType\":\"OIDC_APP_TYPE_WEB\",\"authMethodType\":\"OIDC_AUTH_METHOD_TYPE_BASIC\",\"version\":\"OIDC_VERSION_1_0\",\"devMode\":false}"
)" || return
client_id="$(jq -er .clientId <<< "$app_response")" || return
client_secret="$(jq -er .clientSecret <<< "$app_response")" || return
for username in "nuthatch-$lane-a" "nuthatch-$lane-b"; do
user_response="$(
zitadel_api \
/bootstrap/admin.pat \
POST \
/management/v1/users/human \
"{\"userName\":\"$username\",\"profile\":{\"firstName\":\"Nuthatch\",\"lastName\":\"$username\",\"displayName\":\"$username\",\"preferredLanguage\":\"en\"},\"email\":{\"email\":\"$username@example.test\",\"isEmailVerified\":true},\"initialPassword\":\"Synthetic-OIDC-1!\"}"
)" || return
user_ids+=("$(jq -er .userId <<< "$user_response")") || return
done
printf -v "$client_id_var" '%s' "$client_id"
printf -v "$client_secret_var" '%s' "$client_secret"
printf -v "$additional_audiences_var" '%s' "$project_id"
printf -v "$user_a_var" '%s' "${user_ids[0]}"
printf -v "$user_b_var" '%s' "${user_ids[1]}"
}
create_oidc_automation_secret() {
local lane="$1"
local provider="$2"
local login_pat=unused
if [[ "$provider" = zitadel ]]; then
login_pat="$(
kube --namespace "$FIXTURE_NAMESPACE" exec \
deployment/zitadel \
--container oidc-provisioner \
-- sh -ec 'tr -d "\r\n" < /bootstrap/login-client.pat'
)" || return
fi
kube create secret generic "nuthatch-oidc-automation-$lane" \
--namespace "$FIXTURE_NAMESPACE" \
--from-literal="NUTHATCH_OIDC_LOGIN_PAT=$login_pat" \
--dry-run=client \
--output yaml |
kube apply --filename - >/dev/null
}
write_lane_profile() {
local lane="$1"
local coordination="$2"
local oidc="$3"
local target="$4"
local profile="nuthatch-fixture-lane-$lane"
local profile_file="$RUNTIME/$profile.env"
local database=
local coordination_database=
local oidc_issuer=
local oidc_client_id=
local oidc_client_secret=
local oidc_additional_audiences=
local oidc_redirect_uri=
local oidc_user_a=
local oidc_user_b=
local target_url=
local target_bucket=
local target_region=
local target_access_key=
local target_secret_key=
if fixture_has_family durable; then
database="$(reset_lane_durable_store "$lane")" || return
fi
if fixture_has_family coordination; then
coordination_database="$(reset_lane_coordination_store "$coordination" "$lane")" || return
fi
if fixture_has_family oidc; then
oidc_redirect_uri="https://nuthatch-$lane:8080/oidc/callback"
case "$oidc" in
keycloak) oidc_issuer=http://keycloak.nuthatch.test:8080/realms/master ;;
zitadel) oidc_issuer=http://zitadel.nuthatch.test:8080 ;;
*) return 1 ;;
esac
case "$oidc" in
keycloak)
provision_lane_keycloak_registration \
"$lane" \
"$oidc_redirect_uri" \
oidc_client_id \
oidc_client_secret \
oidc_additional_audiences \
oidc_user_a \
oidc_user_b || return
;;
zitadel)
provision_lane_zitadel_registration \
"$lane" \
"$oidc_redirect_uri" \
oidc_client_id \
oidc_client_secret \
oidc_additional_audiences \
oidc_user_a \
oidc_user_b || return
;;
*) return 1 ;;
esac
create_oidc_automation_secret "$lane" "$oidc" || return
fi
if fixture_has_family target; then
target_bucket="nuthatch-lane-$lane"
target_url="$(target_endpoint "$target")" || return
ensure_target_scope_canary "$target" || return
provision_lane_target_principal \
"$target" \
"$lane" \
"$target_bucket" \
target_access_key \
target_secret_key || return
case "$target" in
garage) target_region=garage ;;
versitygw) target_region=us-east-1 ;;
*) return 1 ;;
esac
fi
{
printf 'NUTHATCH_FIXTURE_PROFILE=%s\n' "$profile"
printf 'NUTHATCH_FIXTURE_LANE=%s\n' "$lane"
printf 'NUTHATCH_COORDINATION_PROVIDER=%s\n' "$coordination"
printf 'NUTHATCH_OIDC_PROVIDER=%s\n' "$oidc"
printf 'NUTHATCH_TARGET_PROVIDER=%s\n' "$target"
printf 'NUTHATCH_PG_CONN=host=postgres port=5432 user=nuthatch password=synthetic-postgres dbname=%s\n' "$database"
printf 'NUTHATCH_REDIS_URL=redis://%s:6379/%s\n' "$coordination" "$coordination_database"
printf 'NUTHATCH_OIDC_ISSUER=%s\n' "$oidc_issuer"
printf 'NUTHATCH_OIDC_CLIENT_ID=%s\n' "$oidc_client_id"
printf 'NUTHATCH_OIDC_ADDITIONAL_AUDIENCES=%s\n' "$oidc_additional_audiences"
printf 'NUTHATCH_OIDC_REDIRECT_URI=%s\n' "$oidc_redirect_uri"
printf 'NUTHATCH_OIDC_USER_A=%s\n' "$oidc_user_a"
printf 'NUTHATCH_OIDC_USER_B=%s\n' "$oidc_user_b"
printf 'NUTHATCH_OIDC_USER_PASSWORD=Synthetic-OIDC-1!\n'
printf 'NUTHATCH_S3_ENDPOINT=%s\n' "$target_url"
printf 'NUTHATCH_S3_BUCKET=%s\n' "$target_bucket"
printf 'NUTHATCH_S3_REGION=%s\n' "$target_region"
printf 'NUTHATCH_S3_ACCESS_KEY=%s\n' "$target_access_key"
printf 'NUTHATCH_S3_SECRET_KEY=%s\n' "$target_secret_key"
printf 'NUTHATCH_S3_PATH_STYLE=true\n'
} > "$profile_file" || return
if fixture_has_family oidc; then
printf '%s' "$oidc_client_secret" > "$profile_file.oidc-secret" || return
chmod 0600 "$profile_file.oidc-secret" || return
fi
kube create configmap "$profile" \
--namespace "$FIXTURE_NAMESPACE" \
--from-env-file="$profile_file" \
--dry-run=client \
--output yaml |
kube apply --filename - >/dev/null || return
kube create configmap "$profile-file" \
--namespace "$FIXTURE_NAMESPACE" \
--from-file="profile.env=$profile_file" \
--dry-run=client \
--output yaml |
kube apply --filename - >/dev/null || return
printf '%s\n' "$profile"
}
render_lane_job() {
local job="$1"
local manifest="$2"
local image="$3"
local profile="$4"
source "$RUNTIME/fixture.env" || return
local coordination_image="$REDIS_LOCAL_IMAGE"
sed \
-e "s|@@JOB_NAME@@|$job|g" \
-e "s|@@PROFILE_NAME@@|$profile|g" \
-e "s|@@TEST_IMAGE@@|$image|g" \
-e "s|@@POSTGRES_IMAGE@@|$POSTGRES_LOCAL_IMAGE|g" \
-e "s|@@COORDINATION_IMAGE@@|$coordination_image|g" \
-e "s|@@CURL_IMAGE@@|$CURL_LOCAL_IMAGE|g" \
-e "s|@@AWS_CLI_IMAGE@@|$AWS_CLI_LOCAL_IMAGE|g" \
"$manifest" > "$RUNTIME/$job.yaml"
}
run_job_manifest() {
local job="$1"
local manifest="$2"
kube --namespace "$FIXTURE_NAMESPACE" delete "job/$job" --ignore-not-found >/dev/null || return
kube apply --filename "$manifest" >/dev/null || return
wait_for_job "$job"
}
cleanup_lane_durable_store() {
local lane="$1"
local database="nuthatch_lane_$lane"
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
env PGPASSWORD=synthetic-postgres psql \
--username nuthatch \
--dbname postgres \
--set ON_ERROR_STOP=1 \
--command "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$database'" \
--command "DROP DATABASE IF EXISTS $database" >/dev/null
}
cleanup_lane_coordination_store() {
local provider="$1"
local lane="$2"
local database=$((10#$lane))
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" -- \
redis-cli --raw --no-auth-warning -n "$database" FLUSHDB >/dev/null
}
cleanup_lane_target_store() {
local provider="$1"
local bucket="$2"
local endpoint
endpoint="$(target_loopback_endpoint "$provider")" || return
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
aws --endpoint-url "$endpoint" s3 rb "s3://$bucket" --force >/dev/null
}
cleanup_lane_state() {
local lane="$1"
local coordination="$2"
local target="$4"
local status=0
if fixture_has_family durable; then
cleanup_lane_durable_store "$lane" || status=1
fi
if fixture_has_family coordination; then
cleanup_lane_coordination_store "$coordination" "$lane" || status=1
fi
if fixture_has_family target; then
cleanup_lane_target_store "$target" "nuthatch-lane-$lane" || status=1
fi
return "$status"
}
cleanup_lane_resources() {
local job="$1"
local profile="$2"
kube --namespace "$FIXTURE_NAMESPACE" delete \
"job/$job" \
"configmap/$profile" \
"configmap/$profile-file" \
"secret/nuthatch-oidc-automation-${profile##*-}" \
--ignore-not-found >/dev/null || return
rm -f \
"$RUNTIME/$profile.env" \
"$RUNTIME/$profile.env.oidc-secret" \
"$RUNTIME/$job.yaml"
}
run_job_matrix() {
local base_job="$1"
local manifest="$2"
local image="$3"
shift 3
local lane
local coordination
local oidc
local target
local profile
local job
select_fixture_families "$@" || return
while IFS=$'\t' read -r lane coordination oidc target; do
profile="$(write_lane_profile "$lane" "$coordination" "$oidc" "$target")" || return
job="$base_job-$lane"
render_lane_job "$job" "$manifest" "$image" "$profile" || return
printf 'Running lane %s: coordination=%s oidc=%s target=%s\n' \
"$lane" "$coordination" "$oidc" "$target"
run_job_manifest "$job" "$RUNTIME/$job.yaml" || return
cleanup_lane_state "$lane" "$coordination" "$oidc" "$target" || return
cleanup_lane_resources "$job" "$profile" || return
done < <(expand_fixture_matrix "$@")
}
destroy_fixture() {
local status=0
configure_provider || return
delete_cluster || status=1
remove_runtime_namespace || status=1
rm -rf "$RUNTIME/image-archives" || status=1
rm -f \
"$KUBECONFIG_PATH" \
"$RUNTIME/dependencies.yaml" \
"$RUNTIME/probe.yaml" \
"$RUNTIME/fixture.env" \
"$RUNTIME"/nuthatch-fixture-lane-*.env \
"$RUNTIME"/nuthatch-fixture-lane-*.env.oidc-secret \
"$RUNTIME"/product-app-*.yaml || status=1
return "$status"
}
View git blame Copy permalink