ms2k/nuthatch
SHA256
Watch
0
Code Releases Activity

Split kind tests into component-owned probes

Browse source
This commit is contained in:
ms2k 2026-07-19 23:57:21 +09:00
commit 691ad92e30
27 changed files with 4269 additions and 3108 deletions
Download patch file Download diff file Expand all files Collapse all files

47
tests/kind/application-probe-job.yaml.in Normal file
View file

@ -0,0 +1,47 @@
apiVersion: batch/v1
kind: Job
metadata:
name: @@JOB_NAME@@
namespace: nuthatch-testbed
spec:
activeDeadlineSeconds: @@DEADLINE_SECONDS@@
backoffLimit: 0
ttlSecondsAfterFinished: 300
template:
spec:
automountServiceAccountToken: false
enableServiceLinks: false
dnsPolicy: None
dnsConfig:
nameservers: [10.96.0.53]
searches:
- nuthatch-testbed.svc.cluster.local
- svc.cluster.local
- cluster.local
options:
- {name: ndots, value: "5"}
restartPolicy: Never
containers:
- name: @@COMPONENT@@-probe
image: @@TEST_IMAGE@@
imagePullPolicy: Never
command: [@@PROBE@@]
args: [@@PROBE_ARGS@@]
envFrom:
- configMapRef: {name: @@PROFILE_NAME@@}
env:
- name: GHCRTS
value: "-N2 -A1m -M192m"
- name: NUTHATCH_OIDC_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: nuthatch-@@LANE@@
key: NUTHATCH_OIDC_CLIENT_SECRET
- name: NUTHATCH_OIDC_LOGIN_PAT
valueFrom:
secretKeyRef:
name: nuthatch-oidc-automation-@@LANE@@
key: NUTHATCH_OIDC_LOGIN_PAT
resources:
requests: {cpu: 50m, memory: 32Mi}
limits: {cpu: "1", memory: 256Mi}

115
tests/kind/auth-run.sh Executable file
View file

@ -0,0 +1,115 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
source "$ROOT/tests/kind/lib.sh"
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
SUCCEEDED=false
OWNS_FIXTURE=false
APP_IMAGE=nuthatch-auth:local
cleanup() {
local status=$?
trap - EXIT INT TERM
if [[ "${NUTHATCH_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
printf 'retained context: %s\n' "$CONTEXT"
exit "$status"
fi
if [[ "$OWNS_FIXTURE" = true ]]; then
destroy_fixture || status=1
fi
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
printf 'user kubeconfig changed unexpectedly\n' >&2
status=1
fi
exit "$status"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
render_application() {
local lane="$1"
local profile_file="$2"
local oidc_secret
oidc_secret="$(<"$profile_file.oidc-secret")"
sed \
-e "s|@@LANE@@|$lane|g" \
-e "s|@@APP_IMAGE@@|$APP_IMAGE|g" \
-e "s|@@PG_CONN@@|$(sed -n 's/^NUTHATCH_PG_CONN=//p' "$profile_file")|g" \
-e "s|@@REDIS_URL@@|$(sed -n 's/^NUTHATCH_REDIS_URL=//p' "$profile_file")|g" \
-e "s|@@OIDC_ISSUER@@|$(sed -n 's/^NUTHATCH_OIDC_ISSUER=//p' "$profile_file")|g" \
-e "s|@@OIDC_CLIENT_ID@@|$(sed -n 's/^NUTHATCH_OIDC_CLIENT_ID=//p' "$profile_file")|g" \
-e "s|@@OIDC_ADDITIONAL_AUDIENCES@@|$(sed -n 's/^NUTHATCH_OIDC_ADDITIONAL_AUDIENCES=//p' "$profile_file")|g" \
-e "s|@@OIDC_CLIENT_SECRET@@|$oidc_secret|g" \
-e "s|@@S3_ENDPOINT@@|$(sed -n 's/^NUTHATCH_S3_ENDPOINT=//p' "$profile_file")|g" \
-e "s|@@S3_REGION@@|$(sed -n 's/^NUTHATCH_S3_REGION=//p' "$profile_file")|g" \
-e "s|@@S3_BUCKET@@|$(sed -n 's/^NUTHATCH_S3_BUCKET=//p' "$profile_file")|g" \
-e "s|@@S3_ACCESS_KEY@@|$(sed -n 's/^NUTHATCH_S3_ACCESS_KEY=//p' "$profile_file")|g" \
-e "s|@@S3_SECRET_KEY@@|$(sed -n 's/^NUTHATCH_S3_SECRET_KEY=//p' "$profile_file")|g" \
"$ROOT/deploy/kubernetes/nuthatch.yaml.in" >"$RUNTIME/auth-app-$lane.yaml"
}
cleanup_application() {
local lane="$1"
kube --namespace "$FIXTURE_NAMESPACE" delete \
"deployment/nuthatch-$lane" \
"service/nuthatch-$lane" \
"configmap/nuthatch-$lane" \
"secret/nuthatch-$lane" \
--ignore-not-found >/dev/null
rm -f "$RUNTIME/auth-app-$lane.yaml"
}
acquire_fixture_lock
OWNS_FIXTURE=true
setup_fixture durable coordination oidc target
source "$RUNTIME/fixture.env"
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" build \
--file "$ROOT/Dockerfile" \
--target test-runtime \
--tag "$APP_IMAGE" \
"$ROOT"
load_local_image "$APP_IMAGE" "$NERDCTL_NAMESPACE"
while IFS=$'\t' read -r lane coordination oidc target; do
profile="$(write_lane_profile "$lane" "$coordination" "$oidc" "$target")"
render_application "$lane" "$RUNTIME/$profile.env"
kube apply --filename "$RUNTIME/auth-app-$lane.yaml" >/dev/null
kube --namespace "$FIXTURE_NAMESPACE" rollout status \
"deployment/nuthatch-$lane" --timeout=180s
if [[ "${NUTHATCH_APP_INTERACTIVE:-0}" = 1 ]]; then
SUCCEEDED=true
OWNS_FIXTURE=false
printf 'interactive kubeconfig: %s\n' "$KUBECONFIG_PATH"
printf 'interactive context: %s\n' "$CONTEXT"
printf 'interactive namespace: %s\n' "$FIXTURE_NAMESPACE"
printf 'interactive service: nuthatch-%s:8080\n' "$lane"
exit 0
fi
job="auth-app-$lane"
render_lane_job \
"$job" \
"$TESTBED/application-probe-job.yaml.in" \
"$APP_IMAGE" \
"$profile"
sed -i \
-e "s|@@LANE@@|$lane|g" \
-e 's|@@COMPONENT@@|auth|g' \
-e 's|@@PROBE@@|nuthatch-auth-probe|g' \
-e 's|@@PROBE_ARGS@@||g' \
-e 's|@@DEADLINE_SECONDS@@|300|g' \
"$RUNTIME/$job.yaml"
printf 'Running auth lane %s: coordination=%s oidc=%s target=%s\n' \
"$lane" "$coordination" "$oidc" "$target"
run_job_manifest "$job" "$RUNTIME/$job.yaml"
cleanup_application "$lane"
cleanup_lane_state "$lane" "$coordination" "$oidc" "$target"
cleanup_lane_resources "$job" "$profile"
done < <(expand_fixture_matrix coordination oidc target)
SUCCEEDED=true
printf 'Nuthatch authentication gate passed\n'

16
tests/kind/fixture/coordination.sh Normal file
View file

@ -0,0 +1,16 @@
reset_lane_coordination_store() {
local provider="$1"
local lane="$2"
local database=$((10#$lane))
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" -- \
redis-cli --raw --no-auth-warning -n "$database" FLUSHDB >/dev/null || return
printf '%s\n' "$database"
}
cleanup_lane_coordination_store() {
local provider="$1"
local lane="$2"
local database=$((10#$lane))
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" -- \
redis-cli --raw --no-auth-warning -n "$database" FLUSHDB >/dev/null
}

100
tests/kind/fixture/core.sh Normal file
View file

@ -0,0 +1,100 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
TESTBED="$ROOT/tests/kind"
RUNTIME="$TESTBED/.runtime"
BIN="$RUNTIME/bin"
KUBECONFIG_PATH="$RUNTIME/kubeconfig"
FIXTURE_LOCK_DIR="${XDG_RUNTIME_DIR:-/tmp/nuthatch-$(id -u)}"
FIXTURE_LOCK_PATH="$FIXTURE_LOCK_DIR/nuthatch-testbed-kind.lock"
CONTEXT=kind-nuthatch-testbed
CLUSTER=nuthatch-testbed
NERDCTL_NAMESPACE=nuthatch-testbed
FIXTURE_NAMESPACE=nuthatch-testbed
FIXTURE_MODULES="$TESTBED/fixture"
set -a
source "$TESTBED/versions.env"
set +a
user_kubeconfig_digest() {
local config_value="${KUBECONFIG:-${HOME:?}/.kube/config}"
local path
local paths
IFS=: read -r -a paths <<< "$config_value"
{
for path in "${paths[@]}"; do
if [[ -f "$path" ]]; then
sha256sum "$path"
else
printf 'missing %s\n' "$path"
fi
done
} | sha256sum | cut -d' ' -f1
}
acquire_fixture_lock() {
mkdir -p "$RUNTIME" "$FIXTURE_LOCK_DIR"
exec 9>"$FIXTURE_LOCK_PATH"
if ! flock --nonblock 9; then
printf 'kind test fixture is already in use\n' >&2
return 1
fi
}
kube() {
"$BIN/kubectl" \
--kubeconfig "$KUBECONFIG_PATH" \
--context "$CONTEXT" \
"$@"
}
wait_for_job() {
local job="$1"
local deadline=$((SECONDS + 300))
local complete
local failed
while ((SECONDS < deadline)); do
complete="$(kube --namespace "$FIXTURE_NAMESPACE" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Complete")].status}')"
if [[ "$complete" = True ]]; then
kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" --all-containers
return
fi
failed="$(kube --namespace "$FIXTURE_NAMESPACE" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Failed")].status}')"
if [[ "$failed" = True ]]; then
kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" --all-containers >&2 || true
return 1
fi
sleep 1
done
printf 'job/%s did not complete within 300 seconds\n' "$job" >&2
return 1
}
source "$FIXTURE_MODULES/runtime.sh"
source "$FIXTURE_MODULES/matrix.sh"
source "$FIXTURE_MODULES/durable.sh"
source "$FIXTURE_MODULES/coordination.sh"
source "$FIXTURE_MODULES/target.sh"
source "$FIXTURE_MODULES/oidc.sh"
source "$FIXTURE_MODULES/dependencies.sh"
source "$FIXTURE_MODULES/profile.sh"
source "$FIXTURE_MODULES/lifecycle.sh"
destroy_fixture() {
local status=0
configure_provider || return
delete_cluster || status=1
remove_runtime_namespace || status=1
rm -rf "$RUNTIME/image-archives" || status=1
rm -f \
"$KUBECONFIG_PATH" \
"$RUNTIME/dependencies.yaml" \
"$RUNTIME/probe.yaml" \
"$RUNTIME/fixture.env" \
"$RUNTIME"/nuthatch-fixture-lane-*.env \
"$RUNTIME"/nuthatch-fixture-lane-*.env.oidc-secret \
"$RUNTIME"/*-app-*.yaml || status=1
return "$status"
}

130
tests/kind/fixture/dependencies.sh Normal file
View file

@ -0,0 +1,130 @@
render_manifests() {
local postgres="$1" redis="$2" dragonfly="$3" keycloak="$4" zitadel="$5"
local garage="$6" versitygw="$7" coredns="$8" curl="$9" aws_cli="${10}"
local postgres_replicas="${11}" redis_replicas="${12}" dragonfly_replicas="${13}"
local keycloak_replicas="${14}" zitadel_replicas="${15}"
local garage_replicas="${16}" versitygw_replicas="${17}" coredns_replicas="${18}"
sed \
-e "s|@@POSTGRES_IMAGE@@|$postgres|g" \
-e "s|@@REDIS_IMAGE@@|$redis|g" \
-e "s|@@DRAGONFLY_IMAGE@@|$dragonfly|g" \
-e "s|@@KEYCLOAK_IMAGE@@|$keycloak|g" \
-e "s|@@ZITADEL_IMAGE@@|$zitadel|g" \
-e "s|@@GARAGE_IMAGE@@|$garage|g" \
-e "s|@@VERSITYGW_IMAGE@@|$versitygw|g" \
-e "s|@@COREDNS_IMAGE@@|$coredns|g" \
-e "s|@@CURL_IMAGE@@|$curl|g" \
-e "s|@@AWS_CLI_IMAGE@@|$aws_cli|g" \
-e "s|@@POSTGRES_REPLICAS@@|$postgres_replicas|g" \
-e "s|@@REDIS_REPLICAS@@|$redis_replicas|g" \
-e "s|@@DRAGONFLY_REPLICAS@@|$dragonfly_replicas|g" \
-e "s|@@KEYCLOAK_REPLICAS@@|$keycloak_replicas|g" \
-e "s|@@ZITADEL_REPLICAS@@|$zitadel_replicas|g" \
-e "s|@@GARAGE_REPLICAS@@|$garage_replicas|g" \
-e "s|@@VERSITYGW_REPLICAS@@|$versitygw_replicas|g" \
-e "s|@@COREDNS_REPLICAS@@|$coredns_replicas|g" \
"$TESTBED/dependencies.yaml.in" > "$RUNTIME/dependencies.yaml"
{
printf 'POSTGRES_LOCAL_IMAGE=%q\n' "$postgres"
printf 'REDIS_LOCAL_IMAGE=%q\n' "$redis"
printf 'DRAGONFLY_LOCAL_IMAGE=%q\n' "$dragonfly"
printf 'KEYCLOAK_LOCAL_IMAGE=%q\n' "$keycloak"
printf 'ZITADEL_LOCAL_IMAGE=%q\n' "$zitadel"
printf 'GARAGE_LOCAL_IMAGE=%q\n' "$garage"
printf 'VERSITYGW_LOCAL_IMAGE=%q\n' "$versitygw"
printf 'COREDNS_LOCAL_IMAGE=%q\n' "$coredns"
printf 'CURL_LOCAL_IMAGE=%q\n' "$curl"
printf 'AWS_CLI_LOCAL_IMAGE=%q\n' "$aws_cli"
} > "$RUNTIME/fixture.env"
}
setup_fixture() {
select_fixture_families "$@" || return
mkdir -p "$RUNTIME"
rm -rf "$RUNTIME/image-archives"
mkdir -p "$RUNTIME/image-archives"
prepare_tools
prepare_provider
"$BIN/kind" create cluster \
--name "$CLUSTER" \
--image "$NODE_IMAGE" \
--config "$TESTBED/cluster.yaml" \
--kubeconfig "$KUBECONFIG_PATH" \
--wait 180s
local postgres=nuthatch-testbed/unused:postgres
local redis=nuthatch-testbed/unused:redis
local dragonfly=nuthatch-testbed/unused:dragonfly
local keycloak=nuthatch-testbed/unused:keycloak
local zitadel=nuthatch-testbed/unused:zitadel
local garage=nuthatch-testbed/unused:garage
local versitygw=nuthatch-testbed/unused:versitygw
local coredns=nuthatch-testbed/unused:coredns
local curl=nuthatch-testbed/unused:curl
local aws_cli=nuthatch-testbed/unused:aws-cli
local postgres_replicas=0 redis_replicas=0 dragonfly_replicas=0
local keycloak_replicas=0 zitadel_replicas=0
local garage_replicas=0 versitygw_replicas=0 coredns_replicas=0
if fixture_has_family durable || fixture_has_family oidc; then
postgres="$(load_external_image postgres "$POSTGRES_IMAGE")"
postgres_replicas=1
fi
if fixture_has_family coordination; then
redis="$(load_external_image redis "$REDIS_IMAGE")"
dragonfly="$(load_external_image dragonfly "$DRAGONFLY_IMAGE")"
redis_replicas=1
dragonfly_replicas=1
fi
if fixture_has_family oidc; then
keycloak="$(load_external_image keycloak "$KEYCLOAK_IMAGE")"
zitadel="$(load_external_image zitadel "$ZITADEL_IMAGE")"
coredns="$(load_external_image coredns "$COREDNS_IMAGE")"
curl="$(load_external_image curl "$CURL_IMAGE")"
keycloak_replicas=1
zitadel_replicas=1
coredns_replicas=1
fi
if fixture_has_family target; then
garage="$(load_external_image garage "$GARAGE_IMAGE")"
versitygw="$(load_external_image versitygw "$VERSITYGW_IMAGE")"
aws_cli="$(load_external_image aws-cli "$AWS_CLI_IMAGE")"
if [[ "$curl" = nuthatch-testbed/unused:curl ]]; then
curl="$(load_external_image curl "$CURL_IMAGE")"
fi
garage_replicas=1
versitygw_replicas=1
fi
render_manifests "$postgres" "$redis" "$dragonfly" "$keycloak" "$zitadel" \
"$garage" "$versitygw" "$coredns" "$curl" "$aws_cli" \
"$postgres_replicas" "$redis_replicas" "$dragonfly_replicas" \
"$keycloak_replicas" "$zitadel_replicas" \
"$garage_replicas" "$versitygw_replicas" "$coredns_replicas"
kube apply --filename "$RUNTIME/dependencies.yaml"
if fixture_has_family durable || fixture_has_family oidc; then
wait_for_deployment postgres
fi
if fixture_has_family coordination; then
wait_for_deployment redis
wait_for_deployment dragonfly
fi
if fixture_has_family oidc; then
wait_for_deployment keycloak
wait_for_deployment zitadel
wait_for_deployment controlled-dns
fi
if fixture_has_family target; then
wait_for_deployment garage
wait_for_deployment versitygw
fi
}
wait_for_deployment() {
kube --namespace "$FIXTURE_NAMESPACE" rollout status "deployment/$1" --timeout 240s
}
verify_dependencies() {
local test_image="$1"
sed "s|@@TEST_IMAGE@@|$test_image|g" \
"$TESTBED/probe.yaml.in" >"$RUNTIME/probe.yaml"
kube apply --filename "$RUNTIME/probe.yaml"
wait_for_job dependency-probe
}

25
tests/kind/fixture/durable.sh Normal file
View file

@ -0,0 +1,25 @@
reset_lane_durable_store() {
local lane="$1"
local database="nuthatch_lane_$lane"
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
env PGPASSWORD=synthetic-postgres psql \
--username nuthatch \
--dbname postgres \
--set ON_ERROR_STOP=1 \
--command "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$database'" \
--command "DROP DATABASE IF EXISTS $database" \
--command "CREATE DATABASE $database" >&2 || return
printf '%s\n' "$database"
}
cleanup_lane_durable_store() {
local lane="$1"
local database="nuthatch_lane_$lane"
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
env PGPASSWORD=synthetic-postgres psql \
--username nuthatch \
--dbname postgres \
--set ON_ERROR_STOP=1 \
--command "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$database'" \
--command "DROP DATABASE IF EXISTS $database" >/dev/null
}

55
tests/kind/fixture/lifecycle.sh Normal file
View file

@ -0,0 +1,55 @@
cleanup_lane_state() {
local lane="$1"
local coordination="$2"
local target="$4"
local status=0
if fixture_has_family durable; then
cleanup_lane_durable_store "$lane" || status=1
fi
if fixture_has_family coordination; then
cleanup_lane_coordination_store "$coordination" "$lane" || status=1
fi
if fixture_has_family target; then
cleanup_lane_target_store "$target" "nuthatch-lane-$lane" || status=1
fi
return "$status"
}
cleanup_lane_resources() {
local job="$1"
local profile="$2"
kube --namespace "$FIXTURE_NAMESPACE" delete \
"job/$job" \
"configmap/$profile" \
"configmap/$profile-file" \
"secret/nuthatch-oidc-automation-${profile##*-}" \
--ignore-not-found >/dev/null || return
rm -f \
"$RUNTIME/$profile.env" \
"$RUNTIME/$profile.env.oidc-secret" \
"$RUNTIME/$job.yaml"
}
run_job_matrix() {
local base_job="$1"
local manifest="$2"
local image="$3"
shift 3
local lane
local coordination
local oidc
local target
local profile
local job
select_fixture_families "$@" || return
while IFS=$'\t' read -r lane coordination oidc target; do
profile="$(write_lane_profile "$lane" "$coordination" "$oidc" "$target")" || return
job="$base_job-$lane"
render_lane_job "$job" "$manifest" "$image" "$profile" || return
printf 'Running lane %s: coordination=%s oidc=%s target=%s\n' \
"$lane" "$coordination" "$oidc" "$target"
run_job_manifest "$job" "$RUNTIME/$job.yaml" || return
cleanup_lane_state "$lane" "$coordination" "$oidc" "$target" || return
cleanup_lane_resources "$job" "$profile" || return
done < <(expand_fixture_matrix "$@")
}

74
tests/kind/fixture/matrix.sh Normal file
View file

@ -0,0 +1,74 @@
select_fixture_families() {
local requested=("$@")
local family
local candidate
local found
local use_all=false
FIXTURE_FAMILIES=()
if [[ "${#requested[@]}" = 0 ]]; then
requested=(all)
fi
for family in "${requested[@]}"; do
case "$family" in
all) use_all=true ;;
base | durable | coordination | oidc | target) ;;
*)
printf 'unknown fixture family: %s\n' "$family" >&2
return 1
;;
esac
done
if [[ "$use_all" = true ]]; then
requested=(durable coordination oidc target)
fi
for candidate in durable coordination oidc target; do
found=false
for family in "${requested[@]}"; do
if [[ "$family" = "$candidate" ]]; then
found=true
fi
done
if [[ "$found" = true ]]; then
FIXTURE_FAMILIES+=("$candidate")
fi
done
}
fixture_has_family() {
local expected="$1"
local family
for family in "${FIXTURE_FAMILIES[@]}"; do
if [[ "$family" = "$expected" ]]; then
return
fi
done
return 1
}
expand_fixture_matrix() {
local coordination_providers=(-)
local oidc_providers=(-)
local target_providers=(-)
local coordination
local oidc
local target
local lane=0
select_fixture_families "$@" || return
if fixture_has_family coordination; then
coordination_providers=(redis dragonfly)
fi
if fixture_has_family oidc; then
oidc_providers=(keycloak zitadel)
fi
if fixture_has_family target; then
target_providers=(garage versitygw)
fi
for coordination in "${coordination_providers[@]}"; do
for oidc in "${oidc_providers[@]}"; do
for target in "${target_providers[@]}"; do
lane=$((lane + 1))
printf '%03d\t%s\t%s\t%s\n' "$lane" "$coordination" "$oidc" "$target"
done
done
done
}

146
tests/kind/fixture/oidc.sh Normal file
View file

@ -0,0 +1,146 @@
keycloak_admin_token() {
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
sh -ec '
/opt/keycloak/bin/kcadm.sh config credentials \
--server http://127.0.0.1:8080 \
--realm master \
--user nuthatch-admin \
--password synthetic-keycloak >/dev/null
/opt/keycloak/bin/kcadm.sh get serverinfo >/dev/null
' >/dev/null
}
provision_lane_keycloak_registration() {
local lane="$1"
local redirect_uri="$2"
local client_id_var="$3"
local client_secret_var="$4"
local additional_audiences_var="$5"
local user_a_var="$6"
local user_b_var="$7"
local client_id="nuthatch-$lane"
local client_secret="synthetic-keycloak-client-secret-$lane"
local username
keycloak_admin_token || return
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
/opt/keycloak/bin/kcadm.sh create clients \
--realm master \
-s "clientId=$client_id" \
-s enabled=true \
-s protocol=openid-connect \
-s publicClient=false \
-s clientAuthenticatorType=client-secret \
-s standardFlowEnabled=true \
-s directAccessGrantsEnabled=false \
-s "secret=$client_secret" \
-s "redirectUris=[\"$redirect_uri\"]" \
-s 'attributes={"pkce.code.challenge.method":"S256"}' >/dev/null || return
for username in "nuthatch-$lane-a" "nuthatch-$lane-b"; do
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
/opt/keycloak/bin/kcadm.sh create users \
--realm master \
-s "username=$username" \
-s enabled=true \
-s "firstName=$username" >/dev/null || return
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
/opt/keycloak/bin/kcadm.sh set-password \
--realm master \
--username "$username" \
--new-password Synthetic-OIDC-1! >/dev/null || return
done
printf -v "$client_id_var" '%s' "$client_id"
printf -v "$client_secret_var" '%s' "$client_secret"
printf -v "$additional_audiences_var" ''
printf -v "$user_a_var" '%s' "nuthatch-$lane-a"
printf -v "$user_b_var" '%s' "nuthatch-$lane-b"
}
zitadel_api() {
local token_path="$1"
local method="$2"
local path="$3"
local body="$4"
kube --namespace "$FIXTURE_NAMESPACE" exec \
deployment/zitadel \
--container oidc-provisioner \
-- sh -ec '
token="$(tr -d "\r\n" < "$1")"
curl --fail --silent --show-error \
--request "$2" \
--header "Host: zitadel.nuthatch.test" \
--header "Authorization: Bearer $token" \
--header "Content-Type: application/json" \
--data "$4" \
"http://127.0.0.1:8080$3"
' sh "$token_path" "$method" "$path" "$body"
}
provision_lane_zitadel_registration() {
local lane="$1"
local redirect_uri="$2"
local client_id_var="$3"
local client_secret_var="$4"
local additional_audiences_var="$5"
local user_a_var="$6"
local user_b_var="$7"
local project_response
local project_id
local app_response
local client_id
local client_secret
local username
local user_response
local user_ids=()
project_response="$(
zitadel_api \
/bootstrap/admin.pat \
POST \
/management/v1/projects \
"{\"name\":\"Nuthatch lane $lane\"}"
)" || return
project_id="$(jq -er .id <<< "$project_response")" || return
app_response="$(
zitadel_api \
/bootstrap/admin.pat \
POST \
"/management/v1/projects/$project_id/apps/oidc" \
"{\"projectId\":\"$project_id\",\"name\":\"Nuthatch lane $lane\",\"redirectUris\":[\"$redirect_uri\"],\"responseTypes\":[\"OIDC_RESPONSE_TYPE_CODE\"],\"grantTypes\":[\"OIDC_GRANT_TYPE_AUTHORIZATION_CODE\"],\"appType\":\"OIDC_APP_TYPE_WEB\",\"authMethodType\":\"OIDC_AUTH_METHOD_TYPE_BASIC\",\"version\":\"OIDC_VERSION_1_0\",\"devMode\":false}"
)" || return
client_id="$(jq -er .clientId <<< "$app_response")" || return
client_secret="$(jq -er .clientSecret <<< "$app_response")" || return
for username in "nuthatch-$lane-a" "nuthatch-$lane-b"; do
user_response="$(
zitadel_api \
/bootstrap/admin.pat \
POST \
/management/v1/users/human \
"{\"userName\":\"$username\",\"profile\":{\"firstName\":\"Nuthatch\",\"lastName\":\"$username\",\"displayName\":\"$username\",\"preferredLanguage\":\"en\"},\"email\":{\"email\":\"$username@example.test\",\"isEmailVerified\":true},\"initialPassword\":\"Synthetic-OIDC-1!\"}"
)" || return
user_ids+=("$(jq -er .userId <<< "$user_response")") || return
done
printf -v "$client_id_var" '%s' "$client_id"
printf -v "$client_secret_var" '%s' "$client_secret"
printf -v "$additional_audiences_var" '%s' "$project_id"
printf -v "$user_a_var" '%s' "${user_ids[0]}"
printf -v "$user_b_var" '%s' "${user_ids[1]}"
}
create_oidc_automation_secret() {
local lane="$1"
local provider="$2"
local login_pat=unused
if [[ "$provider" = zitadel ]]; then
login_pat="$(
kube --namespace "$FIXTURE_NAMESPACE" exec \
deployment/zitadel \
--container oidc-provisioner \
-- sh -ec 'tr -d "\r\n" < /bootstrap/login-client.pat'
)" || return
fi
kube create secret generic "nuthatch-oidc-automation-$lane" \
--namespace "$FIXTURE_NAMESPACE" \
--from-literal="NUTHATCH_OIDC_LOGIN_PAT=$login_pat" \
--dry-run=client \
--output yaml |
kube apply --filename - >/dev/null
}

136
tests/kind/fixture/profile.sh Normal file
View file

@ -0,0 +1,136 @@
write_lane_profile() {
local lane="$1"
local coordination="$2"
local oidc="$3"
local target="$4"
local profile="nuthatch-fixture-lane-$lane"
local profile_file="$RUNTIME/$profile.env"
local database=
local coordination_database=
local oidc_issuer=
local oidc_client_id=
local oidc_client_secret=
local oidc_additional_audiences=
local oidc_redirect_uri=
local oidc_user_a=
local oidc_user_b=
local target_url=
local target_bucket=
local target_region=
local target_access_key=
local target_secret_key=
if fixture_has_family durable; then
database="$(reset_lane_durable_store "$lane")" || return
fi
if fixture_has_family coordination; then
coordination_database="$(reset_lane_coordination_store "$coordination" "$lane")" || return
fi
if fixture_has_family oidc; then
oidc_redirect_uri="https://nuthatch-$lane:8080/oidc/callback"
case "$oidc" in
keycloak) oidc_issuer=http://keycloak.nuthatch.test:8080/realms/master ;;
zitadel) oidc_issuer=http://zitadel.nuthatch.test:8080 ;;
*) return 1 ;;
esac
case "$oidc" in
keycloak)
provision_lane_keycloak_registration \
"$lane" \
"$oidc_redirect_uri" \
oidc_client_id \
oidc_client_secret \
oidc_additional_audiences \
oidc_user_a \
oidc_user_b || return
;;
zitadel)
provision_lane_zitadel_registration \
"$lane" \
"$oidc_redirect_uri" \
oidc_client_id \
oidc_client_secret \
oidc_additional_audiences \
oidc_user_a \
oidc_user_b || return
;;
*) return 1 ;;
esac
create_oidc_automation_secret "$lane" "$oidc" || return
fi
if fixture_has_family target; then
target_bucket="nuthatch-lane-$lane"
target_url="$(target_endpoint "$target")" || return
ensure_target_scope_canary "$target" || return
provision_lane_target_principal \
"$target" \
"$lane" \
"$target_bucket" \
target_access_key \
target_secret_key || return
case "$target" in
garage) target_region=garage ;;
versitygw) target_region=us-east-1 ;;
*) return 1 ;;
esac
fi
{
printf 'NUTHATCH_FIXTURE_PROFILE=%s\n' "$profile"
printf 'NUTHATCH_FIXTURE_LANE=%s\n' "$lane"
printf 'NUTHATCH_COORDINATION_PROVIDER=%s\n' "$coordination"
printf 'NUTHATCH_OIDC_PROVIDER=%s\n' "$oidc"
printf 'NUTHATCH_TARGET_PROVIDER=%s\n' "$target"
printf 'NUTHATCH_PG_CONN=host=postgres port=5432 user=nuthatch password=synthetic-postgres dbname=%s\n' "$database"
printf 'NUTHATCH_REDIS_URL=redis://%s:6379/%s\n' "$coordination" "$coordination_database"
printf 'NUTHATCH_OIDC_ISSUER=%s\n' "$oidc_issuer"
printf 'NUTHATCH_OIDC_CLIENT_ID=%s\n' "$oidc_client_id"
printf 'NUTHATCH_OIDC_ADDITIONAL_AUDIENCES=%s\n' "$oidc_additional_audiences"
printf 'NUTHATCH_OIDC_REDIRECT_URI=%s\n' "$oidc_redirect_uri"
printf 'NUTHATCH_OIDC_USER_A=%s\n' "$oidc_user_a"
printf 'NUTHATCH_OIDC_USER_B=%s\n' "$oidc_user_b"
printf 'NUTHATCH_OIDC_USER_PASSWORD=Synthetic-OIDC-1!\n'
printf 'NUTHATCH_S3_ENDPOINT=%s\n' "$target_url"
printf 'NUTHATCH_S3_BUCKET=%s\n' "$target_bucket"
printf 'NUTHATCH_S3_REGION=%s\n' "$target_region"
printf 'NUTHATCH_S3_ACCESS_KEY=%s\n' "$target_access_key"
printf 'NUTHATCH_S3_SECRET_KEY=%s\n' "$target_secret_key"
printf 'NUTHATCH_S3_PATH_STYLE=true\n'
} > "$profile_file" || return
if fixture_has_family oidc; then
printf '%s' "$oidc_client_secret" > "$profile_file.oidc-secret" || return
chmod 0600 "$profile_file.oidc-secret" || return
fi
kube create configmap "$profile" \
--namespace "$FIXTURE_NAMESPACE" \
--from-env-file="$profile_file" \
--dry-run=client \
--output yaml |
kube apply --filename - >/dev/null || return
kube create configmap "$profile-file" \
--namespace "$FIXTURE_NAMESPACE" \
--from-file="profile.env=$profile_file" \
--dry-run=client \
--output yaml |
kube apply --filename - >/dev/null || return
printf '%s\n' "$profile"
}
render_lane_job() {
local job="$1"
local manifest="$2"
local image="$3"
local profile="$4"
source "$RUNTIME/fixture.env" || return
sed \
-e "s|@@JOB_NAME@@|$job|g" \
-e "s|@@PROFILE_NAME@@|$profile|g" \
-e "s|@@TEST_IMAGE@@|$image|g" \
"$manifest" > "$RUNTIME/$job.yaml"
}
run_job_manifest() {
local job="$1"
local manifest="$2"
kube --namespace "$FIXTURE_NAMESPACE" delete "job/$job" --ignore-not-found >/dev/null || return
kube apply --filename "$manifest" >/dev/null || return
wait_for_job "$job"
}

139
tests/kind/fixture/runtime.sh Normal file
View file

@ -0,0 +1,139 @@
download_tool() {
local destination="$1"
local checksum="$2"
local url="$3"
if [[ -x "$destination" ]] &&
[[ "$(sha256sum "$destination" | cut -d' ' -f1)" = "$checksum" ]]; then
return
fi
curl --fail --location --silent --show-error "$url" --output "$destination.tmp"
test "$(sha256sum "$destination.tmp" | cut -d' ' -f1)" = "$checksum"
chmod 0755 "$destination.tmp"
mv "$destination.tmp" "$destination"
}
prepare_tools() {
mkdir -p "$BIN"
download_tool \
"$BIN/kind" \
"$KIND_SHA256" \
"https://github.com/kubernetes-sigs/kind/releases/download/$KIND_VERSION/kind-linux-amd64"
download_tool \
"$BIN/kubectl" \
"$KUBECTL_SHA256" \
"https://dl.k8s.io/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl"
test "$("$BIN/kind" version | awk '{print $2}')" = "$KIND_VERSION"
test "$("$BIN/kubectl" --kubeconfig "$KUBECONFIG_PATH" --context "$CONTEXT" version --client=true -o json | jq -r .clientVersion.gitVersion)" = "$KUBECTL_VERSION"
test "$(nerdctl --version | awk '{print $3}')" = "$NERDCTL_VERSION"
test "$(runtime_ctr version | awk '/Client:/{client=1} client && $1=="Version:"{print $2; exit}')" = "v$CONTAINERD_VERSION"
test "$(runtime_ctr version | awk '/Server:/{server=1} server && $1=="Version:"{print $2; exit}')" = "v$CONTAINERD_VERSION"
}
configure_provider() {
if [[ -z "${NUTHATCH_REAL_NERDCTL:-}" ]]; then
NUTHATCH_REAL_NERDCTL="$(command -v nerdctl)" || return
fi
export NUTHATCH_REAL_NERDCTL
export CONTAINERD_NAMESPACE="$NERDCTL_NAMESPACE"
export KIND_EXPERIMENTAL_PROVIDER=nerdctl
export PATH="$TESTBED/provider:$PATH"
}
runtime_ctr() {
ctr --address "${CONTAINERD_ADDRESS:-${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/containerd/containerd.sock}" "$@"
}
runtime_namespace_exists() {
"$NUTHATCH_REAL_NERDCTL" namespace list --quiet | grep -Fx "$NERDCTL_NAMESPACE" >/dev/null
}
cluster_exists() {
runtime_namespace_exists &&
"$BIN/kind" get clusters | grep -Fx "$CLUSTER" >/dev/null
}
delete_cluster() {
local container
local containers
if cluster_exists; then
"$BIN/kind" delete cluster --name "$CLUSTER" || true
fi
containers="$("$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" ps \
--all \
--quiet \
--filter "label=io.x-k8s.kind.cluster=$CLUSTER")" || return
while IFS= read -r container; do
[[ -z "$container" ]] ||
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" rm --force "$container" ||
return
done <<< "$containers"
if cluster_exists; then
"$BIN/kind" delete cluster --name "$CLUSTER" || true
fi
if cluster_exists; then
printf 'kind cluster %s still exists after teardown\n' "$CLUSTER" >&2
return 1
fi
}
remove_runtime_namespace() {
local status=0
local digest
local digests
local lease
local leases
local network
for network in kind nuthatch-kind; do
if "$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" network inspect "$network" >/dev/null 2>&1; then
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" network remove "$network" >/dev/null || status=1
fi
done
if ! runtime_namespace_exists; then
return "$status"
fi
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" image prune --all --force >/dev/null || status=1
leases="$(runtime_ctr --namespace "$NERDCTL_NAMESPACE" leases list --quiet)" || status=1
while IFS= read -r lease; do
[[ -z "$lease" ]] || runtime_ctr --namespace "$NERDCTL_NAMESPACE" leases delete "$lease" >/dev/null || status=1
done <<< "$leases"
digests="$(runtime_ctr --namespace "$NERDCTL_NAMESPACE" content list --quiet)" || status=1
while IFS= read -r digest; do
[[ -z "$digest" ]] || runtime_ctr --namespace "$NERDCTL_NAMESPACE" content remove "$digest" >/dev/null || status=1
done <<< "$digests"
"$NUTHATCH_REAL_NERDCTL" namespace remove "$NERDCTL_NAMESPACE" >/dev/null || status=1
runtime_namespace_exists && status=1
return "$status"
}
prepare_provider() {
configure_provider
delete_cluster
remove_runtime_namespace
"$NUTHATCH_REAL_NERDCTL" namespace create "$NERDCTL_NAMESPACE" >/dev/null
}
load_external_image() {
local name="$1"
local source="$2"
local digest="${source##*@}"
local image="nuthatch-testbed/$name:sha256-${digest#sha256:}"
local archive="$RUNTIME/image-archives/$digest.tar"
"$NUTHATCH_REAL_NERDCTL" image pull --platform linux/amd64 --unpack=false --quiet "$source" >&2
"$NUTHATCH_REAL_NERDCTL" image tag "$source" "$image"
"$NUTHATCH_REAL_NERDCTL" image save --platform linux/amd64 --output "$archive" "$image"
printf 'Loading %s from %s\n' "$image" "$source" >&2
"$BIN/kind" load image-archive "$archive" --name "$CLUSTER" >&2
printf '%s\n' "$image"
}
load_local_image() {
local image="$1"
local namespace="${2:-default}"
local archive="$RUNTIME/image-archives/nuthatch-app.tar"
rm -f "$archive"
"$NUTHATCH_REAL_NERDCTL" --namespace "$namespace" image save \
--platform linux/amd64 \
--output "$archive" \
"$image"
"$BIN/kind" load image-archive "$archive" --name "$CLUSTER"
}

91
tests/kind/fixture/target.sh Normal file
View file

@ -0,0 +1,91 @@
target_endpoint() {
case "$1" in
garage) printf 'http://garage:3900\n' ;;
versitygw) printf 'http://versitygw:10000\n' ;;
*) return 1 ;;
esac
}
target_loopback_endpoint() {
case "$1" in
garage) printf 'http://127.0.0.1:3900\n' ;;
versitygw) printf 'http://127.0.0.1:10000\n' ;;
*) return 1 ;;
esac
}
ensure_target_scope_canary() {
local provider="$1"
local endpoint
endpoint="$(target_loopback_endpoint "$provider")" || return
if ! kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
aws --endpoint-url "$endpoint" s3api create-bucket --bucket nuthatch-fixture >&2; then
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
aws --endpoint-url "$endpoint" s3api head-bucket --bucket nuthatch-fixture >/dev/null || return
fi
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
sh -ec 'printf scope-canary >/tmp/scope-canary && aws --endpoint-url "$1" s3api put-object --bucket nuthatch-fixture --key scope-canary --body /tmp/scope-canary >/dev/null' \
sh "$endpoint"
}
provision_lane_target_principal() {
local provider="$1"
local lane="$2"
local bucket="$3"
local access_key_var="$4"
local secret_key_var="$5"
local endpoint
local access_key
local secret_key
endpoint="$(target_loopback_endpoint "$provider")" || return
case "$provider" in
garage)
printf -v access_key 'GKNUTHATCH%024d' "$((10#$lane))"
printf -v secret_key '%064d' "$((10#$lane))"
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/garage --container garage -- \
/garage bucket create "$bucket" >/dev/null || return
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/garage --container garage -- \
/garage bucket allow \
--read \
--write \
--owner \
"$bucket" \
--key GK00000000000000000000000000000000 >/dev/null || return
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/garage --container garage -- \
/garage key import --yes -n "nuthatch-$lane" "$access_key" "$secret_key" >/dev/null || return
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/garage --container garage -- \
/garage bucket allow --read --write "$bucket" --key "$access_key" >/dev/null || return
;;
versitygw)
access_key="nuthatch-lane-$lane"
printf -v secret_key 'synthetic-lane-%049d' "$((10#$lane))"
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/versitygw --container versitygw -- \
/usr/local/bin/versitygw admin \
--endpoint-url http://127.0.0.1:10001 \
create-user \
--access "$access_key" \
--secret "$secret_key" \
--role user \
--user-id "$((10#$lane))" \
--group-id "$((10#$lane))" >/dev/null || return
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/versitygw --container versitygw -- \
/usr/local/bin/versitygw admin \
--endpoint-url http://127.0.0.1:10001 \
create-bucket \
--bucket "$bucket" \
--owner "$access_key" >/dev/null || return
;;
*) return 1 ;;
esac
printf -v "$access_key_var" '%s' "$access_key"
printf -v "$secret_key_var" '%s' "$secret_key"
}
cleanup_lane_target_store() {
local provider="$1"
local bucket="$2"
local endpoint
endpoint="$(target_loopback_endpoint "$provider")" || return
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
aws --endpoint-url "$endpoint" s3 rb "s3://$bucket" --force >/dev/null
}

13
tests/kind/idempotency-run.sh
View file

@ -147,6 +147,7 @@ setup_fixture durable coordination oidc target
source "$RUNTIME/fixture.env"
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" build \
--file "$ROOT/Dockerfile" \
--target test-runtime \
--tag "$APP_IMAGE" \
"$ROOT"
load_local_image "$APP_IMAGE" "$NERDCTL_NAMESPACE"
@ -170,10 +171,16 @@ while IFS=$'\t' read -r lane coordination oidc target; do
job="idempotency-epoch-$lane"
render_lane_job \
"$job" \
"$TESTBED/product-matrix-job.yaml.in" \
"$CURL_LOCAL_IMAGE" \
"$TESTBED/application-probe-job.yaml.in" \
"$APP_IMAGE" \
"$profile"
sed -i "s|@@LANE@@|$lane|g" "$RUNTIME/$job.yaml"
sed -i \
-e "s|@@LANE@@|$lane|g" \
-e 's|@@COMPONENT@@|idempotency|g' \
-e 's|@@PROBE@@|nuthatch-idempotency-probe|g' \
-e 's|@@PROBE_ARGS@@||g' \
-e 's|@@DEADLINE_SECONDS@@|300|g' \
"$RUNTIME/$job.yaml"
run_job_manifest "$job" "$RUNTIME/$job.yaml"
cleanup_application "$lane"

919
tests/kind/lib.sh
View file

@ -1,921 +1,4 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
TESTBED="$ROOT/tests/kind"
RUNTIME="$TESTBED/.runtime"
BIN="$RUNTIME/bin"
KUBECONFIG_PATH="$RUNTIME/kubeconfig"
FIXTURE_LOCK_DIR="${XDG_RUNTIME_DIR:-/tmp/nuthatch-$(id -u)}"
FIXTURE_LOCK_PATH="$FIXTURE_LOCK_DIR/nuthatch-testbed-kind.lock"
CONTEXT=kind-nuthatch-testbed
CLUSTER=nuthatch-testbed
NERDCTL_NAMESPACE=nuthatch-testbed
FIXTURE_NAMESPACE=nuthatch-testbed
set -a
source "$TESTBED/versions.env"
set +a
user_kubeconfig_digest() {
local config_value="${KUBECONFIG:-${HOME:?}/.kube/config}"
local path
local paths
IFS=: read -r -a paths <<< "$config_value"
{
for path in "${paths[@]}"; do
if [[ -f "$path" ]]; then
sha256sum "$path"
else
printf 'missing %s\n' "$path"
fi
done
} | sha256sum | cut -d' ' -f1
}
download_tool() {
local destination="$1"
local checksum="$2"
local url="$3"
if [[ -x "$destination" ]] &&
[[ "$(sha256sum "$destination" | cut -d' ' -f1)" = "$checksum" ]]; then
return
fi
curl --fail --location --silent --show-error "$url" --output "$destination.tmp"
test "$(sha256sum "$destination.tmp" | cut -d' ' -f1)" = "$checksum"
chmod 0755 "$destination.tmp"
mv "$destination.tmp" "$destination"
}
prepare_tools() {
mkdir -p "$BIN"
download_tool \
"$BIN/kind" \
"$KIND_SHA256" \
"https://github.com/kubernetes-sigs/kind/releases/download/$KIND_VERSION/kind-linux-amd64"
download_tool \
"$BIN/kubectl" \
"$KUBECTL_SHA256" \
"https://dl.k8s.io/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl"
test "$("$BIN/kind" version | awk '{print $2}')" = "$KIND_VERSION"
test "$("$BIN/kubectl" --kubeconfig "$KUBECONFIG_PATH" --context "$CONTEXT" version --client=true -o json | jq -r .clientVersion.gitVersion)" = "$KUBECTL_VERSION"
test "$(nerdctl --version | awk '{print $3}')" = "$NERDCTL_VERSION"
test "$(runtime_ctr version | awk '/Client:/{client=1} client && $1=="Version:"{print $2; exit}')" = "v$CONTAINERD_VERSION"
test "$(runtime_ctr version | awk '/Server:/{server=1} server && $1=="Version:"{print $2; exit}')" = "v$CONTAINERD_VERSION"
}
acquire_fixture_lock() {
mkdir -p "$RUNTIME" "$FIXTURE_LOCK_DIR"
exec 9>"$FIXTURE_LOCK_PATH"
if ! flock --nonblock 9; then
printf 'kind test fixture is already in use\n' >&2
return 1
fi
}
configure_provider() {
if [[ -z "${NUTHATCH_REAL_NERDCTL:-}" ]]; then
NUTHATCH_REAL_NERDCTL="$(command -v nerdctl)" || return
fi
export NUTHATCH_REAL_NERDCTL
export CONTAINERD_NAMESPACE="$NERDCTL_NAMESPACE"
export KIND_EXPERIMENTAL_PROVIDER=nerdctl
export PATH="$TESTBED/provider:$PATH"
}
runtime_ctr() {
ctr --address "${CONTAINERD_ADDRESS:-${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/containerd/containerd.sock}" "$@"
}
runtime_namespace_exists() {
"$NUTHATCH_REAL_NERDCTL" namespace list --quiet | grep -Fx "$NERDCTL_NAMESPACE" >/dev/null
}
cluster_exists() {
runtime_namespace_exists &&
"$BIN/kind" get clusters | grep -Fx "$CLUSTER" >/dev/null
}
delete_cluster() {
local container
local containers
if cluster_exists; then
"$BIN/kind" delete cluster --name "$CLUSTER" || true
fi
containers="$("$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" ps \
--all \
--quiet \
--filter "label=io.x-k8s.kind.cluster=$CLUSTER")" || return
while IFS= read -r container; do
[[ -z "$container" ]] ||
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" rm --force "$container" ||
return
done <<< "$containers"
if cluster_exists; then
"$BIN/kind" delete cluster --name "$CLUSTER" || true
fi
if cluster_exists; then
printf 'kind cluster %s still exists after teardown\n' "$CLUSTER" >&2
return 1
fi
}
remove_runtime_namespace() {
local status=0
local digest
local digests
local lease
local leases
local network
for network in kind nuthatch-kind; do
if "$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" network inspect "$network" >/dev/null 2>&1; then
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" network remove "$network" >/dev/null || status=1
fi
done
if ! runtime_namespace_exists; then
return "$status"
fi
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" image prune --all --force >/dev/null || status=1
leases="$(runtime_ctr --namespace "$NERDCTL_NAMESPACE" leases list --quiet)" || status=1
while IFS= read -r lease; do
[[ -z "$lease" ]] || runtime_ctr --namespace "$NERDCTL_NAMESPACE" leases delete "$lease" >/dev/null || status=1
done <<< "$leases"
digests="$(runtime_ctr --namespace "$NERDCTL_NAMESPACE" content list --quiet)" || status=1
while IFS= read -r digest; do
[[ -z "$digest" ]] || runtime_ctr --namespace "$NERDCTL_NAMESPACE" content remove "$digest" >/dev/null || status=1
done <<< "$digests"
"$NUTHATCH_REAL_NERDCTL" namespace remove "$NERDCTL_NAMESPACE" >/dev/null || status=1
runtime_namespace_exists && status=1
return "$status"
}
prepare_provider() {
configure_provider
delete_cluster
remove_runtime_namespace
"$NUTHATCH_REAL_NERDCTL" namespace create "$NERDCTL_NAMESPACE" >/dev/null
}
load_external_image() {
local name="$1"
local source="$2"
local digest="${source##*@}"
local image="nuthatch-testbed/$name:sha256-${digest#sha256:}"
local archive="$RUNTIME/image-archives/$digest.tar"
"$NUTHATCH_REAL_NERDCTL" image pull --platform linux/amd64 --unpack=false --quiet "$source" >&2
"$NUTHATCH_REAL_NERDCTL" image tag "$source" "$image"
"$NUTHATCH_REAL_NERDCTL" image save --platform linux/amd64 --output "$archive" "$image"
printf 'Loading %s from %s\n' "$image" "$source" >&2
"$BIN/kind" load image-archive "$archive" --name "$CLUSTER" >&2
printf '%s\n' "$image"
}
load_local_image() {
local image="$1"
local namespace="${2:-default}"
local archive="$RUNTIME/image-archives/nuthatch-app.tar"
rm -f "$archive"
"$NUTHATCH_REAL_NERDCTL" --namespace "$namespace" image save \
--platform linux/amd64 \
--output "$archive" \
"$image"
"$BIN/kind" load image-archive "$archive" --name "$CLUSTER"
}
render_manifests() {
local postgres="$1" redis="$2" dragonfly="$3" keycloak="$4" zitadel="$5"
local garage="$6" versitygw="$7" coredns="$8" curl="$9" aws_cli="${10}"
local postgres_replicas="${11}" redis_replicas="${12}" dragonfly_replicas="${13}"
local keycloak_replicas="${14}" zitadel_replicas="${15}"
local garage_replicas="${16}" versitygw_replicas="${17}" coredns_replicas="${18}"
sed \
-e "s|@@POSTGRES_IMAGE@@|$postgres|g" \
-e "s|@@REDIS_IMAGE@@|$redis|g" \
-e "s|@@DRAGONFLY_IMAGE@@|$dragonfly|g" \
-e "s|@@KEYCLOAK_IMAGE@@|$keycloak|g" \
-e "s|@@ZITADEL_IMAGE@@|$zitadel|g" \
-e "s|@@GARAGE_IMAGE@@|$garage|g" \
-e "s|@@VERSITYGW_IMAGE@@|$versitygw|g" \
-e "s|@@COREDNS_IMAGE@@|$coredns|g" \
-e "s|@@CURL_IMAGE@@|$curl|g" \
-e "s|@@AWS_CLI_IMAGE@@|$aws_cli|g" \
-e "s|@@POSTGRES_REPLICAS@@|$postgres_replicas|g" \
-e "s|@@REDIS_REPLICAS@@|$redis_replicas|g" \
-e "s|@@DRAGONFLY_REPLICAS@@|$dragonfly_replicas|g" \
-e "s|@@KEYCLOAK_REPLICAS@@|$keycloak_replicas|g" \
-e "s|@@ZITADEL_REPLICAS@@|$zitadel_replicas|g" \
-e "s|@@GARAGE_REPLICAS@@|$garage_replicas|g" \
-e "s|@@VERSITYGW_REPLICAS@@|$versitygw_replicas|g" \
-e "s|@@COREDNS_REPLICAS@@|$coredns_replicas|g" \
"$TESTBED/dependencies.yaml.in" > "$RUNTIME/dependencies.yaml"
sed \
-e "s|@@POSTGRES_IMAGE@@|$postgres|g" \
-e "s|@@REDIS_IMAGE@@|$redis|g" \
-e "s|@@CURL_IMAGE@@|$curl|g" \
-e "s|@@AWS_CLI_IMAGE@@|$aws_cli|g" \
"$TESTBED/probe.yaml.in" > "$RUNTIME/probe.yaml"
{
printf 'POSTGRES_LOCAL_IMAGE=%q\n' "$postgres"
printf 'REDIS_LOCAL_IMAGE=%q\n' "$redis"
printf 'DRAGONFLY_LOCAL_IMAGE=%q\n' "$dragonfly"
printf 'KEYCLOAK_LOCAL_IMAGE=%q\n' "$keycloak"
printf 'ZITADEL_LOCAL_IMAGE=%q\n' "$zitadel"
printf 'GARAGE_LOCAL_IMAGE=%q\n' "$garage"
printf 'VERSITYGW_LOCAL_IMAGE=%q\n' "$versitygw"
printf 'COREDNS_LOCAL_IMAGE=%q\n' "$coredns"
printf 'CURL_LOCAL_IMAGE=%q\n' "$curl"
printf 'AWS_CLI_LOCAL_IMAGE=%q\n' "$aws_cli"
} > "$RUNTIME/fixture.env"
}
kube() {
"$BIN/kubectl" \
--kubeconfig "$KUBECONFIG_PATH" \
--context "$CONTEXT" \
"$@"
}
wait_for_job() {
local job="$1"
local deadline=$((SECONDS + 300))
local complete
local failed
while ((SECONDS < deadline)); do
complete="$(kube --namespace "$FIXTURE_NAMESPACE" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Complete")].status}')"
if [[ "$complete" = True ]]; then
kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" --all-containers
return
fi
failed="$(kube --namespace "$FIXTURE_NAMESPACE" get "job/$job" --output jsonpath='{.status.conditions[?(@.type=="Failed")].status}')"
if [[ "$failed" = True ]]; then
kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" --all-containers >&2 || true
return 1
fi
sleep 1
done
printf 'job/%s did not complete within 300 seconds\n' "$job" >&2
return 1
}
select_fixture_families() {
local requested=("$@")
local family
local candidate
local found
local use_all=false
FIXTURE_FAMILIES=()
if [[ "${#requested[@]}" = 0 ]]; then
requested=(all)
fi
for family in "${requested[@]}"; do
case "$family" in
all) use_all=true ;;
base | durable | coordination | oidc | target) ;;
*)
printf 'unknown fixture family: %s\n' "$family" >&2
return 1
;;
esac
done
if [[ "$use_all" = true ]]; then
requested=(durable coordination oidc target)
fi
for candidate in durable coordination oidc target; do
found=false
for family in "${requested[@]}"; do
if [[ "$family" = "$candidate" ]]; then
found=true
fi
done
if [[ "$found" = true ]]; then
FIXTURE_FAMILIES+=("$candidate")
fi
done
}
fixture_has_family() {
local expected="$1"
local family
for family in "${FIXTURE_FAMILIES[@]}"; do
if [[ "$family" = "$expected" ]]; then
return
fi
done
return 1
}
expand_fixture_matrix() {
local coordination_providers=(-)
local oidc_providers=(-)
local target_providers=(-)
local coordination
local oidc
local target
local lane=0
select_fixture_families "$@" || return
if fixture_has_family coordination; then
coordination_providers=(redis dragonfly)
fi
if fixture_has_family oidc; then
oidc_providers=(keycloak zitadel)
fi
if fixture_has_family target; then
target_providers=(garage versitygw)
fi
for coordination in "${coordination_providers[@]}"; do
for oidc in "${oidc_providers[@]}"; do
for target in "${target_providers[@]}"; do
lane=$((lane + 1))
printf '%03d\t%s\t%s\t%s\n' "$lane" "$coordination" "$oidc" "$target"
done
done
done
}
setup_fixture() {
select_fixture_families "$@" || return
mkdir -p "$RUNTIME"
rm -rf "$RUNTIME/image-archives"
mkdir -p "$RUNTIME/image-archives"
prepare_tools
prepare_provider
"$BIN/kind" create cluster \
--name "$CLUSTER" \
--image "$NODE_IMAGE" \
--config "$TESTBED/cluster.yaml" \
--kubeconfig "$KUBECONFIG_PATH" \
--wait 180s
local postgres=nuthatch-testbed/unused:postgres
local redis=nuthatch-testbed/unused:redis
local dragonfly=nuthatch-testbed/unused:dragonfly
local keycloak=nuthatch-testbed/unused:keycloak
local zitadel=nuthatch-testbed/unused:zitadel
local garage=nuthatch-testbed/unused:garage
local versitygw=nuthatch-testbed/unused:versitygw
local coredns=nuthatch-testbed/unused:coredns
local curl=nuthatch-testbed/unused:curl
local aws_cli=nuthatch-testbed/unused:aws-cli
local postgres_replicas=0 redis_replicas=0 dragonfly_replicas=0
local keycloak_replicas=0 zitadel_replicas=0
local garage_replicas=0 versitygw_replicas=0 coredns_replicas=0
if fixture_has_family durable || fixture_has_family oidc; then
postgres="$(load_external_image postgres "$POSTGRES_IMAGE")"
postgres_replicas=1
fi
if fixture_has_family coordination; then
redis="$(load_external_image redis "$REDIS_IMAGE")"
dragonfly="$(load_external_image dragonfly "$DRAGONFLY_IMAGE")"
redis_replicas=1
dragonfly_replicas=1
fi
if fixture_has_family oidc; then
keycloak="$(load_external_image keycloak "$KEYCLOAK_IMAGE")"
zitadel="$(load_external_image zitadel "$ZITADEL_IMAGE")"
coredns="$(load_external_image coredns "$COREDNS_IMAGE")"
curl="$(load_external_image curl "$CURL_IMAGE")"
keycloak_replicas=1
zitadel_replicas=1
coredns_replicas=1
fi
if fixture_has_family target; then
garage="$(load_external_image garage "$GARAGE_IMAGE")"
versitygw="$(load_external_image versitygw "$VERSITYGW_IMAGE")"
aws_cli="$(load_external_image aws-cli "$AWS_CLI_IMAGE")"
if [[ "$curl" = nuthatch-testbed/unused:curl ]]; then
curl="$(load_external_image curl "$CURL_IMAGE")"
fi
garage_replicas=1
versitygw_replicas=1
fi
render_manifests "$postgres" "$redis" "$dragonfly" "$keycloak" "$zitadel" \
"$garage" "$versitygw" "$coredns" "$curl" "$aws_cli" \
"$postgres_replicas" "$redis_replicas" "$dragonfly_replicas" \
"$keycloak_replicas" "$zitadel_replicas" \
"$garage_replicas" "$versitygw_replicas" "$coredns_replicas"
kube apply --filename "$RUNTIME/dependencies.yaml"
if fixture_has_family durable || fixture_has_family oidc; then
wait_for_deployment postgres
fi
if fixture_has_family coordination; then
wait_for_deployment redis
wait_for_deployment dragonfly
fi
if fixture_has_family oidc; then
wait_for_deployment keycloak
wait_for_deployment zitadel
wait_for_deployment controlled-dns
fi
if fixture_has_family target; then
wait_for_deployment garage
wait_for_deployment versitygw
fi
}
wait_for_deployment() {
kube --namespace "$FIXTURE_NAMESPACE" rollout status "deployment/$1" --timeout 240s
}
verify_dependencies() {
kube apply --filename "$RUNTIME/probe.yaml"
local job
for job in state-probe oidc-probe garage-probe versitygw-probe; do
wait_for_job "$job"
done
}
reset_lane_durable_store() {
local lane="$1"
local database="nuthatch_lane_$lane"
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
env PGPASSWORD=synthetic-postgres psql \
--username nuthatch \
--dbname postgres \
--set ON_ERROR_STOP=1 \
--command "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$database'" \
--command "DROP DATABASE IF EXISTS $database" \
--command "CREATE DATABASE $database" >&2 || return
printf '%s\n' "$database"
}
reset_lane_coordination_store() {
local provider="$1"
local lane="$2"
local database=$((10#$lane))
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" -- \
redis-cli --raw --no-auth-warning -n "$database" FLUSHDB >/dev/null || return
printf '%s\n' "$database"
}
target_endpoint() {
case "$1" in
garage) printf 'http://garage:3900\n' ;;
versitygw) printf 'http://versitygw:10000\n' ;;
*) return 1 ;;
esac
}
target_loopback_endpoint() {
case "$1" in
garage) printf 'http://127.0.0.1:3900\n' ;;
versitygw) printf 'http://127.0.0.1:10000\n' ;;
*) return 1 ;;
esac
}
ensure_target_scope_canary() {
local provider="$1"
local endpoint
endpoint="$(target_loopback_endpoint "$provider")" || return
if ! kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
aws --endpoint-url "$endpoint" s3api create-bucket --bucket nuthatch-fixture >&2; then
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
aws --endpoint-url "$endpoint" s3api head-bucket --bucket nuthatch-fixture >/dev/null || return
fi
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
sh -ec 'printf scope-canary >/tmp/scope-canary && aws --endpoint-url "$1" s3api put-object --bucket nuthatch-fixture --key scope-canary --body /tmp/scope-canary >/dev/null' \
sh "$endpoint"
}
provision_lane_target_principal() {
local provider="$1"
local lane="$2"
local bucket="$3"
local access_key_var="$4"
local secret_key_var="$5"
local endpoint
local access_key
local secret_key
endpoint="$(target_loopback_endpoint "$provider")" || return
case "$provider" in
garage)
printf -v access_key 'GKNUTHATCH%024d' "$((10#$lane))"
printf -v secret_key '%064d' "$((10#$lane))"
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/garage --container garage -- \
/garage bucket create "$bucket" >/dev/null || return
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/garage --container garage -- \
/garage bucket allow \
--read \
--write \
--owner \
"$bucket" \
--key GK00000000000000000000000000000000 >/dev/null || return
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/garage --container garage -- \
/garage key import --yes -n "nuthatch-$lane" "$access_key" "$secret_key" >/dev/null || return
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/garage --container garage -- \
/garage bucket allow --read --write "$bucket" --key "$access_key" >/dev/null || return
;;
versitygw)
access_key="nuthatch-lane-$lane"
printf -v secret_key 'synthetic-lane-%049d' "$((10#$lane))"
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/versitygw --container versitygw -- \
/usr/local/bin/versitygw admin \
--endpoint-url http://127.0.0.1:10001 \
create-user \
--access "$access_key" \
--secret "$secret_key" \
--role user \
--user-id "$((10#$lane))" \
--group-id "$((10#$lane))" >/dev/null || return
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/versitygw --container versitygw -- \
/usr/local/bin/versitygw admin \
--endpoint-url http://127.0.0.1:10001 \
create-bucket \
--bucket "$bucket" \
--owner "$access_key" >/dev/null || return
;;
*) return 1 ;;
esac
printf -v "$access_key_var" '%s' "$access_key"
printf -v "$secret_key_var" '%s' "$secret_key"
}
keycloak_admin_token() {
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
sh -ec '
/opt/keycloak/bin/kcadm.sh config credentials \
--server http://127.0.0.1:8080 \
--realm master \
--user nuthatch-admin \
--password synthetic-keycloak >/dev/null
/opt/keycloak/bin/kcadm.sh get serverinfo >/dev/null
' >/dev/null
}
provision_lane_keycloak_registration() {
local lane="$1"
local redirect_uri="$2"
local client_id_var="$3"
local client_secret_var="$4"
local additional_audiences_var="$5"
local user_a_var="$6"
local user_b_var="$7"
local client_id="nuthatch-$lane"
local client_secret="synthetic-keycloak-client-secret-$lane"
local username
keycloak_admin_token || return
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
/opt/keycloak/bin/kcadm.sh create clients \
--realm master \
-s "clientId=$client_id" \
-s enabled=true \
-s protocol=openid-connect \
-s publicClient=false \
-s clientAuthenticatorType=client-secret \
-s standardFlowEnabled=true \
-s directAccessGrantsEnabled=false \
-s "secret=$client_secret" \
-s "redirectUris=[\"$redirect_uri\"]" \
-s 'attributes={"pkce.code.challenge.method":"S256"}' >/dev/null || return
for username in "nuthatch-$lane-a" "nuthatch-$lane-b"; do
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
/opt/keycloak/bin/kcadm.sh create users \
--realm master \
-s "username=$username" \
-s enabled=true \
-s "firstName=$username" >/dev/null || return
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/keycloak -- \
/opt/keycloak/bin/kcadm.sh set-password \
--realm master \
--username "$username" \
--new-password Synthetic-OIDC-1! >/dev/null || return
done
printf -v "$client_id_var" '%s' "$client_id"
printf -v "$client_secret_var" '%s' "$client_secret"
printf -v "$additional_audiences_var" ''
printf -v "$user_a_var" '%s' "nuthatch-$lane-a"
printf -v "$user_b_var" '%s' "nuthatch-$lane-b"
}
zitadel_api() {
local token_path="$1"
local method="$2"
local path="$3"
local body="$4"
kube --namespace "$FIXTURE_NAMESPACE" exec \
deployment/zitadel \
--container oidc-provisioner \
-- sh -ec '
token="$(tr -d "\r\n" < "$1")"
curl --fail --silent --show-error \
--request "$2" \
--header "Host: zitadel.nuthatch.test" \
--header "Authorization: Bearer $token" \
--header "Content-Type: application/json" \
--data "$4" \
"http://127.0.0.1:8080$3"
' sh "$token_path" "$method" "$path" "$body"
}
provision_lane_zitadel_registration() {
local lane="$1"
local redirect_uri="$2"
local client_id_var="$3"
local client_secret_var="$4"
local additional_audiences_var="$5"
local user_a_var="$6"
local user_b_var="$7"
local project_response
local project_id
local app_response
local client_id
local client_secret
local username
local user_response
local user_ids=()
project_response="$(
zitadel_api \
/bootstrap/admin.pat \
POST \
/management/v1/projects \
"{\"name\":\"Nuthatch lane $lane\"}"
)" || return
project_id="$(jq -er .id <<< "$project_response")" || return
app_response="$(
zitadel_api \
/bootstrap/admin.pat \
POST \
"/management/v1/projects/$project_id/apps/oidc" \
"{\"projectId\":\"$project_id\",\"name\":\"Nuthatch lane $lane\",\"redirectUris\":[\"$redirect_uri\"],\"responseTypes\":[\"OIDC_RESPONSE_TYPE_CODE\"],\"grantTypes\":[\"OIDC_GRANT_TYPE_AUTHORIZATION_CODE\"],\"appType\":\"OIDC_APP_TYPE_WEB\",\"authMethodType\":\"OIDC_AUTH_METHOD_TYPE_BASIC\",\"version\":\"OIDC_VERSION_1_0\",\"devMode\":false}"
)" || return
client_id="$(jq -er .clientId <<< "$app_response")" || return
client_secret="$(jq -er .clientSecret <<< "$app_response")" || return
for username in "nuthatch-$lane-a" "nuthatch-$lane-b"; do
user_response="$(
zitadel_api \
/bootstrap/admin.pat \
POST \
/management/v1/users/human \
"{\"userName\":\"$username\",\"profile\":{\"firstName\":\"Nuthatch\",\"lastName\":\"$username\",\"displayName\":\"$username\",\"preferredLanguage\":\"en\"},\"email\":{\"email\":\"$username@example.test\",\"isEmailVerified\":true},\"initialPassword\":\"Synthetic-OIDC-1!\"}"
)" || return
user_ids+=("$(jq -er .userId <<< "$user_response")") || return
done
printf -v "$client_id_var" '%s' "$client_id"
printf -v "$client_secret_var" '%s' "$client_secret"
printf -v "$additional_audiences_var" '%s' "$project_id"
printf -v "$user_a_var" '%s' "${user_ids[0]}"
printf -v "$user_b_var" '%s' "${user_ids[1]}"
}
create_oidc_automation_secret() {
local lane="$1"
local provider="$2"
local login_pat=unused
if [[ "$provider" = zitadel ]]; then
login_pat="$(
kube --namespace "$FIXTURE_NAMESPACE" exec \
deployment/zitadel \
--container oidc-provisioner \
-- sh -ec 'tr -d "\r\n" < /bootstrap/login-client.pat'
)" || return
fi
kube create secret generic "nuthatch-oidc-automation-$lane" \
--namespace "$FIXTURE_NAMESPACE" \
--from-literal="NUTHATCH_OIDC_LOGIN_PAT=$login_pat" \
--dry-run=client \
--output yaml |
kube apply --filename - >/dev/null
}
write_lane_profile() {
local lane="$1"
local coordination="$2"
local oidc="$3"
local target="$4"
local profile="nuthatch-fixture-lane-$lane"
local profile_file="$RUNTIME/$profile.env"
local database=
local coordination_database=
local oidc_issuer=
local oidc_client_id=
local oidc_client_secret=
local oidc_additional_audiences=
local oidc_redirect_uri=
local oidc_user_a=
local oidc_user_b=
local target_url=
local target_bucket=
local target_region=
local target_access_key=
local target_secret_key=
if fixture_has_family durable; then
database="$(reset_lane_durable_store "$lane")" || return
fi
if fixture_has_family coordination; then
coordination_database="$(reset_lane_coordination_store "$coordination" "$lane")" || return
fi
if fixture_has_family oidc; then
oidc_redirect_uri="https://nuthatch-$lane:8080/oidc/callback"
case "$oidc" in
keycloak) oidc_issuer=http://keycloak.nuthatch.test:8080/realms/master ;;
zitadel) oidc_issuer=http://zitadel.nuthatch.test:8080 ;;
*) return 1 ;;
esac
case "$oidc" in
keycloak)
provision_lane_keycloak_registration \
"$lane" \
"$oidc_redirect_uri" \
oidc_client_id \
oidc_client_secret \
oidc_additional_audiences \
oidc_user_a \
oidc_user_b || return
;;
zitadel)
provision_lane_zitadel_registration \
"$lane" \
"$oidc_redirect_uri" \
oidc_client_id \
oidc_client_secret \
oidc_additional_audiences \
oidc_user_a \
oidc_user_b || return
;;
*) return 1 ;;
esac
create_oidc_automation_secret "$lane" "$oidc" || return
fi
if fixture_has_family target; then
target_bucket="nuthatch-lane-$lane"
target_url="$(target_endpoint "$target")" || return
ensure_target_scope_canary "$target" || return
provision_lane_target_principal \
"$target" \
"$lane" \
"$target_bucket" \
target_access_key \
target_secret_key || return
case "$target" in
garage) target_region=garage ;;
versitygw) target_region=us-east-1 ;;
*) return 1 ;;
esac
fi
{
printf 'NUTHATCH_FIXTURE_PROFILE=%s\n' "$profile"
printf 'NUTHATCH_FIXTURE_LANE=%s\n' "$lane"
printf 'NUTHATCH_COORDINATION_PROVIDER=%s\n' "$coordination"
printf 'NUTHATCH_OIDC_PROVIDER=%s\n' "$oidc"
printf 'NUTHATCH_TARGET_PROVIDER=%s\n' "$target"
printf 'NUTHATCH_PG_CONN=host=postgres port=5432 user=nuthatch password=synthetic-postgres dbname=%s\n' "$database"
printf 'NUTHATCH_REDIS_URL=redis://%s:6379/%s\n' "$coordination" "$coordination_database"
printf 'NUTHATCH_OIDC_ISSUER=%s\n' "$oidc_issuer"
printf 'NUTHATCH_OIDC_CLIENT_ID=%s\n' "$oidc_client_id"
printf 'NUTHATCH_OIDC_ADDITIONAL_AUDIENCES=%s\n' "$oidc_additional_audiences"
printf 'NUTHATCH_OIDC_REDIRECT_URI=%s\n' "$oidc_redirect_uri"
printf 'NUTHATCH_OIDC_USER_A=%s\n' "$oidc_user_a"
printf 'NUTHATCH_OIDC_USER_B=%s\n' "$oidc_user_b"
printf 'NUTHATCH_OIDC_USER_PASSWORD=Synthetic-OIDC-1!\n'
printf 'NUTHATCH_S3_ENDPOINT=%s\n' "$target_url"
printf 'NUTHATCH_S3_BUCKET=%s\n' "$target_bucket"
printf 'NUTHATCH_S3_REGION=%s\n' "$target_region"
printf 'NUTHATCH_S3_ACCESS_KEY=%s\n' "$target_access_key"
printf 'NUTHATCH_S3_SECRET_KEY=%s\n' "$target_secret_key"
printf 'NUTHATCH_S3_PATH_STYLE=true\n'
} > "$profile_file" || return
if fixture_has_family oidc; then
printf '%s' "$oidc_client_secret" > "$profile_file.oidc-secret" || return
chmod 0600 "$profile_file.oidc-secret" || return
fi
kube create configmap "$profile" \
--namespace "$FIXTURE_NAMESPACE" \
--from-env-file="$profile_file" \
--dry-run=client \
--output yaml |
kube apply --filename - >/dev/null || return
kube create configmap "$profile-file" \
--namespace "$FIXTURE_NAMESPACE" \
--from-file="profile.env=$profile_file" \
--dry-run=client \
--output yaml |
kube apply --filename - >/dev/null || return
printf '%s\n' "$profile"
}
render_lane_job() {
local job="$1"
local manifest="$2"
local image="$3"
local profile="$4"
source "$RUNTIME/fixture.env" || return
local coordination_image="$REDIS_LOCAL_IMAGE"
sed \
-e "s|@@JOB_NAME@@|$job|g" \
-e "s|@@PROFILE_NAME@@|$profile|g" \
-e "s|@@TEST_IMAGE@@|$image|g" \
-e "s|@@POSTGRES_IMAGE@@|$POSTGRES_LOCAL_IMAGE|g" \
-e "s|@@COORDINATION_IMAGE@@|$coordination_image|g" \
-e "s|@@CURL_IMAGE@@|$CURL_LOCAL_IMAGE|g" \
-e "s|@@AWS_CLI_IMAGE@@|$AWS_CLI_LOCAL_IMAGE|g" \
"$manifest" > "$RUNTIME/$job.yaml"
}
run_job_manifest() {
local job="$1"
local manifest="$2"
kube --namespace "$FIXTURE_NAMESPACE" delete "job/$job" --ignore-not-found >/dev/null || return
kube apply --filename "$manifest" >/dev/null || return
wait_for_job "$job"
}
cleanup_lane_durable_store() {
local lane="$1"
local database="nuthatch_lane_$lane"
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
env PGPASSWORD=synthetic-postgres psql \
--username nuthatch \
--dbname postgres \
--set ON_ERROR_STOP=1 \
--command "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$database'" \
--command "DROP DATABASE IF EXISTS $database" >/dev/null
}
cleanup_lane_coordination_store() {
local provider="$1"
local lane="$2"
local database=$((10#$lane))
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" -- \
redis-cli --raw --no-auth-warning -n "$database" FLUSHDB >/dev/null
}
cleanup_lane_target_store() {
local provider="$1"
local bucket="$2"
local endpoint
endpoint="$(target_loopback_endpoint "$provider")" || return
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
aws --endpoint-url "$endpoint" s3 rb "s3://$bucket" --force >/dev/null
}
cleanup_lane_state() {
local lane="$1"
local coordination="$2"
local target="$4"
local status=0
if fixture_has_family durable; then
cleanup_lane_durable_store "$lane" || status=1
fi
if fixture_has_family coordination; then
cleanup_lane_coordination_store "$coordination" "$lane" || status=1
fi
if fixture_has_family target; then
cleanup_lane_target_store "$target" "nuthatch-lane-$lane" || status=1
fi
return "$status"
}
cleanup_lane_resources() {
local job="$1"
local profile="$2"
kube --namespace "$FIXTURE_NAMESPACE" delete \
"job/$job" \
"configmap/$profile" \
"configmap/$profile-file" \
"secret/nuthatch-oidc-automation-${profile##*-}" \
--ignore-not-found >/dev/null || return
rm -f \
"$RUNTIME/$profile.env" \
"$RUNTIME/$profile.env.oidc-secret" \
"$RUNTIME/$job.yaml"
}
run_job_matrix() {
local base_job="$1"
local manifest="$2"
local image="$3"
shift 3
local lane
local coordination
local oidc
local target
local profile
local job
select_fixture_families "$@" || return
while IFS=$'\t' read -r lane coordination oidc target; do
profile="$(write_lane_profile "$lane" "$coordination" "$oidc" "$target")" || return
job="$base_job-$lane"
render_lane_job "$job" "$manifest" "$image" "$profile" || return
printf 'Running lane %s: coordination=%s oidc=%s target=%s\n' \
"$lane" "$coordination" "$oidc" "$target"
run_job_manifest "$job" "$RUNTIME/$job.yaml" || return
cleanup_lane_state "$lane" "$coordination" "$oidc" "$target" || return
cleanup_lane_resources "$job" "$profile" || return
done < <(expand_fixture_matrix "$@")
}
destroy_fixture() {
local status=0
configure_provider || return
delete_cluster || status=1
remove_runtime_namespace || status=1
rm -rf "$RUNTIME/image-archives" || status=1
rm -f \
"$KUBECONFIG_PATH" \
"$RUNTIME/dependencies.yaml" \
"$RUNTIME/probe.yaml" \
"$RUNTIME/fixture.env" \
"$RUNTIME"/nuthatch-fixture-lane-*.env \
"$RUNTIME"/nuthatch-fixture-lane-*.env.oidc-secret \
"$RUNTIME"/product-app-*.yaml || status=1
return "$status"
}
source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/fixture/core.sh"

73
tests/kind/matrix-job.yaml.in
View file

@ -18,83 +18,18 @@ spec:
searches: [nuthatch-testbed.svc.cluster.local, svc.cluster.local, cluster.local]
options:
- {name: ndots, value: "5"}
initContainers:
- name: postgres
image: @@POSTGRES_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args: [psql "$NUTHATCH_PG_CONN" --set ON_ERROR_STOP=1 --command "CREATE TABLE lane_canary (value text NOT NULL)" --command "INSERT INTO lane_canary VALUES ('$NUTHATCH_FIXTURE_PROFILE')"]
envFrom:
- configMapRef: {name: @@PROFILE_NAME@@}
resources:
requests: {cpu: 25m, memory: 32Mi}
limits: {cpu: "1", memory: 128Mi}
- name: coordination
image: @@COORDINATION_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args: [redis-cli -u "$NUTHATCH_REDIS_URL" SET lane-canary "$NUTHATCH_FIXTURE_PROFILE" >/dev/null && test "$(redis-cli -u "$NUTHATCH_REDIS_URL" GET lane-canary)" = "$NUTHATCH_FIXTURE_PROFILE"]
envFrom:
- configMapRef: {name: @@PROFILE_NAME@@}
resources:
requests: {cpu: 25m, memory: 32Mi}
limits: {cpu: "1", memory: 128Mi}
- name: oidc
image: @@CURL_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args:
- |
discovery="$(curl --fail --silent --show-error "$NUTHATCH_OIDC_ISSUER/.well-known/openid-configuration")"
printf '%s' "$discovery" | grep -F '"issuer"' >/dev/null
envFrom:
- configMapRef: {name: @@PROFILE_NAME@@}
resources:
requests: {cpu: 25m, memory: 32Mi}
limits: {cpu: "1", memory: 128Mi}
- name: target
image: @@AWS_CLI_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args:
- |
export AWS_ACCESS_KEY_ID="$NUTHATCH_S3_ACCESS_KEY"
export AWS_SECRET_ACCESS_KEY="$NUTHATCH_S3_SECRET_KEY"
export AWS_DEFAULT_REGION="$NUTHATCH_S3_REGION"
printf '%s\n' "$NUTHATCH_FIXTURE_PROFILE" >/tmp/input
aws --endpoint-url "$NUTHATCH_S3_ENDPOINT" s3api put-object --bucket "$NUTHATCH_S3_BUCKET" --key lane-canary --body /tmp/input >/dev/null
aws --endpoint-url "$NUTHATCH_S3_ENDPOINT" s3api get-object --bucket "$NUTHATCH_S3_BUCKET" --key lane-canary /tmp/output >/dev/null
test "$(cat /tmp/input)" = "$(cat /tmp/output)"
envFrom:
- configMapRef: {name: @@PROFILE_NAME@@}
resources:
requests: {cpu: 25m, memory: 64Mi}
limits: {cpu: "1", memory: 256Mi}
containers:
- name: profile
- name: fixture-probe
image: @@TEST_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args:
- |
test -n "$NUTHATCH_FIXTURE_PROFILE"
test "$NUTHATCH_S3_BUCKET" = "nuthatch-lane-$NUTHATCH_FIXTURE_LANE"
case "$NUTHATCH_PG_CONN" in
*"dbname=nuthatch_lane_$NUTHATCH_FIXTURE_LANE") ;;
*) exit 1 ;;
esac
grep -Fx "NUTHATCH_FIXTURE_PROFILE=$NUTHATCH_FIXTURE_PROFILE" /run/nuthatch-fixture/profile.env
grep -Fx "NUTHATCH_COORDINATION_PROVIDER=$NUTHATCH_COORDINATION_PROVIDER" /run/nuthatch-fixture/profile.env
grep -Fx "NUTHATCH_OIDC_PROVIDER=$NUTHATCH_OIDC_PROVIDER" /run/nuthatch-fixture/profile.env
grep -Fx "NUTHATCH_TARGET_PROVIDER=$NUTHATCH_TARGET_PROVIDER" /run/nuthatch-fixture/profile.env
printf 'lane profile injection passed\n'
command: [nuthatch-fixture-probe]
envFrom:
- configMapRef: {name: @@PROFILE_NAME@@}
volumeMounts:
- {name: fixture-profile, mountPath: /run/nuthatch-fixture, readOnly: true}
resources:
requests: {cpu: 25m, memory: 32Mi}
limits: {cpu: "1", memory: 128Mi}
requests: {cpu: 50m, memory: 64Mi}
limits: {cpu: "1", memory: 256Mi}
volumes:
- name: fixture-profile
configMap:

9
tests/kind/matrix-run.sh
View file

@ -7,6 +7,7 @@ source "$ROOT/tests/kind/lib.sh"
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
SUCCEEDED=false
OWNS_FIXTURE=false
TEST_IMAGE=nuthatch-fixture-probe:local
cleanup() {
local status=$?
@ -34,10 +35,16 @@ acquire_fixture_lock
OWNS_FIXTURE=true
setup_fixture all
source "$RUNTIME/fixture.env"
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" build \
--file "$ROOT/Dockerfile" \
--target test-runtime \
--tag "$TEST_IMAGE" \
"$ROOT"
load_local_image "$TEST_IMAGE" "$NERDCTL_NAMESPACE"
run_job_matrix \
fixture-matrix \
"$TESTBED/matrix-job.yaml.in" \
"$CURL_LOCAL_IMAGE" \
"$TEST_IMAGE" \
all
SUCCEEDED=true
printf 'Nuthatch fixture matrix passed\n'

218
tests/kind/probe.yaml.in
View file

@ -1,67 +1,10 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: probe-payload
namespace: nuthatch-testbed
data:
payload: nuthatch-dependency-compatibility-probe
---
apiVersion: batch/v1
kind: Job
metadata:
name: state-probe
namespace: nuthatch-testbed
spec:
backoffLimit: 0
ttlSecondsAfterFinished: 300
template:
spec:
automountServiceAccountToken: false
enableServiceLinks: false
restartPolicy: Never
initContainers:
- name: postgres
image: @@POSTGRES_IMAGE@@
imagePullPolicy: Never
command: [psql]
args:
- -h
- postgres
- -U
- nuthatch
- -d
- nuthatch
- -v
- ON_ERROR_STOP=1
- -c
- "CREATE TABLE dependency_probe (value text NOT NULL)"
- -c
- "INSERT INTO dependency_probe VALUES ('passed')"
- -c
- "SELECT value = 'passed' FROM dependency_probe"
env:
- {name: PGPASSWORD, value: synthetic-postgres}
- name: redis
image: @@REDIS_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args: ["redis-cli -h redis SET nuthatch-probe passed && test \"$(redis-cli -h redis GET nuthatch-probe)\" = passed"]
containers:
- name: dragonfly
image: @@REDIS_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args: ["redis-cli -h dragonfly SET nuthatch-probe passed && test \"$(redis-cli -h dragonfly GET nuthatch-probe)\" = passed && printf 'PostgreSQL and Redis-compatible stores passed\\n'"]
resources:
requests: {cpu: 25m, memory: 32Mi}
limits: {cpu: "1", memory: 128Mi}
---
apiVersion: batch/v1
kind: Job
metadata:
name: oidc-probe
name: dependency-probe
namespace: nuthatch-testbed
spec:
activeDeadlineSeconds: 300
backoffLimit: 0
ttlSecondsAfterFinished: 300
template:
@ -72,155 +15,14 @@ spec:
dnsPolicy: None
dnsConfig:
nameservers: [10.96.0.53]
searches: [nuthatch-testbed.svc.cluster.local, svc.cluster.local, cluster.local]
options:
- {name: ndots, value: "5"}
containers:
- name: curl
image: @@CURL_IMAGE@@
- name: dependency-probe
image: @@TEST_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args:
- |
probe() {
url="$1"
pattern="$2"
attempt=0
until curl --fail --silent --show-error "$url" | grep -q "$pattern"; do
attempt=$((attempt + 1))
test "$attempt" -lt 30
sleep 1
done
}
probe http://keycloak.nuthatch.test:8080/realms/master/.well-known/openid-configuration '"issuer"'
probe http://keycloak.nuthatch.test:8080/realms/master/protocol/openid-connect/certs '"keys"'
probe http://zitadel.nuthatch.test:8080/.well-known/openid-configuration '"issuer"'
probe http://zitadel.nuthatch.test:8080/oauth/v2/keys '"keys"'
printf 'Keycloak and ZITADEL discovery passed through controlled DNS\n'
command: [nuthatch-dependency-probe]
resources:
requests: {cpu: 25m, memory: 32Mi}
limits: {cpu: "1", memory: 128Mi}
---
apiVersion: batch/v1
kind: Job
metadata:
name: garage-probe
namespace: nuthatch-testbed
spec:
backoffLimit: 0
ttlSecondsAfterFinished: 300
template:
spec:
automountServiceAccountToken: false
enableServiceLinks: false
restartPolicy: Never
initContainers:
- name: upload
image: @@AWS_CLI_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args:
- |
attempt=0
until aws --endpoint-url http://garage:3900 s3api put-object --bucket nuthatch-fixture --key probe/payload --body /fixture/payload; do
attempt=$((attempt + 1))
test "$attempt" -lt 30
sleep 1
done
env:
- {name: AWS_ACCESS_KEY_ID, value: GK00000000000000000000000000000000}
- {name: AWS_SECRET_ACCESS_KEY, value: "0000000000000000000000000000000000000000000000000000000000000000"}
- {name: AWS_DEFAULT_REGION, value: garage}
volumeMounts:
- {name: fixture, mountPath: /fixture}
- name: download
image: @@AWS_CLI_IMAGE@@
imagePullPolicy: Never
command: [aws]
args: [--endpoint-url, "http://garage:3900", s3api, get-object, --bucket, nuthatch-fixture, --key, probe/payload, /work/result]
env:
- {name: AWS_ACCESS_KEY_ID, value: GK00000000000000000000000000000000}
- {name: AWS_SECRET_ACCESS_KEY, value: "0000000000000000000000000000000000000000000000000000000000000000"}
- {name: AWS_DEFAULT_REGION, value: garage}
volumeMounts:
- {name: work, mountPath: /work}
containers:
- name: compare
image: @@CURL_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args: ["cmp /fixture/payload /work/result && printf 'Garage put/get passed\\n'"]
volumeMounts:
- {name: fixture, mountPath: /fixture}
- {name: work, mountPath: /work}
resources:
requests: {cpu: 25m, memory: 32Mi}
limits: {cpu: "1", memory: 128Mi}
volumes:
- name: fixture
configMap: {name: probe-payload}
- name: work
emptyDir: {}
---
apiVersion: batch/v1
kind: Job
metadata:
name: versitygw-probe
namespace: nuthatch-testbed
spec:
backoffLimit: 0
ttlSecondsAfterFinished: 300
template:
spec:
automountServiceAccountToken: false
enableServiceLinks: false
restartPolicy: Never
initContainers:
- name: bucket
image: @@AWS_CLI_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args:
- |
aws --endpoint-url http://versitygw:10000 s3api create-bucket --bucket nuthatch-fixture ||
aws --endpoint-url http://versitygw:10000 s3api head-bucket --bucket nuthatch-fixture
env:
- {name: AWS_ACCESS_KEY_ID, value: synthetic-s3-access}
- {name: AWS_SECRET_ACCESS_KEY, value: synthetic-s3-secret}
- {name: AWS_DEFAULT_REGION, value: us-east-1}
- name: upload
image: @@AWS_CLI_IMAGE@@
imagePullPolicy: Never
command: [aws]
args: [--endpoint-url, "http://versitygw:10000", s3api, put-object, --bucket, nuthatch-fixture, --key, probe/payload, --body, /fixture/payload]
env:
- {name: AWS_ACCESS_KEY_ID, value: synthetic-s3-access}
- {name: AWS_SECRET_ACCESS_KEY, value: synthetic-s3-secret}
- {name: AWS_DEFAULT_REGION, value: us-east-1}
volumeMounts:
- {name: fixture, mountPath: /fixture}
- name: download
image: @@AWS_CLI_IMAGE@@
imagePullPolicy: Never
command: [aws]
args: [--endpoint-url, "http://versitygw:10000", s3api, get-object, --bucket, nuthatch-fixture, --key, probe/payload, /work/result]
env:
- {name: AWS_ACCESS_KEY_ID, value: synthetic-s3-access}
- {name: AWS_SECRET_ACCESS_KEY, value: synthetic-s3-secret}
- {name: AWS_DEFAULT_REGION, value: us-east-1}
volumeMounts:
- {name: work, mountPath: /work}
containers:
- name: compare
image: @@CURL_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args: ["cmp /fixture/payload /work/result && printf 'VersityGW put/get passed\\n'"]
volumeMounts:
- {name: fixture, mountPath: /fixture}
- {name: work, mountPath: /work}
resources:
requests: {cpu: 25m, memory: 32Mi}
limits: {cpu: "1", memory: 128Mi}
volumes:
- name: fixture
configMap: {name: probe-payload}
- name: work
emptyDir: {}
requests: {cpu: 50m, memory: 64Mi}
limits: {cpu: "1", memory: 256Mi}

1609
tests/kind/product-matrix-job.yaml.in
View file

File diff suppressed because it is too large Load diff

174
tests/kind/product-run.sh Executable file
View file

@ -0,0 +1,174 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
source "$ROOT/tests/kind/lib.sh"
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
SUCCEEDED=false
OWNS_FIXTURE=false
APP_IMAGE=nuthatch-product:local
cleanup() {
local status=$?
trap - EXIT INT TERM
if [[ "${NUTHATCH_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
printf 'retained context: %s\n' "$CONTEXT"
exit "$status"
fi
if [[ "$OWNS_FIXTURE" = true ]]; then
destroy_fixture || status=1
fi
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
printf 'user kubeconfig changed unexpectedly\n' >&2
status=1
fi
exit "$status"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
render_application() {
local lane="$1"
local profile_file="$2"
local pg_conn
local redis_url
local oidc_issuer
local oidc_client_id
local oidc_additional_audiences
local oidc_client_secret
local s3_endpoint
local s3_region
local s3_bucket
local s3_access_key
local s3_secret_key
pg_conn="$(sed -n 's/^NUTHATCH_PG_CONN=//p' "$profile_file")"
redis_url="$(sed -n 's/^NUTHATCH_REDIS_URL=//p' "$profile_file")"
oidc_issuer="$(sed -n 's/^NUTHATCH_OIDC_ISSUER=//p' "$profile_file")"
oidc_client_id="$(sed -n 's/^NUTHATCH_OIDC_CLIENT_ID=//p' "$profile_file")"
oidc_additional_audiences="$(
sed -n 's/^NUTHATCH_OIDC_ADDITIONAL_AUDIENCES=//p' "$profile_file"
)"
oidc_client_secret="$(<"$profile_file.oidc-secret")"
s3_endpoint="$(sed -n 's/^NUTHATCH_S3_ENDPOINT=//p' "$profile_file")"
s3_region="$(sed -n 's/^NUTHATCH_S3_REGION=//p' "$profile_file")"
s3_bucket="$(sed -n 's/^NUTHATCH_S3_BUCKET=//p' "$profile_file")"
s3_access_key="$(sed -n 's/^NUTHATCH_S3_ACCESS_KEY=//p' "$profile_file")"
s3_secret_key="$(sed -n 's/^NUTHATCH_S3_SECRET_KEY=//p' "$profile_file")"
sed \
-e "s|@@LANE@@|$lane|g" \
-e "s|@@APP_IMAGE@@|$APP_IMAGE|g" \
-e "s|@@PG_CONN@@|$pg_conn|g" \
-e "s|@@REDIS_URL@@|$redis_url|g" \
-e "s|@@OIDC_ISSUER@@|$oidc_issuer|g" \
-e "s|@@OIDC_CLIENT_ID@@|$oidc_client_id|g" \
-e "s|@@OIDC_ADDITIONAL_AUDIENCES@@|$oidc_additional_audiences|g" \
-e "s|@@OIDC_CLIENT_SECRET@@|$oidc_client_secret|g" \
-e "s|@@S3_ENDPOINT@@|$s3_endpoint|g" \
-e "s|@@S3_REGION@@|$s3_region|g" \
-e "s|@@S3_BUCKET@@|$s3_bucket|g" \
-e "s|@@S3_ACCESS_KEY@@|$s3_access_key|g" \
-e "s|@@S3_SECRET_KEY@@|$s3_secret_key|g" \
"$ROOT/deploy/kubernetes/nuthatch.yaml.in" > "$RUNTIME/product-app-$lane.yaml"
sed -i \
-e 's/NUTHATCH_BROWSE_PAGE_SIZE: "200"/NUTHATCH_BROWSE_PAGE_SIZE: "2"/' \
-e 's/NUTHATCH_SEARCH_MAX_KEYS: "500"/NUTHATCH_SEARCH_MAX_KEYS: "3"/' \
-e 's/NUTHATCH_DELETE_PAGE_SIZE: "100"/NUTHATCH_DELETE_PAGE_SIZE: "2"/' \
"$RUNTIME/product-app-$lane.yaml"
}
cleanup_application() {
local lane="$1"
kube --namespace "$FIXTURE_NAMESPACE" delete \
"deployment/nuthatch-$lane" \
"service/nuthatch-$lane" \
"configmap/nuthatch-$lane" \
"secret/nuthatch-$lane" \
--ignore-not-found >/dev/null
rm -f "$RUNTIME/product-app-$lane.yaml"
}
set_product_profile() {
local profile="$1"
local mode="$2"
local profile_file="$RUNTIME/$profile.env"
sed -i '/^NUTHATCH_PRODUCT_MODE=/d' "$profile_file"
printf 'NUTHATCH_PRODUCT_MODE=%s\n' "$mode" >> "$profile_file"
kube create configmap "$profile" \
--namespace "$FIXTURE_NAMESPACE" \
--from-env-file="$profile_file" \
--dry-run=client \
--output yaml |
kube apply --filename - >/dev/null
}
acquire_fixture_lock
OWNS_FIXTURE=true
setup_fixture durable coordination oidc target
source "$RUNTIME/fixture.env"
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" build \
--file "$ROOT/Dockerfile" \
--target test-runtime \
--tag "$APP_IMAGE" \
"$ROOT"
load_local_image "$APP_IMAGE" "$NERDCTL_NAMESPACE"
while IFS=$'\t' read -r lane coordination oidc target; do
profile="$(write_lane_profile "$lane" "$coordination" "$oidc" "$target")"
set_product_profile "$profile" verify
render_application "$lane" "$RUNTIME/$profile.env"
kube apply --filename "$RUNTIME/product-app-$lane.yaml" >/dev/null
kube --namespace "$FIXTURE_NAMESPACE" rollout status \
"deployment/nuthatch-$lane" --timeout=180s
job="product-app-$lane"
render_lane_job \
"$job" \
"$TESTBED/application-probe-job.yaml.in" \
"$APP_IMAGE" \
"$profile"
sed -i \
-e "s|@@LANE@@|$lane|g" \
-e 's|@@COMPONENT@@|product|g' \
-e 's|@@PROBE@@|nuthatch-product-probe|g' \
-e 's|@@PROBE_ARGS@@|"verify"|g' \
-e 's|@@DEADLINE_SECONDS@@|300|g' \
"$RUNTIME/$job.yaml"
printf 'Running lane %s: coordination=%s oidc=%s target=%s\n' \
"$lane" "$coordination" "$oidc" "$target"
run_job_manifest "$job" "$RUNTIME/$job.yaml"
sed -i \
's/NUTHATCH_SEARCH_TIMEOUT_MS: "2000"/NUTHATCH_SEARCH_TIMEOUT_MS: "1"/' \
"$RUNTIME/product-app-$lane.yaml"
kube apply --filename "$RUNTIME/product-app-$lane.yaml" >/dev/null
kube --namespace "$FIXTURE_NAMESPACE" rollout restart \
"deployment/nuthatch-$lane" >/dev/null
kube --namespace "$FIXTURE_NAMESPACE" rollout status \
"deployment/nuthatch-$lane" --timeout=180s
set_product_profile "$profile" timeout
timeout_job="product-timeout-$lane"
render_lane_job \
"$timeout_job" \
"$TESTBED/application-probe-job.yaml.in" \
"$APP_IMAGE" \
"$profile"
sed -i \
-e "s|@@LANE@@|$lane|g" \
-e 's|@@COMPONENT@@|product|g' \
-e 's|@@PROBE@@|nuthatch-product-probe|g' \
-e 's|@@PROBE_ARGS@@|"timeout"|g' \
-e 's|@@DEADLINE_SECONDS@@|300|g' \
"$RUNTIME/$timeout_job.yaml"
run_job_manifest "$timeout_job" "$RUNTIME/$timeout_job.yaml"
kube --namespace "$FIXTURE_NAMESPACE" delete \
"job/$timeout_job" --ignore-not-found >/dev/null
rm -f "$RUNTIME/$timeout_job.yaml"
cleanup_application "$lane"
cleanup_lane_state "$lane" "$coordination" "$oidc" "$target"
cleanup_lane_resources "$job" "$profile"
done < <(expand_fixture_matrix coordination oidc target)
SUCCEEDED=true
printf 'Nuthatch product matrix gate passed\n'

41
tests/kind/recovery-log-job.yaml.in Normal file
View file

@ -0,0 +1,41 @@
apiVersion: batch/v1
kind: Job
metadata:
name: @@JOB_NAME@@
namespace: nuthatch-testbed
spec:
activeDeadlineSeconds: 120
backoffLimit: 0
ttlSecondsAfterFinished: 300
template:
spec:
automountServiceAccountToken: false
enableServiceLinks: false
restartPolicy: Never
containers:
- name: recovery-log-probe
image: @@TEST_IMAGE@@
imagePullPolicy: Never
command: [nuthatch-recovery-probe]
args: [log-exhausted]
envFrom:
- configMapRef: {name: @@PROFILE_NAME@@}
env:
- {name: NUTHATCH_RECOVERY_GRACE_SEC, value: "1"}
- {name: NUTHATCH_OBJECT_REQUEST_TIMEOUT_SEC, value: "2"}
- {name: NUTHATCH_JWT_SECRET, value: synthetic-recovery-session-jwt-secret}
- {name: NUTHATCH_PUBLIC_BASE_URL, value: "https://recovery.invalid"}
- {name: NUTHATCH_OIDC_ISSUER, value: "https://issuer.invalid"}
- {name: NUTHATCH_OIDC_CLIENT_ID, value: recovery}
- {name: NUTHATCH_OIDC_ADDITIONAL_AUDIENCES, value: ""}
- {name: NUTHATCH_OIDC_CLIENT_SECRET, value: synthetic-recovery-oidc-client-secret}
- {name: NUTHATCH_OIDC_REDIRECT_URI, value: "https://recovery.invalid/oidc/callback"}
volumeMounts:
- {name: recovery-log, mountPath: /run/nuthatch-recovery, readOnly: true}
resources:
requests: {cpu: 50m, memory: 64Mi}
limits: {cpu: "1", memory: 256Mi}
volumes:
- name: recovery-log
configMap:
name: @@LOG_CONFIG_MAP@@

Some files were not shown because too many files have changed in this diff Show more