#!/usr/bin/env bash set -euo pipefail ROOT="$(cd "$(dirname "$0")/../.." && pwd)" source "$ROOT/tests/kind/lib.sh" USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)" SUCCEEDED=false OWNS_FIXTURE=false APP_IMAGE=nuthatch-idempotency:local cleanup() { local status=$? trap - EXIT INT TERM if [[ "${NUTHATCH_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH" printf 'retained context: %s\n' "$CONTEXT" exit "$status" fi if [[ "$OWNS_FIXTURE" = true ]]; then destroy_fixture || status=1 fi if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then printf 'user kubeconfig changed unexpectedly\n' >&2 status=1 fi exit "$status" } trap cleanup EXIT trap 'exit 130' INT trap 'exit 143' TERM render_application() { local lane="$1" local profile_file="$2" local pg_conn local redis_url local oidc_issuer local oidc_client_id local oidc_additional_audiences local oidc_client_secret local s3_endpoint local s3_region local s3_bucket local s3_access_key local s3_secret_key pg_conn="$(sed -n 's/^NUTHATCH_PG_CONN=//p' "$profile_file")" redis_url="$(sed -n 's/^NUTHATCH_REDIS_URL=//p' "$profile_file")" oidc_issuer="$(sed -n 's/^NUTHATCH_OIDC_ISSUER=//p' "$profile_file")" oidc_client_id="$(sed -n 's/^NUTHATCH_OIDC_CLIENT_ID=//p' "$profile_file")" oidc_additional_audiences="$( sed -n 's/^NUTHATCH_OIDC_ADDITIONAL_AUDIENCES=//p' "$profile_file" )" oidc_client_secret="$(<"$profile_file.oidc-secret")" s3_endpoint="$(sed -n 's/^NUTHATCH_S3_ENDPOINT=//p' "$profile_file")" s3_region="$(sed -n 's/^NUTHATCH_S3_REGION=//p' "$profile_file")" s3_bucket="$(sed -n 's/^NUTHATCH_S3_BUCKET=//p' "$profile_file")" s3_access_key="$(sed -n 's/^NUTHATCH_S3_ACCESS_KEY=//p' "$profile_file")" s3_secret_key="$(sed -n 's/^NUTHATCH_S3_SECRET_KEY=//p' "$profile_file")" sed \ -e "s|@@LANE@@|$lane|g" \ -e "s|@@APP_IMAGE@@|$APP_IMAGE|g" \ -e "s|@@PG_CONN@@|$pg_conn|g" \ -e "s|@@REDIS_URL@@|$redis_url|g" \ -e "s|@@OIDC_ISSUER@@|$oidc_issuer|g" \ -e "s|@@OIDC_CLIENT_ID@@|$oidc_client_id|g" \ -e "s|@@OIDC_ADDITIONAL_AUDIENCES@@|$oidc_additional_audiences|g" \ -e "s|@@OIDC_CLIENT_SECRET@@|$oidc_client_secret|g" \ -e "s|@@S3_ENDPOINT@@|$s3_endpoint|g" \ -e "s|@@S3_REGION@@|$s3_region|g" \ -e "s|@@S3_BUCKET@@|$s3_bucket|g" \ -e "s|@@S3_ACCESS_KEY@@|$s3_access_key|g" \ -e "s|@@S3_SECRET_KEY@@|$s3_secret_key|g" \ "$ROOT/deploy/kubernetes/nuthatch.yaml.in" >"$RUNTIME/idempotency-app-$lane.yaml" sed -i \ 's/NUTHATCH_IDEMPOTENCY_WINDOW_SEC: "86400"/NUTHATCH_IDEMPOTENCY_WINDOW_SEC: "2"/' \ "$RUNTIME/idempotency-app-$lane.yaml" sed -i 's/ replicas: 1/ replicas: 2/' "$RUNTIME/idempotency-app-$lane.yaml" } cleanup_application() { local lane="$1" kube --namespace "$FIXTURE_NAMESPACE" delete \ "deployment/nuthatch-$lane" \ "service/nuthatch-$lane" \ service/nuthatch-idempotency-a \ service/nuthatch-idempotency-b \ "configmap/nuthatch-$lane" \ "secret/nuthatch-$lane" \ --ignore-not-found >/dev/null rm -f "$RUNTIME/idempotency-app-$lane.yaml" } create_process_services() { local lane="$1" local profile="$2" local pods mapfile -t pods < <( kube --namespace "$FIXTURE_NAMESPACE" get pods \ --selector="app=nuthatch-$lane" \ --field-selector=status.phase=Running \ --output jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | sort ) test "${#pods[@]}" = 2 kube --namespace "$FIXTURE_NAMESPACE" label pod "${pods[0]}" \ nuthatch-idempotency-process=a >/dev/null kube --namespace "$FIXTURE_NAMESPACE" label pod "${pods[1]}" \ nuthatch-idempotency-process=b >/dev/null kube --namespace "$FIXTURE_NAMESPACE" expose pod "${pods[0]}" \ --name=nuthatch-idempotency-a \ --port=8080 \ --target-port=8080 >/dev/null kube --namespace "$FIXTURE_NAMESPACE" expose pod "${pods[1]}" \ --name=nuthatch-idempotency-b \ --port=8080 \ --target-port=8080 >/dev/null local ip_a local ip_b ip_a="$( kube --namespace "$FIXTURE_NAMESPACE" get service nuthatch-idempotency-a \ --output jsonpath='{.spec.clusterIP}' )" ip_b="$( kube --namespace "$FIXTURE_NAMESPACE" get service nuthatch-idempotency-b \ --output jsonpath='{.spec.clusterIP}' )" local profile_file="$RUNTIME/$profile.env" sed -i '/^NUTHATCH_IDEMPOTENCY_/d' "$profile_file" { printf 'NUTHATCH_IDEMPOTENCY_MODE=verify\n' printf 'NUTHATCH_IDEMPOTENCY_IP_A=%s\n' "$ip_a" printf 'NUTHATCH_IDEMPOTENCY_IP_B=%s\n' "$ip_b" } >>"$profile_file" kube create configmap "$profile" \ --namespace "$FIXTURE_NAMESPACE" \ --from-env-file="$profile_file" \ --dry-run=client \ --output yaml | kube apply --filename - >/dev/null } acquire_fixture_lock OWNS_FIXTURE=true setup_fixture durable coordination oidc target source "$RUNTIME/fixture.env" "$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" build \ --file "$ROOT/Dockerfile" \ --tag "$APP_IMAGE" \ "$ROOT" load_local_image "$APP_IMAGE" "$NERDCTL_NAMESPACE" while IFS=$'\t' read -r lane coordination oidc target; do if [[ -n "${NUTHATCH_IDEMPOTENCY_LANE:-}" ]] && [[ "$lane" != "$NUTHATCH_IDEMPOTENCY_LANE" ]]; then continue fi profile="$(write_lane_profile "$lane" "$coordination" "$oidc" "$target")" printf 'Running idempotency lane %s: coordination=%s oidc=%s target=%s\n' \ "$lane" "$coordination" "$oidc" "$target" cleanup_application "$lane" render_application "$lane" "$RUNTIME/$profile.env" kube apply --filename "$RUNTIME/idempotency-app-$lane.yaml" >/dev/null kube --namespace "$FIXTURE_NAMESPACE" rollout status \ "deployment/nuthatch-$lane" --timeout=180s create_process_services "$lane" "$profile" job="idempotency-epoch-$lane" render_lane_job \ "$job" \ "$TESTBED/product-matrix-job.yaml.in" \ "$CURL_LOCAL_IMAGE" \ "$profile" sed -i "s|@@LANE@@|$lane|g" "$RUNTIME/$job.yaml" run_job_manifest "$job" "$RUNTIME/$job.yaml" cleanup_application "$lane" cleanup_lane_state "$lane" "$coordination" "$oidc" "$target" cleanup_lane_resources "$job" "$profile" if [[ "${NUTHATCH_IDEMPOTENCY_FIRST_LANE_ONLY:-0}" = 1 ]]; then break fi done < <(expand_fixture_matrix coordination oidc target) SUCCEEDED=true printf 'Nuthatch idempotency gate passed\n'