ms2k/nuthatch
SHA256
Watch
0
Code Releases Activity

Replace planning-era names and squash initial schema

Browse source 
Use descriptive test and executable names so implementation surfaces do not depend on journal organization. Consolidate the unreleased migration history into the authoritative initial schema because no deployed database requires an upgrade path.
This commit is contained in:
ms2k 2026-07-19 15:21:24 +09:00
commit 15eb0038d2
14 changed files with 299 additions and 436 deletions
Download patch file Download diff file Expand all files Collapse all files

188
tests/kind/idempotency-run.sh Executable file
View file

@ -0,0 +1,188 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
source "$ROOT/tests/kind/lib.sh"
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
SUCCEEDED=false
OWNS_FIXTURE=false
APP_IMAGE=nuthatch-idempotency:local
cleanup() {
local status=$?
trap - EXIT INT TERM
if [[ "${NUTHATCH_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
printf 'retained context: %s\n' "$CONTEXT"
exit "$status"
fi
if [[ "$OWNS_FIXTURE" = true ]]; then
destroy_fixture || status=1
fi
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
printf 'user kubeconfig changed unexpectedly\n' >&2
status=1
fi
exit "$status"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
render_application() {
local lane="$1"
local profile_file="$2"
local pg_conn
local redis_url
local oidc_issuer
local oidc_client_id
local oidc_additional_audiences
local oidc_client_secret
local s3_endpoint
local s3_region
local s3_bucket
local s3_access_key
local s3_secret_key
pg_conn="$(sed -n 's/^NUTHATCH_PG_CONN=//p' "$profile_file")"
redis_url="$(sed -n 's/^NUTHATCH_REDIS_URL=//p' "$profile_file")"
oidc_issuer="$(sed -n 's/^NUTHATCH_OIDC_ISSUER=//p' "$profile_file")"
oidc_client_id="$(sed -n 's/^NUTHATCH_OIDC_CLIENT_ID=//p' "$profile_file")"
oidc_additional_audiences="$(
sed -n 's/^NUTHATCH_OIDC_ADDITIONAL_AUDIENCES=//p' "$profile_file"
)"
oidc_client_secret="$(<"$profile_file.oidc-secret")"
s3_endpoint="$(sed -n 's/^NUTHATCH_S3_ENDPOINT=//p' "$profile_file")"
s3_region="$(sed -n 's/^NUTHATCH_S3_REGION=//p' "$profile_file")"
s3_bucket="$(sed -n 's/^NUTHATCH_S3_BUCKET=//p' "$profile_file")"
s3_access_key="$(sed -n 's/^NUTHATCH_S3_ACCESS_KEY=//p' "$profile_file")"
s3_secret_key="$(sed -n 's/^NUTHATCH_S3_SECRET_KEY=//p' "$profile_file")"
sed \
-e "s|@@LANE@@|$lane|g" \
-e "s|@@APP_IMAGE@@|$APP_IMAGE|g" \
-e "s|@@PG_CONN@@|$pg_conn|g" \
-e "s|@@REDIS_URL@@|$redis_url|g" \
-e "s|@@OIDC_ISSUER@@|$oidc_issuer|g" \
-e "s|@@OIDC_CLIENT_ID@@|$oidc_client_id|g" \
-e "s|@@OIDC_ADDITIONAL_AUDIENCES@@|$oidc_additional_audiences|g" \
-e "s|@@OIDC_CLIENT_SECRET@@|$oidc_client_secret|g" \
-e "s|@@S3_ENDPOINT@@|$s3_endpoint|g" \
-e "s|@@S3_REGION@@|$s3_region|g" \
-e "s|@@S3_BUCKET@@|$s3_bucket|g" \
-e "s|@@S3_ACCESS_KEY@@|$s3_access_key|g" \
-e "s|@@S3_SECRET_KEY@@|$s3_secret_key|g" \
"$ROOT/deploy/kubernetes/nuthatch.yaml.in" >"$RUNTIME/idempotency-app-$lane.yaml"
sed -i \
's/NUTHATCH_IDEMPOTENCY_WINDOW_SEC: "86400"/NUTHATCH_IDEMPOTENCY_WINDOW_SEC: "2"/' \
"$RUNTIME/idempotency-app-$lane.yaml"
sed -i 's/ replicas: 1/ replicas: 2/' "$RUNTIME/idempotency-app-$lane.yaml"
}
cleanup_application() {
local lane="$1"
kube --namespace "$FIXTURE_NAMESPACE" delete \
"deployment/nuthatch-$lane" \
"service/nuthatch-$lane" \
service/nuthatch-idempotency-a \
service/nuthatch-idempotency-b \
"configmap/nuthatch-$lane" \
"secret/nuthatch-$lane" \
--ignore-not-found >/dev/null
rm -f "$RUNTIME/idempotency-app-$lane.yaml"
}
create_process_services() {
local lane="$1"
local profile="$2"
local pods
mapfile -t pods < <(
kube --namespace "$FIXTURE_NAMESPACE" get pods \
--selector="app=nuthatch-$lane" \
--field-selector=status.phase=Running \
--output jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' |
sort
)
test "${#pods[@]}" = 2
kube --namespace "$FIXTURE_NAMESPACE" label pod "${pods[0]}" \
nuthatch-idempotency-process=a >/dev/null
kube --namespace "$FIXTURE_NAMESPACE" label pod "${pods[1]}" \
nuthatch-idempotency-process=b >/dev/null
kube --namespace "$FIXTURE_NAMESPACE" expose pod "${pods[0]}" \
--name=nuthatch-idempotency-a \
--port=8080 \
--target-port=8080 >/dev/null
kube --namespace "$FIXTURE_NAMESPACE" expose pod "${pods[1]}" \
--name=nuthatch-idempotency-b \
--port=8080 \
--target-port=8080 >/dev/null
local ip_a
local ip_b
ip_a="$(
kube --namespace "$FIXTURE_NAMESPACE" get service nuthatch-idempotency-a \
--output jsonpath='{.spec.clusterIP}'
)"
ip_b="$(
kube --namespace "$FIXTURE_NAMESPACE" get service nuthatch-idempotency-b \
--output jsonpath='{.spec.clusterIP}'
)"
local profile_file="$RUNTIME/$profile.env"
sed -i '/^NUTHATCH_IDEMPOTENCY_/d' "$profile_file"
{
printf 'NUTHATCH_IDEMPOTENCY_MODE=verify\n'
printf 'NUTHATCH_IDEMPOTENCY_IP_A=%s\n' "$ip_a"
printf 'NUTHATCH_IDEMPOTENCY_IP_B=%s\n' "$ip_b"
} >>"$profile_file"
kube create configmap "$profile" \
--namespace "$FIXTURE_NAMESPACE" \
--from-env-file="$profile_file" \
--dry-run=client \
--output yaml |
kube apply --filename - >/dev/null
}
acquire_fixture_lock
OWNS_FIXTURE=true
setup_fixture durable coordination oidc target
source "$RUNTIME/fixture.env"
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" build \
--file "$ROOT/Dockerfile" \
--tag "$APP_IMAGE" \
"$ROOT"
load_local_image "$APP_IMAGE" "$NERDCTL_NAMESPACE"
while IFS=$'\t' read -r lane coordination oidc target; do
if [[ -n "${NUTHATCH_IDEMPOTENCY_LANE:-}" ]] &&
[[ "$lane" != "$NUTHATCH_IDEMPOTENCY_LANE" ]]; then
continue
fi
profile="$(write_lane_profile "$lane" "$coordination" "$oidc" "$target")"
printf 'Running idempotency lane %s: coordination=%s oidc=%s target=%s\n' \
"$lane" "$coordination" "$oidc" "$target"
cleanup_application "$lane"
render_application "$lane" "$RUNTIME/$profile.env"
kube apply --filename "$RUNTIME/idempotency-app-$lane.yaml" >/dev/null
kube --namespace "$FIXTURE_NAMESPACE" rollout status \
"deployment/nuthatch-$lane" --timeout=180s
create_process_services "$lane" "$profile"
job="idempotency-epoch-$lane"
render_lane_job \
"$job" \
"$TESTBED/product-matrix-job.yaml.in" \
"$CURL_LOCAL_IMAGE" \
"$profile"
sed -i "s|@@LANE@@|$lane|g" "$RUNTIME/$job.yaml"
run_job_manifest "$job" "$RUNTIME/$job.yaml"
cleanup_application "$lane"
cleanup_lane_state "$lane" "$coordination" "$oidc" "$target"
cleanup_lane_resources "$job" "$profile"
if [[ "${NUTHATCH_IDEMPOTENCY_FIRST_LANE_ONLY:-0}" = 1 ]]; then
break
fi
done < <(expand_fixture_matrix coordination oidc target)
SUCCEEDED=true
printf 'Nuthatch idempotency gate passed\n'

8
tests/kind/lib.sh
View file

@ -488,7 +488,7 @@ provision_lane_target_principal() {
endpoint="$(target_loopback_endpoint "$provider")" || return
case "$provider" in
garage)
printf -v access_key 'GKWP05%028d' "$((10#$lane))"
printf -v access_key 'GKNUTHATCH%024d' "$((10#$lane))"
printf -v secret_key '%064d' "$((10#$lane))"
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/garage --container garage -- \
/garage bucket create "$bucket" >/dev/null || return
@ -505,8 +505,8 @@ provision_lane_target_principal() {
/garage bucket allow --read --write "$bucket" --key "$access_key" >/dev/null || return
;;
versitygw)
access_key="nuthatch-wp05-$lane"
printf -v secret_key 'synthetic-wp05-%049d' "$((10#$lane))"
access_key="nuthatch-lane-$lane"
printf -v secret_key 'synthetic-lane-%049d' "$((10#$lane))"
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/versitygw --container versitygw -- \
/usr/local/bin/versitygw admin \
--endpoint-url http://127.0.0.1:10001 \
@ -916,6 +916,6 @@ destroy_fixture() {
"$RUNTIME/fixture.env" \
"$RUNTIME"/nuthatch-fixture-lane-*.env \
"$RUNTIME"/nuthatch-fixture-lane-*.env.oidc-secret \
"$RUNTIME"/wp05-app-*.yaml || status=1
"$RUNTIME"/product-app-*.yaml || status=1
return "$status"
}

1615
tests/kind/product-matrix-job.yaml.in Normal file
View file

File diff suppressed because it is too large Load diff

176
tests/kind/product-matrix-run.sh Executable file
View file

@ -0,0 +1,176 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
source "$ROOT/tests/kind/lib.sh"
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
SUCCEEDED=false
OWNS_FIXTURE=false
APP_IMAGE=nuthatch-product-matrix:local
cleanup() {
local status=$?
trap - EXIT INT TERM
if [[ "${NUTHATCH_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
printf 'retained context: %s\n' "$CONTEXT"
exit "$status"
fi
if [[ "$OWNS_FIXTURE" = true ]]; then
destroy_fixture || status=1
fi
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
printf 'user kubeconfig changed unexpectedly\n' >&2
status=1
fi
exit "$status"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
render_application() {
local lane="$1"
local profile_file="$2"
local pg_conn
local redis_url
local oidc_issuer
local oidc_client_id
local oidc_additional_audiences
local oidc_client_secret
local s3_endpoint
local s3_region
local s3_bucket
local s3_access_key
local s3_secret_key
pg_conn="$(sed -n 's/^NUTHATCH_PG_CONN=//p' "$profile_file")"
redis_url="$(sed -n 's/^NUTHATCH_REDIS_URL=//p' "$profile_file")"
oidc_issuer="$(sed -n 's/^NUTHATCH_OIDC_ISSUER=//p' "$profile_file")"
oidc_client_id="$(sed -n 's/^NUTHATCH_OIDC_CLIENT_ID=//p' "$profile_file")"
oidc_additional_audiences="$(
sed -n 's/^NUTHATCH_OIDC_ADDITIONAL_AUDIENCES=//p' "$profile_file"
)"
oidc_client_secret="$(<"$profile_file.oidc-secret")"
s3_endpoint="$(sed -n 's/^NUTHATCH_S3_ENDPOINT=//p' "$profile_file")"
s3_region="$(sed -n 's/^NUTHATCH_S3_REGION=//p' "$profile_file")"
s3_bucket="$(sed -n 's/^NUTHATCH_S3_BUCKET=//p' "$profile_file")"
s3_access_key="$(sed -n 's/^NUTHATCH_S3_ACCESS_KEY=//p' "$profile_file")"
s3_secret_key="$(sed -n 's/^NUTHATCH_S3_SECRET_KEY=//p' "$profile_file")"
sed \
-e "s|@@LANE@@|$lane|g" \
-e "s|@@APP_IMAGE@@|$APP_IMAGE|g" \
-e "s|@@PG_CONN@@|$pg_conn|g" \
-e "s|@@REDIS_URL@@|$redis_url|g" \
-e "s|@@OIDC_ISSUER@@|$oidc_issuer|g" \
-e "s|@@OIDC_CLIENT_ID@@|$oidc_client_id|g" \
-e "s|@@OIDC_ADDITIONAL_AUDIENCES@@|$oidc_additional_audiences|g" \
-e "s|@@OIDC_CLIENT_SECRET@@|$oidc_client_secret|g" \
-e "s|@@S3_ENDPOINT@@|$s3_endpoint|g" \
-e "s|@@S3_REGION@@|$s3_region|g" \
-e "s|@@S3_BUCKET@@|$s3_bucket|g" \
-e "s|@@S3_ACCESS_KEY@@|$s3_access_key|g" \
-e "s|@@S3_SECRET_KEY@@|$s3_secret_key|g" \
"$ROOT/deploy/kubernetes/nuthatch.yaml.in" > "$RUNTIME/product-app-$lane.yaml"
if [[ "${NUTHATCH_PRODUCT_GATE:-0}" = 1 ]]; then
sed -i \
-e 's/NUTHATCH_BROWSE_PAGE_SIZE: "200"/NUTHATCH_BROWSE_PAGE_SIZE: "2"/' \
-e 's/NUTHATCH_SEARCH_MAX_KEYS: "500"/NUTHATCH_SEARCH_MAX_KEYS: "3"/' \
-e 's/NUTHATCH_DELETE_PAGE_SIZE: "100"/NUTHATCH_DELETE_PAGE_SIZE: "2"/' \
"$RUNTIME/product-app-$lane.yaml"
fi
}
cleanup_application() {
local lane="$1"
kube --namespace "$FIXTURE_NAMESPACE" delete \
"deployment/nuthatch-$lane" \
"service/nuthatch-$lane" \
"configmap/nuthatch-$lane" \
"secret/nuthatch-$lane" \
--ignore-not-found >/dev/null
rm -f "$RUNTIME/product-app-$lane.yaml"
}
set_product_profile() {
local profile="$1"
local mode="$2"
local profile_file="$RUNTIME/$profile.env"
sed -i '/^NUTHATCH_PRODUCT_MODE=/d' "$profile_file"
printf 'NUTHATCH_PRODUCT_MODE=%s\n' "$mode" >> "$profile_file"
kube create configmap "$profile" \
--namespace "$FIXTURE_NAMESPACE" \
--from-env-file="$profile_file" \
--dry-run=client \
--output yaml |
kube apply --filename - >/dev/null
}
acquire_fixture_lock
OWNS_FIXTURE=true
setup_fixture durable coordination oidc target
source "$RUNTIME/fixture.env"
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" build \
--file "$ROOT/Dockerfile" \
--tag "$APP_IMAGE" \
"$ROOT"
load_local_image "$APP_IMAGE" "$NERDCTL_NAMESPACE"
while IFS=$'\t' read -r lane coordination oidc target; do
profile="$(write_lane_profile "$lane" "$coordination" "$oidc" "$target")"
if [[ "${NUTHATCH_PRODUCT_GATE:-0}" = 1 ]]; then
set_product_profile "$profile" verify
fi
render_application "$lane" "$RUNTIME/$profile.env"
kube apply --filename "$RUNTIME/product-app-$lane.yaml" >/dev/null
kube --namespace "$FIXTURE_NAMESPACE" rollout status \
"deployment/nuthatch-$lane" --timeout=180s
if [[ "${NUTHATCH_APP_INTERACTIVE:-0}" = 1 ]]; then
SUCCEEDED=true
OWNS_FIXTURE=false
printf 'interactive kubeconfig: %s\n' "$KUBECONFIG_PATH"
printf 'interactive context: %s\n' "$CONTEXT"
printf 'interactive namespace: %s\n' "$FIXTURE_NAMESPACE"
printf 'interactive service: nuthatch-%s:8080\n' "$lane"
exit 0
fi
job="product-app-$lane"
render_lane_job \
"$job" \
"$TESTBED/product-matrix-job.yaml.in" \
"$CURL_LOCAL_IMAGE" \
"$profile"
sed -i "s|@@LANE@@|$lane|g" "$RUNTIME/$job.yaml"
printf 'Running lane %s: coordination=%s oidc=%s target=%s\n' \
"$lane" "$coordination" "$oidc" "$target"
run_job_manifest "$job" "$RUNTIME/$job.yaml"
if [[ "${NUTHATCH_PRODUCT_GATE:-0}" = 1 ]]; then
sed -i \
's/NUTHATCH_SEARCH_TIMEOUT_MS: "2000"/NUTHATCH_SEARCH_TIMEOUT_MS: "1"/' \
"$RUNTIME/product-app-$lane.yaml"
kube apply --filename "$RUNTIME/product-app-$lane.yaml" >/dev/null
kube --namespace "$FIXTURE_NAMESPACE" rollout restart \
"deployment/nuthatch-$lane" >/dev/null
kube --namespace "$FIXTURE_NAMESPACE" rollout status \
"deployment/nuthatch-$lane" --timeout=180s
set_product_profile "$profile" timeout
timeout_job="product-timeout-$lane"
render_lane_job \
"$timeout_job" \
"$TESTBED/product-matrix-job.yaml.in" \
"$CURL_LOCAL_IMAGE" \
"$profile"
sed -i "s|@@LANE@@|$lane|g" "$RUNTIME/$timeout_job.yaml"
run_job_manifest "$timeout_job" "$RUNTIME/$timeout_job.yaml"
kube --namespace "$FIXTURE_NAMESPACE" delete \
"job/$timeout_job" --ignore-not-found >/dev/null
rm -f "$RUNTIME/$timeout_job.yaml"
fi
cleanup_application "$lane"
cleanup_lane_state "$lane" "$coordination" "$oidc" "$target"
cleanup_lane_resources "$job" "$profile"
done < <(expand_fixture_matrix coordination oidc target)
SUCCEEDED=true
printf 'Nuthatch product matrix gate passed\n'

33
tests/kind/protocol-conformance-job.yaml.in Normal file
View file

@ -0,0 +1,33 @@
apiVersion: batch/v1
kind: Job
metadata:
name: @@JOB_NAME@@
namespace: nuthatch-testbed
spec:
activeDeadlineSeconds: 300
backoffLimit: 0
template:
spec:
automountServiceAccountToken: false
enableServiceLinks: false
restartPolicy: Never
containers:
- name: protocol-conformance
image: @@TEST_IMAGE@@
imagePullPolicy: Never
command: [nuthatch-protocol-conformance]
args: ["@@MODE@@"]
envFrom:
- configMapRef: {name: @@PROFILE_NAME@@}
env:
- {name: NUTHATCH_OBJECT_REQUEST_TIMEOUT_SEC, value: "3"}
- {name: NUTHATCH_JWT_SECRET, value: synthetic-protocol-session-jwt-secret}
- {name: NUTHATCH_PUBLIC_BASE_URL, value: "https://protocol.invalid"}
- {name: NUTHATCH_OIDC_ISSUER, value: "https://issuer.invalid"}
- {name: NUTHATCH_OIDC_CLIENT_ID, value: protocol-conformance}
- {name: NUTHATCH_OIDC_ADDITIONAL_AUDIENCES, value: ""}
- {name: NUTHATCH_OIDC_CLIENT_SECRET, value: synthetic-protocol-oidc-client-secret}
- {name: NUTHATCH_OIDC_REDIRECT_URI, value: "https://protocol.invalid/oidc/callback"}
resources:
requests: {cpu: 100m, memory: 96Mi}
limits: {cpu: "2", memory: 256Mi}

238
tests/kind/protocol-conformance-run.sh Executable file
View file

@ -0,0 +1,238 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
source "$ROOT/tests/kind/lib.sh"
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
SUCCEEDED=false
OWNS_FIXTURE=false
APP_IMAGE=nuthatch-protocol-conformance:local
cleanup() {
local status=$?
trap - EXIT INT TERM
if [[ "${NUTHATCH_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
printf 'retained context: %s\n' "$CONTEXT"
exit "$status"
fi
if [[ "$OWNS_FIXTURE" = true ]]; then
destroy_fixture || status=1
fi
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
printf 'user kubeconfig changed unexpectedly\n' >&2
status=1
fi
exit "$status"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
set_target_available() {
local target="$1"
local selector="$target"
if [[ "$2" = false ]]; then
selector=protocol-unavailable
fi
kube --namespace "$FIXTURE_NAMESPACE" patch service "$target" \
--type merge \
--patch "{\"spec\":{\"selector\":{\"app\":\"$selector\"}}}" >/dev/null
}
set_postgres_available() {
local selector=postgres
if [[ "$1" = false ]]; then
selector=protocol-unavailable
fi
kube --namespace "$FIXTURE_NAMESPACE" patch service postgres \
--type merge \
--patch "{\"spec\":{\"selector\":{\"app\":\"$selector\"}}}" >/dev/null
}
database_connection_count() {
local database="$1"
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
env PGPASSWORD=synthetic-postgres psql \
--username nuthatch \
--dbname postgres \
--tuples-only \
--no-align \
--set ON_ERROR_STOP=1 \
--command \
"SELECT count(*) FROM pg_stat_activity WHERE datname = '$database' AND pid <> pg_backend_pid()" |
tr -d '[:space:]'
}
terminate_database_connections() {
local database="$1"
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
env PGPASSWORD=synthetic-postgres psql \
--username nuthatch \
--dbname postgres \
--set ON_ERROR_STOP=1 \
--command \
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$database' AND pid <> pg_backend_pid()" \
>/dev/null
}
render_mode() {
local lane="$1"
local profile="$2"
local mode="$3"
local job="protocol-$mode-$lane"
render_lane_job \
"$job" \
"$TESTBED/protocol-conformance-job.yaml.in" \
"$APP_IMAGE" \
"$profile"
sed -i "s|@@MODE@@|$mode|g" "$RUNTIME/$job.yaml"
}
run_mode() {
local lane="$1"
local profile="$2"
local mode="$3"
local job="protocol-$mode-$lane"
render_mode "$lane" "$profile" "$mode"
run_job_manifest "$job" "$RUNTIME/$job.yaml"
}
start_mode() {
local lane="$1"
local profile="$2"
local mode="$3"
local job="protocol-$mode-$lane"
render_mode "$lane" "$profile" "$mode"
kube apply --filename "$RUNTIME/$job.yaml" >/dev/null
}
wait_for_mode_log() {
local lane="$1"
local mode="$2"
local message="$3"
local job="protocol-$mode-$lane"
for _ in $(seq 1 120); do
if kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" 2>/dev/null |
grep -F "$message" >/dev/null; then
return
fi
sleep 0.25
done
printf 'timed out waiting for %s\n' "$message" >&2
return 1
}
stop_mode() {
local lane="$1"
local mode="$2"
kube --namespace "$FIXTURE_NAMESPACE" delete "job/protocol-$mode-$lane" \
--ignore-not-found >/dev/null
}
verify_mapping() {
local lane="$1"
local log="$RUNTIME/protocol-full-$lane.log"
kube --namespace "$FIXTURE_NAMESPACE" logs "job/protocol-full-$lane" >"$log"
local actions=(
FirstRequest ReplayRequest ConflictRequest PrepareEffect DispatchEffect
LoseUploadBody SettleLostUpload ExternalEffectApplied
ExternalEffectNotApplied FinalizeSuccess FinalizeFailure ControllerAdvance
StoreSettlement
)
local invariants=(
TypeOK RecordIdentityConsistent PersonalRequestsUseOwnPool
PersonalPoolsDisjoint FingerprintBindsRequest ReservationsConsistent
LocksConsistent ActiveObjectConsistent ExternalStateConsistent
UploadBodyConsistent QuotaReservationBound ObjectStateCovered
ObjectQuotaBound AccountingAtQuiescence ObservationConsistent
RecordsNeverForgotten RequestBindingsStable TerminalRecordsStable
SettledEffectsDoNotMutate UnfinishedEventuallyTerminal
LostUploadEventuallyTerminal
)
local name
for name in "${actions[@]}"; do
grep -F "protocol action covered: $name" "$log" >/dev/null
rg -q "^$name(\\(| ==)" "$ROOT/docs/protocol/Nuthatch.tla"
done
for name in "${invariants[@]}"; do
grep -F "protocol invariant checked: $name" "$log" >/dev/null
rg -q "^$name(\\(| ==)" "$ROOT/docs/protocol/Nuthatch.tla"
done
}
acquire_fixture_lock
OWNS_FIXTURE=true
setup_fixture durable coordination target
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" build \
--file "$ROOT/Dockerfile" \
--target test-runtime \
--tag "$APP_IMAGE" \
"$ROOT"
load_local_image "$APP_IMAGE" "$NERDCTL_NAMESPACE"
while IFS=$'\t' read -r lane coordination oidc target; do
if [[ -n "${NUTHATCH_PROTOCOL_TARGET:-}" ]] &&
[[ "$target" != "$NUTHATCH_PROTOCOL_TARGET" ]]; then
continue
fi
profile="$(write_lane_profile "$lane" redis - "$target")"
printf 'Running protocol lane %s: target=%s\n' "$lane" "$target"
run_mode "$lane" "$profile" full
verify_mapping "$lane"
start_mode "$lane" "$profile" pg-prepare
wait_for_mode_log "$lane" pg-prepare "PostgreSQL transition fixture ready"
printf 'Checking PostgreSQL loss and restoration for %s\n' "$target"
set_postgres_available false
active_connections="$(database_connection_count "nuthatch_lane_$lane")"
test "$active_connections" -gt 0
terminate_database_connections "nuthatch_lane_$lane" || true
remaining_connections="$(database_connection_count "nuthatch_lane_$lane")"
test "$remaining_connections" = 0
stop_mode "$lane" pg-prepare
run_mode "$lane" "$profile" pg-fence
set_postgres_available true
run_mode "$lane" "$profile" pg-restored
kube --namespace "$FIXTURE_NAMESPACE" logs "job/protocol-pg-fence-$lane" |
grep -F "protocol action covered: LosePostgres" >/dev/null
kube --namespace "$FIXTURE_NAMESPACE" logs "job/protocol-pg-restored-$lane" |
grep -F "protocol action covered: RestorePostgres" >/dev/null
kube --namespace "$FIXTURE_NAMESPACE" logs "job/protocol-pg-restored-$lane" |
grep -F "protocol invariant checked: PostgresFailureFencesDurableState" \
>/dev/null
rg -q '^LosePostgres(\(| ==)' "$ROOT/docs/protocol/Nuthatch.tla"
rg -q '^RestorePostgres(\(| ==)' "$ROOT/docs/protocol/Nuthatch.tla"
rg -q '^PostgresFailureFencesDurableState(\(| ==)' \
"$ROOT/docs/protocol/Nuthatch.tla"
run_mode "$lane" "$profile" store-prepare
printf 'Checking store-down admission fence for %s\n' "$target"
set_target_available "$target" false
run_mode "$lane" "$profile" fence
printf 'Checking restored store observation for %s\n' "$target"
set_target_available "$target" true
run_mode "$lane" "$profile" restored
kube --namespace "$FIXTURE_NAMESPACE" logs "job/protocol-fence-$lane" |
grep -F "store-down rejection creates no protocol record" >/dev/null
kube --namespace "$FIXTURE_NAMESPACE" logs "job/protocol-restored-$lane" |
grep -F "protocol action covered: RestoreObjectStore" >/dev/null
kube --namespace "$FIXTURE_NAMESPACE" logs "job/protocol-restored-$lane" |
grep -F "protocol invariant checked: ObjectStoreFailureFencesObjects" >/dev/null
rg -q '^LoseObjectStore(\(| ==)' "$ROOT/docs/protocol/Nuthatch.tla"
rg -q '^RestoreObjectStore(\(| ==)' "$ROOT/docs/protocol/Nuthatch.tla"
rg -q '^ObjectStoreFailureFencesObjects(\(| ==)' \
"$ROOT/docs/protocol/Nuthatch.tla"
cleanup_lane_state "$lane" redis - "$target"
cleanup_lane_resources "protocol-full-$lane" "$profile"
cleanup_lane_resources "protocol-pg-prepare-$lane" "$profile"
cleanup_lane_resources "protocol-pg-fence-$lane" "$profile"
cleanup_lane_resources "protocol-pg-restored-$lane" "$profile"
cleanup_lane_resources "protocol-store-prepare-$lane" "$profile"
cleanup_lane_resources "protocol-fence-$lane" "$profile"
cleanup_lane_resources "protocol-restored-$lane" "$profile"
rm -f "$RUNTIME/protocol-full-$lane.log"
done < <(expand_fixture_matrix target)
SUCCEEDED=true
printf 'Nuthatch protocol conformance gate passed\n'

34
tests/kind/recovery-phase-job.yaml.in Normal file
View file

@ -0,0 +1,34 @@
apiVersion: batch/v1
kind: Job
metadata:
name: @@JOB_NAME@@
namespace: nuthatch-testbed
spec:
activeDeadlineSeconds: 300
backoffLimit: 0
template:
spec:
automountServiceAccountToken: false
enableServiceLinks: false
restartPolicy: Never
containers:
- name: phase
image: @@TEST_IMAGE@@
imagePullPolicy: Never
command: [nuthatch-recovery-probe]
envFrom:
- configMapRef: {name: @@PROFILE_NAME@@}
env:
- {name: NUTHATCH_RECOVERY_PHASE, value: "@@PHASE@@"}
- {name: NUTHATCH_RECOVERY_GRACE_SEC, value: "1"}
- {name: NUTHATCH_OBJECT_REQUEST_TIMEOUT_SEC, value: "2"}
- {name: NUTHATCH_JWT_SECRET, value: synthetic-recovery-session-jwt-secret}
- {name: NUTHATCH_PUBLIC_BASE_URL, value: "https://recovery.invalid"}
- {name: NUTHATCH_OIDC_ISSUER, value: "https://issuer.invalid"}
- {name: NUTHATCH_OIDC_CLIENT_ID, value: recovery}
- {name: NUTHATCH_OIDC_ADDITIONAL_AUDIENCES, value: ""}
- {name: NUTHATCH_OIDC_CLIENT_SECRET, value: synthetic-recovery-oidc-client-secret}
- {name: NUTHATCH_OIDC_REDIRECT_URI, value: "https://recovery.invalid/oidc/callback"}
resources:
requests: {cpu: 100m, memory: 64Mi}
limits: {cpu: "2", memory: 256Mi}

408
tests/kind/recovery-run.sh Executable file
View file

@ -0,0 +1,408 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
source "$ROOT/tests/kind/lib.sh"
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
SUCCEEDED=false
OWNS_FIXTURE=false
APP_IMAGE=nuthatch-recovery:local
cleanup() {
local status=$?
trap - EXIT INT TERM
if [[ "${NUTHATCH_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
printf 'retained context: %s\n' "$CONTEXT"
exit "$status"
fi
if [[ "$OWNS_FIXTURE" = true ]]; then
destroy_fixture || status=1
fi
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
printf 'user kubeconfig changed unexpectedly\n' >&2
status=1
fi
exit "$status"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
pg_value() {
local database="$1"
local query="$2"
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
env PGPASSWORD=synthetic-postgres psql \
--username nuthatch \
--dbname "$database" \
--tuples-only \
--no-align \
--set ON_ERROR_STOP=1 \
--command "$query" |
tr -d '[:space:]'
}
wait_for_value() {
local database="$1"
local query="$2"
local expected="$3"
local description="$4"
for _ in $(seq 1 120); do
if [[ "$(pg_value "$database" "$query" 2>/dev/null || true)" = "$expected" ]]; then
return
fi
sleep 0.25
done
printf 'timed out waiting for %s\n' "$description" >&2
return 1
}
render_application() {
local lane="$1"
local profile_file="$2"
local pg_conn
local redis_url
local s3_endpoint
local s3_region
local s3_bucket
local s3_access_key
local s3_secret_key
pg_conn="$(sed -n 's/^NUTHATCH_PG_CONN=//p' "$profile_file")"
redis_url="$(sed -n 's/^NUTHATCH_REDIS_URL=//p' "$profile_file")"
s3_endpoint="$(sed -n 's/^NUTHATCH_S3_ENDPOINT=//p' "$profile_file")"
s3_region="$(sed -n 's/^NUTHATCH_S3_REGION=//p' "$profile_file")"
s3_bucket="$(sed -n 's/^NUTHATCH_S3_BUCKET=//p' "$profile_file")"
s3_access_key="$(sed -n 's/^NUTHATCH_S3_ACCESS_KEY=//p' "$profile_file")"
s3_secret_key="$(sed -n 's/^NUTHATCH_S3_SECRET_KEY=//p' "$profile_file")"
sed \
-e "s|@@LANE@@|$lane|g" \
-e "s|@@APP_IMAGE@@|$APP_IMAGE|g" \
-e "s|@@PG_CONN@@|$pg_conn|g" \
-e "s|@@REDIS_URL@@|$redis_url|g" \
-e "s|@@OIDC_ISSUER@@|https://issuer.invalid|g" \
-e "s|@@OIDC_CLIENT_ID@@|recovery|g" \
-e "s|@@OIDC_ADDITIONAL_AUDIENCES@@||g" \
-e "s|@@OIDC_CLIENT_SECRET@@|synthetic-recovery-oidc-client-secret|g" \
-e "s|@@S3_ENDPOINT@@|$s3_endpoint|g" \
-e "s|@@S3_REGION@@|$s3_region|g" \
-e "s|@@S3_BUCKET@@|$s3_bucket|g" \
-e "s|@@S3_ACCESS_KEY@@|$s3_access_key|g" \
-e "s|@@S3_SECRET_KEY@@|$s3_secret_key|g" \
"$ROOT/deploy/kubernetes/nuthatch.yaml.in" >"$RUNTIME/recovery-app-$lane.yaml"
sed -i \
's/NUTHATCH_OBJECT_REQUEST_TIMEOUT_SEC: "300"/NUTHATCH_OBJECT_REQUEST_TIMEOUT_SEC: "2"/' \
"$RUNTIME/recovery-app-$lane.yaml"
sed -i \
'/NUTHATCH_OBJECT_REQUEST_TIMEOUT_SEC/a\ NUTHATCH_RECOVERY_GRACE_SEC: "1"' \
"$RUNTIME/recovery-app-$lane.yaml"
sed -i 's/nameservers: \[10.96.0.53\]/nameservers: [10.96.0.10]/' \
"$RUNTIME/recovery-app-$lane.yaml"
sed -i 's/ replicas: 1/ replicas: 2/' "$RUNTIME/recovery-app-$lane.yaml"
}
deploy_recovery() {
local lane="$1"
local profile="$2"
render_application "$lane" "$RUNTIME/$profile.env"
kube apply --filename "$RUNTIME/recovery-app-$lane.yaml" >/dev/null
kube --namespace "$FIXTURE_NAMESPACE" rollout status \
"deployment/nuthatch-$lane" --timeout=180s
}
cleanup_recovery() {
local lane="$1"
kube --namespace "$FIXTURE_NAMESPACE" delete \
"deployment/nuthatch-$lane" \
"service/nuthatch-$lane" \
"configmap/nuthatch-$lane" \
"secret/nuthatch-$lane" \
--ignore-not-found >/dev/null
kube --namespace "$FIXTURE_NAMESPACE" wait \
--for=delete pod \
--selector="app=nuthatch-$lane" \
--timeout=60s >/dev/null
rm -f "$RUNTIME/recovery-app-$lane.yaml"
}
start_phase() {
local lane="$1"
local profile="$2"
local phase="$3"
local job="recovery-$phase-$lane"
render_lane_job \
"$job" \
"$TESTBED/recovery-phase-job.yaml.in" \
"$APP_IMAGE" \
"$profile"
sed -i "s|@@PHASE@@|$phase|g" "$RUNTIME/$job.yaml"
kube apply --filename "$RUNTIME/$job.yaml" >/dev/null
}
kill_phase() {
local lane="$1"
local phase="$2"
local job="recovery-$phase-$lane"
kube --namespace "$FIXTURE_NAMESPACE" delete pod \
--selector="job-name=$job" \
--grace-period=0 \
--force >/dev/null
kube --namespace "$FIXTURE_NAMESPACE" delete "job/$job" \
--ignore-not-found >/dev/null
}
wait_for_phase_checkpoint() {
local lane="$1"
local phase="$2"
local operation="recovery-$phase"
local job="recovery-$phase-$lane"
for _ in $(seq 1 120); do
if kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" 2>/dev/null |
grep -F "recovery checkpoint $operation" >/dev/null; then
return
fi
sleep 0.25
done
printf 'timed out waiting for %s checkpoint\n' "$phase" >&2
return 1
}
assert_terminal_authority() {
local database="$1"
local operation="$2"
local result="$3"
wait_for_value \
"$database" \
"SELECT state FROM mutations WHERE operation_id = '$operation'" \
"$result" \
"$operation terminal result"
test "$(
pg_value "$database" \
"SELECT count(*) FROM key_locks WHERE operation_id = '$operation'"
)" = 0
test "$(
pg_value "$database" \
"SELECT reserved_bytes FROM pools WHERE pool_id = 'shared'"
)" = 0
}
set_service_available() {
local service="$1"
local available="$2"
local selector="$service"
if [[ "$available" = false ]]; then
selector=recovery-unavailable
fi
kube --namespace "$FIXTURE_NAMESPACE" patch service "$service" \
--type merge \
--patch "{\"spec\":{\"selector\":{\"app\":\"$selector\"}}}" >/dev/null
}
terminate_database_connections() {
local database="$1"
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
env PGPASSWORD=synthetic-postgres psql \
--username nuthatch \
--dbname postgres \
--set ON_ERROR_STOP=1 \
--command \
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$database' AND pid <> pg_backend_pid()" \
>/dev/null
}
assert_object_absent() {
local target="$1"
local bucket="$2"
local key="$3"
local endpoint
endpoint="$(target_loopback_endpoint "$target")"
if kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$target" \
--container s3-readiness -- \
aws --endpoint-url "$endpoint" s3api head-object \
--bucket "$bucket" \
--key "$key" >/dev/null 2>&1; then
return 1
fi
}
assert_object_present() {
local target="$1"
local bucket="$2"
local key="$3"
local endpoint
endpoint="$(target_loopback_endpoint "$target")"
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$target" \
--container s3-readiness -- \
aws --endpoint-url "$endpoint" s3api head-object \
--bucket "$bucket" \
--key "$key" >/dev/null
}
assert_cache_invalidated() {
local lane="$1"
local value
value="$(
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/redis -- \
redis-cli --raw -n "$((10#$lane))" \
GET nuthatch:list:recovery-user:shared:: |
tr -d '[:space:]'
)"
test -z "$value"
}
run_phase_recovery() {
local lane="$1"
local profile="$2"
local database="$3"
local target="$4"
local phase="$5"
local durable_state="$6"
local terminal_state="$7"
local operation="recovery-$phase"
start_phase "$lane" "$profile" "$phase"
wait_for_value \
"$database" \
"SELECT state || ':' || external_state FROM mutations WHERE operation_id = '$operation'" \
"$durable_state" \
"$phase durable checkpoint"
if [[ "$phase" = during-put ]]; then
sleep 1
fi
kill_phase "$lane" "$phase"
deploy_recovery "$lane" "$profile"
assert_terminal_authority "$database" "$operation" "$terminal_state"
cleanup_recovery "$lane"
if [[ "$terminal_state" = succeeded ]]; then
assert_cache_invalidated "$lane"
fi
}
acquire_fixture_lock
OWNS_FIXTURE=true
setup_fixture durable coordination target
source "$RUNTIME/fixture.env"
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" build \
--file "$ROOT/Dockerfile" \
--target test-runtime \
--tag "$APP_IMAGE" \
"$ROOT"
load_local_image "$APP_IMAGE" "$NERDCTL_NAMESPACE"
while IFS=$'\t' read -r lane coordination oidc target; do
if [[ -n "${NUTHATCH_RECOVERY_TARGET:-}" ]] &&
[[ "$target" != "$NUTHATCH_RECOVERY_TARGET" ]]; then
continue
fi
profile="$(write_lane_profile "$lane" redis - "$target")"
database="nuthatch_lane_$lane"
bucket="nuthatch-lane-$lane"
printf 'Running recovery lane %s: target=%s\n' "$lane" "$target"
run_phase_recovery \
"$lane" "$profile" "$database" "$target" \
before-dispatch prepared:idle failed
run_phase_recovery \
"$lane" "$profile" "$database" "$target" \
during-put applying:pending failed
sleep 5
assert_object_absent "$target" "$bucket" \
shared/recovery-during-put.bin
start_phase "$lane" "$profile" live-timeout
wait_for_phase_checkpoint "$lane" live-timeout
test "$(
pg_value "$database" \
"SELECT count(*) FROM key_locks WHERE operation_id = 'recovery-live-timeout'"
)" = 1
test "$(
pg_value "$database" \
"SELECT reserved_bytes FROM pools WHERE pool_id = 'shared'"
)" = "$((20 * 1024 * 1024))"
kill_phase "$lane" live-timeout
deploy_recovery "$lane" "$profile"
assert_terminal_authority "$database" recovery-live-timeout failed
cleanup_recovery "$lane"
sleep 5
assert_object_absent "$target" "$bucket" \
shared/recovery-live-timeout.bin
start_phase "$lane" "$profile" after-dispatch
wait_for_value \
"$database" \
"SELECT state || ':' || external_state FROM mutations WHERE operation_id = 'recovery-after-dispatch'" \
applying:pending \
"after-dispatch checkpoint"
kill_phase "$lane" after-dispatch
pg_value \
"$database" \
"UPDATE mutations SET request_deadline_at = now() + interval '60 seconds', worker_lease_until = now() + interval '60 seconds' WHERE operation_id = 'recovery-after-dispatch'; SELECT 1" \
>/dev/null
deploy_recovery "$lane" "$profile"
before_pg_restarts="$(
kube --namespace "$FIXTURE_NAMESPACE" get pods \
--selector="app=nuthatch-$lane" \
--output json |
jq '[.items[].status.containerStatuses[].restartCount] | add'
)"
set_service_available postgres false
terminate_database_connections "$database"
sleep 6
test "$(
kube --namespace "$FIXTURE_NAMESPACE" get pods \
--selector="app=nuthatch-$lane" \
--output json |
jq '[.items[].status.containerStatuses[].restartCount] | add'
)" = "$before_pg_restarts"
set_service_available postgres true
wait_for_value "$database" "SELECT 1" 1 "PostgreSQL recovery"
pg_value \
"$database" \
"UPDATE mutations SET request_deadline_at = now() - interval '1 second', worker_lease_until = now() - interval '1 second' WHERE operation_id = 'recovery-after-dispatch'; SELECT 1" \
>/dev/null
assert_terminal_authority "$database" recovery-after-dispatch succeeded
cleanup_recovery "$lane"
assert_cache_invalidated "$lane"
assert_object_present "$target" "$bucket" \
shared/recovery-after-dispatch.bin
start_phase "$lane" "$profile" after-settlement
wait_for_value \
"$database" \
"SELECT state || ':' || external_state FROM mutations WHERE operation_id = 'recovery-after-settlement'" \
applying:settled \
"after-settlement checkpoint"
kill_phase "$lane" after-settlement
set_service_available "$target" false
deploy_recovery "$lane" "$profile"
sleep 3
kube --namespace "$FIXTURE_NAMESPACE" logs \
--selector="app=nuthatch-$lane" \
--prefix=true |
grep -F '"result":"exhausted"'
set_service_available "$target" true
assert_terminal_authority "$database" recovery-after-settlement succeeded
cleanup_recovery "$lane"
assert_cache_invalidated "$lane"
run_phase_recovery \
"$lane" "$profile" "$database" "$target" \
delete-settled applying:settled succeeded
assert_object_absent "$target" "$bucket" \
shared/recovery-delete-settled.bin
cleanup_lane_state "$lane" redis - "$target"
for phase in \
before-dispatch \
during-put \
live-timeout \
after-dispatch \
after-settlement \
delete-settled; do
cleanup_lane_resources "recovery-$phase-$lane" "$profile"
done
done < <(expand_fixture_matrix target)
SUCCEEDED=true
printf 'Nuthatch recovery gate passed\n'

39
tests/kind/runtime-health-job.yaml.in Normal file
View file

@ -0,0 +1,39 @@
apiVersion: batch/v1
kind: Job
metadata:
name: @@JOB_NAME@@
namespace: nuthatch-testbed
spec:
activeDeadlineSeconds: 60
backoffLimit: 0
template:
spec:
automountServiceAccountToken: false
enableServiceLinks: false
restartPolicy: Never
containers:
- name: health
image: @@CURL_IMAGE@@
imagePullPolicy: Never
command: [/bin/sh, -ec]
args:
- |
base="https://nuthatch-@@LANE@@:8080"
test "$(
curl --insecure --silent --output /tmp/health \
--write-out '%{http_code}' "$base/healthz"
)" = 200
grep -F '"ok":true' /tmp/health
test "$(
curl --insecure --silent --output /tmp/ready \
--write-out '%{http_code}' "$base/readyz"
)" = "@@READY_STATUS@@"
curl --insecure --silent --output /dev/null "$base/app"
test "$(
curl --insecure --silent --output /dev/null \
--write-out '%{http_code}' "$base/api/session"
)" = 303
printf 'runtime health semantics passed\n'
resources:
requests: {cpu: 25m, memory: 16Mi}
limits: {cpu: 250m, memory: 64Mi}

32
tests/kind/runtime-probe-job.yaml.in Normal file
View file

@ -0,0 +1,32 @@
apiVersion: batch/v1
kind: Job
metadata:
name: @@JOB_NAME@@
namespace: nuthatch-testbed
spec:
activeDeadlineSeconds: 120
backoffLimit: 0
template:
spec:
automountServiceAccountToken: false
enableServiceLinks: false
restartPolicy: Never
containers:
- name: runtime-probe
image: @@TEST_IMAGE@@
imagePullPolicy: Never
command: [nuthatch-runtime-probe, "@@MODE@@"]
envFrom:
- configMapRef: {name: @@PROFILE_NAME@@}
env:
- {name: NUTHATCH_LIST_CACHE_TTL_SEC, value: "1"}
- {name: NUTHATCH_JWT_SECRET, value: synthetic-runtime-session-jwt-secret}
- {name: NUTHATCH_PUBLIC_BASE_URL, value: "https://runtime.invalid"}
- {name: NUTHATCH_OIDC_ISSUER, value: "https://issuer.invalid"}
- {name: NUTHATCH_OIDC_CLIENT_ID, value: runtime-probe}
- {name: NUTHATCH_OIDC_ADDITIONAL_AUDIENCES, value: ""}
- {name: NUTHATCH_OIDC_CLIENT_SECRET, value: synthetic-runtime-oidc-client-secret}
- {name: NUTHATCH_OIDC_REDIRECT_URI, value: "https://runtime.invalid/oidc/callback"}
resources:
requests: {cpu: 50m, memory: 32Mi}
limits: {cpu: "1", memory: 128Mi}

363
tests/kind/runtime-resilience-run.sh Executable file
View file

@ -0,0 +1,363 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
source "$ROOT/tests/kind/lib.sh"
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
SUCCEEDED=false
OWNS_FIXTURE=false
APP_IMAGE=nuthatch-runtime-resilience:local
cleanup() {
local status=$?
trap - EXIT INT TERM
if [[ "${NUTHATCH_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
printf 'retained context: %s\n' "$CONTEXT"
exit "$status"
fi
if [[ "$OWNS_FIXTURE" = true ]]; then
destroy_fixture || status=1
fi
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
printf 'user kubeconfig changed unexpectedly\n' >&2
status=1
fi
exit "$status"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
set_service_available() {
local service="$1"
local available="$2"
local selector="$service"
if [[ "$available" = false ]]; then
selector=runtime-unavailable
fi
kube --namespace "$FIXTURE_NAMESPACE" patch service "$service" \
--type merge \
--patch "{\"spec\":{\"selector\":{\"app\":\"$selector\"}}}" >/dev/null
}
render_application() {
local lane="$1"
local profile_file="$2"
local oidc_secret
oidc_secret="$(<"$profile_file.oidc-secret")"
sed \
-e "s|@@LANE@@|$lane|g" \
-e "s|@@APP_IMAGE@@|$APP_IMAGE|g" \
-e "s|@@PG_CONN@@|$(sed -n 's/^NUTHATCH_PG_CONN=//p' "$profile_file")|g" \
-e "s|@@REDIS_URL@@|$(sed -n 's/^NUTHATCH_REDIS_URL=//p' "$profile_file")|g" \
-e "s|@@OIDC_ISSUER@@|$(sed -n 's/^NUTHATCH_OIDC_ISSUER=//p' "$profile_file")|g" \
-e "s|@@OIDC_CLIENT_ID@@|$(sed -n 's/^NUTHATCH_OIDC_CLIENT_ID=//p' "$profile_file")|g" \
-e "s|@@OIDC_ADDITIONAL_AUDIENCES@@|$(sed -n 's/^NUTHATCH_OIDC_ADDITIONAL_AUDIENCES=//p' "$profile_file")|g" \
-e "s|@@OIDC_CLIENT_SECRET@@|$oidc_secret|g" \
-e "s|@@S3_ENDPOINT@@|$(sed -n 's/^NUTHATCH_S3_ENDPOINT=//p' "$profile_file")|g" \
-e "s|@@S3_REGION@@|$(sed -n 's/^NUTHATCH_S3_REGION=//p' "$profile_file")|g" \
-e "s|@@S3_BUCKET@@|$(sed -n 's/^NUTHATCH_S3_BUCKET=//p' "$profile_file")|g" \
-e "s|@@S3_ACCESS_KEY@@|$(sed -n 's/^NUTHATCH_S3_ACCESS_KEY=//p' "$profile_file")|g" \
-e "s|@@S3_SECRET_KEY@@|$(sed -n 's/^NUTHATCH_S3_SECRET_KEY=//p' "$profile_file")|g" \
"$ROOT/deploy/kubernetes/nuthatch.yaml.in" >"$RUNTIME/runtime-app-$lane.yaml"
sed -i \
-e 's/ replicas: 1/ replicas: 2/' \
-e 's/NUTHATCH_LIST_CACHE_TTL_SEC: "30"/NUTHATCH_LIST_CACHE_TTL_SEC: "1"/' \
-e 's/NUTHATCH_STARTUP_ATTEMPTS: "60"/NUTHATCH_STARTUP_ATTEMPTS: "4"/' \
-e 's/NUTHATCH_STARTUP_DELAY_MS: "1000"/NUTHATCH_STARTUP_DELAY_MS: "250"/' \
"$RUNTIME/runtime-app-$lane.yaml"
}
wait_for_startup_exhaustion() {
local lane="$1"
for _ in $(seq 1 120); do
if kube --namespace "$FIXTURE_NAMESPACE" logs \
--selector="app=nuthatch-$lane" --all-containers=true \
--prefix=true 2>/dev/null |
grep -F '"action":"dependency_startup"' |
grep -F '"result":"exhausted"' >/dev/null; then
return
fi
sleep 0.25
done
printf 'timed out waiting for bounded startup exhaustion\n' >&2
return 1
}
run_runtime_probe() {
local lane="$1"
local profile="$2"
local mode="$3"
local job="runtime-probe-$mode-$lane"
render_lane_job \
"$job" \
"$TESTBED/runtime-probe-job.yaml.in" \
"$APP_IMAGE" \
"$profile"
sed -i "s|@@MODE@@|$mode|g" "$RUNTIME/$job.yaml"
run_job_manifest "$job" "$RUNTIME/$job.yaml"
}
set_profile_mode() {
local profile="$1"
local name="$2"
local value="$3"
local profile_file="$RUNTIME/$profile.env"
sed -i "/^$name=/d" "$profile_file"
if [[ -n "$value" ]]; then
printf '%s=%s\n' "$name" "$value" >>"$profile_file"
fi
kube create configmap "$profile" \
--namespace "$FIXTURE_NAMESPACE" \
--from-env-file="$profile_file" \
--dry-run=client \
--output yaml |
kube apply --filename - >/dev/null
}
run_product_probe() {
local lane="$1"
local profile="$2"
local suffix="$3"
local job="runtime-product-$suffix-$lane"
render_lane_job \
"$job" \
"$TESTBED/product-matrix-job.yaml.in" \
"$CURL_LOCAL_IMAGE" \
"$profile"
sed -i "s|@@LANE@@|$lane|g" "$RUNTIME/$job.yaml"
run_job_manifest "$job" "$RUNTIME/$job.yaml"
}
run_database_failure_probe() {
local lane="$1"
local profile="$2"
local database="$3"
local job="runtime-product-database-unavailable-$lane"
render_lane_job \
"$job" \
"$TESTBED/product-matrix-job.yaml.in" \
"$CURL_LOCAL_IMAGE" \
"$profile"
sed -i "s|@@LANE@@|$lane|g" "$RUNTIME/$job.yaml"
kube apply --filename "$RUNTIME/$job.yaml" >/dev/null
for _ in $(seq 1 240); do
if kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" 2>/dev/null |
grep -F 'Database fault checkpoint' >/dev/null; then
break
fi
sleep 0.25
done
kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" |
grep -F 'Database fault checkpoint' >/dev/null
kube --namespace "$FIXTURE_NAMESPACE" patch service "nuthatch-$lane" \
--type merge \
--patch '{"spec":{"publishNotReadyAddresses":true}}' >/dev/null
set_service_available postgres false
terminate_database_connections "$database"
local status=0
wait_for_job "$job" || status=$?
set_service_available postgres true
kube --namespace "$FIXTURE_NAMESPACE" patch service "nuthatch-$lane" \
--type merge \
--patch '{"spec":{"publishNotReadyAddresses":false}}' >/dev/null
local restored=false
for _ in $(seq 1 120); do
if run_health_probe \
"$lane" "$profile" 200 database-mutation-restored 2>/dev/null; then
restored=true
break
fi
sleep 0.25
done
test "$restored" = true
printf 'PostgreSQL mutation recovery passed status=%s\n' "$status"
return "$status"
}
run_health_probe() {
local lane="$1"
local profile="$2"
local status="$3"
local suffix="$4"
local job="runtime-health-$suffix-$lane"
render_lane_job \
"$job" \
"$TESTBED/runtime-health-job.yaml.in" \
"$CURL_LOCAL_IMAGE" \
"$profile"
sed -i \
-e "s|@@LANE@@|$lane|g" \
-e "s|@@READY_STATUS@@|$status|g" \
"$RUNTIME/$job.yaml"
run_job_manifest "$job" "$RUNTIME/$job.yaml"
}
restart_count() {
local lane="$1"
kube --namespace "$FIXTURE_NAMESPACE" get pods \
--selector="app=nuthatch-$lane" \
--output json |
jq '[.items[].status.containerStatuses[]?.restartCount] | add // 0'
}
terminate_database_connections() {
local database="$1"
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
env PGPASSWORD=synthetic-postgres psql \
--username nuthatch \
--dbname postgres \
--set ON_ERROR_STOP=1 \
--command \
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$database' AND pid <> pg_backend_pid()" \
>/dev/null
}
acquire_fixture_lock
OWNS_FIXTURE=true
setup_fixture durable coordination oidc target
source "$RUNTIME/fixture.env"
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" build \
--file "$ROOT/Dockerfile" \
--target test-runtime \
--tag "$APP_IMAGE" \
"$ROOT"
load_local_image "$APP_IMAGE" "$NERDCTL_NAMESPACE"
lane=01
coordination=redis
oidc=keycloak
target=garage
profile="$(write_lane_profile "$lane" "$coordination" "$oidc" "$target")"
database="nuthatch_lane_$lane"
render_application "$lane" "$RUNTIME/$profile.env"
run_runtime_probe "$lane" "$profile" config
set_service_available postgres false
kube apply --filename "$RUNTIME/runtime-app-$lane.yaml" >/dev/null
wait_for_startup_exhaustion "$lane"
set_service_available postgres true
set_service_available redis false
set_service_available "$target" false
kube --namespace "$FIXTURE_NAMESPACE" rollout restart \
"deployment/nuthatch-$lane" >/dev/null
kube --namespace "$FIXTURE_NAMESPACE" rollout status \
"deployment/nuthatch-$lane" --timeout=180s
run_health_probe "$lane" "$profile" 200 redis-down
run_runtime_probe "$lane" "$profile" unavailable
set_service_available redis true
set_service_available "$target" true
run_runtime_probe "$lane" "$profile" available
run_health_probe "$lane" "$profile" 200 restored
run_product_probe "$lane" "$profile" baseline
set_profile_mode "$profile" NUTHATCH_RUNTIME_MODE coherence
run_product_probe "$lane" "$profile" coherence
set_profile_mode "$profile" NUTHATCH_RUNTIME_MODE ""
set_service_available redis false
kube --namespace "$FIXTURE_NAMESPACE" rollout restart deployment/redis >/dev/null
kube --namespace "$FIXTURE_NAMESPACE" rollout status \
deployment/redis --timeout=120s
set_profile_mode "$profile" NUTHATCH_STREAMING_MODE unavailable
run_product_probe "$lane" "$profile" redis-unavailable
set_profile_mode "$profile" NUTHATCH_STREAMING_MODE ""
set_service_available redis true
set_profile_mode "$profile" NUTHATCH_RUNTIME_MODE database-unavailable
run_database_failure_probe "$lane" "$profile" "$database"
set_profile_mode "$profile" NUTHATCH_RUNTIME_MODE ""
printf 'Database mutation audit lane passed\n'
before_s3="$(restart_count "$lane")"
printf 'S3 runtime fault starting restarts=%s\n' "$before_s3"
set_service_available "$target" false
run_health_probe "$lane" "$profile" 200 s3-down
sleep 6
test "$(restart_count "$lane")" = "$before_s3"
set_service_available "$target" true
before_pg="$(restart_count "$lane")"
set_service_available postgres false
terminate_database_connections "$database"
run_health_probe "$lane" "$profile" 503 postgres-down
sleep 6
test "$(restart_count "$lane")" = "$before_pg"
set_service_available postgres true
restored=false
for _ in $(seq 1 120); do
if run_health_probe "$lane" "$profile" 200 postgres-restored 2>/dev/null; then
restored=true
break
fi
sleep 0.25
done
test "$restored" = true
kube --namespace "$FIXTURE_NAMESPACE" logs \
--selector="app=nuthatch-$lane" \
--container nuthatch \
--tail=-1 >"$RUNTIME/runtime-app.log"
grep '^{' "$RUNTIME/runtime-app.log" >"$RUNTIME/runtime-audit.jsonl"
test -s "$RUNTIME/runtime-audit.jsonl"
jq --slurp -e '
length > 0
and all(.[]; has("ts") and has("action") and has("result"))
' \
"$RUNTIME/runtime-audit.jsonl" >/dev/null
for action in authentication login logout authorization put delete; do
grep -F "\"action\":\"$action\"" "$RUNTIME/runtime-audit.jsonl" >/dev/null
done
jq --slurp -e '
[.[] | select(.action == "put")] as $records
| ($records | length) > 0
and all($records[]; has("user_id") and has("root") and has("bytes"))
' "$RUNTIME/runtime-audit.jsonl" >/dev/null
jq --slurp -e '
[.[] |
select(
(.action == "put" or .action == "delete")
and (.result != "ok")
)
] as $records
| ($records | length) > 0
and all($records[]; has("reason"))
' "$RUNTIME/runtime-audit.jsonl" >/dev/null
if grep -Ei 'Bearer |"authorization"[[:space:]]*:|client_secret|NUTHATCH_' \
"$RUNTIME/runtime-app.log" >/dev/null; then
printf 'audit output exposed prohibited data\n' >&2
exit 1
fi
while IFS= read -r secret_value; do
if [[ -n "$secret_value" ]] &&
grep -F -- "$secret_value" "$RUNTIME/runtime-app.log" >/dev/null; then
printf 'audit output exposed a configured secret\n' >&2
exit 1
fi
done < <(
{
printf '%s\n' synthetic-runtime-session-jwt-secret
sed -n 's/^NUTHATCH_PG_CONN=//p' "$RUNTIME/$profile.env"
sed -n 's/^NUTHATCH_S3_ACCESS_KEY=//p' "$RUNTIME/$profile.env"
sed -n 's/^NUTHATCH_S3_SECRET_KEY=//p' "$RUNTIME/$profile.env"
sed -n '1p' "$RUNTIME/$profile.env.oidc-secret"
}
)
for body_value in private-a runtime-coherence first-personal second-personal; do
if grep -F -- "$body_value" "$RUNTIME/runtime-app.log" >/dev/null; then
printf 'audit output exposed request body data\n' >&2
exit 1
fi
done
test "$(kube --namespace "$FIXTURE_NAMESPACE" get deployment \
"nuthatch-$lane" --output jsonpath='{.status.readyReplicas}')" = 2
cleanup_lane_state "$lane" "$coordination" "$oidc" "$target"
cleanup_lane_resources "runtime-probe-available-$lane" "$profile"
SUCCEEDED=true
printf 'Nuthatch runtime resilience gate passed\n'

34
tests/kind/storage-foundation-job.yaml.in Normal file
View file

@ -0,0 +1,34 @@
apiVersion: batch/v1
kind: Job
metadata:
name: @@JOB_NAME@@
namespace: nuthatch-testbed
spec:
activeDeadlineSeconds: 300
backoffLimit: 0
ttlSecondsAfterFinished: 300
template:
spec:
automountServiceAccountToken: false
enableServiceLinks: false
restartPolicy: Never
containers:
- name: storage-foundation
image: @@TEST_IMAGE@@
imagePullPolicy: Never
command: [nuthatch-storage-integration]
args: ["+RTS", "-N2", "-RTS"]
envFrom:
- configMapRef: {name: @@PROFILE_NAME@@}
env:
- {name: NUTHATCH_JWT_SECRET, value: synthetic-storage-session-jwt-secret}
- {name: NUTHATCH_REDIS_URL, value: "redis://127.0.0.1:6379/0"}
- {name: NUTHATCH_PUBLIC_BASE_URL, value: "https://storage.invalid"}
- {name: NUTHATCH_OIDC_ISSUER, value: "https://issuer.invalid"}
- {name: NUTHATCH_OIDC_CLIENT_ID, value: storage-foundation}
- {name: NUTHATCH_OIDC_ADDITIONAL_AUDIENCES, value: ""}
- {name: NUTHATCH_OIDC_CLIENT_SECRET, value: synthetic-storage-oidc-client-secret}
- {name: NUTHATCH_OIDC_REDIRECT_URI, value: "https://storage.invalid/oidc/callback"}
resources:
requests: {cpu: 100m, memory: 32Mi}
limits: {cpu: "2", memory: 96Mi}

49
tests/kind/storage-foundation-run.sh Executable file
View file

@ -0,0 +1,49 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
source "$ROOT/tests/kind/lib.sh"
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
SUCCEEDED=false
OWNS_FIXTURE=false
APP_IMAGE=nuthatch-storage-foundation:local
cleanup() {
local status=$?
trap - EXIT INT TERM
if [[ "${NUTHATCH_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
printf 'retained context: %s\n' "$CONTEXT"
exit "$status"
fi
if [[ "$OWNS_FIXTURE" = true ]]; then
destroy_fixture || status=1
fi
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
printf 'user kubeconfig changed unexpectedly\n' >&2
status=1
fi
exit "$status"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
acquire_fixture_lock
OWNS_FIXTURE=true
setup_fixture durable target
"$NUTHATCH_REAL_NERDCTL" --namespace default build \
--file "$ROOT/Dockerfile" \
--target test-runtime \
--tag "$APP_IMAGE" \
"$ROOT"
load_local_image "$APP_IMAGE" default
run_job_matrix \
storage-foundation \
"$TESTBED/storage-foundation-job.yaml.in" \
"$APP_IMAGE" \
durable target
SUCCEEDED=true
printf 'Nuthatch storage foundation gate passed\n'

217
tests/kind/streaming-run.sh Executable file
View file

@ -0,0 +1,217 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
source "$ROOT/tests/kind/lib.sh"
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
SUCCEEDED=false
OWNS_FIXTURE=false
APP_IMAGE=nuthatch-streaming:local
cleanup() {
local status=$?
trap - EXIT INT TERM
if [[ "${NUTHATCH_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
printf 'retained context: %s\n' "$CONTEXT"
exit "$status"
fi
if [[ "$OWNS_FIXTURE" = true ]]; then
destroy_fixture || status=1
fi
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
printf 'user kubeconfig changed unexpectedly\n' >&2
status=1
fi
exit "$status"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
render_application() {
local lane="$1"
local profile_file="$2"
local rate="$3"
local burst="$4"
local pg_conn
local redis_url
local oidc_issuer
local oidc_client_id
local oidc_additional_audiences
local oidc_client_secret
local s3_endpoint
local s3_region
local s3_bucket
local s3_access_key
local s3_secret_key
pg_conn="$(sed -n 's/^NUTHATCH_PG_CONN=//p' "$profile_file")"
redis_url="$(sed -n 's/^NUTHATCH_REDIS_URL=//p' "$profile_file")"
oidc_issuer="$(sed -n 's/^NUTHATCH_OIDC_ISSUER=//p' "$profile_file")"
oidc_client_id="$(sed -n 's/^NUTHATCH_OIDC_CLIENT_ID=//p' "$profile_file")"
oidc_additional_audiences="$(
sed -n 's/^NUTHATCH_OIDC_ADDITIONAL_AUDIENCES=//p' "$profile_file"
)"
oidc_client_secret="$(<"$profile_file.oidc-secret")"
s3_endpoint="$(sed -n 's/^NUTHATCH_S3_ENDPOINT=//p' "$profile_file")"
s3_region="$(sed -n 's/^NUTHATCH_S3_REGION=//p' "$profile_file")"
s3_bucket="$(sed -n 's/^NUTHATCH_S3_BUCKET=//p' "$profile_file")"
s3_access_key="$(sed -n 's/^NUTHATCH_S3_ACCESS_KEY=//p' "$profile_file")"
s3_secret_key="$(sed -n 's/^NUTHATCH_S3_SECRET_KEY=//p' "$profile_file")"
sed \
-e "s|@@LANE@@|$lane|g" \
-e "s|@@APP_IMAGE@@|$APP_IMAGE|g" \
-e "s|@@PG_CONN@@|$pg_conn|g" \
-e "s|@@REDIS_URL@@|$redis_url|g" \
-e "s|@@OIDC_ISSUER@@|$oidc_issuer|g" \
-e "s|@@OIDC_CLIENT_ID@@|$oidc_client_id|g" \
-e "s|@@OIDC_ADDITIONAL_AUDIENCES@@|$oidc_additional_audiences|g" \
-e "s|@@OIDC_CLIENT_SECRET@@|$oidc_client_secret|g" \
-e "s|@@S3_ENDPOINT@@|$s3_endpoint|g" \
-e "s|@@S3_REGION@@|$s3_region|g" \
-e "s|@@S3_BUCKET@@|$s3_bucket|g" \
-e "s|@@S3_ACCESS_KEY@@|$s3_access_key|g" \
-e "s|@@S3_SECRET_KEY@@|$s3_secret_key|g" \
"$ROOT/deploy/kubernetes/nuthatch.yaml.in" > "$RUNTIME/streaming-app-$lane.yaml"
sed -i \
"/NUTHATCH_S3_PATH_STYLE/a\\ NUTHATCH_MAX_OBJECT_BYTES: \"201326592\"\\n NUTHATCH_THROTTLE_BYTES_PER_SEC: \"$rate\"\\n NUTHATCH_THROTTLE_BURST_BYTES: \"$burst\"" \
"$RUNTIME/streaming-app-$lane.yaml"
sed -i 's/ replicas: 1/ replicas: 2/' "$RUNTIME/streaming-app-$lane.yaml"
sed -i \
's/limits: {cpu: "2", memory: 512Mi}/limits: {cpu: "2", memory: 128Mi}/' \
"$RUNTIME/streaming-app-$lane.yaml"
sed -i \
'/ envFrom:/i\ env:\n - {name: GHCRTS, value: "-N2 -A1m -M96m"}' \
"$RUNTIME/streaming-app-$lane.yaml"
}
cleanup_application() {
local lane="$1"
kube --namespace "$FIXTURE_NAMESPACE" delete \
"deployment/nuthatch-$lane" \
"service/nuthatch-$lane" \
"configmap/nuthatch-$lane" \
"secret/nuthatch-$lane" \
--ignore-not-found >/dev/null
rm -f "$RUNTIME/streaming-app-$lane.yaml"
}
set_streaming_profile() {
local profile="$1"
local mode="$2"
local throttle_burst=262144
local profile_file="$RUNTIME/$profile.env"
if [[ "$mode" = refill ]]; then
throttle_burst=65536
fi
sed -i '/^NUTHATCH_STREAMING_/d' "$profile_file"
{
printf 'NUTHATCH_STREAMING_MODE=%s\n' "$mode"
printf 'NUTHATCH_STREAMING_LARGE_BYTES=167772160\n'
printf 'NUTHATCH_STREAMING_OVER_LIMIT_BYTES=201326593\n'
printf 'NUTHATCH_STREAMING_THROTTLE_BURST_BYTES=%s\n' "$throttle_burst"
} >> "$profile_file"
kube create configmap "$profile" \
--namespace "$FIXTURE_NAMESPACE" \
--from-env-file="$profile_file" \
--dry-run=client \
--output yaml |
kube apply --filename - >/dev/null
}
run_streaming_job() {
local lane="$1"
local profile="$2"
local mode="$3"
local job="streaming-$mode-$lane"
set_streaming_profile "$profile" "$mode"
render_lane_job \
"$job" \
"$TESTBED/product-matrix-job.yaml.in" \
"$CURL_LOCAL_IMAGE" \
"$profile"
sed -i "s|@@LANE@@|$lane|g" "$RUNTIME/$job.yaml"
run_job_manifest "$job" "$RUNTIME/$job.yaml"
}
scale_coordination() {
local provider="$1"
local replicas="$2"
kube --namespace "$FIXTURE_NAMESPACE" scale \
"deployment/$provider" \
--replicas="$replicas" >/dev/null
if [[ "$replicas" = 0 ]]; then
kube --namespace "$FIXTURE_NAMESPACE" wait \
--for=delete \
"pod" \
--selector="app=$provider" \
--timeout=120s >/dev/null
else
kube --namespace "$FIXTURE_NAMESPACE" rollout status \
"deployment/$provider" \
--timeout=180s >/dev/null
fi
}
deploy_application() {
local lane="$1"
local profile="$2"
local rate="$3"
local burst="$4"
cleanup_application "$lane"
render_application "$lane" "$RUNTIME/$profile.env" "$rate" "$burst"
kube apply --filename "$RUNTIME/streaming-app-$lane.yaml" >/dev/null
kube --namespace "$FIXTURE_NAMESPACE" rollout status \
"deployment/nuthatch-$lane" --timeout=180s
}
acquire_fixture_lock
OWNS_FIXTURE=true
setup_fixture durable coordination oidc target
source "$RUNTIME/fixture.env"
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" build \
--file "$ROOT/Dockerfile" \
--tag "$APP_IMAGE" \
"$ROOT"
load_local_image "$APP_IMAGE" "$NERDCTL_NAMESPACE"
while IFS=$'\t' read -r lane coordination oidc target; do
if [[ -n "${NUTHATCH_STREAMING_LANE:-}" ]] &&
[[ "$lane" != "$NUTHATCH_STREAMING_LANE" ]]; then
continue
fi
profile="$(write_lane_profile "$lane" "$coordination" "$oidc" "$target")"
printf 'Running streaming lane %s: coordination=%s oidc=%s target=%s\n' \
"$lane" "$coordination" "$oidc" "$target"
deploy_application "$lane" "$profile" 536870912 536870912
run_streaming_job "$lane" "$profile" large
cleanup_application "$lane"
scale_coordination "$coordination" 0
scale_coordination "$coordination" 1
deploy_application "$lane" "$profile" 1 262144
run_streaming_job "$lane" "$profile" throttle
deploy_application "$lane" "$profile" 65536 65536
run_streaming_job "$lane" "$profile" refill
deploy_application "$lane" "$profile" 536870912 536870912
scale_coordination "$coordination" 0
run_streaming_job "$lane" "$profile" unavailable
cleanup_application "$lane"
scale_coordination "$coordination" 1
cleanup_lane_state "$lane" "$coordination" "$oidc" "$target"
for mode in large throttle refill unavailable; do
cleanup_lane_resources "streaming-$mode-$lane" "$profile"
done
if [[ "${NUTHATCH_STREAMING_FIRST_LANE_ONLY:-0}" = 1 ]]; then
break
fi
done < <(expand_fixture_matrix coordination oidc target)
SUCCEEDED=true
printf 'Nuthatch streaming gate passed\n'