Make object recovery settlement-safe
This commit is contained in:
parent
cc23211e63
commit
cd83841520
2 changed files with 1054 additions and 90 deletions
34
tests/kind/wp09-phase.yaml.in
Normal file
34
tests/kind/wp09-phase.yaml.in
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: @@JOB_NAME@@
|
||||
namespace: nuthatch-testbed
|
||||
spec:
|
||||
activeDeadlineSeconds: 300
|
||||
backoffLimit: 0
|
||||
template:
|
||||
spec:
|
||||
automountServiceAccountToken: false
|
||||
enableServiceLinks: false
|
||||
restartPolicy: Never
|
||||
containers:
|
||||
- name: phase
|
||||
image: @@TEST_IMAGE@@
|
||||
imagePullPolicy: Never
|
||||
command: [nuthatch-wp09-phase]
|
||||
envFrom:
|
||||
- configMapRef: {name: @@PROFILE_NAME@@}
|
||||
env:
|
||||
- {name: NUTHATCH_RECOVERY_PHASE, value: "@@PHASE@@"}
|
||||
- {name: NUTHATCH_RECOVERY_GRACE_SEC, value: "1"}
|
||||
- {name: NUTHATCH_OBJECT_REQUEST_TIMEOUT_SEC, value: "2"}
|
||||
- {name: NUTHATCH_JWT_SECRET, value: synthetic-wp09-session-jwt-secret}
|
||||
- {name: NUTHATCH_PUBLIC_BASE_URL, value: "https://wp09.invalid"}
|
||||
- {name: NUTHATCH_OIDC_ISSUER, value: "https://issuer.invalid"}
|
||||
- {name: NUTHATCH_OIDC_CLIENT_ID, value: wp09}
|
||||
- {name: NUTHATCH_OIDC_ADDITIONAL_AUDIENCES, value: ""}
|
||||
- {name: NUTHATCH_OIDC_CLIENT_SECRET, value: synthetic-wp09-oidc-client-secret}
|
||||
- {name: NUTHATCH_OIDC_REDIRECT_URI, value: "https://wp09.invalid/oidc/callback"}
|
||||
resources:
|
||||
requests: {cpu: 100m, memory: 64Mi}
|
||||
limits: {cpu: "2", memory: 256Mi}
|
||||
402
tests/kind/wp09-run.sh
Executable file
402
tests/kind/wp09-run.sh
Executable file
|
|
@ -0,0 +1,402 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
source "$ROOT/tests/kind/lib.sh"
|
||||
|
||||
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
|
||||
SUCCEEDED=false
|
||||
OWNS_FIXTURE=false
|
||||
APP_IMAGE=nuthatch-wp09:local
|
||||
|
||||
cleanup() {
|
||||
local status=$?
|
||||
trap - EXIT INT TERM
|
||||
if [[ "${NUTHATCH_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
|
||||
printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
|
||||
printf 'retained context: %s\n' "$CONTEXT"
|
||||
exit "$status"
|
||||
fi
|
||||
if [[ "$OWNS_FIXTURE" = true ]]; then
|
||||
destroy_fixture || status=1
|
||||
fi
|
||||
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
|
||||
printf 'user kubeconfig changed unexpectedly\n' >&2
|
||||
status=1
|
||||
fi
|
||||
exit "$status"
|
||||
}
|
||||
|
||||
trap cleanup EXIT
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
|
||||
pg_value() {
|
||||
local database="$1"
|
||||
local query="$2"
|
||||
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
|
||||
env PGPASSWORD=synthetic-postgres psql \
|
||||
--username nuthatch \
|
||||
--dbname "$database" \
|
||||
--tuples-only \
|
||||
--no-align \
|
||||
--set ON_ERROR_STOP=1 \
|
||||
--command "$query" |
|
||||
tr -d '[:space:]'
|
||||
}
|
||||
|
||||
wait_for_value() {
|
||||
local database="$1"
|
||||
local query="$2"
|
||||
local expected="$3"
|
||||
local description="$4"
|
||||
for _ in $(seq 1 120); do
|
||||
if [[ "$(pg_value "$database" "$query" 2>/dev/null || true)" = "$expected" ]]; then
|
||||
return
|
||||
fi
|
||||
sleep 0.25
|
||||
done
|
||||
printf 'timed out waiting for %s\n' "$description" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
render_application() {
|
||||
local lane="$1"
|
||||
local profile_file="$2"
|
||||
local pg_conn
|
||||
local redis_url
|
||||
local s3_endpoint
|
||||
local s3_region
|
||||
local s3_bucket
|
||||
local s3_access_key
|
||||
local s3_secret_key
|
||||
pg_conn="$(sed -n 's/^NUTHATCH_PG_CONN=//p' "$profile_file")"
|
||||
redis_url="$(sed -n 's/^NUTHATCH_REDIS_URL=//p' "$profile_file")"
|
||||
s3_endpoint="$(sed -n 's/^NUTHATCH_S3_ENDPOINT=//p' "$profile_file")"
|
||||
s3_region="$(sed -n 's/^NUTHATCH_S3_REGION=//p' "$profile_file")"
|
||||
s3_bucket="$(sed -n 's/^NUTHATCH_S3_BUCKET=//p' "$profile_file")"
|
||||
s3_access_key="$(sed -n 's/^NUTHATCH_S3_ACCESS_KEY=//p' "$profile_file")"
|
||||
s3_secret_key="$(sed -n 's/^NUTHATCH_S3_SECRET_KEY=//p' "$profile_file")"
|
||||
sed \
|
||||
-e "s|@@LANE@@|$lane|g" \
|
||||
-e "s|@@APP_IMAGE@@|$APP_IMAGE|g" \
|
||||
-e "s|@@PG_CONN@@|$pg_conn|g" \
|
||||
-e "s|@@REDIS_URL@@|$redis_url|g" \
|
||||
-e "s|@@OIDC_ISSUER@@|https://issuer.invalid|g" \
|
||||
-e "s|@@OIDC_CLIENT_ID@@|wp09|g" \
|
||||
-e "s|@@OIDC_ADDITIONAL_AUDIENCES@@||g" \
|
||||
-e "s|@@OIDC_CLIENT_SECRET@@|synthetic-wp09-oidc-client-secret|g" \
|
||||
-e "s|@@S3_ENDPOINT@@|$s3_endpoint|g" \
|
||||
-e "s|@@S3_REGION@@|$s3_region|g" \
|
||||
-e "s|@@S3_BUCKET@@|$s3_bucket|g" \
|
||||
-e "s|@@S3_ACCESS_KEY@@|$s3_access_key|g" \
|
||||
-e "s|@@S3_SECRET_KEY@@|$s3_secret_key|g" \
|
||||
"$ROOT/deploy/kubernetes/nuthatch.yaml.in" >"$RUNTIME/wp09-app-$lane.yaml"
|
||||
sed -i \
|
||||
's/NUTHATCH_OBJECT_REQUEST_TIMEOUT_SEC: "300"/NUTHATCH_OBJECT_REQUEST_TIMEOUT_SEC: "2"/' \
|
||||
"$RUNTIME/wp09-app-$lane.yaml"
|
||||
sed -i \
|
||||
'/NUTHATCH_OBJECT_REQUEST_TIMEOUT_SEC/a\ NUTHATCH_RECOVERY_GRACE_SEC: "1"' \
|
||||
"$RUNTIME/wp09-app-$lane.yaml"
|
||||
sed -i 's/nameservers: \[10.96.0.53\]/nameservers: [10.96.0.10]/' \
|
||||
"$RUNTIME/wp09-app-$lane.yaml"
|
||||
sed -i 's/ replicas: 1/ replicas: 2/' "$RUNTIME/wp09-app-$lane.yaml"
|
||||
}
|
||||
|
||||
deploy_recovery() {
|
||||
local lane="$1"
|
||||
local profile="$2"
|
||||
render_application "$lane" "$RUNTIME/$profile.env"
|
||||
kube apply --filename "$RUNTIME/wp09-app-$lane.yaml" >/dev/null
|
||||
kube --namespace "$FIXTURE_NAMESPACE" rollout status \
|
||||
"deployment/nuthatch-$lane" --timeout=180s
|
||||
}
|
||||
|
||||
cleanup_recovery() {
|
||||
local lane="$1"
|
||||
kube --namespace "$FIXTURE_NAMESPACE" delete \
|
||||
"deployment/nuthatch-$lane" \
|
||||
"service/nuthatch-$lane" \
|
||||
"configmap/nuthatch-$lane" \
|
||||
"secret/nuthatch-$lane" \
|
||||
--ignore-not-found >/dev/null
|
||||
kube --namespace "$FIXTURE_NAMESPACE" wait \
|
||||
--for=delete pod \
|
||||
--selector="app=nuthatch-$lane" \
|
||||
--timeout=60s >/dev/null
|
||||
rm -f "$RUNTIME/wp09-app-$lane.yaml"
|
||||
}
|
||||
|
||||
start_phase() {
|
||||
local lane="$1"
|
||||
local profile="$2"
|
||||
local phase="$3"
|
||||
local job="wp09-$phase-$lane"
|
||||
render_lane_job \
|
||||
"$job" \
|
||||
"$TESTBED/wp09-phase.yaml.in" \
|
||||
"$APP_IMAGE" \
|
||||
"$profile"
|
||||
sed -i "s|@@PHASE@@|$phase|g" "$RUNTIME/$job.yaml"
|
||||
kube apply --filename "$RUNTIME/$job.yaml" >/dev/null
|
||||
}
|
||||
|
||||
kill_phase() {
|
||||
local lane="$1"
|
||||
local phase="$2"
|
||||
local job="wp09-$phase-$lane"
|
||||
kube --namespace "$FIXTURE_NAMESPACE" delete pod \
|
||||
--selector="job-name=$job" \
|
||||
--grace-period=0 \
|
||||
--force >/dev/null
|
||||
kube --namespace "$FIXTURE_NAMESPACE" delete "job/$job" \
|
||||
--ignore-not-found >/dev/null
|
||||
}
|
||||
|
||||
wait_for_phase_checkpoint() {
|
||||
local lane="$1"
|
||||
local phase="$2"
|
||||
local operation="recovery-$phase"
|
||||
local job="wp09-$phase-$lane"
|
||||
for _ in $(seq 1 120); do
|
||||
if kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" 2>/dev/null |
|
||||
grep -F "recovery checkpoint $operation" >/dev/null; then
|
||||
return
|
||||
fi
|
||||
sleep 0.25
|
||||
done
|
||||
printf 'timed out waiting for %s checkpoint\n' "$phase" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
assert_terminal_authority() {
|
||||
local database="$1"
|
||||
local operation="$2"
|
||||
local result="$3"
|
||||
wait_for_value \
|
||||
"$database" \
|
||||
"SELECT state FROM mutations WHERE operation_id = '$operation'" \
|
||||
"$result" \
|
||||
"$operation terminal result"
|
||||
test "$(
|
||||
pg_value "$database" \
|
||||
"SELECT count(*) FROM key_locks WHERE operation_id = '$operation'"
|
||||
)" = 0
|
||||
test "$(
|
||||
pg_value "$database" \
|
||||
"SELECT reserved_bytes FROM pools WHERE pool_id = 'shared'"
|
||||
)" = 0
|
||||
}
|
||||
|
||||
set_service_available() {
|
||||
local service="$1"
|
||||
local available="$2"
|
||||
local selector="$service"
|
||||
if [[ "$available" = false ]]; then
|
||||
selector=wp09-unavailable
|
||||
fi
|
||||
kube --namespace "$FIXTURE_NAMESPACE" patch service "$service" \
|
||||
--type merge \
|
||||
--patch "{\"spec\":{\"selector\":{\"app\":\"$selector\"}}}" >/dev/null
|
||||
}
|
||||
|
||||
terminate_database_connections() {
|
||||
local database="$1"
|
||||
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
|
||||
env PGPASSWORD=synthetic-postgres psql \
|
||||
--username nuthatch \
|
||||
--dbname postgres \
|
||||
--set ON_ERROR_STOP=1 \
|
||||
--command \
|
||||
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$database' AND pid <> pg_backend_pid()" \
|
||||
>/dev/null
|
||||
}
|
||||
|
||||
assert_object_absent() {
|
||||
local target="$1"
|
||||
local bucket="$2"
|
||||
local key="$3"
|
||||
local endpoint
|
||||
endpoint="$(target_loopback_endpoint "$target")"
|
||||
if kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$target" \
|
||||
--container s3-readiness -- \
|
||||
aws --endpoint-url "$endpoint" s3api head-object \
|
||||
--bucket "$bucket" \
|
||||
--key "$key" >/dev/null 2>&1; then
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
assert_object_present() {
|
||||
local target="$1"
|
||||
local bucket="$2"
|
||||
local key="$3"
|
||||
local endpoint
|
||||
endpoint="$(target_loopback_endpoint "$target")"
|
||||
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$target" \
|
||||
--container s3-readiness -- \
|
||||
aws --endpoint-url "$endpoint" s3api head-object \
|
||||
--bucket "$bucket" \
|
||||
--key "$key" >/dev/null
|
||||
}
|
||||
|
||||
assert_cache_invalidated() {
|
||||
local lane="$1"
|
||||
local value
|
||||
value="$(
|
||||
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/redis -- \
|
||||
redis-cli --raw -n "$((10#$lane))" \
|
||||
GET nuthatch:list:recovery-user:shared:: |
|
||||
tr -d '[:space:]'
|
||||
)"
|
||||
test -z "$value"
|
||||
}
|
||||
|
||||
run_phase_recovery() {
|
||||
local lane="$1"
|
||||
local profile="$2"
|
||||
local database="$3"
|
||||
local target="$4"
|
||||
local phase="$5"
|
||||
local durable_state="$6"
|
||||
local terminal_state="$7"
|
||||
local operation="recovery-$phase"
|
||||
start_phase "$lane" "$profile" "$phase"
|
||||
wait_for_value \
|
||||
"$database" \
|
||||
"SELECT state || ':' || external_state FROM mutations WHERE operation_id = '$operation'" \
|
||||
"$durable_state" \
|
||||
"$phase durable checkpoint"
|
||||
if [[ "$phase" = during-put ]]; then
|
||||
sleep 1
|
||||
fi
|
||||
kill_phase "$lane" "$phase"
|
||||
deploy_recovery "$lane" "$profile"
|
||||
assert_terminal_authority "$database" "$operation" "$terminal_state"
|
||||
cleanup_recovery "$lane"
|
||||
if [[ "$terminal_state" = succeeded ]]; then
|
||||
assert_cache_invalidated "$lane"
|
||||
fi
|
||||
}
|
||||
|
||||
acquire_fixture_lock
|
||||
OWNS_FIXTURE=true
|
||||
setup_fixture durable coordination target
|
||||
source "$RUNTIME/fixture.env"
|
||||
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" build \
|
||||
--file "$ROOT/Dockerfile" \
|
||||
--target test-runtime \
|
||||
--tag "$APP_IMAGE" \
|
||||
"$ROOT"
|
||||
load_local_image "$APP_IMAGE" "$NERDCTL_NAMESPACE"
|
||||
|
||||
while IFS=$'\t' read -r lane coordination oidc target; do
|
||||
if [[ -n "${NUTHATCH_WP09_TARGET:-}" ]] &&
|
||||
[[ "$target" != "$NUTHATCH_WP09_TARGET" ]]; then
|
||||
continue
|
||||
fi
|
||||
profile="$(write_lane_profile "$lane" redis - "$target")"
|
||||
database="nuthatch_lane_$lane"
|
||||
bucket="nuthatch-lane-$lane"
|
||||
printf 'Running WP09 lane %s: target=%s\n' "$lane" "$target"
|
||||
|
||||
run_phase_recovery \
|
||||
"$lane" "$profile" "$database" "$target" \
|
||||
before-dispatch prepared:idle failed
|
||||
run_phase_recovery \
|
||||
"$lane" "$profile" "$database" "$target" \
|
||||
during-put applying:pending failed
|
||||
sleep 5
|
||||
assert_object_absent "$target" "$bucket" \
|
||||
shared/recovery-during-put.bin
|
||||
|
||||
start_phase "$lane" "$profile" live-timeout
|
||||
wait_for_phase_checkpoint "$lane" live-timeout
|
||||
test "$(
|
||||
pg_value "$database" \
|
||||
"SELECT count(*) FROM key_locks WHERE operation_id = 'recovery-live-timeout'"
|
||||
)" = 1
|
||||
test "$(
|
||||
pg_value "$database" \
|
||||
"SELECT reserved_bytes FROM pools WHERE pool_id = 'shared'"
|
||||
)" = "$((20 * 1024 * 1024))"
|
||||
kill_phase "$lane" live-timeout
|
||||
deploy_recovery "$lane" "$profile"
|
||||
assert_terminal_authority "$database" recovery-live-timeout failed
|
||||
cleanup_recovery "$lane"
|
||||
sleep 5
|
||||
assert_object_absent "$target" "$bucket" \
|
||||
shared/recovery-live-timeout.bin
|
||||
|
||||
start_phase "$lane" "$profile" after-dispatch
|
||||
wait_for_value \
|
||||
"$database" \
|
||||
"SELECT state || ':' || external_state FROM mutations WHERE operation_id = 'recovery-after-dispatch'" \
|
||||
applying:pending \
|
||||
"after-dispatch checkpoint"
|
||||
kill_phase "$lane" after-dispatch
|
||||
pg_value \
|
||||
"$database" \
|
||||
"UPDATE mutations SET request_deadline_at = now() + interval '60 seconds', worker_lease_until = now() + interval '60 seconds' WHERE operation_id = 'recovery-after-dispatch'; SELECT 1" \
|
||||
>/dev/null
|
||||
deploy_recovery "$lane" "$profile"
|
||||
set_service_available postgres false
|
||||
terminate_database_connections "$database"
|
||||
sleep 6
|
||||
test "$(
|
||||
kube --namespace "$FIXTURE_NAMESPACE" get pods \
|
||||
--selector="app=nuthatch-$lane" \
|
||||
--output json |
|
||||
jq '[.items[].status.containerStatuses[].restartCount] | add'
|
||||
)" -gt 0
|
||||
set_service_available postgres true
|
||||
wait_for_value "$database" "SELECT 1" 1 "PostgreSQL recovery"
|
||||
pg_value \
|
||||
"$database" \
|
||||
"UPDATE mutations SET request_deadline_at = now() - interval '1 second', worker_lease_until = now() - interval '1 second' WHERE operation_id = 'recovery-after-dispatch'; SELECT 1" \
|
||||
>/dev/null
|
||||
assert_terminal_authority "$database" recovery-after-dispatch succeeded
|
||||
cleanup_recovery "$lane"
|
||||
assert_cache_invalidated "$lane"
|
||||
assert_object_present "$target" "$bucket" \
|
||||
shared/recovery-after-dispatch.bin
|
||||
|
||||
start_phase "$lane" "$profile" after-settlement
|
||||
wait_for_value \
|
||||
"$database" \
|
||||
"SELECT state || ':' || external_state FROM mutations WHERE operation_id = 'recovery-after-settlement'" \
|
||||
applying:settled \
|
||||
"after-settlement checkpoint"
|
||||
kill_phase "$lane" after-settlement
|
||||
set_service_available "$target" false
|
||||
deploy_recovery "$lane" "$profile"
|
||||
sleep 3
|
||||
kube --namespace "$FIXTURE_NAMESPACE" logs \
|
||||
--selector="app=nuthatch-$lane" \
|
||||
--prefix=true |
|
||||
grep -F '"result":"exhausted"'
|
||||
set_service_available "$target" true
|
||||
assert_terminal_authority "$database" recovery-after-settlement succeeded
|
||||
cleanup_recovery "$lane"
|
||||
assert_cache_invalidated "$lane"
|
||||
|
||||
run_phase_recovery \
|
||||
"$lane" "$profile" "$database" "$target" \
|
||||
delete-settled applying:settled succeeded
|
||||
assert_object_absent "$target" "$bucket" \
|
||||
shared/recovery-delete-settled.bin
|
||||
|
||||
cleanup_lane_state "$lane" redis - "$target"
|
||||
for phase in \
|
||||
before-dispatch \
|
||||
during-put \
|
||||
live-timeout \
|
||||
after-dispatch \
|
||||
after-settlement \
|
||||
delete-settled; do
|
||||
cleanup_lane_resources "wp09-$phase-$lane" "$profile"
|
||||
done
|
||||
done < <(expand_fixture_matrix target)
|
||||
|
||||
SUCCEEDED=true
|
||||
printf 'Nuthatch WP-09 recovery gate passed\n'
|
||||
Loading…
Reference in a new issue