Split kind tests into component-owned probes
This commit is contained in:
parent
b3c0f6d58a
commit
691ad92e30
1 changed files with 4269 additions and 3108 deletions
|
|
@ -45,21 +45,6 @@ pg_value() {
|
|||
tr -d '[:space:]'
|
||||
}
|
||||
|
||||
wait_for_value() {
|
||||
local database="$1"
|
||||
local query="$2"
|
||||
local expected="$3"
|
||||
local description="$4"
|
||||
for _ in $(seq 1 120); do
|
||||
if [[ "$(pg_value "$database" "$query" 2>/dev/null || true)" = "$expected" ]]; then
|
||||
return
|
||||
fi
|
||||
sleep 0.25
|
||||
done
|
||||
printf 'timed out waiting for %s\n' "$description" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
render_application() {
|
||||
local lane="$1"
|
||||
local profile_file="$2"
|
||||
|
|
@ -137,10 +122,70 @@ start_phase() {
|
|||
"$TESTBED/recovery-phase-job.yaml.in" \
|
||||
"$APP_IMAGE" \
|
||||
"$profile"
|
||||
sed -i "s|@@PHASE@@|$phase|g" "$RUNTIME/$job.yaml"
|
||||
sed -i \
|
||||
-e "s|@@PHASE@@|$phase|g" \
|
||||
-e 's|@@PROBE_ARGS@@||g' \
|
||||
"$RUNTIME/$job.yaml"
|
||||
kube apply --filename "$RUNTIME/$job.yaml" >/dev/null
|
||||
}
|
||||
|
||||
run_recovery_probe() {
|
||||
local lane="$1"
|
||||
local profile="$2"
|
||||
local suffix="$3"
|
||||
shift 3
|
||||
local probe_args
|
||||
probe_args="$(
|
||||
printf '"%s",' "$@"
|
||||
)"
|
||||
probe_args="${probe_args%,}"
|
||||
local job="recovery-check-$suffix-$lane"
|
||||
render_lane_job \
|
||||
"$job" \
|
||||
"$TESTBED/recovery-phase-job.yaml.in" \
|
||||
"$APP_IMAGE" \
|
||||
"$profile"
|
||||
sed -i \
|
||||
-e 's|@@PHASE@@|observer|g' \
|
||||
-e "s|@@PROBE_ARGS@@|$probe_args|g" \
|
||||
"$RUNTIME/$job.yaml"
|
||||
run_job_manifest "$job" "$RUNTIME/$job.yaml"
|
||||
kube --namespace "$FIXTURE_NAMESPACE" delete \
|
||||
"job/$job" \
|
||||
--ignore-not-found >/dev/null
|
||||
rm -f "$RUNTIME/$job.yaml"
|
||||
}
|
||||
|
||||
run_recovery_log_probe() {
|
||||
local lane="$1"
|
||||
local profile="$2"
|
||||
local log_file="$RUNTIME/recovery-app-$lane.log"
|
||||
local log_config_map="recovery-log-$lane"
|
||||
local job="recovery-check-log-$lane"
|
||||
kube --namespace "$FIXTURE_NAMESPACE" logs \
|
||||
--selector="app=nuthatch-$lane" \
|
||||
--prefix=true >"$log_file"
|
||||
kube --namespace "$FIXTURE_NAMESPACE" create configmap "$log_config_map" \
|
||||
--from-file="app.log=$log_file" \
|
||||
--dry-run=client \
|
||||
--output yaml |
|
||||
kube apply --filename - >/dev/null
|
||||
render_lane_job \
|
||||
"$job" \
|
||||
"$TESTBED/recovery-log-job.yaml.in" \
|
||||
"$APP_IMAGE" \
|
||||
"$profile"
|
||||
sed -i \
|
||||
-e "s|@@LOG_CONFIG_MAP@@|$log_config_map|g" \
|
||||
"$RUNTIME/$job.yaml"
|
||||
run_job_manifest "$job" "$RUNTIME/$job.yaml"
|
||||
kube --namespace "$FIXTURE_NAMESPACE" delete \
|
||||
"job/$job" \
|
||||
"configmap/$log_config_map" \
|
||||
--ignore-not-found >/dev/null
|
||||
rm -f "$RUNTIME/$job.yaml" "$log_file"
|
||||
}
|
||||
|
||||
kill_phase() {
|
||||
local lane="$1"
|
||||
local phase="$2"
|
||||
|
|
@ -169,25 +214,6 @@ wait_for_phase_checkpoint() {
|
|||
return 1
|
||||
}
|
||||
|
||||
assert_terminal_authority() {
|
||||
local database="$1"
|
||||
local operation="$2"
|
||||
local result="$3"
|
||||
wait_for_value \
|
||||
"$database" \
|
||||
"SELECT state FROM mutations WHERE operation_id = '$operation'" \
|
||||
"$result" \
|
||||
"$operation terminal result"
|
||||
test "$(
|
||||
pg_value "$database" \
|
||||
"SELECT count(*) FROM key_locks WHERE operation_id = '$operation'"
|
||||
)" = 0
|
||||
test "$(
|
||||
pg_value "$database" \
|
||||
"SELECT reserved_bytes FROM pools WHERE pool_id = 'shared'"
|
||||
)" = 0
|
||||
}
|
||||
|
||||
set_service_available() {
|
||||
local service="$1"
|
||||
local available="$2"
|
||||
|
|
@ -212,70 +238,30 @@ terminate_database_connections() {
|
|||
>/dev/null
|
||||
}
|
||||
|
||||
assert_object_absent() {
|
||||
local target="$1"
|
||||
local bucket="$2"
|
||||
local key="$3"
|
||||
local endpoint
|
||||
endpoint="$(target_loopback_endpoint "$target")"
|
||||
if kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$target" \
|
||||
--container s3-readiness -- \
|
||||
aws --endpoint-url "$endpoint" s3api head-object \
|
||||
--bucket "$bucket" \
|
||||
--key "$key" >/dev/null 2>&1; then
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
assert_object_present() {
|
||||
local target="$1"
|
||||
local bucket="$2"
|
||||
local key="$3"
|
||||
local endpoint
|
||||
endpoint="$(target_loopback_endpoint "$target")"
|
||||
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$target" \
|
||||
--container s3-readiness -- \
|
||||
aws --endpoint-url "$endpoint" s3api head-object \
|
||||
--bucket "$bucket" \
|
||||
--key "$key" >/dev/null
|
||||
}
|
||||
|
||||
assert_cache_invalidated() {
|
||||
local lane="$1"
|
||||
local value
|
||||
value="$(
|
||||
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/redis -- \
|
||||
redis-cli --raw -n "$((10#$lane))" \
|
||||
GET nuthatch:list:recovery-user:shared:: |
|
||||
tr -d '[:space:]'
|
||||
)"
|
||||
test -z "$value"
|
||||
}
|
||||
|
||||
run_phase_recovery() {
|
||||
local lane="$1"
|
||||
local profile="$2"
|
||||
local database="$3"
|
||||
local target="$4"
|
||||
local phase="$5"
|
||||
local durable_state="$6"
|
||||
local terminal_state="$7"
|
||||
local phase="$3"
|
||||
local durable_state="$4"
|
||||
local terminal_state="$5"
|
||||
local operation="recovery-$phase"
|
||||
start_phase "$lane" "$profile" "$phase"
|
||||
wait_for_value \
|
||||
"$database" \
|
||||
"SELECT state || ':' || external_state FROM mutations WHERE operation_id = '$operation'" \
|
||||
"$durable_state" \
|
||||
"$phase durable checkpoint"
|
||||
run_recovery_probe \
|
||||
"$lane" "$profile" "phase-$phase" \
|
||||
wait-phase "$operation" "$durable_state"
|
||||
if [[ "$phase" = during-put ]]; then
|
||||
sleep 1
|
||||
fi
|
||||
kill_phase "$lane" "$phase"
|
||||
deploy_recovery "$lane" "$profile"
|
||||
assert_terminal_authority "$database" "$operation" "$terminal_state"
|
||||
run_recovery_probe \
|
||||
"$lane" "$profile" "terminal-$phase" \
|
||||
terminal "$operation" "$terminal_state"
|
||||
cleanup_recovery "$lane"
|
||||
if [[ "$terminal_state" = succeeded ]]; then
|
||||
assert_cache_invalidated "$lane"
|
||||
run_recovery_probe \
|
||||
"$lane" "$profile" "cache-$phase" \
|
||||
cache-invalidated
|
||||
fi
|
||||
}
|
||||
|
||||
|
|
@ -297,43 +283,39 @@ while IFS=$'\t' read -r lane coordination oidc target; do
|
|||
fi
|
||||
profile="$(write_lane_profile "$lane" redis - "$target")"
|
||||
database="nuthatch_lane_$lane"
|
||||
bucket="nuthatch-lane-$lane"
|
||||
printf 'Running recovery lane %s: target=%s\n' "$lane" "$target"
|
||||
|
||||
run_phase_recovery \
|
||||
"$lane" "$profile" "$database" "$target" \
|
||||
"$lane" "$profile" \
|
||||
before-dispatch prepared:idle failed
|
||||
run_phase_recovery \
|
||||
"$lane" "$profile" "$database" "$target" \
|
||||
"$lane" "$profile" \
|
||||
during-put applying:pending failed
|
||||
sleep 5
|
||||
assert_object_absent "$target" "$bucket" \
|
||||
shared/recovery-during-put.bin
|
||||
run_recovery_probe \
|
||||
"$lane" "$profile" during-put-object \
|
||||
object absent shared/recovery-during-put.bin
|
||||
|
||||
start_phase "$lane" "$profile" live-timeout
|
||||
wait_for_phase_checkpoint "$lane" live-timeout
|
||||
test "$(
|
||||
pg_value "$database" \
|
||||
"SELECT count(*) FROM key_locks WHERE operation_id = 'recovery-live-timeout'"
|
||||
)" = 1
|
||||
test "$(
|
||||
pg_value "$database" \
|
||||
"SELECT reserved_bytes FROM pools WHERE pool_id = 'shared'"
|
||||
)" = "$((20 * 1024 * 1024))"
|
||||
run_recovery_probe \
|
||||
"$lane" "$profile" live-timeout-active \
|
||||
active recovery-live-timeout "$((20 * 1024 * 1024))"
|
||||
kill_phase "$lane" live-timeout
|
||||
deploy_recovery "$lane" "$profile"
|
||||
assert_terminal_authority "$database" recovery-live-timeout failed
|
||||
run_recovery_probe \
|
||||
"$lane" "$profile" live-timeout-terminal \
|
||||
terminal recovery-live-timeout failed
|
||||
cleanup_recovery "$lane"
|
||||
sleep 5
|
||||
assert_object_absent "$target" "$bucket" \
|
||||
shared/recovery-live-timeout.bin
|
||||
run_recovery_probe \
|
||||
"$lane" "$profile" live-timeout-object \
|
||||
object absent shared/recovery-live-timeout.bin
|
||||
|
||||
start_phase "$lane" "$profile" after-dispatch
|
||||
wait_for_value \
|
||||
"$database" \
|
||||
"SELECT state || ':' || external_state FROM mutations WHERE operation_id = 'recovery-after-dispatch'" \
|
||||
applying:pending \
|
||||
"after-dispatch checkpoint"
|
||||
run_recovery_probe \
|
||||
"$lane" "$profile" after-dispatch-phase \
|
||||
wait-phase recovery-after-dispatch applying:pending
|
||||
kill_phase "$lane" after-dispatch
|
||||
pg_value \
|
||||
"$database" \
|
||||
|
|
@ -356,41 +338,48 @@ while IFS=$'\t' read -r lane coordination oidc target; do
|
|||
jq '[.items[].status.containerStatuses[].restartCount] | add'
|
||||
)" = "$before_pg_restarts"
|
||||
set_service_available postgres true
|
||||
wait_for_value "$database" "SELECT 1" 1 "PostgreSQL recovery"
|
||||
run_recovery_probe \
|
||||
"$lane" "$profile" database-available \
|
||||
database-available
|
||||
pg_value \
|
||||
"$database" \
|
||||
"UPDATE mutations SET request_deadline_at = now() - interval '1 second', worker_lease_until = now() - interval '1 second' WHERE operation_id = 'recovery-after-dispatch'; SELECT 1" \
|
||||
>/dev/null
|
||||
assert_terminal_authority "$database" recovery-after-dispatch succeeded
|
||||
run_recovery_probe \
|
||||
"$lane" "$profile" after-dispatch-terminal \
|
||||
terminal recovery-after-dispatch succeeded
|
||||
cleanup_recovery "$lane"
|
||||
assert_cache_invalidated "$lane"
|
||||
assert_object_present "$target" "$bucket" \
|
||||
shared/recovery-after-dispatch.bin
|
||||
run_recovery_probe \
|
||||
"$lane" "$profile" after-dispatch-cache \
|
||||
cache-invalidated
|
||||
run_recovery_probe \
|
||||
"$lane" "$profile" after-dispatch-object \
|
||||
object present shared/recovery-after-dispatch.bin
|
||||
|
||||
start_phase "$lane" "$profile" after-settlement
|
||||
wait_for_value \
|
||||
"$database" \
|
||||
"SELECT state || ':' || external_state FROM mutations WHERE operation_id = 'recovery-after-settlement'" \
|
||||
applying:settled \
|
||||
"after-settlement checkpoint"
|
||||
run_recovery_probe \
|
||||
"$lane" "$profile" after-settlement-phase \
|
||||
wait-phase recovery-after-settlement applying:settled
|
||||
kill_phase "$lane" after-settlement
|
||||
set_service_available "$target" false
|
||||
deploy_recovery "$lane" "$profile"
|
||||
sleep 3
|
||||
kube --namespace "$FIXTURE_NAMESPACE" logs \
|
||||
--selector="app=nuthatch-$lane" \
|
||||
--prefix=true |
|
||||
grep -F '"result":"exhausted"'
|
||||
run_recovery_log_probe "$lane" "$profile"
|
||||
set_service_available "$target" true
|
||||
assert_terminal_authority "$database" recovery-after-settlement succeeded
|
||||
run_recovery_probe \
|
||||
"$lane" "$profile" after-settlement-terminal \
|
||||
terminal recovery-after-settlement succeeded
|
||||
cleanup_recovery "$lane"
|
||||
assert_cache_invalidated "$lane"
|
||||
run_recovery_probe \
|
||||
"$lane" "$profile" after-settlement-cache \
|
||||
cache-invalidated
|
||||
|
||||
run_phase_recovery \
|
||||
"$lane" "$profile" "$database" "$target" \
|
||||
"$lane" "$profile" \
|
||||
delete-settled applying:settled succeeded
|
||||
assert_object_absent "$target" "$bucket" \
|
||||
shared/recovery-delete-settled.bin
|
||||
run_recovery_probe \
|
||||
"$lane" "$profile" delete-settled-object \
|
||||
object absent shared/recovery-delete-settled.bin
|
||||
|
||||
cleanup_lane_state "$lane" redis - "$target"
|
||||
for phase in \
|
||||
|
|
|
|||
Loading…
Reference in a new issue