ms2k
15eb0038d2
Use descriptive test and executable names so implementation surfaces do not depend on journal organization. Consolidate the unreleased migration history into the authoritative initial schema because no deployed database requires an upgrade path.
238 lines
8.2 KiB
Shell
Executable file
238 lines
8.2 KiB
Shell
Executable file
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
|
source "$ROOT/tests/kind/lib.sh"
|
|
|
|
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
|
|
SUCCEEDED=false
|
|
OWNS_FIXTURE=false
|
|
APP_IMAGE=nuthatch-protocol-conformance:local
|
|
|
|
cleanup() {
|
|
local status=$?
|
|
trap - EXIT INT TERM
|
|
if [[ "${NUTHATCH_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
|
|
printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
|
|
printf 'retained context: %s\n' "$CONTEXT"
|
|
exit "$status"
|
|
fi
|
|
if [[ "$OWNS_FIXTURE" = true ]]; then
|
|
destroy_fixture || status=1
|
|
fi
|
|
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
|
|
printf 'user kubeconfig changed unexpectedly\n' >&2
|
|
status=1
|
|
fi
|
|
exit "$status"
|
|
}
|
|
|
|
trap cleanup EXIT
|
|
trap 'exit 130' INT
|
|
trap 'exit 143' TERM
|
|
|
|
set_target_available() {
|
|
local target="$1"
|
|
local selector="$target"
|
|
if [[ "$2" = false ]]; then
|
|
selector=protocol-unavailable
|
|
fi
|
|
kube --namespace "$FIXTURE_NAMESPACE" patch service "$target" \
|
|
--type merge \
|
|
--patch "{\"spec\":{\"selector\":{\"app\":\"$selector\"}}}" >/dev/null
|
|
}
|
|
|
|
set_postgres_available() {
|
|
local selector=postgres
|
|
if [[ "$1" = false ]]; then
|
|
selector=protocol-unavailable
|
|
fi
|
|
kube --namespace "$FIXTURE_NAMESPACE" patch service postgres \
|
|
--type merge \
|
|
--patch "{\"spec\":{\"selector\":{\"app\":\"$selector\"}}}" >/dev/null
|
|
}
|
|
|
|
database_connection_count() {
|
|
local database="$1"
|
|
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
|
|
env PGPASSWORD=synthetic-postgres psql \
|
|
--username nuthatch \
|
|
--dbname postgres \
|
|
--tuples-only \
|
|
--no-align \
|
|
--set ON_ERROR_STOP=1 \
|
|
--command \
|
|
"SELECT count(*) FROM pg_stat_activity WHERE datname = '$database' AND pid <> pg_backend_pid()" |
|
|
tr -d '[:space:]'
|
|
}
|
|
|
|
terminate_database_connections() {
|
|
local database="$1"
|
|
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
|
|
env PGPASSWORD=synthetic-postgres psql \
|
|
--username nuthatch \
|
|
--dbname postgres \
|
|
--set ON_ERROR_STOP=1 \
|
|
--command \
|
|
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$database' AND pid <> pg_backend_pid()" \
|
|
>/dev/null
|
|
}
|
|
|
|
render_mode() {
|
|
local lane="$1"
|
|
local profile="$2"
|
|
local mode="$3"
|
|
local job="protocol-$mode-$lane"
|
|
render_lane_job \
|
|
"$job" \
|
|
"$TESTBED/protocol-conformance-job.yaml.in" \
|
|
"$APP_IMAGE" \
|
|
"$profile"
|
|
sed -i "s|@@MODE@@|$mode|g" "$RUNTIME/$job.yaml"
|
|
}
|
|
|
|
run_mode() {
|
|
local lane="$1"
|
|
local profile="$2"
|
|
local mode="$3"
|
|
local job="protocol-$mode-$lane"
|
|
render_mode "$lane" "$profile" "$mode"
|
|
run_job_manifest "$job" "$RUNTIME/$job.yaml"
|
|
}
|
|
|
|
start_mode() {
|
|
local lane="$1"
|
|
local profile="$2"
|
|
local mode="$3"
|
|
local job="protocol-$mode-$lane"
|
|
render_mode "$lane" "$profile" "$mode"
|
|
kube apply --filename "$RUNTIME/$job.yaml" >/dev/null
|
|
}
|
|
|
|
wait_for_mode_log() {
|
|
local lane="$1"
|
|
local mode="$2"
|
|
local message="$3"
|
|
local job="protocol-$mode-$lane"
|
|
for _ in $(seq 1 120); do
|
|
if kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" 2>/dev/null |
|
|
grep -F "$message" >/dev/null; then
|
|
return
|
|
fi
|
|
sleep 0.25
|
|
done
|
|
printf 'timed out waiting for %s\n' "$message" >&2
|
|
return 1
|
|
}
|
|
|
|
stop_mode() {
|
|
local lane="$1"
|
|
local mode="$2"
|
|
kube --namespace "$FIXTURE_NAMESPACE" delete "job/protocol-$mode-$lane" \
|
|
--ignore-not-found >/dev/null
|
|
}
|
|
|
|
verify_mapping() {
|
|
local lane="$1"
|
|
local log="$RUNTIME/protocol-full-$lane.log"
|
|
kube --namespace "$FIXTURE_NAMESPACE" logs "job/protocol-full-$lane" >"$log"
|
|
local actions=(
|
|
FirstRequest ReplayRequest ConflictRequest PrepareEffect DispatchEffect
|
|
LoseUploadBody SettleLostUpload ExternalEffectApplied
|
|
ExternalEffectNotApplied FinalizeSuccess FinalizeFailure ControllerAdvance
|
|
StoreSettlement
|
|
)
|
|
local invariants=(
|
|
TypeOK RecordIdentityConsistent PersonalRequestsUseOwnPool
|
|
PersonalPoolsDisjoint FingerprintBindsRequest ReservationsConsistent
|
|
LocksConsistent ActiveObjectConsistent ExternalStateConsistent
|
|
UploadBodyConsistent QuotaReservationBound ObjectStateCovered
|
|
ObjectQuotaBound AccountingAtQuiescence ObservationConsistent
|
|
RecordsNeverForgotten RequestBindingsStable TerminalRecordsStable
|
|
SettledEffectsDoNotMutate UnfinishedEventuallyTerminal
|
|
LostUploadEventuallyTerminal
|
|
)
|
|
local name
|
|
for name in "${actions[@]}"; do
|
|
grep -F "protocol action covered: $name" "$log" >/dev/null
|
|
rg -q "^$name(\\(| ==)" "$ROOT/docs/protocol/Nuthatch.tla"
|
|
done
|
|
for name in "${invariants[@]}"; do
|
|
grep -F "protocol invariant checked: $name" "$log" >/dev/null
|
|
rg -q "^$name(\\(| ==)" "$ROOT/docs/protocol/Nuthatch.tla"
|
|
done
|
|
}
|
|
|
|
acquire_fixture_lock
|
|
OWNS_FIXTURE=true
|
|
setup_fixture durable coordination target
|
|
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" build \
|
|
--file "$ROOT/Dockerfile" \
|
|
--target test-runtime \
|
|
--tag "$APP_IMAGE" \
|
|
"$ROOT"
|
|
load_local_image "$APP_IMAGE" "$NERDCTL_NAMESPACE"
|
|
|
|
while IFS=$'\t' read -r lane coordination oidc target; do
|
|
if [[ -n "${NUTHATCH_PROTOCOL_TARGET:-}" ]] &&
|
|
[[ "$target" != "$NUTHATCH_PROTOCOL_TARGET" ]]; then
|
|
continue
|
|
fi
|
|
profile="$(write_lane_profile "$lane" redis - "$target")"
|
|
printf 'Running protocol lane %s: target=%s\n' "$lane" "$target"
|
|
run_mode "$lane" "$profile" full
|
|
verify_mapping "$lane"
|
|
start_mode "$lane" "$profile" pg-prepare
|
|
wait_for_mode_log "$lane" pg-prepare "PostgreSQL transition fixture ready"
|
|
printf 'Checking PostgreSQL loss and restoration for %s\n' "$target"
|
|
set_postgres_available false
|
|
active_connections="$(database_connection_count "nuthatch_lane_$lane")"
|
|
test "$active_connections" -gt 0
|
|
terminate_database_connections "nuthatch_lane_$lane" || true
|
|
remaining_connections="$(database_connection_count "nuthatch_lane_$lane")"
|
|
test "$remaining_connections" = 0
|
|
stop_mode "$lane" pg-prepare
|
|
run_mode "$lane" "$profile" pg-fence
|
|
set_postgres_available true
|
|
run_mode "$lane" "$profile" pg-restored
|
|
kube --namespace "$FIXTURE_NAMESPACE" logs "job/protocol-pg-fence-$lane" |
|
|
grep -F "protocol action covered: LosePostgres" >/dev/null
|
|
kube --namespace "$FIXTURE_NAMESPACE" logs "job/protocol-pg-restored-$lane" |
|
|
grep -F "protocol action covered: RestorePostgres" >/dev/null
|
|
kube --namespace "$FIXTURE_NAMESPACE" logs "job/protocol-pg-restored-$lane" |
|
|
grep -F "protocol invariant checked: PostgresFailureFencesDurableState" \
|
|
>/dev/null
|
|
rg -q '^LosePostgres(\(| ==)' "$ROOT/docs/protocol/Nuthatch.tla"
|
|
rg -q '^RestorePostgres(\(| ==)' "$ROOT/docs/protocol/Nuthatch.tla"
|
|
rg -q '^PostgresFailureFencesDurableState(\(| ==)' \
|
|
"$ROOT/docs/protocol/Nuthatch.tla"
|
|
run_mode "$lane" "$profile" store-prepare
|
|
printf 'Checking store-down admission fence for %s\n' "$target"
|
|
set_target_available "$target" false
|
|
run_mode "$lane" "$profile" fence
|
|
printf 'Checking restored store observation for %s\n' "$target"
|
|
set_target_available "$target" true
|
|
run_mode "$lane" "$profile" restored
|
|
kube --namespace "$FIXTURE_NAMESPACE" logs "job/protocol-fence-$lane" |
|
|
grep -F "store-down rejection creates no protocol record" >/dev/null
|
|
kube --namespace "$FIXTURE_NAMESPACE" logs "job/protocol-restored-$lane" |
|
|
grep -F "protocol action covered: RestoreObjectStore" >/dev/null
|
|
kube --namespace "$FIXTURE_NAMESPACE" logs "job/protocol-restored-$lane" |
|
|
grep -F "protocol invariant checked: ObjectStoreFailureFencesObjects" >/dev/null
|
|
rg -q '^LoseObjectStore(\(| ==)' "$ROOT/docs/protocol/Nuthatch.tla"
|
|
rg -q '^RestoreObjectStore(\(| ==)' "$ROOT/docs/protocol/Nuthatch.tla"
|
|
rg -q '^ObjectStoreFailureFencesObjects(\(| ==)' \
|
|
"$ROOT/docs/protocol/Nuthatch.tla"
|
|
cleanup_lane_state "$lane" redis - "$target"
|
|
cleanup_lane_resources "protocol-full-$lane" "$profile"
|
|
cleanup_lane_resources "protocol-pg-prepare-$lane" "$profile"
|
|
cleanup_lane_resources "protocol-pg-fence-$lane" "$profile"
|
|
cleanup_lane_resources "protocol-pg-restored-$lane" "$profile"
|
|
cleanup_lane_resources "protocol-store-prepare-$lane" "$profile"
|
|
cleanup_lane_resources "protocol-fence-$lane" "$profile"
|
|
cleanup_lane_resources "protocol-restored-$lane" "$profile"
|
|
rm -f "$RUNTIME/protocol-full-$lane.log"
|
|
done < <(expand_fixture_matrix target)
|
|
|
|
SUCCEEDED=true
|
|
printf 'Nuthatch protocol conformance gate passed\n'
|