ms2k/nuthatch
SHA256
Watch
0
Code Releases Activity

Replace planning-era names and squash initial schema

Browse source 
Use descriptive test and executable names so implementation surfaces do not depend on journal organization. Consolidate the unreleased migration history into the authoritative initial schema because no deployed database requires an upgrade path.
This commit is contained in:
ms2k 2026-07-19 15:21:24 +09:00
commit 15eb0038d2
29 changed files with 299 additions and 436 deletions
Download patch file Download diff file Expand all files Collapse all files

18
Dockerfile
View file

@ -27,13 +27,13 @@ COPY app ./app
COPY src ./src COPY src ./src
COPY test ./test COPY test ./test
RUN cabal build exe:nuthatch exe:nuthatch-wp04-integration exe:nuthatch-wp09-phase exe:nuthatch-wp10-conformance exe:nuthatch-wp12-runtime \ RUN cabal build exe:nuthatch exe:nuthatch-storage-integration exe:nuthatch-recovery-probe exe:nuthatch-protocol-conformance exe:nuthatch-runtime-probe \
&& mkdir -p /out \ && mkdir -p /out \
&& cp "$(cabal list-bin exe:nuthatch)" /out/nuthatch \ && cp "$(cabal list-bin exe:nuthatch)" /out/nuthatch \
&& cp "$(cabal list-bin exe:nuthatch-wp04-integration)" /out/nuthatch-wp04-integration \ && cp "$(cabal list-bin exe:nuthatch-storage-integration)" /out/nuthatch-storage-integration \
&& cp "$(cabal list-bin exe:nuthatch-wp09-phase)" /out/nuthatch-wp09-phase \ && cp "$(cabal list-bin exe:nuthatch-recovery-probe)" /out/nuthatch-recovery-probe \
&& cp "$(cabal list-bin exe:nuthatch-wp10-conformance)" /out/nuthatch-wp10-conformance \ && cp "$(cabal list-bin exe:nuthatch-protocol-conformance)" /out/nuthatch-protocol-conformance \
&& cp "$(cabal list-bin exe:nuthatch-wp12-runtime)" /out/nuthatch-wp12-runtime && cp "$(cabal list-bin exe:nuthatch-runtime-probe)" /out/nuthatch-runtime-probe
FROM debian:bookworm-slim AS runtime-base FROM debian:bookworm-slim AS runtime-base
@ -55,10 +55,10 @@ USER 10001:10001
FROM runtime-base AS test-runtime FROM runtime-base AS test-runtime
COPY --from=build /out/nuthatch /usr/local/bin/nuthatch COPY --from=build /out/nuthatch /usr/local/bin/nuthatch
COPY --from=build /out/nuthatch-wp04-integration /usr/local/bin/nuthatch-wp04-integration COPY --from=build /out/nuthatch-storage-integration /usr/local/bin/nuthatch-storage-integration
COPY --from=build /out/nuthatch-wp09-phase /usr/local/bin/nuthatch-wp09-phase COPY --from=build /out/nuthatch-recovery-probe /usr/local/bin/nuthatch-recovery-probe
COPY --from=build /out/nuthatch-wp10-conformance /usr/local/bin/nuthatch-wp10-conformance COPY --from=build /out/nuthatch-protocol-conformance /usr/local/bin/nuthatch-protocol-conformance
COPY --from=build /out/nuthatch-wp12-runtime /usr/local/bin/nuthatch-wp12-runtime COPY --from=build /out/nuthatch-runtime-probe /usr/local/bin/nuthatch-runtime-probe
CMD ["nuthatch"] CMD ["nuthatch"]
FROM runtime-base AS runtime FROM runtime-base AS runtime

20
Makefile
View file

@ -13,32 +13,32 @@ check-kind-matrix:
./tests/kind/matrix-run.sh ./tests/kind/matrix-run.sh
check-storage-foundation: check-storage-foundation:
./tests/kind/wp04-run.sh ./tests/kind/storage-foundation-run.sh
check-auth: check-auth:
./tests/kind/wp05-run.sh ./tests/kind/product-matrix-run.sh
check-app: check-app:
./tests/kind/wp05-run.sh ./tests/kind/product-matrix-run.sh
check-streaming: check-streaming:
./tests/kind/wp07-run.sh ./tests/kind/streaming-run.sh
check-idempotency: check-idempotency:
./tests/kind/wp08-run.sh ./tests/kind/idempotency-run.sh
check-recovery: check-recovery:
./tests/kind/wp09-run.sh ./tests/kind/recovery-run.sh
check-protocol: check-protocol:
./tests/kind/wp10-run.sh ./tests/kind/protocol-conformance-run.sh
check-product: test-frontend check-frontend build-frontend check-product: test-frontend check-frontend build-frontend
NUTHATCH_PRODUCT_GATE=1 ./tests/kind/wp05-run.sh NUTHATCH_PRODUCT_GATE=1 ./tests/kind/product-matrix-run.sh
check-runtime: test check-recovery check-runtime: test check-recovery
./tests/runtime-static.sh ./tests/runtime-static.sh
./tests/kind/wp12-run.sh ./tests/kind/runtime-resilience-run.sh
check-v1: check-v1:
./tests/check-v1.sh ./tests/check-v1.sh
@ -47,7 +47,7 @@ check-specs:
./tests/spec-contracts.sh ./tests/spec-contracts.sh
app-up: app-up:
NUTHATCH_APP_INTERACTIVE=1 ./tests/kind/wp05-run.sh NUTHATCH_APP_INTERACTIVE=1 ./tests/kind/product-matrix-run.sh
app-down: app-down:
./tests/kind/destroy.sh ./tests/kind/destroy.sh

16
docs/spec/contracts.md
View file

@ -55,13 +55,12 @@ certificate and key paths.
## SQL contract ## SQL contract
The ordered files `sql/001_schema.sql`, `sql/002_pool_limit_guard.sql`, and `sql/001_initial_schema.sql` is the authoritative initial schema.
`sql/003_recovery_settlement.sql` are the authoritative schema history. `postgresql-migration` initializes, applies, records, and validates it at
`postgresql-migration` initializes, applies, records, and validates that process startup. A fixed PostgreSQL advisory lock serializes migration across
history at process startup. A fixed PostgreSQL advisory lock serializes replicas, and startup fails if migration or validation fails. After the first
migration across replicas, and startup fails if migration or validation fails. deployment, schema changes are new ordered migration files and deployed
Schema changes are new ordered migration files; deployed migration contents do migration contents do not change.
not change.
`pools` owns quota limits, used bytes, and reservations. `pools` owns quota limits, used bytes, and reservations.
`mutations` owns idempotency identity, request fingerprint data, protocol `mutations` owns idempotency identity, request fingerprint data, protocol
@ -183,7 +182,8 @@ credentials.
The deployment's object-store IAM or policy system enforces that scope. The The deployment's object-store IAM or policy system enforces that scope. The
Kubernetes manifest supplies the scoped credential through the secret boundary; Kubernetes manifest supplies the scoped credential through the secret boundary;
`tests/kind/lib.sh` provisions equivalent fixture principals, and `tests/kind/lib.sh` provisions equivalent fixture principals, and
`tests/kind/wp05-job.yaml.in` verifies denial against a different bucket. `tests/kind/product-matrix-job.yaml.in` verifies denial against a different
bucket.
Nuthatch is the exclusive writer to `users/` and `shared/` in the configured Nuthatch is the exclusive writer to `users/` and `shared/` in the configured
bucket while the service is enabled. Operators may inspect those prefixes, but bucket while the service is enabled. Operators may inspect those prefixes, but

4
docs/spec/testing.md
View file

@ -42,7 +42,7 @@ All fixture credentials and data are synthetic. Test workloads use ClusterIP
service names and do not require host port forwarding. The fixture does not service names and do not require host port forwarding. The fixture does not
read or change the user's Kubernetes context. read or change the user's Kubernetes context.
Package runners select the `base`, `durable`, `coordination`, `oidc`, and Test runners select the `base`, `durable`, `coordination`, `oidc`, and
`target` fixture families; `all` selects every dependency family. The selected `target` fixture families; `all` selects every dependency family. The selected
families expand into provider lanes, which run sequentially and stop at the families expand into provider lanes, which run sequentially and stop at the
first failure. Each lane receives an isolated PostgreSQL database, first failure. Each lane receives an isolated PostgreSQL database,
@ -52,7 +52,7 @@ Lane configuration is available both as environment variables from a
lane-specific ConfigMap and as `/run/nuthatch-fixture/profile.env`. Successful lane-specific ConfigMap and as `/run/nuthatch-fixture/profile.env`. Successful
lanes delete their Job, profile, database, coordination state, and bucket lanes delete their Job, profile, database, coordination state, and bucket
before the next lane. Failed lanes remain intact when the enclosing runner is before the next lane. Failed lanes remain intact when the enclosing runner is
retained. Package-owned application or test images are loaded with retained. Locally built application or test images are loaded with
`load_local_image`; released dependencies always use the digest-pinned images `load_local_image`; released dependencies always use the digest-pinned images
from `tests/kind/versions.env`. from `tests/kind/versions.env`.

12
nuthatch.cabal
View file

@ -8,9 +8,7 @@ maintainer: nuthatch@localhost
category: Network category: Network
build-type: Simple build-type: Simple
extra-source-files: extra-source-files:
sql/001_schema.sql sql/001_initial_schema.sql
sql/002_pool_limit_guard.sql
sql/003_recovery_settlement.sql
common shared common shared
default-language: Haskell2010 default-language: Haskell2010
@ -129,7 +127,7 @@ executable nuthatch
-rtsopts -rtsopts
-with-rtsopts=-N -with-rtsopts=-N
executable nuthatch-wp04-integration executable nuthatch-storage-integration
import: shared import: shared
main-is: Integration.hs main-is: Integration.hs
hs-source-dirs: test hs-source-dirs: test
@ -144,7 +142,7 @@ executable nuthatch-wp04-integration
-rtsopts -rtsopts
-with-rtsopts=-N -with-rtsopts=-N
executable nuthatch-wp09-phase executable nuthatch-recovery-probe
import: shared import: shared
main-is: RecoveryPhase.hs main-is: RecoveryPhase.hs
hs-source-dirs: test hs-source-dirs: test
@ -158,7 +156,7 @@ executable nuthatch-wp09-phase
-rtsopts -rtsopts
-with-rtsopts=-N -with-rtsopts=-N
executable nuthatch-wp10-conformance executable nuthatch-protocol-conformance
import: shared import: shared
main-is: ProtocolConformance.hs main-is: ProtocolConformance.hs
hs-source-dirs: test hs-source-dirs: test
@ -175,7 +173,7 @@ executable nuthatch-wp10-conformance
-rtsopts -rtsopts
-with-rtsopts=-N -with-rtsopts=-N
executable nuthatch-wp12-runtime executable nuthatch-runtime-probe
import: shared import: shared
main-is: RuntimeProbe.hs main-is: RuntimeProbe.hs
hs-source-dirs: test hs-source-dirs: test

86
sql/001_initial_schema.sql Normal file
View file

@ -0,0 +1,86 @@
CREATE TABLE pools (
pool_id TEXT PRIMARY KEY,
limit_bytes BIGINT NOT NULL CHECK (limit_bytes > 0),
used_bytes BIGINT NOT NULL DEFAULT 0 CHECK (used_bytes >= 0),
reserved_bytes BIGINT NOT NULL DEFAULT 0 CHECK (reserved_bytes >= 0),
CONSTRAINT pools_live_bytes_within_limit
CHECK (used_bytes + reserved_bytes <= limit_bytes)
);
INSERT INTO pools (pool_id, limit_bytes, used_bytes, reserved_bytes)
VALUES ('shared', 10737418240, 0, 0);
CREATE TABLE mutations (
user_id TEXT NOT NULL,
kind TEXT NOT NULL,
operation_id TEXT NOT NULL,
pool_id TEXT NOT NULL REFERENCES pools (pool_id),
object_key TEXT NOT NULL,
relative_path TEXT NOT NULL,
declared_size BIGINT NOT NULL,
content_sha256 TEXT,
prior_size BIGINT NOT NULL,
prior_sha256 TEXT,
growth_bytes BIGINT NOT NULL,
state TEXT NOT NULL,
external_state TEXT NOT NULL DEFAULT 'idle',
dispatch_started_at TIMESTAMPTZ,
request_deadline_at TIMESTAMPTZ,
worker_lease_until TIMESTAMPTZ,
settled_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (user_id, kind, operation_id),
CONSTRAINT mutations_kind_check
CHECK (kind IN ('put', 'delete')),
CONSTRAINT mutations_declared_size_check
CHECK (declared_size >= 0),
CONSTRAINT mutations_prior_size_check
CHECK (prior_size >= 0),
CONSTRAINT mutations_growth_bytes_check
CHECK (growth_bytes >= 0),
CONSTRAINT mutations_state_check
CHECK (state IN ('prepared', 'applying', 'succeeded', 'failed', 'rejected')),
CONSTRAINT mutations_external_state_check
CHECK (external_state IN ('idle', 'pending', 'settled')),
CONSTRAINT mutations_dispatch_timing_check CHECK (
external_state <> 'pending'
OR (
dispatch_started_at IS NOT NULL
AND request_deadline_at IS NOT NULL
AND worker_lease_until IS NOT NULL
)
)
);
CREATE INDEX mutations_active_key_idx
ON mutations (object_key)
WHERE state IN ('prepared', 'applying');
CREATE INDEX mutations_active_idx
ON mutations (state)
WHERE state IN ('prepared', 'applying');
CREATE INDEX mutations_recovery_idx
ON mutations (worker_lease_until, request_deadline_at, updated_at)
WHERE state IN ('prepared', 'applying');
CREATE TABLE key_locks (
object_key TEXT PRIMARY KEY,
user_id TEXT NOT NULL,
kind TEXT NOT NULL,
operation_id TEXT NOT NULL,
acquired_at TIMESTAMPTZ NOT NULL DEFAULT now(),
FOREIGN KEY (user_id, kind, operation_id)
REFERENCES mutations (user_id, kind, operation_id)
);
CREATE TABLE objects_meta (
object_key TEXT PRIMARY KEY,
pool_id TEXT NOT NULL REFERENCES pools (pool_id),
size_bytes BIGINT NOT NULL,
content_sha256 TEXT NOT NULL,
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
CONSTRAINT objects_meta_size_bytes_check
CHECK (size_bytes >= 0)
);

61
sql/001_schema.sql
View file

@ -1,61 +0,0 @@
-- Durable quota and mutation protocol state for Nuthatch.
CREATE TABLE IF NOT EXISTS pools (
pool_id TEXT PRIMARY KEY,
limit_bytes BIGINT NOT NULL CHECK (limit_bytes > 0),
used_bytes BIGINT NOT NULL DEFAULT 0 CHECK (used_bytes >= 0),
reserved_bytes BIGINT NOT NULL DEFAULT 0 CHECK (reserved_bytes >= 0)
);
INSERT INTO pools (pool_id, limit_bytes, used_bytes, reserved_bytes)
VALUES ('shared', 10737418240, 0, 0)
ON CONFLICT (pool_id) DO NOTHING;
CREATE TABLE IF NOT EXISTS mutations (
user_id TEXT NOT NULL,
kind TEXT NOT NULL CHECK (kind IN ('put', 'delete')),
operation_id TEXT NOT NULL,
pool_id TEXT NOT NULL REFERENCES pools (pool_id),
object_key TEXT NOT NULL,
relative_path TEXT NOT NULL,
declared_size BIGINT NOT NULL CHECK (declared_size >= 0),
content_sha256 TEXT,
prior_size BIGINT NOT NULL CHECK (prior_size >= 0),
prior_sha256 TEXT,
growth_bytes BIGINT NOT NULL CHECK (growth_bytes >= 0),
state TEXT NOT NULL CHECK (
state IN ('prepared', 'applying', 'succeeded', 'failed', 'rejected')
),
external_state TEXT NOT NULL DEFAULT 'idle' CHECK (
external_state IN ('idle', 'pending', 'settled')
),
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (user_id, kind, operation_id)
);
CREATE INDEX IF NOT EXISTS mutations_active_key_idx
ON mutations (object_key)
WHERE state IN ('prepared', 'applying');
CREATE INDEX IF NOT EXISTS mutations_active_idx
ON mutations (state)
WHERE state IN ('prepared', 'applying');
CREATE TABLE IF NOT EXISTS key_locks (
object_key TEXT PRIMARY KEY,
user_id TEXT NOT NULL,
kind TEXT NOT NULL,
operation_id TEXT NOT NULL,
acquired_at TIMESTAMPTZ NOT NULL DEFAULT now(),
FOREIGN KEY (user_id, kind, operation_id)
REFERENCES mutations (user_id, kind, operation_id)
);
CREATE TABLE IF NOT EXISTS objects_meta (
object_key TEXT PRIMARY KEY,
pool_id TEXT NOT NULL REFERENCES pools (pool_id),
size_bytes BIGINT NOT NULL CHECK (size_bytes >= 0),
content_sha256 TEXT NOT NULL,
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
);

56
sql/002_pool_limit_guard.sql
View file

@ -1,56 +0,0 @@
ALTER TABLE mutations
DROP CONSTRAINT IF EXISTS mutations_kind_check,
DROP CONSTRAINT IF EXISTS mutations_declared_size_check,
DROP CONSTRAINT IF EXISTS mutations_prior_size_check,
DROP CONSTRAINT IF EXISTS mutations_growth_bytes_check,
DROP CONSTRAINT IF EXISTS mutations_state_check,
DROP CONSTRAINT IF EXISTS mutations_external_state_check;
ALTER TABLE mutations
ADD CONSTRAINT mutations_kind_check
CHECK (kind IN ('put', 'delete')),
ADD CONSTRAINT mutations_declared_size_check
CHECK (declared_size >= 0),
ADD CONSTRAINT mutations_prior_size_check
CHECK (prior_size >= 0),
ADD CONSTRAINT mutations_growth_bytes_check
CHECK (growth_bytes >= 0),
ADD CONSTRAINT mutations_state_check
CHECK (state IN ('prepared', 'applying', 'succeeded', 'failed', 'rejected')),
ADD CONSTRAINT mutations_external_state_check
CHECK (external_state IN ('idle', 'pending', 'settled'));
ALTER TABLE key_locks
ADD COLUMN IF NOT EXISTS acquired_at TIMESTAMPTZ NOT NULL DEFAULT now(),
DROP CONSTRAINT IF EXISTS key_locks_user_id_kind_operation_id_fkey;
ALTER TABLE key_locks
ADD CONSTRAINT key_locks_user_id_kind_operation_id_fkey
FOREIGN KEY (user_id, kind, operation_id)
REFERENCES mutations (user_id, kind, operation_id);
ALTER TABLE objects_meta
DROP CONSTRAINT IF EXISTS objects_meta_size_bytes_check;
ALTER TABLE objects_meta
ADD CONSTRAINT objects_meta_size_bytes_check
CHECK (size_bytes >= 0);
CREATE INDEX IF NOT EXISTS mutations_active_key_idx
ON mutations (object_key)
WHERE state IN ('prepared', 'applying');
CREATE INDEX IF NOT EXISTS mutations_active_idx
ON mutations (state)
WHERE state IN ('prepared', 'applying');
UPDATE pools
SET limit_bytes = used_bytes + reserved_bytes
WHERE used_bytes + reserved_bytes > limit_bytes;
ALTER TABLE pools
DROP CONSTRAINT IF EXISTS pools_live_bytes_within_limit;
ALTER TABLE pools
ADD CONSTRAINT pools_live_bytes_within_limit
CHECK (used_bytes + reserved_bytes <= limit_bytes);

33
sql/003_recovery_settlement.sql
View file

@ -1,33 +0,0 @@
ALTER TABLE mutations
ADD COLUMN IF NOT EXISTS dispatch_started_at TIMESTAMPTZ,
ADD COLUMN IF NOT EXISTS request_deadline_at TIMESTAMPTZ,
ADD COLUMN IF NOT EXISTS worker_lease_until TIMESTAMPTZ,
ADD COLUMN IF NOT EXISTS settled_at TIMESTAMPTZ;
DO $$
BEGIN
IF EXISTS (
SELECT 1
FROM mutations
WHERE external_state = 'pending'
) THEN
RAISE EXCEPTION
'recovery settlement migration requires no pending mutations';
END IF;
END
$$;
ALTER TABLE mutations
ADD CONSTRAINT mutations_dispatch_timing_check CHECK (
(external_state <> 'pending') OR
(dispatch_started_at IS NOT NULL AND
request_deadline_at IS NOT NULL AND
worker_lease_until IS NOT NULL)
) NOT VALID;
ALTER TABLE mutations
VALIDATE CONSTRAINT mutations_dispatch_timing_check;
CREATE INDEX IF NOT EXISTS mutations_recovery_idx
ON mutations (worker_lease_until, request_deadline_at, updated_at)
WHERE state IN ('prepared', 'applying');

74
test/Integration.hs
View file

@ -36,19 +36,11 @@ main = do
withDb config $ \database -> do withDb config $ \database -> do
migrate database "sql" config migrate database "sql" config
migrate database "sql" config migrate database "sql" config
emptyCatalog <- schemaCatalog database
resetSchema database
installLegacySchema database
migrate database "sql" config
migrate database "sql" config
adoptedCatalog <- schemaCatalog database
assertEqual "legacy and empty migration schema equivalence"
emptyCatalog adoptedCatalog
verifySchema database verifySchema database
verifyAdmission database config verifyAdmission database config
store <- newS3 config store <- newS3 config
verifyStorage config store verifyStorage config store
putStrLn "WP-04 integration gate passed" putStrLn "Storage foundation integration gate passed"
verifySchema :: Db -> IO () verifySchema :: Db -> IO ()
verifySchema database = do verifySchema database = do
@ -71,71 +63,9 @@ verifySchema database = do
"SELECT filename FROM schema_migrations ORDER BY filename" "SELECT filename FROM schema_migrations ORDER BY filename"
:: IO [Only Text] :: IO [Only Text]
assertEqual "migration history" assertEqual "migration history"
[ "001_schema.sql" ["001_initial_schema.sql"]
, "002_pool_limit_guard.sql"
, "003_recovery_settlement.sql"
]
(map fromOnly migrations) (map fromOnly migrations)
schemaCatalog :: Db -> IO [Text]
schemaCatalog database = do
rows <- Db.query_ database
"SELECT description FROM ( \
\SELECT 'column|' || table_name || '|' || ordinal_position::text || '|' \
\ || column_name || '|' || data_type || '|' || is_nullable || '|' \
\ || COALESCE(column_default, '') AS description \
\FROM information_schema.columns \
\WHERE table_schema = 'public' \
\UNION ALL \
\SELECT 'constraint|' || c.relname || '|' || con.conname || '|' \
\ || con.contype::text || '|' || pg_get_constraintdef(con.oid) \
\FROM pg_constraint con \
\JOIN pg_class c ON c.oid = con.conrelid \
\JOIN pg_namespace n ON n.oid = c.relnamespace \
\WHERE n.nspname = 'public' \
\UNION ALL \
\SELECT 'index|' || tablename || '|' || indexname || '|' || indexdef \
\FROM pg_indexes WHERE schemaname = 'public' \
\) catalog ORDER BY description"
:: IO [Only Text]
pure (map fromOnly rows)
resetSchema :: Db -> IO ()
resetSchema database = do
_ <- Db.execute_ database
"DROP TABLE IF EXISTS schema_migrations, key_locks, objects_meta, \
\mutations, pools CASCADE"
pure ()
installLegacySchema :: Db -> IO ()
installLegacySchema database = do
mapM_ (Db.execute_ database)
[ "CREATE TABLE pools ( \
\pool_id TEXT PRIMARY KEY, \
\limit_bytes BIGINT NOT NULL CHECK (limit_bytes > 0), \
\used_bytes BIGINT NOT NULL DEFAULT 0 CHECK (used_bytes >= 0), \
\reserved_bytes BIGINT NOT NULL DEFAULT 0 CHECK (reserved_bytes >= 0))"
, "CREATE TABLE mutations ( \
\user_id TEXT NOT NULL, kind TEXT NOT NULL, operation_id TEXT NOT NULL, \
\pool_id TEXT NOT NULL REFERENCES pools (pool_id), object_key TEXT NOT NULL, \
\relative_path TEXT NOT NULL, declared_size BIGINT NOT NULL, \
\content_sha256 TEXT, prior_size BIGINT NOT NULL, prior_sha256 TEXT, \
\growth_bytes BIGINT NOT NULL, state TEXT NOT NULL, \
\external_state TEXT NOT NULL DEFAULT 'idle', \
\created_at TIMESTAMPTZ NOT NULL DEFAULT now(), \
\updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), \
\PRIMARY KEY (user_id, kind, operation_id))"
, "CREATE TABLE key_locks ( \
\object_key TEXT PRIMARY KEY, user_id TEXT NOT NULL, kind TEXT NOT NULL, \
\operation_id TEXT NOT NULL)"
, "CREATE TABLE objects_meta ( \
\object_key TEXT PRIMARY KEY, \
\pool_id TEXT NOT NULL REFERENCES pools (pool_id), \
\size_bytes BIGINT NOT NULL, content_sha256 TEXT NOT NULL, \
\updated_at TIMESTAMPTZ NOT NULL DEFAULT now())"
]
pure ()
verifyAdmission :: Db -> Config -> IO () verifyAdmission :: Db -> Config -> IO ()
verifyAdmission database config = do verifyAdmission database config = do
resetAuthority database 100 resetAuthority database 100

4
test/ProtocolConformance.hs
View file

@ -60,7 +60,7 @@ verifyConformance cfg database store = do
effectScenarios cfg database store effectScenarios cfg database store
targetContract cfg database store targetContract cfg database store
checkedInvariant "TypeOK" checkedInvariant "TypeOK"
putStrLn "WP-10 protocol conformance passed" putStrLn "Protocol conformance passed"
admissionScenarios :: Config -> Db -> IO () admissionScenarios :: Config -> Db -> IO ()
admissionScenarios cfg database = do admissionScenarios cfg database = do
@ -250,7 +250,7 @@ targetContract cfg database store = do
verifyStoreFence :: Config -> Db -> Redis -> S3 -> IO () verifyStoreFence :: Config -> Db -> Redis -> S3 -> IO ()
verifyStoreFence cfg database redis store = do verifyStoreFence cfg database redis store = do
let env = newEnv cfg database redis store let env = newEnv cfg database redis store
user = SessionUser "wp10" "fence" protocolUser (Just "Protocol") user = SessionUser "protocol" "fence" protocolUser (Just "Protocol")
expiredDelete <- prepared expiredDelete <- prepared
=<< admitDelete =<< admitDelete
cfg database "expired-delete-store-down" "expired-delete-store-down.bin" cfg database "expired-delete-store-down" "expired-delete-store-down.bin"

2
test/RecoveryPhase.hs
View file

@ -217,7 +217,7 @@ recoveryUser = UserId "recovery-user"
recoverySession :: SessionUser recoverySession :: SessionUser
recoverySession = recoverySession =
SessionUser "wp09" "recovery" recoveryUser (Just "Recovery") SessionUser "recovery" "recovery" recoveryUser (Just "Recovery")
slowSize :: Integer slowSize :: Integer
slowSize = 20 * 1024 * 1024 slowSize = 20 * 1024 * 1024

12
test/RuntimeProbe.hs
View file

@ -63,7 +63,7 @@ verifyConfiguration = do
expectRejected expectRejected
(load (load
[ ( "NUTHATCH_PUBLIC_BASE_URL" [ ( "NUTHATCH_PUBLIC_BASE_URL"
, Just "https://user:password@wp12.invalid" , Just "https://user:password@runtime.invalid"
) )
]) ])
"credential-bearing URI configuration" "credential-bearing URI configuration"
@ -91,10 +91,10 @@ setVariable name = maybe (unsetEnv name) (setEnv name)
verifyAvailable :: Config -> Redis -> IO () verifyAvailable :: Config -> Redis -> IO ()
verifyAvailable config redis = do verifyAvailable config redis = do
cacheSet redis "nuthatch:wp12:ttl" (cfgListCacheTtlSec config) "cached" cacheSet redis "nuthatch:runtime:ttl" (cfgListCacheTtlSec config) "cached"
cacheGet redis "nuthatch:wp12:ttl" >>= requireEqual (Just "cached") cacheGet redis "nuthatch:runtime:ttl" >>= requireEqual (Just "cached")
threadDelay (fromInteger ((cfgListCacheTtlSec config + 1) * 1000000)) threadDelay (fromInteger ((cfgListCacheTtlSec config + 1) * 1000000))
cacheGet redis "nuthatch:wp12:ttl" >>= requireEqual Nothing cacheGet redis "nuthatch:runtime:ttl" >>= requireEqual Nothing
let userA = UserId "runtime-user-a" let userA = UserId "runtime-user-a"
userB = UserId "runtime-user-b" userB = UserId "runtime-user-b"
sharedBeforeA <- cacheGeneration redis userA Shared sharedBeforeA <- cacheGeneration redis userA Shared
@ -112,8 +112,8 @@ verifyAvailable config redis = do
verifyUnavailable :: Config -> Redis -> IO () verifyUnavailable :: Config -> Redis -> IO ()
verifyUnavailable config redis = do verifyUnavailable config redis = do
cacheGet redis "nuthatch:wp12:missing" >>= requireEqual Nothing cacheGet redis "nuthatch:runtime:missing" >>= requireEqual Nothing
cacheSet redis "nuthatch:wp12:missing" 1 "ignored" cacheSet redis "nuthatch:runtime:missing" 1 "ignored"
cacheInvalidatePrefixes redis (UserId "runtime-user") Shared [""] cacheInvalidatePrefixes redis (UserId "runtime-user") Shared [""]
throttled <- throttleCheck redis config (UserId "runtime-user") 1 throttled <- throttleCheck redis config (UserId "runtime-user") 1
case throttled of case throttled of

13
tests/check-v1.sh
View file

@ -10,12 +10,12 @@ fi
unset NUTHATCH_APP_INTERACTIVE unset NUTHATCH_APP_INTERACTIVE
unset NUTHATCH_PRODUCT_GATE unset NUTHATCH_PRODUCT_GATE
unset NUTHATCH_WP07_LANE unset NUTHATCH_STREAMING_LANE
unset NUTHATCH_WP07_FIRST_LANE_ONLY unset NUTHATCH_STREAMING_FIRST_LANE_ONLY
unset NUTHATCH_WP08_LANE unset NUTHATCH_IDEMPOTENCY_LANE
unset NUTHATCH_WP08_FIRST_LANE_ONLY unset NUTHATCH_IDEMPOTENCY_FIRST_LANE_ONLY
unset NUTHATCH_WP09_TARGET unset NUTHATCH_RECOVERY_TARGET
unset NUTHATCH_WP10_TARGET unset NUTHATCH_PROTOCOL_TARGET
targets=( targets=(
test test
@ -30,6 +30,7 @@ targets=(
check-protocol check-protocol
check-product check-product
check-runtime check-runtime
check-specs
) )
for target in "${targets[@]}"; do for target in "${targets[@]}"; do

188
tests/kind/idempotency-run.sh Executable file
View file

@ -0,0 +1,188 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
source "$ROOT/tests/kind/lib.sh"
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
SUCCEEDED=false
OWNS_FIXTURE=false
APP_IMAGE=nuthatch-idempotency:local
cleanup() {
local status=$?
trap - EXIT INT TERM
if [[ "${NUTHATCH_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
printf 'retained context: %s\n' "$CONTEXT"
exit "$status"
fi
if [[ "$OWNS_FIXTURE" = true ]]; then
destroy_fixture || status=1
fi
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
printf 'user kubeconfig changed unexpectedly\n' >&2
status=1
fi
exit "$status"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
render_application() {
local lane="$1"
local profile_file="$2"
local pg_conn
local redis_url
local oidc_issuer
local oidc_client_id
local oidc_additional_audiences
local oidc_client_secret
local s3_endpoint
local s3_region
local s3_bucket
local s3_access_key
local s3_secret_key
pg_conn="$(sed -n 's/^NUTHATCH_PG_CONN=//p' "$profile_file")"
redis_url="$(sed -n 's/^NUTHATCH_REDIS_URL=//p' "$profile_file")"
oidc_issuer="$(sed -n 's/^NUTHATCH_OIDC_ISSUER=//p' "$profile_file")"
oidc_client_id="$(sed -n 's/^NUTHATCH_OIDC_CLIENT_ID=//p' "$profile_file")"
oidc_additional_audiences="$(
sed -n 's/^NUTHATCH_OIDC_ADDITIONAL_AUDIENCES=//p' "$profile_file"
)"
oidc_client_secret="$(<"$profile_file.oidc-secret")"
s3_endpoint="$(sed -n 's/^NUTHATCH_S3_ENDPOINT=//p' "$profile_file")"
s3_region="$(sed -n 's/^NUTHATCH_S3_REGION=//p' "$profile_file")"
s3_bucket="$(sed -n 's/^NUTHATCH_S3_BUCKET=//p' "$profile_file")"
s3_access_key="$(sed -n 's/^NUTHATCH_S3_ACCESS_KEY=//p' "$profile_file")"
s3_secret_key="$(sed -n 's/^NUTHATCH_S3_SECRET_KEY=//p' "$profile_file")"
sed \
-e "s|@@LANE@@|$lane|g" \
-e "s|@@APP_IMAGE@@|$APP_IMAGE|g" \
-e "s|@@PG_CONN@@|$pg_conn|g" \
-e "s|@@REDIS_URL@@|$redis_url|g" \
-e "s|@@OIDC_ISSUER@@|$oidc_issuer|g" \
-e "s|@@OIDC_CLIENT_ID@@|$oidc_client_id|g" \
-e "s|@@OIDC_ADDITIONAL_AUDIENCES@@|$oidc_additional_audiences|g" \
-e "s|@@OIDC_CLIENT_SECRET@@|$oidc_client_secret|g" \
-e "s|@@S3_ENDPOINT@@|$s3_endpoint|g" \
-e "s|@@S3_REGION@@|$s3_region|g" \
-e "s|@@S3_BUCKET@@|$s3_bucket|g" \
-e "s|@@S3_ACCESS_KEY@@|$s3_access_key|g" \
-e "s|@@S3_SECRET_KEY@@|$s3_secret_key|g" \
"$ROOT/deploy/kubernetes/nuthatch.yaml.in" >"$RUNTIME/idempotency-app-$lane.yaml"
sed -i \
's/NUTHATCH_IDEMPOTENCY_WINDOW_SEC: "86400"/NUTHATCH_IDEMPOTENCY_WINDOW_SEC: "2"/' \
"$RUNTIME/idempotency-app-$lane.yaml"
sed -i 's/ replicas: 1/ replicas: 2/' "$RUNTIME/idempotency-app-$lane.yaml"
}
cleanup_application() {
local lane="$1"
kube --namespace "$FIXTURE_NAMESPACE" delete \
"deployment/nuthatch-$lane" \
"service/nuthatch-$lane" \
service/nuthatch-idempotency-a \
service/nuthatch-idempotency-b \
"configmap/nuthatch-$lane" \
"secret/nuthatch-$lane" \
--ignore-not-found >/dev/null
rm -f "$RUNTIME/idempotency-app-$lane.yaml"
}
create_process_services() {
local lane="$1"
local profile="$2"
local pods
mapfile -t pods < <(
kube --namespace "$FIXTURE_NAMESPACE" get pods \
--selector="app=nuthatch-$lane" \
--field-selector=status.phase=Running \
--output jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' |
sort
)
test "${#pods[@]}" = 2
kube --namespace "$FIXTURE_NAMESPACE" label pod "${pods[0]}" \
nuthatch-idempotency-process=a >/dev/null
kube --namespace "$FIXTURE_NAMESPACE" label pod "${pods[1]}" \
nuthatch-idempotency-process=b >/dev/null
kube --namespace "$FIXTURE_NAMESPACE" expose pod "${pods[0]}" \
--name=nuthatch-idempotency-a \
--port=8080 \
--target-port=8080 >/dev/null
kube --namespace "$FIXTURE_NAMESPACE" expose pod "${pods[1]}" \
--name=nuthatch-idempotency-b \
--port=8080 \
--target-port=8080 >/dev/null
local ip_a
local ip_b
ip_a="$(
kube --namespace "$FIXTURE_NAMESPACE" get service nuthatch-idempotency-a \
--output jsonpath='{.spec.clusterIP}'
)"
ip_b="$(
kube --namespace "$FIXTURE_NAMESPACE" get service nuthatch-idempotency-b \
--output jsonpath='{.spec.clusterIP}'
)"
local profile_file="$RUNTIME/$profile.env"
sed -i '/^NUTHATCH_IDEMPOTENCY_/d' "$profile_file"
{
printf 'NUTHATCH_IDEMPOTENCY_MODE=verify\n'
printf 'NUTHATCH_IDEMPOTENCY_IP_A=%s\n' "$ip_a"
printf 'NUTHATCH_IDEMPOTENCY_IP_B=%s\n' "$ip_b"
} >>"$profile_file"
kube create configmap "$profile" \
--namespace "$FIXTURE_NAMESPACE" \
--from-env-file="$profile_file" \
--dry-run=client \
--output yaml |
kube apply --filename - >/dev/null
}
acquire_fixture_lock
OWNS_FIXTURE=true
setup_fixture durable coordination oidc target
source "$RUNTIME/fixture.env"
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" build \
--file "$ROOT/Dockerfile" \
--tag "$APP_IMAGE" \
"$ROOT"
load_local_image "$APP_IMAGE" "$NERDCTL_NAMESPACE"
while IFS=$'\t' read -r lane coordination oidc target; do
if [[ -n "${NUTHATCH_IDEMPOTENCY_LANE:-}" ]] &&
[[ "$lane" != "$NUTHATCH_IDEMPOTENCY_LANE" ]]; then
continue
fi
profile="$(write_lane_profile "$lane" "$coordination" "$oidc" "$target")"
printf 'Running idempotency lane %s: coordination=%s oidc=%s target=%s\n' \
"$lane" "$coordination" "$oidc" "$target"
cleanup_application "$lane"
render_application "$lane" "$RUNTIME/$profile.env"
kube apply --filename "$RUNTIME/idempotency-app-$lane.yaml" >/dev/null
kube --namespace "$FIXTURE_NAMESPACE" rollout status \
"deployment/nuthatch-$lane" --timeout=180s
create_process_services "$lane" "$profile"
job="idempotency-epoch-$lane"
render_lane_job \
"$job" \
"$TESTBED/product-matrix-job.yaml.in" \
"$CURL_LOCAL_IMAGE" \
"$profile"
sed -i "s|@@LANE@@|$lane|g" "$RUNTIME/$job.yaml"
run_job_manifest "$job" "$RUNTIME/$job.yaml"
cleanup_application "$lane"
cleanup_lane_state "$lane" "$coordination" "$oidc" "$target"
cleanup_lane_resources "$job" "$profile"
if [[ "${NUTHATCH_IDEMPOTENCY_FIRST_LANE_ONLY:-0}" = 1 ]]; then
break
fi
done < <(expand_fixture_matrix coordination oidc target)
SUCCEEDED=true
printf 'Nuthatch idempotency gate passed\n'

8
tests/kind/lib.sh
View file

@ -488,7 +488,7 @@ provision_lane_target_principal() {
endpoint="$(target_loopback_endpoint "$provider")" || return endpoint="$(target_loopback_endpoint "$provider")" || return
case "$provider" in case "$provider" in
garage) garage)
printf -v access_key 'GKWP05%028d' "$((10#$lane))" printf -v access_key 'GKNUTHATCH%024d' "$((10#$lane))"
printf -v secret_key '%064d' "$((10#$lane))" printf -v secret_key '%064d' "$((10#$lane))"
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/garage --container garage -- \ kube --namespace "$FIXTURE_NAMESPACE" exec deployment/garage --container garage -- \
/garage bucket create "$bucket" >/dev/null || return /garage bucket create "$bucket" >/dev/null || return
@ -505,8 +505,8 @@ provision_lane_target_principal() {
/garage bucket allow --read --write "$bucket" --key "$access_key" >/dev/null || return /garage bucket allow --read --write "$bucket" --key "$access_key" >/dev/null || return
;; ;;
versitygw) versitygw)
access_key="nuthatch-wp05-$lane" access_key="nuthatch-lane-$lane"
printf -v secret_key 'synthetic-wp05-%049d' "$((10#$lane))" printf -v secret_key 'synthetic-lane-%049d' "$((10#$lane))"
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/versitygw --container versitygw -- \ kube --namespace "$FIXTURE_NAMESPACE" exec deployment/versitygw --container versitygw -- \
/usr/local/bin/versitygw admin \ /usr/local/bin/versitygw admin \
--endpoint-url http://127.0.0.1:10001 \ --endpoint-url http://127.0.0.1:10001 \
@ -916,6 +916,6 @@ destroy_fixture() {
"$RUNTIME/fixture.env" \ "$RUNTIME/fixture.env" \
"$RUNTIME"/nuthatch-fixture-lane-*.env \ "$RUNTIME"/nuthatch-fixture-lane-*.env \
"$RUNTIME"/nuthatch-fixture-lane-*.env.oidc-secret \ "$RUNTIME"/nuthatch-fixture-lane-*.env.oidc-secret \
"$RUNTIME"/wp05-app-*.yaml || status=1 "$RUNTIME"/product-app-*.yaml || status=1
return "$status" return "$status"
} }

1615
tests/kind/product-matrix-job.yaml.in Normal file
View file

File diff suppressed because it is too large Load diff

176
tests/kind/product-matrix-run.sh Executable file
View file

@ -0,0 +1,176 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
source "$ROOT/tests/kind/lib.sh"
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
SUCCEEDED=false
OWNS_FIXTURE=false
APP_IMAGE=nuthatch-product-matrix:local
cleanup() {
local status=$?
trap - EXIT INT TERM
if [[ "${NUTHATCH_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
printf 'retained context: %s\n' "$CONTEXT"
exit "$status"
fi
if [[ "$OWNS_FIXTURE" = true ]]; then
destroy_fixture || status=1
fi
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
printf 'user kubeconfig changed unexpectedly\n' >&2
status=1
fi
exit "$status"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
render_application() {
local lane="$1"
local profile_file="$2"
local pg_conn
local redis_url
local oidc_issuer
local oidc_client_id
local oidc_additional_audiences
local oidc_client_secret
local s3_endpoint
local s3_region
local s3_bucket
local s3_access_key
local s3_secret_key
pg_conn="$(sed -n 's/^NUTHATCH_PG_CONN=//p' "$profile_file")"
redis_url="$(sed -n 's/^NUTHATCH_REDIS_URL=//p' "$profile_file")"
oidc_issuer="$(sed -n 's/^NUTHATCH_OIDC_ISSUER=//p' "$profile_file")"
oidc_client_id="$(sed -n 's/^NUTHATCH_OIDC_CLIENT_ID=//p' "$profile_file")"
oidc_additional_audiences="$(
sed -n 's/^NUTHATCH_OIDC_ADDITIONAL_AUDIENCES=//p' "$profile_file"
)"
oidc_client_secret="$(<"$profile_file.oidc-secret")"
s3_endpoint="$(sed -n 's/^NUTHATCH_S3_ENDPOINT=//p' "$profile_file")"
s3_region="$(sed -n 's/^NUTHATCH_S3_REGION=//p' "$profile_file")"
s3_bucket="$(sed -n 's/^NUTHATCH_S3_BUCKET=//p' "$profile_file")"
s3_access_key="$(sed -n 's/^NUTHATCH_S3_ACCESS_KEY=//p' "$profile_file")"
s3_secret_key="$(sed -n 's/^NUTHATCH_S3_SECRET_KEY=//p' "$profile_file")"
sed \
-e "s|@@LANE@@|$lane|g" \
-e "s|@@APP_IMAGE@@|$APP_IMAGE|g" \
-e "s|@@PG_CONN@@|$pg_conn|g" \
-e "s|@@REDIS_URL@@|$redis_url|g" \
-e "s|@@OIDC_ISSUER@@|$oidc_issuer|g" \
-e "s|@@OIDC_CLIENT_ID@@|$oidc_client_id|g" \
-e "s|@@OIDC_ADDITIONAL_AUDIENCES@@|$oidc_additional_audiences|g" \
-e "s|@@OIDC_CLIENT_SECRET@@|$oidc_client_secret|g" \
-e "s|@@S3_ENDPOINT@@|$s3_endpoint|g" \
-e "s|@@S3_REGION@@|$s3_region|g" \
-e "s|@@S3_BUCKET@@|$s3_bucket|g" \
-e "s|@@S3_ACCESS_KEY@@|$s3_access_key|g" \
-e "s|@@S3_SECRET_KEY@@|$s3_secret_key|g" \
"$ROOT/deploy/kubernetes/nuthatch.yaml.in" > "$RUNTIME/product-app-$lane.yaml"
if [[ "${NUTHATCH_PRODUCT_GATE:-0}" = 1 ]]; then
sed -i \
-e 's/NUTHATCH_BROWSE_PAGE_SIZE: "200"/NUTHATCH_BROWSE_PAGE_SIZE: "2"/' \
-e 's/NUTHATCH_SEARCH_MAX_KEYS: "500"/NUTHATCH_SEARCH_MAX_KEYS: "3"/' \
-e 's/NUTHATCH_DELETE_PAGE_SIZE: "100"/NUTHATCH_DELETE_PAGE_SIZE: "2"/' \
"$RUNTIME/product-app-$lane.yaml"
fi
}
cleanup_application() {
local lane="$1"
kube --namespace "$FIXTURE_NAMESPACE" delete \
"deployment/nuthatch-$lane" \
"service/nuthatch-$lane" \
"configmap/nuthatch-$lane" \
"secret/nuthatch-$lane" \
--ignore-not-found >/dev/null
rm -f "$RUNTIME/product-app-$lane.yaml"
}
set_product_profile() {
local profile="$1"
local mode="$2"
local profile_file="$RUNTIME/$profile.env"
sed -i '/^NUTHATCH_PRODUCT_MODE=/d' "$profile_file"
printf 'NUTHATCH_PRODUCT_MODE=%s\n' "$mode" >> "$profile_file"
kube create configmap "$profile" \
--namespace "$FIXTURE_NAMESPACE" \
--from-env-file="$profile_file" \
--dry-run=client \
--output yaml |
kube apply --filename - >/dev/null
}
acquire_fixture_lock
OWNS_FIXTURE=true
setup_fixture durable coordination oidc target
source "$RUNTIME/fixture.env"
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" build \
--file "$ROOT/Dockerfile" \
--tag "$APP_IMAGE" \
"$ROOT"
load_local_image "$APP_IMAGE" "$NERDCTL_NAMESPACE"
while IFS=$'\t' read -r lane coordination oidc target; do
profile="$(write_lane_profile "$lane" "$coordination" "$oidc" "$target")"
if [[ "${NUTHATCH_PRODUCT_GATE:-0}" = 1 ]]; then
set_product_profile "$profile" verify
fi
render_application "$lane" "$RUNTIME/$profile.env"
kube apply --filename "$RUNTIME/product-app-$lane.yaml" >/dev/null
kube --namespace "$FIXTURE_NAMESPACE" rollout status \
"deployment/nuthatch-$lane" --timeout=180s
if [[ "${NUTHATCH_APP_INTERACTIVE:-0}" = 1 ]]; then
SUCCEEDED=true
OWNS_FIXTURE=false
printf 'interactive kubeconfig: %s\n' "$KUBECONFIG_PATH"
printf 'interactive context: %s\n' "$CONTEXT"
printf 'interactive namespace: %s\n' "$FIXTURE_NAMESPACE"
printf 'interactive service: nuthatch-%s:8080\n' "$lane"
exit 0
fi
job="product-app-$lane"
render_lane_job \
"$job" \
"$TESTBED/product-matrix-job.yaml.in" \
"$CURL_LOCAL_IMAGE" \
"$profile"
sed -i "s|@@LANE@@|$lane|g" "$RUNTIME/$job.yaml"
printf 'Running lane %s: coordination=%s oidc=%s target=%s\n' \
"$lane" "$coordination" "$oidc" "$target"
run_job_manifest "$job" "$RUNTIME/$job.yaml"
if [[ "${NUTHATCH_PRODUCT_GATE:-0}" = 1 ]]; then
sed -i \
's/NUTHATCH_SEARCH_TIMEOUT_MS: "2000"/NUTHATCH_SEARCH_TIMEOUT_MS: "1"/' \
"$RUNTIME/product-app-$lane.yaml"
kube apply --filename "$RUNTIME/product-app-$lane.yaml" >/dev/null
kube --namespace "$FIXTURE_NAMESPACE" rollout restart \
"deployment/nuthatch-$lane" >/dev/null
kube --namespace "$FIXTURE_NAMESPACE" rollout status \
"deployment/nuthatch-$lane" --timeout=180s
set_product_profile "$profile" timeout
timeout_job="product-timeout-$lane"
render_lane_job \
"$timeout_job" \
"$TESTBED/product-matrix-job.yaml.in" \
"$CURL_LOCAL_IMAGE" \
"$profile"
sed -i "s|@@LANE@@|$lane|g" "$RUNTIME/$timeout_job.yaml"
run_job_manifest "$timeout_job" "$RUNTIME/$timeout_job.yaml"
kube --namespace "$FIXTURE_NAMESPACE" delete \
"job/$timeout_job" --ignore-not-found >/dev/null
rm -f "$RUNTIME/$timeout_job.yaml"
fi
cleanup_application "$lane"
cleanup_lane_state "$lane" "$coordination" "$oidc" "$target"
cleanup_lane_resources "$job" "$profile"
done < <(expand_fixture_matrix coordination oidc target)
SUCCEEDED=true
printf 'Nuthatch product matrix gate passed\n'

33
tests/kind/protocol-conformance-job.yaml.in Normal file
View file

@ -0,0 +1,33 @@
apiVersion: batch/v1
kind: Job
metadata:
name: @@JOB_NAME@@
namespace: nuthatch-testbed
spec:
activeDeadlineSeconds: 300
backoffLimit: 0
template:
spec:
automountServiceAccountToken: false
enableServiceLinks: false
restartPolicy: Never
containers:
- name: protocol-conformance
image: @@TEST_IMAGE@@
imagePullPolicy: Never
command: [nuthatch-protocol-conformance]
args: ["@@MODE@@"]
envFrom:
- configMapRef: {name: @@PROFILE_NAME@@}
env:
- {name: NUTHATCH_OBJECT_REQUEST_TIMEOUT_SEC, value: "3"}
- {name: NUTHATCH_JWT_SECRET, value: synthetic-protocol-session-jwt-secret}
- {name: NUTHATCH_PUBLIC_BASE_URL, value: "https://protocol.invalid"}
- {name: NUTHATCH_OIDC_ISSUER, value: "https://issuer.invalid"}
- {name: NUTHATCH_OIDC_CLIENT_ID, value: protocol-conformance}
- {name: NUTHATCH_OIDC_ADDITIONAL_AUDIENCES, value: ""}
- {name: NUTHATCH_OIDC_CLIENT_SECRET, value: synthetic-protocol-oidc-client-secret}
- {name: NUTHATCH_OIDC_REDIRECT_URI, value: "https://protocol.invalid/oidc/callback"}
resources:
requests: {cpu: 100m, memory: 96Mi}
limits: {cpu: "2", memory: 256Mi}

238
tests/kind/protocol-conformance-run.sh Executable file
View file

@ -0,0 +1,238 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
source "$ROOT/tests/kind/lib.sh"
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
SUCCEEDED=false
OWNS_FIXTURE=false
APP_IMAGE=nuthatch-protocol-conformance:local
cleanup() {
local status=$?
trap - EXIT INT TERM
if [[ "${NUTHATCH_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
printf 'retained context: %s\n' "$CONTEXT"
exit "$status"
fi
if [[ "$OWNS_FIXTURE" = true ]]; then
destroy_fixture || status=1
fi
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
printf 'user kubeconfig changed unexpectedly\n' >&2
status=1
fi
exit "$status"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
set_target_available() {
local target="$1"
local selector="$target"
if [[ "$2" = false ]]; then
selector=protocol-unavailable
fi
kube --namespace "$FIXTURE_NAMESPACE" patch service "$target" \
--type merge \
--patch "{\"spec\":{\"selector\":{\"app\":\"$selector\"}}}" >/dev/null
}
set_postgres_available() {
local selector=postgres
if [[ "$1" = false ]]; then
selector=protocol-unavailable
fi
kube --namespace "$FIXTURE_NAMESPACE" patch service postgres \
--type merge \
--patch "{\"spec\":{\"selector\":{\"app\":\"$selector\"}}}" >/dev/null
}
database_connection_count() {
local database="$1"
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
env PGPASSWORD=synthetic-postgres psql \
--username nuthatch \
--dbname postgres \
--tuples-only \
--no-align \
--set ON_ERROR_STOP=1 \
--command \
"SELECT count(*) FROM pg_stat_activity WHERE datname = '$database' AND pid <> pg_backend_pid()" |
tr -d '[:space:]'
}
terminate_database_connections() {
local database="$1"
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
env PGPASSWORD=synthetic-postgres psql \
--username nuthatch \
--dbname postgres \
--set ON_ERROR_STOP=1 \
--command \
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$database' AND pid <> pg_backend_pid()" \
>/dev/null
}
render_mode() {
local lane="$1"
local profile="$2"
local mode="$3"
local job="protocol-$mode-$lane"
render_lane_job \
"$job" \
"$TESTBED/protocol-conformance-job.yaml.in" \
"$APP_IMAGE" \
"$profile"
sed -i "s|@@MODE@@|$mode|g" "$RUNTIME/$job.yaml"
}
run_mode() {
local lane="$1"
local profile="$2"
local mode="$3"
local job="protocol-$mode-$lane"
render_mode "$lane" "$profile" "$mode"
run_job_manifest "$job" "$RUNTIME/$job.yaml"
}
start_mode() {
local lane="$1"
local profile="$2"
local mode="$3"
local job="protocol-$mode-$lane"
render_mode "$lane" "$profile" "$mode"
kube apply --filename "$RUNTIME/$job.yaml" >/dev/null
}
wait_for_mode_log() {
local lane="$1"
local mode="$2"
local message="$3"
local job="protocol-$mode-$lane"
for _ in $(seq 1 120); do
if kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" 2>/dev/null |
grep -F "$message" >/dev/null; then
return
fi
sleep 0.25
done
printf 'timed out waiting for %s\n' "$message" >&2
return 1
}
stop_mode() {
local lane="$1"
local mode="$2"
kube --namespace "$FIXTURE_NAMESPACE" delete "job/protocol-$mode-$lane" \
--ignore-not-found >/dev/null
}
verify_mapping() {
local lane="$1"
local log="$RUNTIME/protocol-full-$lane.log"
kube --namespace "$FIXTURE_NAMESPACE" logs "job/protocol-full-$lane" >"$log"
local actions=(
FirstRequest ReplayRequest ConflictRequest PrepareEffect DispatchEffect
LoseUploadBody SettleLostUpload ExternalEffectApplied
ExternalEffectNotApplied FinalizeSuccess FinalizeFailure ControllerAdvance
StoreSettlement
)
local invariants=(
TypeOK RecordIdentityConsistent PersonalRequestsUseOwnPool
PersonalPoolsDisjoint FingerprintBindsRequest ReservationsConsistent
LocksConsistent ActiveObjectConsistent ExternalStateConsistent
UploadBodyConsistent QuotaReservationBound ObjectStateCovered
ObjectQuotaBound AccountingAtQuiescence ObservationConsistent
RecordsNeverForgotten RequestBindingsStable TerminalRecordsStable
SettledEffectsDoNotMutate UnfinishedEventuallyTerminal
LostUploadEventuallyTerminal
)
local name
for name in "${actions[@]}"; do
grep -F "protocol action covered: $name" "$log" >/dev/null
rg -q "^$name(\\(| ==)" "$ROOT/docs/protocol/Nuthatch.tla"
done
for name in "${invariants[@]}"; do
grep -F "protocol invariant checked: $name" "$log" >/dev/null
rg -q "^$name(\\(| ==)" "$ROOT/docs/protocol/Nuthatch.tla"
done
}
acquire_fixture_lock
OWNS_FIXTURE=true
setup_fixture durable coordination target
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" build \
--file "$ROOT/Dockerfile" \
--target test-runtime \
--tag "$APP_IMAGE" \
"$ROOT"
load_local_image "$APP_IMAGE" "$NERDCTL_NAMESPACE"
while IFS=$'\t' read -r lane coordination oidc target; do
if [[ -n "${NUTHATCH_PROTOCOL_TARGET:-}" ]] &&
[[ "$target" != "$NUTHATCH_PROTOCOL_TARGET" ]]; then
continue
fi
profile="$(write_lane_profile "$lane" redis - "$target")"
printf 'Running protocol lane %s: target=%s\n' "$lane" "$target"
run_mode "$lane" "$profile" full
verify_mapping "$lane"
start_mode "$lane" "$profile" pg-prepare
wait_for_mode_log "$lane" pg-prepare "PostgreSQL transition fixture ready"
printf 'Checking PostgreSQL loss and restoration for %s\n' "$target"
set_postgres_available false
active_connections="$(database_connection_count "nuthatch_lane_$lane")"
test "$active_connections" -gt 0
terminate_database_connections "nuthatch_lane_$lane" || true
remaining_connections="$(database_connection_count "nuthatch_lane_$lane")"
test "$remaining_connections" = 0
stop_mode "$lane" pg-prepare
run_mode "$lane" "$profile" pg-fence
set_postgres_available true
run_mode "$lane" "$profile" pg-restored
kube --namespace "$FIXTURE_NAMESPACE" logs "job/protocol-pg-fence-$lane" |
grep -F "protocol action covered: LosePostgres" >/dev/null
kube --namespace "$FIXTURE_NAMESPACE" logs "job/protocol-pg-restored-$lane" |
grep -F "protocol action covered: RestorePostgres" >/dev/null
kube --namespace "$FIXTURE_NAMESPACE" logs "job/protocol-pg-restored-$lane" |
grep -F "protocol invariant checked: PostgresFailureFencesDurableState" \
>/dev/null
rg -q '^LosePostgres(\(| ==)' "$ROOT/docs/protocol/Nuthatch.tla"
rg -q '^RestorePostgres(\(| ==)' "$ROOT/docs/protocol/Nuthatch.tla"
rg -q '^PostgresFailureFencesDurableState(\(| ==)' \
"$ROOT/docs/protocol/Nuthatch.tla"
run_mode "$lane" "$profile" store-prepare
printf 'Checking store-down admission fence for %s\n' "$target"
set_target_available "$target" false
run_mode "$lane" "$profile" fence
printf 'Checking restored store observation for %s\n' "$target"
set_target_available "$target" true
run_mode "$lane" "$profile" restored
kube --namespace "$FIXTURE_NAMESPACE" logs "job/protocol-fence-$lane" |
grep -F "store-down rejection creates no protocol record" >/dev/null
kube --namespace "$FIXTURE_NAMESPACE" logs "job/protocol-restored-$lane" |
grep -F "protocol action covered: RestoreObjectStore" >/dev/null
kube --namespace "$FIXTURE_NAMESPACE" logs "job/protocol-restored-$lane" |
grep -F "protocol invariant checked: ObjectStoreFailureFencesObjects" >/dev/null
rg -q '^LoseObjectStore(\(| ==)' "$ROOT/docs/protocol/Nuthatch.tla"
rg -q '^RestoreObjectStore(\(| ==)' "$ROOT/docs/protocol/Nuthatch.tla"
rg -q '^ObjectStoreFailureFencesObjects(\(| ==)' \
"$ROOT/docs/protocol/Nuthatch.tla"
cleanup_lane_state "$lane" redis - "$target"
cleanup_lane_resources "protocol-full-$lane" "$profile"
cleanup_lane_resources "protocol-pg-prepare-$lane" "$profile"
cleanup_lane_resources "protocol-pg-fence-$lane" "$profile"
cleanup_lane_resources "protocol-pg-restored-$lane" "$profile"
cleanup_lane_resources "protocol-store-prepare-$lane" "$profile"
cleanup_lane_resources "protocol-fence-$lane" "$profile"
cleanup_lane_resources "protocol-restored-$lane" "$profile"
rm -f "$RUNTIME/protocol-full-$lane.log"
done < <(expand_fixture_matrix target)
SUCCEEDED=true
printf 'Nuthatch protocol conformance gate passed\n'

Some files were not shown because too many files have changed in this diff Show more