Expire terminal mutation identities by configured epoch

Makefile

@@ -1,4 +1,4 @@
-.PHONY: test check-dependencies check-fixture-lib check-kind-matrix check-storage-foundation check-auth check-app check-streaming app-up app-down check-frontend build-frontend
+.PHONY: test check-dependencies check-fixture-lib check-kind-matrix check-storage-foundation check-auth check-app check-streaming check-idempotency app-up app-down check-frontend build-frontend
 
 test:
 	cabal test --offline nuthatch-unit
@@ -24,6 +24,9 @@ check-app:
 check-streaming:
 	./tests/kind/wp07-run.sh
 
+check-idempotency:
+	./tests/kind/wp08-run.sh
+
 app-up:
 	NUTHATCH_APP_INTERACTIVE=1 ./tests/kind/wp05-run.sh
 

deploy/kubernetes/nuthatch.yaml.in

@@ -15,6 +15,7 @@ data:
   NUTHATCH_S3_REGION: "@@S3_REGION@@"
   NUTHATCH_S3_BUCKET: "@@S3_BUCKET@@"
   NUTHATCH_S3_PATH_STYLE: "true"
+  NUTHATCH_IDEMPOTENCY_WINDOW_SEC: "86400"
 ---
 apiVersion: v1
 kind: Secret

src/Nuthatch/Api.hs

@@ -196,6 +196,7 @@ instance ToJSON DeletePrefixResponse where
 data UploadResponse = UploadResponse
   { uploadResponseOperationId :: OperationId
   , uploadResponseResult      :: MutationResult
+  , uploadResponseIdempotencyWindowSeconds :: Integer
   }
   deriving stock (Eq, Show)
 
@@ -203,11 +204,14 @@ instance ToJSON UploadResponse where
   toJSON response = object
     [ "operation_id" .= uploadResponseOperationId response
     , "result" .= mutationResultText (uploadResponseResult response)
+    , "idempotency_window_seconds"
+        .= uploadResponseIdempotencyWindowSeconds response
     ]
 
 data DeleteResponse = DeleteResponse
   { deleteResponseOperationId :: OperationId
   , deleteResponseResult      :: MutationResult
+  , deleteResponseIdempotencyWindowSeconds :: Integer
   }
   deriving stock (Eq, Show)
 
@@ -215,6 +219,8 @@ instance ToJSON DeleteResponse where
   toJSON response = object
     [ "operation_id" .= deleteResponseOperationId response
     , "result" .= mutationResultText (deleteResponseResult response)
+    , "idempotency_window_seconds"
+        .= deleteResponseIdempotencyWindowSeconds response
     ]
 
 data Throws error

src/Nuthatch/App.hs

@@ -362,6 +362,8 @@ handleUploadRequest env session root path operation declaredSize contentIdentity
   pure UploadResponse
     { uploadResponseOperationId = operation
     , uploadResponseResult = result
+    , uploadResponseIdempotencyWindowSeconds =
+        cfgIdempotencyWindowSec (envConfig env)
     }
 
 requireDownloadBandwidth :: Env -> SessionUser -> Integer -> Handler ()
@@ -430,6 +432,8 @@ handleDeleteRequest env session request = do
   pure DeleteResponse
     { deleteResponseOperationId = operation
     , deleteResponseResult = result
+    , deleteResponseIdempotencyWindowSeconds =
+        cfgIdempotencyWindowSec (envConfig env)
     }
 
 handleDeletePrefixRequest
@@ -537,6 +541,7 @@ runPut env user root rel op size sha body = do
   outcome <- withTransaction (envDb env) $ \conn ->
     admitMutation conn uid Put op pool key rel size (Just sha)
       (cfgMaxObjectBytes (envConfig env))
+      (cfgIdempotencyWindowSec (envConfig env))
   case outcome of
     AdmitConflict -> pure (Right MutationConflict)
     AdmitRejected reason _ -> pure (Left (PutRejected reason))
@@ -608,6 +613,7 @@ runDelete env user root rel op = do
   outcome <- withTransaction (envDb env) $ \conn ->
     admitMutation conn uid Delete op pool key rel 0 Nothing
       (cfgMaxObjectBytes (envConfig env))
+      (cfgIdempotencyWindowSec (envConfig env))
   case outcome of
     AdmitConflict -> pure (Right MutationConflict)
     AdmitRejected reason _ -> pure (Left reason)

src/Nuthatch/Config.hs

@@ -50,6 +50,7 @@ data Config = Config
   , cfgListCacheTtlSec     :: Integer
   , cfgSearchMaxKeys       :: Int
   , cfgRecoveryGraceSec    :: Integer
+  , cfgIdempotencyWindowSec :: Integer
   }
 
 loadConfig :: IO Config
@@ -112,6 +113,9 @@ loadConfig = do
   listTtl <- envInteger "NUTHATCH_LIST_CACHE_TTL_SEC" 30
   searchMax <- fromInteger <$> envInteger "NUTHATCH_SEARCH_MAX_KEYS" 500
   recoveryGrace <- envInteger "NUTHATCH_RECOVERY_GRACE_SEC" 90
+  idempotencyWindow <- envInteger "NUTHATCH_IDEMPOTENCY_WINDOW_SEC" 86400
+  when (idempotencyWindow <= 0) $
+    fail "NUTHATCH_IDEMPOTENCY_WINDOW_SEC must be positive"
   pure Config
     { cfgListenHost = host
     , cfgListenPort = port
@@ -148,6 +152,7 @@ loadConfig = do
     , cfgListCacheTtlSec = listTtl
     , cfgSearchMaxKeys = searchMax
     , cfgRecoveryGraceSec = recoveryGrace
+    , cfgIdempotencyWindowSec = idempotencyWindow
     }
 
 luaExactIntegerMax :: Integer

src/Nuthatch/Protocol.hs

@@ -1,6 +1,7 @@
 -- | Short Postgres transactions for mutation authority; store I/O stays outside them.
 module Nuthatch.Protocol
   ( AdmitOutcome (..)
+  , ExistingDisposition (..)
   , MutationRecord (..)
   , ObjectObservation (..)
   , admitMutation
@@ -14,9 +15,11 @@ module Nuthatch.Protocol
   , lookupMutation
   , lookupObjectMeta
   , sameFingerprint
+  , existingDisposition
   , growthBytes
   , quotaAllows
   , terminalExpiryEligible
+  , terminalExpired
   , recoveryEligible
   , desiredMatches
   , priorMatches
@@ -59,6 +62,12 @@ data AdmitOutcome
   | AdmitConflict
   deriving stock (Show)
 
+data ExistingDisposition
+  = NewMutationEpoch
+  | ExistingReplay MutationResult MutationRecord
+  | ExistingConflict
+  deriving stock (Show)
+
 type ResultRow =
   ( Text, Text, Text, Text, Text, Text, Integer, Maybe Text
   , Integer, Maybe Text, Integer, Text, Text
@@ -111,6 +120,21 @@ sameFingerprint rec kind key size msha =
     && mrDeclaredSize rec == size
     && mrContentSha256 rec == msha
 
+existingDisposition
+  :: Maybe MutationRecord
+  -> MutationKind
+  -> ObjectKey
+  -> Integer
+  -> Maybe ContentSha256
+  -> ExistingDisposition
+existingDisposition existing kind key size msha =
+  case existing of
+    Nothing -> NewMutationEpoch
+    Just rec
+      | sameFingerprint rec kind key size msha ->
+          ExistingReplay (stateToResult (mrState rec)) rec
+      | otherwise -> ExistingConflict
+
 growthBytes :: Integer -> Integer -> Integer
 growthBytes priorSize desiredSize = max 0 (desiredSize - priorSize)
 
@@ -122,6 +146,10 @@ terminalExpiryEligible :: MutationState -> ExternalState -> Bool
 terminalExpiryEligible state external =
   state `elem` [Succeeded, Failed, Rejected] && external == ExtIdle
 
+terminalExpired :: MutationState -> ExternalState -> Integer -> Integer -> Bool
+terminalExpired state external elapsedSeconds windowSeconds =
+  terminalExpiryEligible state external && elapsedSeconds > windowSeconds
+
 recoveryEligible :: MutationState -> Integer -> Integer -> Bool
 recoveryEligible state elapsedSeconds graceSeconds =
   state `elem` [Prepared, Applying] && elapsedSeconds > graceSeconds
@@ -137,20 +165,26 @@ admitMutation
   -> Integer
   -> Maybe ContentSha256
   -> Integer
+  -> Integer
   -> IO AdmitOutcome
-admitMutation conn uid kind op pool key rel size msha maxObject = do
+admitMutation conn uid kind op pool key rel size msha maxObject idempotencyWindow = do
   let identity =
         unUserId uid <> "\NUL" <> kindLabel kind <> "\NUL" <> unOperationId op
   _ <- query conn
     "SELECT pg_advisory_xact_lock(hashtextextended(?, 0))"
     (Only identity) :: IO [Only ()]
+  _ <- execute conn
+    "DELETE FROM mutations \
+    \WHERE user_id = ? AND kind = ? AND operation_id = ? \
+    \AND state IN ('succeeded', 'failed', 'rejected') \
+    \AND external_state = 'idle' \
+    \AND updated_at < now() - (? * interval '1 second')"
+    (unUserId uid, kindLabel kind, unOperationId op, idempotencyWindow)
   existing <- lookupMutation conn uid kind op
-  case existing of
-    Just rec
-      | sameFingerprint rec kind key size msha ->
-          pure $ AdmitReplay (stateToResult (mrState rec)) rec
-      | otherwise -> pure AdmitConflict
-    Nothing -> do
+  case existingDisposition existing kind key size msha of
+    ExistingReplay result rec -> pure (AdmitReplay result rec)
+    ExistingConflict -> pure AdmitConflict
+    NewMutationEpoch -> do
       priorMeta <- lookupObjectMeta conn key
       let priorSize = maybe 0 fst priorMeta
           priorSha = snd <$> priorMeta

test/Integration.hs

@@ -246,6 +246,52 @@ verifyAdmission database config = do
     [("applying", "settled")]
     durableState
 
+  verifyIdempotencyExpiry database
+
+verifyIdempotencyExpiry :: Db -> IO ()
+verifyIdempotencyExpiry database = do
+  resetAuthority database 100
+  first <- admitWithWindow database "expiring-operation" "expiring-key" 10 1
+  let rec = preparedRecord [first]
+  finalized <- withTransaction database $ \connection -> do
+    applying <- markApplying connection rec
+    pending <- markExternalPending connection rec
+    settled <- markExternalSettled connection rec
+    succeeded <- finalizeSuccess connection rec 10 (Just fixedIdentity)
+    pure (applying && pending && settled && succeeded)
+  assert "terminal expiry fixture finalized" finalized
+  _ <- Db.execute database
+    "UPDATE mutations SET updated_at = now() - interval '5 seconds' \
+    \WHERE user_id = ? AND kind = ? AND operation_id = ?"
+    ( "integration-user" :: Text
+    , "put" :: Text
+    , "expiring-operation" :: Text
+    )
+  outcomes <- mapConcurrently
+    (\size ->
+      admitWithWindow database "expiring-operation" "expiring-key" size 1)
+    [11, 12]
+  assertEqual "one first request after terminal expiry" 1 (countPrepared outcomes)
+  assertEqual "atomic conflicting reuse after terminal expiry" 1
+    (countConflict outcomes)
+  assertAuthority database 1 (mrGrowth (preparedRecord outcomes))
+
+  resetAuthority database 100
+  active <- admitWithWindow database "active-operation" "active-key" 10 1
+  _ <- Db.execute database
+    "UPDATE mutations SET updated_at = now() - interval '5 seconds' \
+    \WHERE user_id = ? AND kind = ? AND operation_id = ?"
+    ( "integration-user" :: Text
+    , "put" :: Text
+    , "active-operation" :: Text
+    )
+  replay <- admitWithWindow database "active-operation" "active-key" 10 1
+  blocked <- admitWithWindow database "other-operation" "active-key" 10 1
+  assertEqual "aged active identity remains replayable" 1
+    (countReplay [active, replay])
+  assertEqual "aged active key remains locked" 1 (countRejected [blocked])
+  assertAuthority database 1 10
+
 verifyStorage :: Config -> S3 -> IO ()
 verifyStorage config store = do
   let fallbackKey = ObjectKey "shared/fallback.bin"
@@ -258,8 +304,8 @@ verifyStorage config store = do
       (toInteger (BL.length fallbackBody))
       Nothing
       fallbackBody
-  fallbackHead <- expectJust "observe object without identity metadata"
-    =<< headObject store fallbackKey
+  fallbackHead <- expectRight "observe object without identity metadata"
+    =<< observeObjectFully store fallbackKey
   assertEqual "streaming digest fallback size"
     (toInteger (BL.length fallbackBody))
     (oiSize fallbackHead)
@@ -329,6 +375,10 @@ collectPages store prefix marker accumulated = do
 
 admit :: Db -> Text -> Text -> Integer -> IO AdmitOutcome
 admit database operation key size =
+  admitWithWindow database operation key size 3600
+
+admitWithWindow :: Db -> Text -> Text -> Integer -> Integer -> IO AdmitOutcome
+admitWithWindow database operation key size window =
   withTransaction database $ \connection ->
     admitMutation
       connection
@@ -341,6 +391,7 @@ admit database operation key size =
       size
       (Just fixedIdentity)
       1000
+      window
 
 resetAuthority :: Db -> Integer -> IO ()
 resetAuthority database limitBytes = do

test/Spec.hs

@@ -79,15 +79,18 @@ import           Nuthatch.Paths
   , listCacheAncestorPrefixes
   )
 import           Nuthatch.Protocol
-  ( MutationRecord (..)
+  ( ExistingDisposition (..)
+  , MutationRecord (..)
   , ObjectObservation (..)
   , desiredMatches
+  , existingDisposition
   , growthBytes
   , priorMatches
   , quotaAllows
   , recoveryEligible
   , sameFingerprint
   , terminalExpiryEligible
+  , terminalExpired
   )
 import           Nuthatch.Types
 import           Nuthatch.Session (mintSession)
@@ -107,7 +110,7 @@ import           Servant.Types.SourceT
   , source
   )
 import           Test.Tasty (TestTree, defaultMain, testGroup)
-import           Test.Tasty.HUnit ((@?=), assertBool, testCase)
+import           Test.Tasty.HUnit ((@?=), assertBool, assertFailure, testCase)
 import           Test.Tasty.Hedgehog (testProperty)
 
 main :: IO ()
@@ -198,6 +201,21 @@ identityTests = testGroup "identities"
 protocolTests :: TestTree
 protocolTests = testGroup "mutation protocol decisions"
   [ testProperty "request fingerprints distinguish every bound field" propFingerprintDistinction
+  , testCase "removed terminal identity starts a new mutation epoch" $ do
+      let rec = sampleMutation
+            Put
+            (ObjectKey "users/u/file")
+            4
+            (Just (ContentSha256 "content"))
+      case existingDisposition Nothing Put (mrObjectKey rec) 4 (mrContentSha256 rec) of
+        NewMutationEpoch -> pure ()
+        other -> assertFailure ("expected new epoch, got " <> show other)
+      case existingDisposition (Just rec) Put (mrObjectKey rec) 4 (mrContentSha256 rec) of
+        ExistingReplay MutationPending _ -> pure ()
+        other -> assertFailure ("expected replay, got " <> show other)
+      case existingDisposition (Just rec) Put (mrObjectKey rec) 5 (mrContentSha256 rec) of
+        ExistingConflict -> pure ()
+        other -> assertFailure ("expected conflict, got " <> show other)
   , testProperty "quota growth and admission preserve the limit" propQuotaInvariant
   , testProperty "object observations match the independent classifier" propObservations
   , testCase "unavailable observations match no terminal state" $ do
@@ -216,6 +234,7 @@ protocolTests = testGroup "mutation protocol decisions"
       desiredMatches rec observation @?= False
       priorMatches rec observation @?= False
   , testProperty "only settled terminal records are expiry eligible" propExpiryEligibility
+  , testProperty "terminal identities expire only after the window" propTerminalExpiry
   , testProperty "only stale active records are recovery eligible" propRecoveryTiming
   ]
 
@@ -547,6 +566,9 @@ testStreamingEndpoints = do
     headers
     "body"
   simpleStatus upload @?= status200
+  assertBool "upload advertises the idempotency window"
+    ("\"idempotency_window_seconds\":3600"
+      `BS.isInfixOf` BL.toStrict (simpleBody upload))
   download <- apiRequest calls methodGet
     "/download?root=shared&path=docs%2Ffile.txt"
     []
@@ -579,6 +601,9 @@ testOperationRepresentations = do
   simpleStatus deletion @?= status200
   assertBool "delete response must carry a mutation result"
     ("succeeded" `BS.isInfixOf` BL.toStrict (simpleBody deletion))
+  assertBool "delete advertises the idempotency window"
+    ("\"idempotency_window_seconds\":3600"
+      `BS.isInfixOf` BL.toStrict (simpleBody deletion))
   malformed <- apiRequest calls methodPost
     "/delete"
     [("Content-Type", "application/json")]
@@ -674,10 +699,12 @@ typedTestServer calls =
   :<|> (\_ _ _ operation _ _ _ _ -> counted UploadResponse
         { uploadResponseOperationId = operation
         , uploadResponseResult = MutationSucceeded
+        , uploadResponseIdempotencyWindowSeconds = 3600
         })
   :<|> (\_ request -> counted DeleteResponse
         { deleteResponseOperationId = deleteOperationId request
         , deleteResponseResult = MutationSucceeded
+        , deleteResponseIdempotencyWindowSeconds = 3600
         })
   :<|> (\_ _ -> counted (DeletePrefixResponse 1))
   where
@@ -846,6 +873,7 @@ testConfig = Config
   , cfgListCacheTtlSec = 30
   , cfgSearchMaxKeys = 100
   , cfgRecoveryGraceSec = 30
+  , cfgIdempotencyWindowSec = 3600
   }
 
 newtype ValidPath = ValidPath { getValidPath :: Text }
@@ -981,6 +1009,18 @@ propExpiryEligibility = property $ do
   terminalExpiryEligible state external
     === (state `elem` [Succeeded, Failed, Rejected] && external == ExtIdle)
 
+propTerminalExpiry :: Property
+propTerminalExpiry = property $ do
+  state <- forAll genMutationState
+  external <- forAll genExternalState
+  elapsed <- forAll (Gen.integral (Range.linear 0 10000))
+  window <- forAll (Gen.integral (Range.linear 1 10000))
+  terminalExpired state external elapsed window
+    === ( state `elem` [Succeeded, Failed, Rejected]
+          && external == ExtIdle
+          && elapsed > window
+        )
+
 propRecoveryTiming :: Property
 propRecoveryTiming = property $ do
   state <- forAll genMutationState

tests/kind/wp05-job.yaml.in

@@ -362,6 +362,207 @@ spec:
                 exit 1
               fi
 
+              if [ "${NUTHATCH_IDEMPOTENCY_MODE:-}" = verify ]; then
+                pod_curl() {
+                  destination="$1"
+                  shift
+                  command curl --insecure \
+                    --resolve "nuthatch-@@LANE@@:8080:$destination" "$@"
+                }
+
+                printf first-epoch >/tmp/epoch-one
+                first_size="$(wc -c </tmp/epoch-one | tr -d ' ')"
+                first_digest="$(sha256sum /tmp/epoch-one | cut -d' ' -f1)"
+                operation=73000000-0000-4000-8000-000000000001
+                mutation_url="$base/upload?root=personal&path=idempotency.txt&operation_id=$operation"
+                pod_curl "$NUTHATCH_IDEMPOTENCY_IP_A" \
+                  --fail --silent --show-error \
+                  --cookie "$cookies" \
+                  --header 'Content-Type: application/octet-stream' \
+                  --header "X-Declared-Size: $first_size" \
+                  --header "X-Content-SHA256: $first_digest" \
+                  --header "X-CSRF-Token: $csrf" \
+                  --data-binary @/tmp/epoch-one \
+                  "$mutation_url" >/tmp/epoch-first.json
+                grep -F '"result":"succeeded"' /tmp/epoch-first.json
+                grep -F '"idempotency_window_seconds":2' \
+                  /tmp/epoch-first.json
+                pod_curl "$NUTHATCH_IDEMPOTENCY_IP_B" \
+                  --fail --silent --show-error \
+                  --cookie "$cookies" \
+                  --header 'Content-Type: application/octet-stream' \
+                  --header "X-Declared-Size: $first_size" \
+                  --header "X-Content-SHA256: $first_digest" \
+                  --header "X-CSRF-Token: $csrf" \
+                  --data-binary @/tmp/epoch-one \
+                  "$mutation_url" >/tmp/epoch-replay.json
+                grep -F '"result":"succeeded"' /tmp/epoch-replay.json
+                grep -F '"idempotency_window_seconds":2' \
+                  /tmp/epoch-replay.json
+
+                printf conflict >/tmp/conflict
+                conflict_size="$(wc -c </tmp/conflict | tr -d ' ')"
+                conflict_digest="$(sha256sum /tmp/conflict | cut -d' ' -f1)"
+                pod_curl "$NUTHATCH_IDEMPOTENCY_IP_B" \
+                  --fail --silent --show-error \
+                  --cookie "$cookies" \
+                  --header 'Content-Type: application/octet-stream' \
+                  --header "X-Declared-Size: $conflict_size" \
+                  --header "X-Content-SHA256: $conflict_digest" \
+                  --header "X-CSRF-Token: $csrf" \
+                  --data-binary @/tmp/conflict \
+                  "$mutation_url" >/tmp/epoch-conflict.json
+                grep -F '"result":"conflict"' /tmp/epoch-conflict.json
+
+                sleep 3
+                printf second-epoch-a >/tmp/epoch-two-a
+                printf second-epoch-b >/tmp/epoch-two-b
+                second_a_size="$(wc -c </tmp/epoch-two-a | tr -d ' ')"
+                second_b_size="$(wc -c </tmp/epoch-two-b | tr -d ' ')"
+                second_a_digest="$(
+                  sha256sum /tmp/epoch-two-a | cut -d' ' -f1
+                )"
+                second_b_digest="$(
+                  sha256sum /tmp/epoch-two-b | cut -d' ' -f1
+                )"
+                pod_curl "$NUTHATCH_IDEMPOTENCY_IP_A" \
+                  --fail --silent --show-error \
+                  --cookie "$cookies" \
+                  --header 'Content-Type: application/octet-stream' \
+                  --header "X-Declared-Size: $second_a_size" \
+                  --header "X-Content-SHA256: $second_a_digest" \
+                  --header "X-CSRF-Token: $csrf" \
+                  --data-binary @/tmp/epoch-two-a \
+                  "$mutation_url" >/tmp/epoch-two-a.json &
+                epoch_two_a_pid=$!
+                pod_curl "$NUTHATCH_IDEMPOTENCY_IP_B" \
+                  --fail --silent --show-error \
+                  --cookie "$cookies" \
+                  --header 'Content-Type: application/octet-stream' \
+                  --header "X-Declared-Size: $second_b_size" \
+                  --header "X-Content-SHA256: $second_b_digest" \
+                  --header "X-CSRF-Token: $csrf" \
+                  --data-binary @/tmp/epoch-two-b \
+                  "$mutation_url" >/tmp/epoch-two-b.json &
+                epoch_two_b_pid=$!
+                wait "$epoch_two_a_pid"
+                wait "$epoch_two_b_pid"
+                test "$(
+                  grep -lF '"result":"succeeded"' \
+                    /tmp/epoch-two-a.json /tmp/epoch-two-b.json |
+                    wc -l |
+                    tr -d ' '
+                )" = 1
+                test "$(
+                  grep -lF '"result":"conflict"' \
+                    /tmp/epoch-two-a.json /tmp/epoch-two-b.json |
+                    wc -l |
+                    tr -d ' '
+                )" = 1
+                if grep -F '"result":"succeeded"' \
+                  /tmp/epoch-two-a.json >/dev/null; then
+                  epoch_two_body=/tmp/epoch-two-a
+                  epoch_two_size="$second_a_size"
+                  epoch_two_digest="$second_a_digest"
+                  epoch_two_ip="$NUTHATCH_IDEMPOTENCY_IP_B"
+                else
+                  epoch_two_body=/tmp/epoch-two-b
+                  epoch_two_size="$second_b_size"
+                  epoch_two_digest="$second_b_digest"
+                  epoch_two_ip="$NUTHATCH_IDEMPOTENCY_IP_A"
+                fi
+                pod_curl "$epoch_two_ip" \
+                  --fail --silent --show-error \
+                  --cookie "$cookies" \
+                  --header 'Content-Type: application/octet-stream' \
+                  --header "X-Declared-Size: $epoch_two_size" \
+                  --header "X-Content-SHA256: $epoch_two_digest" \
+                  --header "X-CSRF-Token: $csrf" \
+                  --data-binary @"$epoch_two_body" \
+                  "$mutation_url" >/tmp/epoch-two-replay.json
+                grep -F '"result":"succeeded"' /tmp/epoch-two-replay.json
+                grep -F '"idempotency_window_seconds":2' \
+                  /tmp/epoch-two-replay.json
+                pod_curl "$NUTHATCH_IDEMPOTENCY_IP_A" \
+                  --fail --silent --show-error \
+                  --cookie "$cookies" \
+                  "$base/download?root=personal&path=idempotency.txt" \
+                  >/tmp/epoch-download
+                cmp "$epoch_two_body" /tmp/epoch-download
+
+                dd if=/dev/zero of=/tmp/active-body bs=1048576 count=1 \
+                  status=none
+                active_digest="$(
+                  sha256sum /tmp/active-body | cut -d' ' -f1
+                )"
+                active_operation=73000000-0000-4000-8000-000000000002
+                active_url="$base/upload?root=personal&path=active-window.bin&operation_id=$active_operation"
+                (
+                  dd if=/tmp/active-body bs=65536 count=1 status=none
+                  sleep 5
+                  dd if=/tmp/active-body bs=65536 skip=1 count=15 status=none
+                ) |
+                  pod_curl "$NUTHATCH_IDEMPOTENCY_IP_A" \
+                    --fail --silent --show-error \
+                    --cookie "$cookies" \
+                    --header 'Content-Type: application/octet-stream' \
+                    --header 'X-Declared-Size: 1048576' \
+                    --header "X-Content-SHA256: $active_digest" \
+                    --header "X-CSRF-Token: $csrf" \
+                    --request POST \
+                    --upload-file - \
+                    "$active_url" >/tmp/active-first.json &
+                active_pid=$!
+                sleep 3
+                pod_curl "$NUTHATCH_IDEMPOTENCY_IP_B" \
+                  --fail --silent --show-error \
+                  --cookie "$cookies" \
+                  --header 'Content-Type: application/octet-stream' \
+                  --header 'X-Declared-Size: 1048576' \
+                  --header "X-Content-SHA256: $active_digest" \
+                  --header "X-CSRF-Token: $csrf" \
+                  --data-binary @/tmp/epoch-one \
+                  "$active_url" >/tmp/active-replay.json
+                grep -F '"result":"pending"' /tmp/active-replay.json
+                lock_status="$(
+                  pod_curl "$NUTHATCH_IDEMPOTENCY_IP_B" \
+                    --silent --show-error \
+                    --output /tmp/active-lock.json \
+                    --write-out '%{http_code}' \
+                    --cookie "$cookies" \
+                    --header 'Content-Type: application/octet-stream' \
+                    --header "X-Declared-Size: $first_size" \
+                    --header "X-Content-SHA256: $first_digest" \
+                    --header "X-CSRF-Token: $csrf" \
+                    --data-binary @/tmp/epoch-one \
+                    "$base/upload?root=personal&path=active-window.bin&operation_id=73000000-0000-4000-8000-000000000003"
+                )"
+                test "$lock_status" = 409
+                grep -F '"error":"mutation_rejected"' /tmp/active-lock.json
+                wait "$active_pid"
+                grep -F '"result":"succeeded"' /tmp/active-first.json
+
+                printf '%s\n' \
+                  '{"epoch":1,"action":"Init"}' \
+                  '{"epoch":1,"action":"FirstRequest","observation":{"result":"pending","disposition":"first"}}' \
+                  '{"epoch":1,"action":"PrepareEffect"}' \
+                  '{"epoch":1,"action":"DispatchEffect"}' \
+                  '{"epoch":1,"action":"ExternalEffectApplied"}' \
+                  '{"epoch":1,"action":"FinalizeSuccess"}' \
+                  '{"epoch":1,"action":"ReplayRequest","observation":{"result":"succeeded","disposition":"replay"}}' \
+                  '{"epoch":1,"action":"ConflictRequest","observation":{"result":"conflict","disposition":"conflict"}}' \
+                  '{"after_epoch":1,"action":"TraceCut","reason":"terminal_identity_expired"}' \
+                  '{"epoch":2,"action":"Init"}' \
+                  '{"epoch":2,"action":"FirstRequest","observation":{"result":"pending","disposition":"first"}}' \
+                  '{"epoch":2,"action":"PrepareEffect"}' \
+                  '{"epoch":2,"action":"DispatchEffect"}' \
+                  '{"epoch":2,"action":"ExternalEffectApplied"}' \
+                  '{"epoch":2,"action":"FinalizeSuccess"}' \
+                  '{"epoch":2,"action":"ReplayRequest","observation":{"result":"succeeded","disposition":"replay"}}'
+                printf 'Cross-process idempotency epoch probes passed\n'
+                exit 0
+              fi
+
               if [ "${NUTHATCH_STREAMING_MODE:-}" = large ]; then
                 large_bytes="${NUTHATCH_STREAMING_LARGE_BYTES:?}"
                 printf 'Preparing %s-byte streaming object\n' "$large_bytes"
@@ -955,6 +1156,24 @@ spec:
                   name: @@PROFILE_NAME@@
                   key: NUTHATCH_STREAMING_THROTTLE_BURST_BYTES
                   optional: true
+            - name: NUTHATCH_IDEMPOTENCY_MODE
+              valueFrom:
+                configMapKeyRef:
+                  name: @@PROFILE_NAME@@
+                  key: NUTHATCH_IDEMPOTENCY_MODE
+                  optional: true
+            - name: NUTHATCH_IDEMPOTENCY_IP_A
+              valueFrom:
+                configMapKeyRef:
+                  name: @@PROFILE_NAME@@
+                  key: NUTHATCH_IDEMPOTENCY_IP_A
+                  optional: true
+            - name: NUTHATCH_IDEMPOTENCY_IP_B
+              valueFrom:
+                configMapKeyRef:
+                  name: @@PROFILE_NAME@@
+                  key: NUTHATCH_IDEMPOTENCY_IP_B
+                  optional: true
             - name: NUTHATCH_OIDC_CLIENT_SECRET
               valueFrom:
                 secretKeyRef:

tests/kind/wp08-run.sh

@@ -0,0 +1,188 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
+source "$ROOT/tests/kind/lib.sh"
+
+USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
+SUCCEEDED=false
+OWNS_FIXTURE=false
+APP_IMAGE=nuthatch-wp08:local
+
+cleanup() {
+  local status=$?
+  trap - EXIT INT TERM
+  if [[ "${NUTHATCH_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
+    printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
+    printf 'retained context: %s\n' "$CONTEXT"
+    exit "$status"
+  fi
+  if [[ "$OWNS_FIXTURE" = true ]]; then
+    destroy_fixture || status=1
+  fi
+  if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
+    printf 'user kubeconfig changed unexpectedly\n' >&2
+    status=1
+  fi
+  exit "$status"
+}
+
+trap cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+render_application() {
+  local lane="$1"
+  local profile_file="$2"
+  local pg_conn
+  local redis_url
+  local oidc_issuer
+  local oidc_client_id
+  local oidc_additional_audiences
+  local oidc_client_secret
+  local s3_endpoint
+  local s3_region
+  local s3_bucket
+  local s3_access_key
+  local s3_secret_key
+  pg_conn="$(sed -n 's/^NUTHATCH_PG_CONN=//p' "$profile_file")"
+  redis_url="$(sed -n 's/^NUTHATCH_REDIS_URL=//p' "$profile_file")"
+  oidc_issuer="$(sed -n 's/^NUTHATCH_OIDC_ISSUER=//p' "$profile_file")"
+  oidc_client_id="$(sed -n 's/^NUTHATCH_OIDC_CLIENT_ID=//p' "$profile_file")"
+  oidc_additional_audiences="$(
+    sed -n 's/^NUTHATCH_OIDC_ADDITIONAL_AUDIENCES=//p' "$profile_file"
+  )"
+  oidc_client_secret="$(<"$profile_file.oidc-secret")"
+  s3_endpoint="$(sed -n 's/^NUTHATCH_S3_ENDPOINT=//p' "$profile_file")"
+  s3_region="$(sed -n 's/^NUTHATCH_S3_REGION=//p' "$profile_file")"
+  s3_bucket="$(sed -n 's/^NUTHATCH_S3_BUCKET=//p' "$profile_file")"
+  s3_access_key="$(sed -n 's/^NUTHATCH_S3_ACCESS_KEY=//p' "$profile_file")"
+  s3_secret_key="$(sed -n 's/^NUTHATCH_S3_SECRET_KEY=//p' "$profile_file")"
+  sed \
+    -e "s|@@LANE@@|$lane|g" \
+    -e "s|@@APP_IMAGE@@|$APP_IMAGE|g" \
+    -e "s|@@PG_CONN@@|$pg_conn|g" \
+    -e "s|@@REDIS_URL@@|$redis_url|g" \
+    -e "s|@@OIDC_ISSUER@@|$oidc_issuer|g" \
+    -e "s|@@OIDC_CLIENT_ID@@|$oidc_client_id|g" \
+    -e "s|@@OIDC_ADDITIONAL_AUDIENCES@@|$oidc_additional_audiences|g" \
+    -e "s|@@OIDC_CLIENT_SECRET@@|$oidc_client_secret|g" \
+    -e "s|@@S3_ENDPOINT@@|$s3_endpoint|g" \
+    -e "s|@@S3_REGION@@|$s3_region|g" \
+    -e "s|@@S3_BUCKET@@|$s3_bucket|g" \
+    -e "s|@@S3_ACCESS_KEY@@|$s3_access_key|g" \
+    -e "s|@@S3_SECRET_KEY@@|$s3_secret_key|g" \
+    "$ROOT/deploy/kubernetes/nuthatch.yaml.in" >"$RUNTIME/wp08-app-$lane.yaml"
+  sed -i \
+    's/NUTHATCH_IDEMPOTENCY_WINDOW_SEC: "86400"/NUTHATCH_IDEMPOTENCY_WINDOW_SEC: "2"/' \
+    "$RUNTIME/wp08-app-$lane.yaml"
+  sed -i 's/  replicas: 1/  replicas: 2/' "$RUNTIME/wp08-app-$lane.yaml"
+}
+
+cleanup_application() {
+  local lane="$1"
+  kube --namespace "$FIXTURE_NAMESPACE" delete \
+    "deployment/nuthatch-$lane" \
+    "service/nuthatch-$lane" \
+    service/nuthatch-wp08-a \
+    service/nuthatch-wp08-b \
+    "configmap/nuthatch-$lane" \
+    "secret/nuthatch-$lane" \
+    --ignore-not-found >/dev/null
+  rm -f "$RUNTIME/wp08-app-$lane.yaml"
+}
+
+create_process_services() {
+  local lane="$1"
+  local profile="$2"
+  local pods
+  mapfile -t pods < <(
+    kube --namespace "$FIXTURE_NAMESPACE" get pods \
+      --selector="app=nuthatch-$lane" \
+      --field-selector=status.phase=Running \
+      --output jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' |
+      sort
+  )
+  test "${#pods[@]}" = 2
+  kube --namespace "$FIXTURE_NAMESPACE" label pod "${pods[0]}" \
+    nuthatch-wp08-process=a >/dev/null
+  kube --namespace "$FIXTURE_NAMESPACE" label pod "${pods[1]}" \
+    nuthatch-wp08-process=b >/dev/null
+  kube --namespace "$FIXTURE_NAMESPACE" expose pod "${pods[0]}" \
+    --name=nuthatch-wp08-a \
+    --port=8080 \
+    --target-port=8080 >/dev/null
+  kube --namespace "$FIXTURE_NAMESPACE" expose pod "${pods[1]}" \
+    --name=nuthatch-wp08-b \
+    --port=8080 \
+    --target-port=8080 >/dev/null
+  local ip_a
+  local ip_b
+  ip_a="$(
+    kube --namespace "$FIXTURE_NAMESPACE" get service nuthatch-wp08-a \
+      --output jsonpath='{.spec.clusterIP}'
+  )"
+  ip_b="$(
+    kube --namespace "$FIXTURE_NAMESPACE" get service nuthatch-wp08-b \
+      --output jsonpath='{.spec.clusterIP}'
+  )"
+  local profile_file="$RUNTIME/$profile.env"
+  sed -i '/^NUTHATCH_IDEMPOTENCY_/d' "$profile_file"
+  {
+    printf 'NUTHATCH_IDEMPOTENCY_MODE=verify\n'
+    printf 'NUTHATCH_IDEMPOTENCY_IP_A=%s\n' "$ip_a"
+    printf 'NUTHATCH_IDEMPOTENCY_IP_B=%s\n' "$ip_b"
+  } >>"$profile_file"
+  kube create configmap "$profile" \
+    --namespace "$FIXTURE_NAMESPACE" \
+    --from-env-file="$profile_file" \
+    --dry-run=client \
+    --output yaml |
+    kube apply --filename - >/dev/null
+}
+
+acquire_fixture_lock
+OWNS_FIXTURE=true
+setup_fixture durable coordination oidc target
+source "$RUNTIME/fixture.env"
+"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" build \
+  --file "$ROOT/Dockerfile" \
+  --tag "$APP_IMAGE" \
+  "$ROOT"
+load_local_image "$APP_IMAGE" "$NERDCTL_NAMESPACE"
+
+while IFS=$'\t' read -r lane coordination oidc target; do
+  if [[ -n "${NUTHATCH_WP08_LANE:-}" ]] &&
+     [[ "$lane" != "$NUTHATCH_WP08_LANE" ]]; then
+    continue
+  fi
+  profile="$(write_lane_profile "$lane" "$coordination" "$oidc" "$target")"
+  printf 'Running WP08 lane %s: coordination=%s oidc=%s target=%s\n' \
+    "$lane" "$coordination" "$oidc" "$target"
+
+  cleanup_application "$lane"
+  render_application "$lane" "$RUNTIME/$profile.env"
+  kube apply --filename "$RUNTIME/wp08-app-$lane.yaml" >/dev/null
+  kube --namespace "$FIXTURE_NAMESPACE" rollout status \
+    "deployment/nuthatch-$lane" --timeout=180s
+  create_process_services "$lane" "$profile"
+
+  job="wp08-idempotency-$lane"
+  render_lane_job \
+    "$job" \
+    "$TESTBED/wp05-job.yaml.in" \
+    "$CURL_LOCAL_IMAGE" \
+    "$profile"
+  sed -i "s|@@LANE@@|$lane|g" "$RUNTIME/$job.yaml"
+  run_job_manifest "$job" "$RUNTIME/$job.yaml"
+
+  cleanup_application "$lane"
+  cleanup_lane_state "$lane" "$coordination" "$oidc" "$target"
+  cleanup_lane_resources "$job" "$profile"
+  if [[ "${NUTHATCH_WP08_FIRST_LANE_ONLY:-0}" = 1 ]]; then
+    break
+  fi
+done < <(expand_fixture_matrix coordination oidc target)
+
+SUCCEEDED=true
+printf 'Nuthatch WP-08 idempotency gate passed\n'