Model durable failure disposition

2026-07-13 16:41:51 +09:00 · 2026-07-13 16:41:51 +09:00 · 5fdb339fe5
commit 5fdb339fe5
parent 8c0ec9627e
12 changed files with 113 additions and 17 deletions
--- a/docs/protocol/Remora.cfg
+++ b/docs/protocol/Remora.cfg
@ -39,6 +39,7 @@ PROPERTIES
    TerminalStateIsImmutable
    FencingTokensMonotonic
    CommitIsFencedCAS
+    FailureDispositionHonored
    EventuallyDispatched
    EventuallyReaped
    EventuallyTerminal
--- a/docs/protocol/Remora.tla
+++ b/docs/protocol/Remora.tla
@ -93,10 +93,12 @@ StageComplete == "StageComplete"
 StageCleaned  == "StageCleaned"
 StageStates   == {StageNone, StagePartial, StageComplete, StageCleaned}

-NoOutcome    == "NoOutcome"
-WantsSuccess == "WantsSuccess"
-WantsFailure == "WantsFailure"
-OutcomeRequests == {NoOutcome, WantsSuccess, WantsFailure}
+NoOutcome             == "NoOutcome"
+WantsSuccess          == "WantsSuccess"
+WantsRetryableFailure == "WantsRetryableFailure"
+WantsPermanentFailure == "WantsPermanentFailure"
+FailureOutcomes == {WantsRetryableFailure, WantsPermanentFailure}
+OutcomeRequests == {NoOutcome, WantsSuccess} \cup FailureOutcomes

 NoAttempt == [job |-> NoJob, token |-> 0]

@ -628,12 +630,12 @@ ReportProgress(a) ==

 SendOutcome(a, outcome) ==
    LET w == attemptWorker[a]
-    IN /\ outcome \in {WantsSuccess, WantsFailure}
+    IN /\ outcome \in {WantsSuccess} \cup FailureOutcomes
       /\ attemptState[a] = AttemptExecuting
       /\ pendingOutcome[a] = NoOutcome
       /\ workerState[w] = WorkerUp
       /\ CanReach(w, DurableSvc)
-       /\ (outcome = WantsFailure \/ stageState[a] = StageComplete)
+       /\ (outcome \in FailureOutcomes \/ stageState[a] = StageComplete)
       /\ pendingOutcome' = [pendingOutcome EXCEPT ![a] = outcome]
       /\ UNCHANGED durableVars
       /\ UNCHANGED coordinationVars
@ -651,7 +653,7 @@ CommitEligible(a) ==

 FailureEligible(a) ==
    LET j == JobOf(a)
-    IN /\ pendingOutcome[a] = WantsFailure
+    IN /\ pendingOutcome[a] \in FailureOutcomes
       /\ CurrentAuthority(a)
       /\ phase[j] = Running
       /\ ~cancelRequested[j]
@ -675,13 +677,16 @@ AcceptSuccess(a) ==
       /\ UNCHANGED <<attemptWorker, attemptProgress, initialProgress,
                       resumedFromCheckpoint, stageState>>

-AcceptFailure(a) ==
+AcceptFailure(a, outcome) ==
    LET j == JobOf(a)
-    IN /\ FailureEligible(a)
+    IN /\ pendingOutcome[a] = outcome
+       /\ FailureEligible(a)
       /\ phase' =
            [phase EXCEPT
-                ![j] = IF currentToken[j] < MaxAttempts
-                        THEN RetryWait ELSE Failed]
+                ![j] = IF outcome = WantsPermanentFailure
+                        THEN Failed
+                        ELSE IF currentToken[j] < MaxAttempts
+                             THEN RetryWait ELSE Failed]
       /\ authority'      = [authority EXCEPT ![j] = NoAttempt]
       /\ attemptState'   =
            [attemptState EXCEPT ![a] = AttemptStopped]
@ -693,6 +698,18 @@ AcceptFailure(a) ==
       /\ UNCHANGED <<attemptWorker, attemptProgress, initialProgress,
                       resumedFromCheckpoint, stageState>>

+AcceptPermanentFailure(a) ==
+    /\ pendingOutcome[a] = WantsPermanentFailure
+    /\ AcceptFailure(a, WantsPermanentFailure)
+
+AcceptRetryableFailureWithBudget(a) ==
+    /\ currentToken[JobOf(a)] < MaxAttempts
+    /\ AcceptFailure(a, WantsRetryableFailure)
+
+AcceptRetryableFailureExhausted(a) ==
+    /\ currentToken[JobOf(a)] = MaxAttempts
+    /\ AcceptFailure(a, WantsRetryableFailure)
+
 RejectOutcome(a) ==
    /\ pendingOutcome[a] # NoOutcome
    /\ ~CommitEligible(a)
@ -730,11 +747,16 @@ ActivateJob(j)  ==
 ExecuteJob(j) ==
    \E a \in AttemptIds :
        /\ JobOf(a) = j
-        /\ (SendOutcome(a, WantsFailure)
+        /\ (SendOutcome(a, WantsRetryableFailure)
+             \/ SendOutcome(a, WantsPermanentFailure)
             \/ SendOutcome(a, WantsSuccess)
             \/ \E amount \in 1..MaxProgress : AdvanceAttempt(a, amount))
 ProcessOutcome(a) ==
-    AcceptSuccess(a) \/ AcceptFailure(a) \/ RejectOutcome(a)
+    \/ AcceptSuccess(a)
+    \/ AcceptPermanentFailure(a)
+    \/ AcceptRetryableFailureWithBudget(a)
+    \/ AcceptRetryableFailureExhausted(a)
+    \/ RejectOutcome(a)
 ReapJob(j) ==
    \E a \in AttemptIds : JobOf(a) = j /\ ReapLostAttempt(a)

@ -765,10 +787,12 @@ Next ==
    \/ \E a \in AttemptIds : PersistCheckpoint(a)
    \/ \E j \in Jobs : InvalidateCheckpoint(j)
    \/ \E a \in AttemptIds : ReportProgress(a)
-    \/ \E a \in AttemptIds, o \in {WantsSuccess, WantsFailure} :
+    \/ \E a \in AttemptIds, o \in {WantsSuccess} \cup FailureOutcomes :
           SendOutcome(a, o)
    \/ \E a \in AttemptIds : AcceptSuccess(a)
-    \/ \E a \in AttemptIds : AcceptFailure(a)
+    \/ \E a \in AttemptIds : AcceptPermanentFailure(a)
+    \/ \E a \in AttemptIds : AcceptRetryableFailureWithBudget(a)
+    \/ \E a \in AttemptIds : AcceptRetryableFailureExhausted(a)
    \/ \E a \in AttemptIds : RejectOutcome(a)
    \/ \E a \in AttemptIds : CleanupStage(a)
    \/ Done
@ -952,17 +976,31 @@ CommitLinearizationStep ==
            /\ ~cancelRequested[j]
            /\ stageState[committedAttempt'[j]] = StageComplete

+FailureDispositionStep ==
+    \A a \in AttemptIds :
+        LET j == JobOf(a)
+        IN /\ FailureEligible(a)
+           /\ attemptState'[a] = AttemptStopped
+           /\ pendingOutcome'[a] = NoOutcome
+           /\ authority'[j] = NoAttempt
+           => phase'[j] =
+                IF pendingOutcome[a] = WantsPermanentFailure
+                THEN Failed
+                ELSE IF currentToken[j] < MaxAttempts
+                     THEN RetryWait ELSE Failed
+
 TerminalStateIsImmutable == [][TerminalStep]_vars
 FencingTokensMonotonic   == [][FenceStep]_vars
 CommitIsFencedCAS        == [][CommitLinearizationStep]_vars
+FailureDispositionHonored == [][FailureDispositionStep]_vars

 \* Liveness is conditional on eventual lasting capacity. Safety needs no recovery.
 \* WorkerAvailable requires Coord, Durable, and Target reachable for some Up worker.
 \* EventuallyDispatched: accepted work leaves Queued (starts Running or ends terminal,
 \* including Cancelled). It is not "successful delivery of bytes."
 \* EventuallyTerminal: some terminal outcome (Succeeded, Failed, or Cancelled).
-\* Permanent source/target failure may yield Failed after the retry budget; success
-\* is not guaranteed under unrestricted WantsFailure nondeterminism.
+\* Permanent failure may yield Failed immediately; retryable failure may yield
+\* Failed after the retry budget. Success is not guaranteed under either choice.
 \* EventuallyNonWinnerStagingCleaned / EventuallySystemQuiescent: model GC goal under
 \* fairness (infallible CleanupStage). Product seeks zero orphans under recovery;
 \* not a safety invariant and not a readiness gate (IF-7).
--- a/docs/protocol/RemoraCombined.cfg
+++ b/docs/protocol/RemoraCombined.cfg
@ -39,3 +39,4 @@ PROPERTIES
    TerminalStateIsImmutable
    FencingTokensMonotonic
    CommitIsFencedCAS
+    FailureDispositionHonored
--- a/docs/protocol/RemoraDisposition.cfg
+++ b/docs/protocol/RemoraDisposition.cfg
@ -0,0 +1,39 @@
+SPECIFICATION SafetySpec
+
+CHECK_DEADLOCK TRUE
+
+CONSTANTS
+    Jobs = {j1}
+    Workers = {w1}
+    InitialWorkers = {w1}
+    NoJob = NoJob
+    NoWorker = NoWorker
+    MaxAttempts = 2
+    MaxProgress = 1
+    LeaseLimit = 1
+    LocalLimit = 1
+    GlobalLimit = 1
+    FaultsEnabled = FALSE
+    FaultServices = {}
+    MaxCoordLosses = 0
+    MaxPartitions = 0
+
+INVARIANTS
+    TypeOK
+    StateConsistency
+    AllocatedTokensUsed
+    LiveProgressWellFormed
+    LiveProgressCompatibleWithLiveLease
+    AtMostOneLiveLeasePerJob
+    AuthorityWellFormed
+    CommittedResultConsistent
+    CancellationSafety
+    PermitSafety
+    ResumeValidity
+    StagingOwnership
+
+PROPERTIES
+    TerminalStateIsImmutable
+    FencingTokensMonotonic
+    CommitIsFencedCAS
+    FailureDispositionHonored
--- a/docs/protocol/RemoraDurableFault.cfg
+++ b/docs/protocol/RemoraDurableFault.cfg
@ -38,3 +38,4 @@ PROPERTIES
    TerminalStateIsImmutable
    FencingTokensMonotonic
    CommitIsFencedCAS
+    FailureDispositionHonored
--- a/docs/protocol/RemoraFencing.cfg
+++ b/docs/protocol/RemoraFencing.cfg
@ -38,3 +38,4 @@ PROPERTIES
    TerminalStateIsImmutable
    FencingTokensMonotonic
    CommitIsFencedCAS
+    FailureDispositionHonored
--- a/docs/protocol/RemoraGlobal.cfg
+++ b/docs/protocol/RemoraGlobal.cfg
@ -37,3 +37,4 @@ PROPERTIES
    TerminalStateIsImmutable
    FencingTokensMonotonic
    CommitIsFencedCAS
+    FailureDispositionHonored
--- a/docs/protocol/RemoraLocal.cfg
+++ b/docs/protocol/RemoraLocal.cfg
@ -37,3 +37,4 @@ PROPERTIES
    TerminalStateIsImmutable
    FencingTokensMonotonic
    CommitIsFencedCAS
+    FailureDispositionHonored
--- a/docs/protocol/RemoraMultiShot.cfg
+++ b/docs/protocol/RemoraMultiShot.cfg
@ -37,3 +37,4 @@ PROPERTIES
    TerminalStateIsImmutable
    FencingTokensMonotonic
    CommitIsFencedCAS
+    FailureDispositionHonored
--- a/docs/protocol/RemoraStress.cfg
+++ b/docs/protocol/RemoraStress.cfg
@ -39,3 +39,4 @@ PROPERTIES
    TerminalStateIsImmutable
    FencingTokensMonotonic
    CommitIsFencedCAS
+    FailureDispositionHonored
--- a/docs/protocol/RemoraTargetFault.cfg
+++ b/docs/protocol/RemoraTargetFault.cfg
@ -38,3 +38,4 @@ PROPERTIES
    TerminalStateIsImmutable
    FencingTokensMonotonic
    CommitIsFencedCAS
+    FailureDispositionHonored
--- a/docs/spec/architecture.md
+++ b/docs/spec/architecture.md
@ -121,6 +121,16 @@ flowchart TD
 - A Hugging Face branch or tag is resolved once to a full commit. The source identity includes repository type and ID, that commit, and the exact file inventory with Git blob or LFS identities. Retries and cache reuse remain pinned to the stored commit and member identities.
 - A checkpoint never authorizes best-effort concatenation. An unavailable explicitly pinned or resolved snapshot produces a structured source-changed outcome rather than silently resolving a newer snapshot.

+## Retry and Structured Failures
+
+- Structured failures use the stable classes `validation`, `authorization`, `source_not_found`, `source_changed`, `transient_source`, `transient_target`, `integrity`, `capacity`, `internal`, `cancelled`, and `lost_attempt`.
+- Attempt-reported runtime failures carry a durable permanent or retryable disposition into `JobProtocol`. The model is the sole definition of how that disposition affects execution phase and attempt budget.
+- Authorization, source-not-found, and source-changed runtime failures are permanent. Transient-source, transient-target, integrity, capacity, and internal failures are retryable. Unrecognized runtime errors map to a redacted internal failure.
+- Validation normally rejects submission before a job exists. Cancellation and lost attempts follow their dedicated `JobProtocol` behavior rather than a worker-selected failure disposition.
+- All retryable work uses the model-wide attempt bound. Source adapters may vary connect, response-start, no-progress, bounded full-jitter, and bounded server-hint timing without changing that bound.
+- Remora has no elapsed wall-clock job deadline. Per-operation timeouts produce a structured runtime failure and remain subject to `JobProtocol`.
+- Public failure details are constructed from per-class allowlists. Credentials, upstream bodies, raw URLs, and internal exception text are excluded.
+
 ## Result Visibility and Deletion

 - `JobProtocol` remains the sole owner of execution phase. Deletion never rewrites a terminal execution outcome and is available only after a job is terminal; cancelling active work remains a separate protocol operation.