ms2k/remora
SHA256
Watch
0
Code Releases Activity

Model checkpoint staging ownership transfer

Browse source
This commit is contained in:
ms2k 2026-07-17 15:01:15 +09:00
commit 16cba3c317
1 changed files with 111 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

66
docs/protocol/Remora.tla
View file

@ -137,7 +137,7 @@ CredentialIntents ==
VARIABLES
\* DurableStore authority.
phase, cancelRequested, currentToken, authority, dispatchNeeded,
committedAttempt, checkpoint,
committedAttempt, checkpoint, checkpointStageOwner,
\* Abstracts the complete source-identity and destination-state proof.
checkpointValid,
@ -157,7 +157,7 @@ VARIABLES
durableVars ==
<<phase, cancelRequested, currentToken, authority, dispatchNeeded,
committedAttempt, checkpoint, checkpointValid>>
committedAttempt, checkpoint, checkpointStageOwner, checkpointValid>>
coordinationVars ==
<<queued, reservation, reservationAge, leaseState, leaseAge, cancelNotice,
liveProgress, liveProgressAttempt, coordLossCount>>
@ -226,6 +226,7 @@ Init ==
/\ dispatchNeeded = [j \in Jobs |-> TRUE]
/\ committedAttempt = [j \in Jobs |-> NoAttempt]
/\ checkpoint = [j \in Jobs |-> 0]
/\ checkpointStageOwner = [j \in Jobs |-> NoAttempt]
/\ checkpointValid = [j \in Jobs |-> FALSE]
/\ queued = [j \in Jobs |-> FALSE]
/\ reservation = [j \in Jobs |-> NoWorker]
@ -291,6 +292,9 @@ AuthorizeAttempt(j, w) ==
/\ CanReach(w, DurableSvc)
/\ currentToken[j] < MaxAttempts
/\ attemptState[a] = AttemptUnused
/\ IF checkpointValid[j]
THEN attemptState[checkpointStageOwner[j]] = AttemptStopped
ELSE TRUE
/\ phase' = [phase EXCEPT ![j] = Running]
/\ currentToken' = [currentToken EXCEPT ![j] = @ + 1]
/\ authority' = [authority EXCEPT ![j] = a]
@ -307,11 +311,15 @@ AuthorizeAttempt(j, w) ==
[resumedFromCheckpoint EXCEPT
![a] = checkpointValid[j] /\ checkpoint[j] > 0]
/\ stageState' =
[stageState EXCEPT
![a] = IF ~checkpointValid[j] \/ checkpoint[j] = 0
THEN StageNone
ELSE IF checkpoint[j] = MaxProgress
THEN StageComplete ELSE StagePartial]
IF checkpointValid[j]
THEN [stageState EXCEPT
![checkpointStageOwner[j]] = StageNone,
![a] = IF checkpoint[j] = MaxProgress
THEN StageComplete ELSE StagePartial]
ELSE [stageState EXCEPT ![a] = StageNone]
/\ checkpointStageOwner' =
[checkpointStageOwner EXCEPT
![j] = IF checkpointValid[j] THEN a ELSE NoAttempt]
/\ UNCHANGED <<cancelRequested, committedAttempt, checkpoint,
checkpointValid>>
/\ UNCHANGED coordinationVars
@ -512,7 +520,8 @@ ReapLostAttempt(a) ==
![a] = IF attemptState[a] = AttemptAuthorized
THEN AttemptStopped ELSE @]
/\ UNCHANGED <<cancelRequested, currentToken, dispatchNeeded,
committedAttempt, checkpoint, checkpointValid>>
committedAttempt, checkpoint, checkpointStageOwner,
checkpointValid>>
/\ UNCHANGED coordinationVars
/\ UNCHANGED workerVars
/\ UNCHANGED <<attemptWorker, attemptProgress, initialProgress,
@ -524,7 +533,8 @@ ScheduleRetry(j) ==
/\ phase' = [phase EXCEPT ![j] = Queued]
/\ dispatchNeeded' = [dispatchNeeded EXCEPT ![j] = TRUE]
/\ UNCHANGED <<cancelRequested, currentToken, authority,
committedAttempt, checkpoint, checkpointValid>>
committedAttempt, checkpoint, checkpointStageOwner,
checkpointValid>>
/\ UNCHANGED coordinationVars
/\ UNCHANGED workerVars
/\ UNCHANGED attemptVars
@ -539,7 +549,8 @@ RequestCancel(j) ==
[authority EXCEPT
![j] = IF phase[j] = Running THEN @ ELSE NoAttempt]
/\ dispatchNeeded' = [dispatchNeeded EXCEPT ![j] = FALSE]
/\ UNCHANGED <<currentToken, committedAttempt, checkpoint, checkpointValid>>
/\ UNCHANGED <<currentToken, committedAttempt, checkpoint,
checkpointStageOwner, checkpointValid>>
/\ UNCHANGED coordinationVars
/\ UNCHANGED workerVars
/\ UNCHANGED attemptVars
@ -556,7 +567,8 @@ FinalizeCancel(j) ==
IF JobOf(a) = j /\ attemptState[a] = AttemptAuthorized
THEN AttemptStopped ELSE attemptState[a]]
/\ UNCHANGED <<cancelRequested, currentToken, dispatchNeeded,
committedAttempt, checkpoint, checkpointValid>>
committedAttempt, checkpoint, checkpointStageOwner,
checkpointValid>>
/\ UNCHANGED coordinationVars
/\ UNCHANGED workerVars
/\ UNCHANGED <<attemptWorker, attemptProgress, initialProgress,
@ -664,6 +676,7 @@ PersistCheckpoint(a) ==
/\ attemptProgress[a] > 0
/\ (~checkpointValid[j] \/ checkpoint[j] # attemptProgress[a])
/\ checkpoint' = [checkpoint EXCEPT ![j] = attemptProgress[a]]
/\ checkpointStageOwner' = [checkpointStageOwner EXCEPT ![j] = a]
/\ checkpointValid' = [checkpointValid EXCEPT ![j] = TRUE]
/\ UNCHANGED <<phase, cancelRequested, currentToken, authority,
dispatchNeeded, committedAttempt>>
@ -674,6 +687,8 @@ PersistCheckpoint(a) ==
InvalidateCheckpoint(j) ==
/\ ~Terminal(j)
/\ checkpointValid[j]
/\ checkpointStageOwner' =
[checkpointStageOwner EXCEPT ![j] = NoAttempt]
/\ checkpointValid' = [checkpointValid EXCEPT ![j] = FALSE]
/\ UNCHANGED <<phase, cancelRequested, currentToken, authority,
dispatchNeeded, committedAttempt, checkpoint>>
@ -740,6 +755,7 @@ AcceptSuccess(a) ==
/\ authority' = [authority EXCEPT ![j] = NoAttempt]
/\ committedAttempt' = [committedAttempt EXCEPT ![j] = a]
/\ checkpoint' = [checkpoint EXCEPT ![j] = MaxProgress]
/\ checkpointStageOwner' = [checkpointStageOwner EXCEPT ![j] = a]
/\ checkpointValid' = [checkpointValid EXCEPT ![j] = TRUE]
/\ attemptState' =
[attemptState EXCEPT ![a] = AttemptCommitted]
@ -765,7 +781,8 @@ AcceptFailure(a, outcome) ==
[attemptState EXCEPT ![a] = AttemptStopped]
/\ pendingOutcome' = [pendingOutcome EXCEPT ![a] = NoOutcome]
/\ UNCHANGED <<cancelRequested, currentToken, dispatchNeeded,
committedAttempt, checkpoint, checkpointValid>>
committedAttempt, checkpoint, checkpointStageOwner,
checkpointValid>>
/\ UNCHANGED coordinationVars
/\ UNCHANGED workerVars
/\ UNCHANGED <<attemptWorker, attemptProgress, initialProgress,
@ -800,6 +817,9 @@ CleanupStage(a) ==
IN /\ stageState[a] \in {StagePartial, StageComplete}
/\ committedAttempt[j] # a
/\ ~CurrentAuthority(a)
/\ (Terminal(j)
\/ ~checkpointValid[j]
\/ checkpointStageOwner[j] # a)
/\ (attemptState[a] = AttemptStopped
\/ workerState[attemptWorker[a]] = WorkerDown)
/\ stageState' = [stageState EXCEPT ![a] = StageCleaned]
@ -1002,6 +1022,7 @@ TypeOK ==
/\ dispatchNeeded \in [Jobs -> BOOLEAN]
/\ committedAttempt \in [Jobs -> AttemptIds \cup {NoAttempt}]
/\ checkpoint \in [Jobs -> 0..MaxProgress]
/\ checkpointStageOwner \in [Jobs -> AttemptIds \cup {NoAttempt}]
/\ checkpointValid \in [Jobs -> BOOLEAN]
/\ queued \in [Jobs -> BOOLEAN]
/\ reservation \in [Jobs -> Workers \cup {NoWorker}]
@ -1154,8 +1175,21 @@ ResumeValidity ==
/\ (initialProgress[a] > 0 => resumedFromCheckpoint[a])
/\ (resumedFromCheckpoint[a] => initialProgress[a] > 0)
\* Uncommitted staging is only retained for live or stopped attempts, not cleaned
\* while still authoritative (cleanup gated in CleanupStage).
CheckpointStagingOwnership ==
\A j \in Jobs :
/\ (~checkpointValid[j] => checkpointStageOwner[j] = NoAttempt)
/\ (checkpointValid[j] =>
/\ checkpoint[j] > 0
/\ checkpointStageOwner[j] \in AttemptIds
/\ JobOf(checkpointStageOwner[j]) = j
/\ attemptState[checkpointStageOwner[j]] # AttemptUnused)
/\ (~Terminal(j) /\ checkpointValid[j] =>
stageState[checkpointStageOwner[j]]
\in {StagePartial, StageComplete})
/\ (phase[j] = Running /\ checkpointValid[j] =>
checkpointStageOwner[j] = authority[j])
\* Transfer removes the predecessor's cleanup eligibility before replacement I/O.
StagingOwnership ==
\A a \in AttemptIds :
/\ (stageState[a] = StageCleaned =>
@ -1174,6 +1208,7 @@ TerminalStep ==
/\ authority'[j] = authority[j]
/\ committedAttempt'[j] = committedAttempt[j]
/\ checkpoint'[j] = checkpoint[j]
/\ checkpointStageOwner'[j] = checkpointStageOwner[j]
/\ checkpointValid'[j] = checkpointValid[j]
FenceStep ==
\A j \in Jobs : currentToken'[j] >= currentToken[j]
@ -1243,6 +1278,9 @@ CleanupEligible(a) ==
/\ stageState[a] \in {StagePartial, StageComplete}
/\ committedAttempt[JobOf(a)] # a
/\ ~CurrentAuthority(a)
/\ (Terminal(JobOf(a))
\/ ~checkpointValid[JobOf(a)]
\/ checkpointStageOwner[JobOf(a)] # a)
/\ (attemptState[a] = AttemptStopped
\/ workerState[attemptWorker[a]] = WorkerDown)