ms2k/remora
SHA256
Watch
0
Code Releases Activity

Specify fenced activation and source credential contracts

Browse source 
Define bounded reservation recovery, exact activation replay fencing, immutable source identities, and deployment-owned secret and target policy boundaries. Align the OpenAPI contract and add a split-store activation safety refinement.
This commit is contained in:
ms2k 2026-07-14 18:12:28 +09:00
commit 033cfeaac6
1 changed files with 877 additions and 49 deletions
Download patch file Download diff file Expand all files Collapse all files

63
docs/protocol/Remora.tla
View file

@ -29,7 +29,7 @@ EXTENDS Naturals, FiniteSets, TLC
CONSTANTS
Jobs, Workers, InitialWorkers, NoJob, NoWorker,
MaxAttempts, MaxProgress, LeaseLimit,
MaxAttempts, MaxProgress, LeaseLimit, ReservationLimit,
LocalLimit, GlobalLimit, FaultsEnabled, FaultServices,
MaxCoordLosses, MaxPartitions
@ -40,6 +40,7 @@ ASSUME /\ Jobs # {}
/\ MaxAttempts \in Nat \ {0}
/\ MaxProgress \in Nat \ {0}
/\ LeaseLimit \in Nat \ {0}
/\ ReservationLimit \in Nat \ {0}
/\ LocalLimit \in Nat \ {0}
/\ GlobalLimit \in Nat \ {0}
/\ FaultsEnabled \in BOOLEAN
@ -119,7 +120,7 @@ VARIABLES
checkpointValid,
\* CoordinationStore (reconstructible).
queued, reservation, leaseState, leaseAge, cancelNotice,
queued, reservation, reservationAge, leaseState, leaseAge, cancelNotice,
liveProgress, liveProgressAttempt, coordLossCount,
\* Worker incarnations, per-service links, partition budget.
@ -133,7 +134,7 @@ durableVars ==
<<phase, cancelRequested, currentToken, authority, dispatchNeeded,
committedAttempt, checkpoint, checkpointValid>>
coordinationVars ==
<<queued, reservation, leaseState, leaseAge, cancelNotice,
<<queued, reservation, reservationAge, leaseState, leaseAge, cancelNotice,
liveProgress, liveProgressAttempt, coordLossCount>>
workerVars == <<workerState, links, partitionCount>>
attemptVars ==
@ -168,6 +169,7 @@ OtherCapacity(w) ==
CoordinationClean(j) ==
/\ ~queued[j]
/\ reservation[j] = NoWorker
/\ reservationAge[j] = 0
/\ ~cancelNotice[j]
/\ liveProgress[j] = 0
/\ liveProgressAttempt[j] = NoAttempt
@ -195,6 +197,7 @@ Init ==
/\ checkpointValid = [j \in Jobs |-> FALSE]
/\ queued = [j \in Jobs |-> FALSE]
/\ reservation = [j \in Jobs |-> NoWorker]
/\ reservationAge = [j \in Jobs |-> 0]
/\ leaseState = [a \in AttemptIds |-> NoLease]
/\ leaseAge = [a \in AttemptIds |-> 0]
/\ cancelNotice = [j \in Jobs |-> FALSE]
@ -222,7 +225,7 @@ Enqueue(j) ==
/\ reservation[j] = NoWorker
/\ queued' = [queued EXCEPT ![j] = TRUE]
/\ UNCHANGED durableVars
/\ UNCHANGED <<reservation, leaseState, leaseAge, cancelNotice,
/\ UNCHANGED <<reservation, reservationAge, leaseState, leaseAge, cancelNotice,
liveProgress, liveProgressAttempt, coordLossCount>>
/\ UNCHANGED workerVars
/\ UNCHANGED attemptVars
@ -235,6 +238,7 @@ Reserve(j, w) ==
/\ CapacityAvailable(w)
/\ queued' = [queued EXCEPT ![j] = FALSE]
/\ reservation' = [reservation EXCEPT ![j] = w]
/\ reservationAge' = [reservationAge EXCEPT ![j] = 0]
/\ UNCHANGED durableVars
/\ UNCHANGED <<leaseState, leaseAge, cancelNotice, liveProgress,
liveProgressAttempt, coordLossCount>>
@ -289,6 +293,7 @@ ActivateAttempt(a) ==
/\ workerState[w] = WorkerUp
/\ CanReach(w, CoordSvc)
/\ reservation' = [reservation EXCEPT ![j] = NoWorker]
/\ reservationAge' = [reservationAge EXCEPT ![j] = 0]
/\ leaseState' = [leaseState EXCEPT ![a] = LeaseLive]
/\ leaseAge' = [leaseAge EXCEPT ![a] = 0]
/\ liveProgress' = [liveProgress EXCEPT ![j] = attemptProgress[a]]
@ -309,24 +314,44 @@ RenewLease(a) ==
/\ CanReach(w, CoordSvc)
/\ leaseAge' = [leaseAge EXCEPT ![a] = 0]
/\ UNCHANGED durableVars
/\ UNCHANGED <<queued, reservation, leaseState, cancelNotice,
/\ UNCHANGED <<queued, reservation, reservationAge, leaseState, cancelNotice,
liveProgress, liveProgressAttempt, coordLossCount>>
/\ UNCHANGED workerVars
/\ UNCHANGED attemptVars
Tick ==
/\ \E a \in AttemptIds :
leaseState[a] = LeaseLive /\ leaseAge[a] < LeaseLimit
/\ (\E a \in AttemptIds :
leaseState[a] = LeaseLive /\ leaseAge[a] < LeaseLimit)
\/ (\E j \in Jobs :
reservation[j] # NoWorker /\ reservationAge[j] < ReservationLimit)
/\ leaseAge' =
[a \in AttemptIds |->
IF leaseState[a] = LeaseLive /\ leaseAge[a] < LeaseLimit
THEN leaseAge[a] + 1 ELSE leaseAge[a]]
/\ reservationAge' =
[j \in Jobs |->
IF reservation[j] # NoWorker
/\ reservationAge[j] < ReservationLimit
THEN reservationAge[j] + 1 ELSE reservationAge[j]]
/\ UNCHANGED durableVars
/\ UNCHANGED <<queued, reservation, leaseState, cancelNotice,
liveProgress, liveProgressAttempt, coordLossCount>>
/\ UNCHANGED workerVars
/\ UNCHANGED attemptVars
\* Reservation expiry is coordination-only. Queued jobs become eligible for
\* Enqueue; authorized attempts become eligible for ReapLostAttempt.
ExpireReservation(j) ==
/\ reservation[j] # NoWorker
/\ reservationAge[j] = ReservationLimit
/\ reservation' = [reservation EXCEPT ![j] = NoWorker]
/\ reservationAge' = [reservationAge EXCEPT ![j] = 0]
/\ UNCHANGED durableVars
/\ UNCHANGED <<queued, leaseState, leaseAge, cancelNotice,
liveProgress, liveProgressAttempt, coordLossCount>>
/\ UNCHANGED workerVars
/\ UNCHANGED attemptVars
\* Lease expiry does not clear live progress: product may retain a stale
\* (attempt, bytes) hint until overwrite, reader discard, or terminal GC.
ExpireLease(a) ==
@ -335,7 +360,7 @@ ExpireLease(a) ==
/\ leaseState' = [leaseState EXCEPT ![a] = LeaseExpired]
/\ leaseAge' = [leaseAge EXCEPT ![a] = 0]
/\ UNCHANGED durableVars
/\ UNCHANGED <<queued, reservation, cancelNotice, liveProgress,
/\ UNCHANGED <<queued, reservation, reservationAge, cancelNotice, liveProgress,
liveProgressAttempt, coordLossCount>>
/\ UNCHANGED workerVars
/\ UNCHANGED attemptVars
@ -347,6 +372,7 @@ LoseCoordinationState ==
/\ coordLossCount < MaxCoordLosses
/\ queued' = [j \in Jobs |-> FALSE]
/\ reservation' = [j \in Jobs |-> NoWorker]
/\ reservationAge' = [j \in Jobs |-> 0]
/\ leaseState' =
[a \in AttemptIds |->
IF leaseState[a] = LeaseLive THEN LeaseExpired
@ -419,6 +445,7 @@ ReleaseReservation(j) ==
\/ (authority[j] # NoAttempt
/\ attemptState[authority[j]] # AttemptAuthorized))
/\ reservation' = [reservation EXCEPT ![j] = NoWorker]
/\ reservationAge' = [reservationAge EXCEPT ![j] = 0]
/\ UNCHANGED durableVars
/\ UNCHANGED <<queued, leaseState, leaseAge, cancelNotice,
liveProgress, liveProgressAttempt, coordLossCount>>
@ -502,7 +529,7 @@ PublishCancelNotice(j) ==
/\ ~cancelNotice[j]
/\ cancelNotice' = [cancelNotice EXCEPT ![j] = TRUE]
/\ UNCHANGED durableVars
/\ UNCHANGED <<queued, reservation, leaseState, leaseAge, liveProgress,
/\ UNCHANGED <<queued, reservation, reservationAge, leaseState, leaseAge, liveProgress,
liveProgressAttempt, coordLossCount>>
/\ UNCHANGED workerVars
/\ UNCHANGED attemptVars
@ -531,7 +558,7 @@ ReleaseLease(a) ==
/\ leaseState' = [leaseState EXCEPT ![a] = LeaseExpired]
/\ leaseAge' = [leaseAge EXCEPT ![a] = 0]
/\ UNCHANGED durableVars
/\ UNCHANGED <<queued, reservation, cancelNotice, liveProgress,
/\ UNCHANGED <<queued, reservation, reservationAge, cancelNotice, liveProgress,
liveProgressAttempt, coordLossCount>>
/\ UNCHANGED workerVars
/\ UNCHANGED attemptVars
@ -546,7 +573,7 @@ ClearStaleLiveProgress(j) ==
/\ liveProgressAttempt' =
[liveProgressAttempt EXCEPT ![j] = NoAttempt]
/\ UNCHANGED durableVars
/\ UNCHANGED <<queued, reservation, leaseState, leaseAge,
/\ UNCHANGED <<queued, reservation, reservationAge, leaseState, leaseAge,
cancelNotice, coordLossCount>>
/\ UNCHANGED workerVars
/\ UNCHANGED attemptVars
@ -557,6 +584,7 @@ CleanTerminalCoordination(j) ==
\/ liveProgress[j] # 0 \/ liveProgressAttempt[j] # NoAttempt)
/\ queued' = [queued EXCEPT ![j] = FALSE]
/\ reservation' = [reservation EXCEPT ![j] = NoWorker]
/\ reservationAge' = [reservationAge EXCEPT ![j] = 0]
/\ cancelNotice' = [cancelNotice EXCEPT ![j] = FALSE]
/\ liveProgress' = [liveProgress EXCEPT ![j] = 0]
/\ liveProgressAttempt' =
@ -628,7 +656,7 @@ ReportProgress(a) ==
/\ liveProgressAttempt' =
[liveProgressAttempt EXCEPT ![j] = a]
/\ UNCHANGED durableVars
/\ UNCHANGED <<queued, reservation, leaseState, leaseAge,
/\ UNCHANGED <<queued, reservation, reservationAge, leaseState, leaseAge,
cancelNotice, coordLossCount>>
/\ UNCHANGED workerVars
/\ UNCHANGED attemptVars
@ -774,6 +802,7 @@ Next ==
\/ \E a \in AttemptIds : RenewLease(a)
\/ Tick
\/ \E a \in AttemptIds : ExpireLease(a)
\/ \E j \in Jobs : ExpireReservation(j)
\/ LoseCoordinationState
\/ \E w \in Workers, s \in Services : DropLink(w, s)
\/ \E w \in Workers, s \in Services : RestoreLink(w, s)
@ -812,6 +841,7 @@ Fairness ==
/\ \A j \in Jobs : SF_vars(ExecuteJob(j))
/\ \A a \in AttemptIds : WF_vars(ProcessOutcome(a))
/\ \A a \in AttemptIds : WF_vars(ExpireLease(a))
/\ \A j \in Jobs : WF_vars(ExpireReservation(j))
/\ \A a \in AttemptIds : WF_vars(CooperativeStop(a))
/\ \A a \in AttemptIds : WF_vars(ReleaseLease(a))
/\ \A a \in AttemptIds : WF_vars(CleanupStage(a))
@ -840,6 +870,7 @@ TypeOK ==
/\ checkpointValid \in [Jobs -> BOOLEAN]
/\ queued \in [Jobs -> BOOLEAN]
/\ reservation \in [Jobs -> Workers \cup {NoWorker}]
/\ reservationAge \in [Jobs -> 0..ReservationLimit]
/\ leaseState \in [AttemptIds -> LeaseStates]
/\ leaseAge \in [AttemptIds -> 0..LeaseLimit]
/\ cancelNotice \in [Jobs -> BOOLEAN]
@ -874,6 +905,7 @@ StateConsistency ==
/\ (committedAttempt[j] # NoAttempt <=> phase[j] = Succeeded)
/\ (phase[j] = Queued => dispatchNeeded[j])
/\ (reservation[j] # NoWorker => ~queued[j])
/\ (reservation[j] = NoWorker => reservationAge[j] = 0)
/\ \A a \in AttemptIds :
/\ (attemptState[a] = AttemptUnused <=>
attemptWorker[a] = NoWorker)
@ -1024,6 +1056,13 @@ EventuallyReaped ==
\A a \in AttemptIds :
(CurrentAuthority(a) /\ leaseState[a] # LeaseLive)
~> (~CurrentAuthority(a) \/ Terminal(JobOf(a)))
EventuallyAuthorizedResolved ==
RecoveryAssumption =>
\A a \in AttemptIds :
(CurrentAuthority(a) /\ attemptState[a] = AttemptAuthorized)
~> (attemptState[a] = AttemptExecuting
\/ ~CurrentAuthority(a)
\/ Terminal(JobOf(a)))
EventuallyTerminal ==
RecoveryAssumption => \A j \in Jobs : <>Terminal(j)