Define Nuthatch architecture and mutation protocol

docs/protocol/Nuthatch.cfg

@@ -0,0 +1,48 @@
+SPECIFICATION Spec
+
+CHECK_DEADLOCK TRUE
+
+CONSTANTS
+    Users = {u1, u2}
+    OperationIds = {op1}
+    EnabledKinds = {"put"}
+    PersonalKeyNames = {personalKey}
+    SharedKeys = {sharedKey}
+    Contents = {content1}
+    MaxObjectSize = 1
+    PersonalLimit = 2
+    SharedLimit = 2
+    MaxPostgresLosses = 1
+    MaxObjectStoreLosses = 1
+    SharedPool = SharedPool
+    NoObject = NoObject
+    NoRecord = NoRecord
+    NoOwner = NoOwner
+    NoObservation = NoObservation
+
+INVARIANTS
+    TypeOK
+    RecordIdentityConsistent
+    PersonalRequestsUseOwnPool
+    PersonalPoolsDisjoint
+    FingerprintBindsRequest
+    ReservationsConsistent
+    LocksConsistent
+    ActiveObjectConsistent
+    ExternalStateConsistent
+    UploadBodyConsistent
+    QuotaReservationBound
+    ObjectStateCovered
+    ObjectQuotaBound
+    AccountingAtQuiescence
+    ObservationConsistent
+
+PROPERTIES
+    RecordsNeverForgotten
+    RequestBindingsStable
+    TerminalRecordsStable
+    PostgresFailureFencesDurableState
+    ObjectStoreFailureFencesObjects
+    SettledEffectsDoNotMutate
+    UnfinishedEventuallyTerminal
+    LostUploadEventuallyTerminal

docs/protocol/Nuthatch.tla

@@ -0,0 +1,662 @@
+----------------------------- MODULE Nuthatch -----------------------------
+EXTENDS FiniteSets, Naturals, TLC
+
+\* This model covers authorized one-key mutations within one idempotency epoch;
+\* lost bodies are not resumed and settle from the observed object state.
+
+CONSTANTS
+    Users, OperationIds, EnabledKinds, PersonalKeyNames, SharedKeys, Contents,
+    MaxObjectSize, PersonalLimit, SharedLimit,
+    MaxPostgresLosses, MaxObjectStoreLosses,
+    SharedPool, NoObject, NoRecord, NoOwner, NoObservation
+
+ASSUME
+    /\ Users # {}
+    /\ OperationIds # {}
+    /\ EnabledKinds # {}
+    /\ EnabledKinds \subseteq {"put", "delete"}
+    /\ PersonalKeyNames # {}
+    /\ SharedKeys # {}
+    /\ (Users \X PersonalKeyNames) \cap SharedKeys = {}
+    /\ Contents # {}
+    /\ SharedPool \notin Users
+    /\ NoObject \notin Contents
+    /\ MaxObjectSize \in Nat \ {0}
+    /\ PersonalLimit \in Nat
+    /\ SharedLimit \in Nat
+    /\ MaxPostgresLosses \in Nat
+    /\ MaxObjectStoreLosses \in Nat
+
+PersonalKeys == Users \X PersonalKeyNames
+Keys == PersonalKeys \cup SharedKeys
+Pools == Users \cup {SharedPool}
+Kinds == EnabledKinds
+Identities == Users \X Kinds \X OperationIds
+Sizes == 1..MaxObjectSize
+
+ObjectValues ==
+    {NoObject} \cup
+    [size : Sizes, content : Contents]
+
+Requests ==
+    [user      : Users,
+     operation : OperationIds,
+     kind      : Kinds,
+     key       : Keys,
+     size      : 0..MaxObjectSize,
+     content   : Contents \cup {NoObject}]
+
+ValidRequests ==
+    {request \in Requests :
+        /\ \/ request.key \in SharedKeys
+           \/ /\ request.key \in PersonalKeys
+              /\ request.key[1] = request.user
+        /\ IF request.kind = "put"
+           THEN /\ request.size \in Sizes
+                /\ request.content \in Contents
+           ELSE /\ request.size = 0
+                /\ request.content = NoObject}
+
+RecordStates == {"prepared", "applying", "succeeded", "failed", "rejected"}
+ActiveStates == {"prepared", "applying"}
+TerminalStates == {"succeeded", "failed", "rejected"}
+ExternalStates == {"idle", "pending", "settled"}
+
+RecordValues ==
+    [request : ValidRequests,
+     prior   : ObjectValues,
+     state   : RecordStates]
+
+Results == {"pending", "succeeded", "failed", "rejected", "conflict"}
+Dispositions == {"first", "replay", "conflict"}
+
+Observations ==
+    [request     : ValidRequests,
+     result      : Results,
+     disposition : Dispositions]
+
+VARIABLES
+    objects,
+    usedBytes,
+    reservedBytes,
+    keyOwner,
+    records,
+    externalState,
+    uploadBodyAvailable,
+    postgresUp,
+    objectStoreUp,
+    postgresLosses,
+    objectStoreLosses,
+    observation
+
+\* externalState is environmental truth: "settled" means no issued request can
+\* later mutate its key. Process loss does not erase a pending store request.
+
+vars == <<
+    objects,
+    usedBytes,
+    reservedBytes,
+    keyOwner,
+    records,
+    externalState,
+    uploadBodyAvailable,
+    postgresUp,
+    objectStoreUp,
+    postgresLosses,
+    objectStoreLosses,
+    observation
+>>
+
+Identity(request) == <<request.user, request.kind, request.operation>>
+
+PoolOf(key) ==
+    IF key \in PersonalKeys THEN key[1] ELSE SharedPool
+
+PoolKeys(pool) ==
+    IF pool = SharedPool
+    THEN SharedKeys
+    ELSE {key \in PersonalKeys : key[1] = pool}
+
+Limit(pool) ==
+    IF pool = SharedPool THEN SharedLimit ELSE PersonalLimit
+
+ObjectSize(object) ==
+    IF object = NoObject THEN 0 ELSE object.size
+
+DesiredObject(request) ==
+    IF request.kind = "delete"
+    THEN NoObject
+    ELSE [size |-> request.size, content |-> request.content]
+
+Growth(prior, desired) ==
+    IF ObjectSize(desired) > ObjectSize(prior)
+    THEN ObjectSize(desired) - ObjectSize(prior)
+    ELSE 0
+
+RequestFingerprint(request) ==
+    <<request.kind, PoolOf(request.key), request.key,
+      request.size, request.content>>
+
+RECURSIVE ObjectBytes(_, _)
+
+ObjectBytes(objectMap, keys) ==
+    IF keys = {}
+    THEN 0
+    ELSE LET key == CHOOSE candidate \in keys : TRUE
+         IN ObjectSize(objectMap[key]) +
+            ObjectBytes(objectMap, keys \ {key})
+
+Active(identity) ==
+    /\ records[identity] # NoRecord
+    /\ records[identity].state \in ActiveStates
+
+Terminal(identity) ==
+    /\ records[identity] # NoRecord
+    /\ records[identity].state \in TerminalStates
+
+Reservation(identity) ==
+    IF Active(identity)
+    THEN Growth(records[identity].prior,
+                DesiredObject(records[identity].request))
+    ELSE 0
+
+RECURSIVE ReservationTotal(_, _)
+
+ReservationTotal(identities, pool) ==
+    IF identities = {}
+    THEN 0
+    ELSE LET identity == CHOOSE candidate \in identities : TRUE
+             remainder == identities \ {identity}
+             amount ==
+                IF Active(identity) /\
+                   PoolOf(records[identity].request.key) = pool
+                THEN Reservation(identity)
+                ELSE 0
+         IN amount + ReservationTotal(remainder, pool)
+
+StateResult(state) ==
+    CASE state \in ActiveStates -> "pending"
+      [] state = "succeeded" -> "succeeded"
+      [] state = "failed" -> "failed"
+      [] state = "rejected" -> "rejected"
+
+CanAdmit(request) ==
+    LET pool == PoolOf(request.key)
+        prior == objects[request.key]
+        desired == DesiredObject(request)
+    IN /\ keyOwner[request.key] = NoOwner
+       /\ usedBytes[pool] + reservedBytes[pool] +
+          Growth(prior, desired) <= Limit(pool)
+
+Init ==
+    /\ objects \in [Keys -> ObjectValues]
+    /\ \A pool \in Pools :
+        ObjectBytes(objects, PoolKeys(pool)) <= Limit(pool)
+    /\ usedBytes =
+        [pool \in Pools |-> ObjectBytes(objects, PoolKeys(pool))]
+    /\ reservedBytes = [pool \in Pools |-> 0]
+    /\ keyOwner = [key \in Keys |-> NoOwner]
+    /\ records = [identity \in Identities |-> NoRecord]
+    /\ externalState = [identity \in Identities |-> "idle"]
+    /\ uploadBodyAvailable = [identity \in Identities |-> FALSE]
+    /\ postgresUp = TRUE
+    /\ objectStoreUp = TRUE
+    /\ postgresLosses = 0
+    /\ objectStoreLosses = 0
+    /\ observation = NoObservation
+
+FirstRequest(request) ==
+    LET identity == Identity(request)
+        pool == PoolOf(request.key)
+        prior == objects[request.key]
+        desired == DesiredObject(request)
+        admitted == CanAdmit(request)
+        nextState == IF admitted THEN "prepared" ELSE "rejected"
+        record ==
+            [request |-> request, prior |-> prior, state |-> nextState]
+        growth == Growth(prior, desired)
+    IN /\ postgresUp
+       /\ records[identity] = NoRecord
+       /\ records' = [records EXCEPT ![identity] = record]
+       /\ reservedBytes' =
+            IF admitted
+            THEN [reservedBytes EXCEPT ![pool] = @ + growth]
+            ELSE reservedBytes
+       /\ keyOwner' =
+            IF admitted
+            THEN [keyOwner EXCEPT ![request.key] = identity]
+            ELSE keyOwner
+       /\ uploadBodyAvailable' =
+            [uploadBodyAvailable EXCEPT
+                ![identity] = admitted /\ request.kind = "put"]
+       /\ observation' =
+            [request |-> request,
+             result |-> IF admitted THEN "pending" ELSE "rejected",
+             disposition |-> "first"]
+       /\ UNCHANGED <<
+            objects, usedBytes, externalState,
+            postgresUp, objectStoreUp,
+            postgresLosses, objectStoreLosses
+          >>
+
+ReplayRequest(request) ==
+    LET identity == Identity(request)
+        record == records[identity]
+    IN /\ postgresUp
+       /\ record # NoRecord
+       /\ record.request = request
+       /\ observation' =
+            [request |-> request,
+             result |-> StateResult(record.state),
+             disposition |-> "replay"]
+       /\ UNCHANGED <<
+            objects, usedBytes, reservedBytes, keyOwner, records,
+            externalState, uploadBodyAvailable,
+            postgresUp, objectStoreUp,
+            postgresLosses, objectStoreLosses
+          >>
+
+ConflictRequest(request) ==
+    LET identity == Identity(request)
+        record == records[identity]
+    IN /\ postgresUp
+       /\ record # NoRecord
+       /\ RequestFingerprint(record.request) # RequestFingerprint(request)
+       /\ observation' =
+            [request |-> request,
+             result |-> "conflict",
+             disposition |-> "conflict"]
+       /\ UNCHANGED <<
+            objects, usedBytes, reservedBytes, keyOwner, records,
+            externalState, uploadBodyAvailable,
+            postgresUp, objectStoreUp,
+            postgresLosses, objectStoreLosses
+          >>
+
+PrepareEffect(identity) ==
+    /\ postgresUp
+    /\ records[identity] # NoRecord
+    /\ records[identity].state = "prepared"
+    /\ records' = [records EXCEPT ![identity].state = "applying"]
+    /\ observation' = NoObservation
+    /\ UNCHANGED <<
+        objects, usedBytes, reservedBytes, keyOwner, externalState,
+        uploadBodyAvailable,
+        postgresUp, objectStoreUp, postgresLosses, objectStoreLosses
+       >>
+
+DispatchEffect(identity) ==
+    /\ postgresUp
+    /\ objectStoreUp
+    /\ records[identity] # NoRecord
+    /\ records[identity].state = "applying"
+    /\ externalState[identity] = "idle"
+    /\ (records[identity].request.kind = "delete" \/
+        uploadBodyAvailable[identity])
+    /\ externalState' =
+        [externalState EXCEPT ![identity] = "pending"]
+    /\ uploadBodyAvailable' =
+        [uploadBodyAvailable EXCEPT ![identity] = FALSE]
+    /\ observation' = NoObservation
+    /\ UNCHANGED <<
+        objects, usedBytes, reservedBytes, keyOwner, records,
+        postgresUp, objectStoreUp, postgresLosses, objectStoreLosses
+       >>
+
+LoseUploadBody(identity) ==
+    /\ records[identity] # NoRecord
+    /\ records[identity].request.kind = "put"
+    /\ Active(identity)
+    /\ externalState[identity] = "idle"
+    /\ uploadBodyAvailable[identity]
+    /\ uploadBodyAvailable' =
+        [uploadBodyAvailable EXCEPT ![identity] = FALSE]
+    /\ observation' = NoObservation
+    /\ UNCHANGED <<
+        objects, usedBytes, reservedBytes, keyOwner, records, externalState,
+        postgresUp, objectStoreUp, postgresLosses, objectStoreLosses
+       >>
+
+SettleLostUpload(identity) ==
+    /\ postgresUp
+    /\ objectStoreUp
+    /\ records[identity] # NoRecord
+    /\ records[identity].state = "applying"
+    /\ records[identity].request.kind = "put"
+    /\ externalState[identity] = "idle"
+    /\ ~uploadBodyAvailable[identity]
+    /\ externalState' =
+        [externalState EXCEPT ![identity] = "settled"]
+    /\ observation' = NoObservation
+    /\ UNCHANGED <<
+        objects, usedBytes, reservedBytes, keyOwner, records,
+        uploadBodyAvailable,
+        postgresUp, objectStoreUp, postgresLosses, objectStoreLosses
+       >>
+
+ExternalEffectApplied(identity) ==
+    LET request == records[identity].request
+    IN /\ objectStoreUp
+       /\ records[identity] # NoRecord
+       /\ records[identity].state = "applying"
+       /\ externalState[identity] = "pending"
+       /\ objects' =
+            [objects EXCEPT ![request.key] = DesiredObject(request)]
+       /\ externalState' =
+            [externalState EXCEPT ![identity] = "settled"]
+       /\ observation' = NoObservation
+       /\ UNCHANGED <<
+            usedBytes, reservedBytes, keyOwner, records,
+            uploadBodyAvailable,
+            postgresUp, objectStoreUp,
+            postgresLosses, objectStoreLosses
+          >>
+
+ExternalEffectNotApplied(identity) ==
+    /\ objectStoreUp
+    /\ records[identity] # NoRecord
+    /\ records[identity].state = "applying"
+    /\ externalState[identity] = "pending"
+    /\ externalState' =
+        [externalState EXCEPT ![identity] = "settled"]
+    /\ observation' = NoObservation
+    /\ UNCHANGED <<
+        objects, usedBytes, reservedBytes, keyOwner, records,
+        uploadBodyAvailable,
+        postgresUp, objectStoreUp, postgresLosses, objectStoreLosses
+       >>
+
+FinalizeSuccess(identity) ==
+    LET record == records[identity]
+        request == record.request
+        key == request.key
+        pool == PoolOf(key)
+        priorSize == ObjectSize(record.prior)
+        desired == DesiredObject(request)
+        desiredSize == ObjectSize(desired)
+        growth == Growth(record.prior, desired)
+    IN /\ postgresUp
+       /\ objectStoreUp
+       /\ record # NoRecord
+       /\ record.state = "applying"
+       /\ externalState[identity] = "settled"
+       /\ objects[key] = desired
+       /\ records' = [records EXCEPT ![identity].state = "succeeded"]
+       /\ usedBytes' =
+            [usedBytes EXCEPT ![pool] = (@ - priorSize) + desiredSize]
+       /\ reservedBytes' =
+            [reservedBytes EXCEPT ![pool] = @ - growth]
+       /\ keyOwner' = [keyOwner EXCEPT ![key] = NoOwner]
+       /\ externalState' =
+            [externalState EXCEPT ![identity] = "idle"]
+       /\ uploadBodyAvailable' =
+            [uploadBodyAvailable EXCEPT ![identity] = FALSE]
+       /\ observation' = NoObservation
+       /\ UNCHANGED <<
+            objects, postgresUp, objectStoreUp,
+            postgresLosses, objectStoreLosses
+          >>
+
+FinalizeFailure(identity) ==
+    LET record == records[identity]
+        request == record.request
+        key == request.key
+        pool == PoolOf(key)
+        desired == DesiredObject(request)
+        growth == Growth(record.prior, desired)
+    IN /\ postgresUp
+       /\ objectStoreUp
+       /\ record # NoRecord
+       /\ record.state = "applying"
+       /\ externalState[identity] = "settled"
+       /\ objects[key] = record.prior
+       /\ record.prior # desired
+       /\ records' = [records EXCEPT ![identity].state = "failed"]
+       /\ reservedBytes' =
+            [reservedBytes EXCEPT ![pool] = @ - growth]
+       /\ keyOwner' = [keyOwner EXCEPT ![key] = NoOwner]
+       /\ externalState' =
+            [externalState EXCEPT ![identity] = "idle"]
+       /\ uploadBodyAvailable' =
+            [uploadBodyAvailable EXCEPT ![identity] = FALSE]
+       /\ observation' = NoObservation
+       /\ UNCHANGED <<
+            objects, usedBytes, postgresUp, objectStoreUp,
+            postgresLosses, objectStoreLosses
+          >>
+
+LosePostgres ==
+    /\ postgresUp
+    /\ postgresLosses < MaxPostgresLosses
+    /\ postgresUp' = FALSE
+    /\ postgresLosses' = postgresLosses + 1
+    /\ observation' = NoObservation
+    /\ UNCHANGED <<
+        objects, usedBytes, reservedBytes, keyOwner, records,
+        externalState, uploadBodyAvailable,
+        objectStoreUp, objectStoreLosses
+       >>
+
+RestorePostgres ==
+    /\ ~postgresUp
+    /\ postgresUp' = TRUE
+    /\ observation' = NoObservation
+    /\ UNCHANGED <<
+        objects, usedBytes, reservedBytes, keyOwner, records,
+        externalState, uploadBodyAvailable, objectStoreUp,
+        postgresLosses, objectStoreLosses
+       >>
+
+LoseObjectStore ==
+    /\ objectStoreUp
+    /\ objectStoreLosses < MaxObjectStoreLosses
+    /\ objectStoreUp' = FALSE
+    /\ objectStoreLosses' = objectStoreLosses + 1
+    /\ observation' = NoObservation
+    /\ UNCHANGED <<
+        objects, usedBytes, reservedBytes, keyOwner, records,
+        externalState, uploadBodyAvailable, postgresUp, postgresLosses
+       >>
+
+RestoreObjectStore ==
+    /\ ~objectStoreUp
+    /\ objectStoreUp' = TRUE
+    /\ observation' = NoObservation
+    /\ UNCHANGED <<
+        objects, usedBytes, reservedBytes, keyOwner, records,
+        externalState, uploadBodyAvailable, postgresUp,
+        postgresLosses, objectStoreLosses
+       >>
+
+ControllerAdvance(identity) ==
+    \/ PrepareEffect(identity)
+    \/ DispatchEffect(identity)
+    \/ LoseUploadBody(identity)
+    \/ SettleLostUpload(identity)
+    \/ FinalizeSuccess(identity)
+    \/ FinalizeFailure(identity)
+
+StoreSettlement(identity) ==
+    \/ ExternalEffectApplied(identity)
+    \/ ExternalEffectNotApplied(identity)
+
+Next ==
+    \/ \E request \in ValidRequests : FirstRequest(request)
+    \/ \E request \in ValidRequests : ReplayRequest(request)
+    \/ \E request \in ValidRequests : ConflictRequest(request)
+    \/ \E identity \in Identities : ControllerAdvance(identity)
+    \/ \E identity \in Identities : StoreSettlement(identity)
+    \/ LosePostgres
+    \/ RestorePostgres
+    \/ LoseObjectStore
+    \/ RestoreObjectStore
+
+Fairness ==
+    /\ WF_vars(RestorePostgres)
+    /\ WF_vars(RestoreObjectStore)
+    /\ \A identity \in Identities :
+        /\ WF_vars(ControllerAdvance(identity))
+        /\ WF_vars(StoreSettlement(identity))
+
+Spec == Init /\ [][Next]_vars /\ Fairness
+
+TypeOK ==
+    /\ objects \in [Keys -> ObjectValues]
+    /\ usedBytes \in [Pools -> Nat]
+    /\ reservedBytes \in [Pools -> Nat]
+    /\ keyOwner \in [Keys -> Identities \cup {NoOwner}]
+    /\ records \in [Identities -> RecordValues \cup {NoRecord}]
+    /\ externalState \in [Identities -> ExternalStates]
+    /\ uploadBodyAvailable \in [Identities -> BOOLEAN]
+    /\ postgresUp \in BOOLEAN
+    /\ objectStoreUp \in BOOLEAN
+    /\ postgresLosses \in 0..MaxPostgresLosses
+    /\ objectStoreLosses \in 0..MaxObjectStoreLosses
+    /\ observation \in Observations \cup {NoObservation}
+
+RecordIdentityConsistent ==
+    \A identity \in Identities :
+        records[identity] # NoRecord =>
+            Identity(records[identity].request) = identity
+
+PersonalRequestsUseOwnPool ==
+    \A request \in ValidRequests :
+        request.key \in PersonalKeys =>
+            PoolOf(request.key) = request.user
+
+PersonalPoolsDisjoint ==
+    \A first, second \in Users :
+        first # second =>
+            PoolKeys(first) \cap PoolKeys(second) = {}
+
+FingerprintBindsRequest ==
+    \A first, second \in ValidRequests :
+        Identity(first) = Identity(second) /\
+        RequestFingerprint(first) = RequestFingerprint(second)
+        => first = second
+
+ReservationsConsistent ==
+    \A pool \in Pools :
+        reservedBytes[pool] = ReservationTotal(Identities, pool)
+
+LocksConsistent ==
+    \A key \in Keys :
+        \/ /\ keyOwner[key] = NoOwner
+           /\ ~\E identity \in Identities :
+                Active(identity) /\
+                records[identity].request.key = key
+        \/ /\ keyOwner[key] \in Identities
+           /\ Active(keyOwner[key])
+           /\ records[keyOwner[key]].request.key = key
+           /\ \A identity \in Identities :
+                Active(identity) /\
+                records[identity].request.key = key
+                => identity = keyOwner[key]
+
+ActiveObjectConsistent ==
+    \A identity \in Identities :
+        Active(identity) =>
+            LET record == records[identity]
+                key == record.request.key
+            IN objects[key] \in
+                {record.prior, DesiredObject(record.request)}
+
+ExternalStateConsistent ==
+    \A identity \in Identities :
+        /\ (externalState[identity] # "idle" =>
+                /\ records[identity] # NoRecord
+                /\ records[identity].state = "applying")
+        /\ (records[identity] # NoRecord /\
+            records[identity].state # "applying" =>
+                externalState[identity] = "idle")
+
+UploadBodyConsistent ==
+    \A identity \in Identities :
+        uploadBodyAvailable[identity] =>
+            /\ Active(identity)
+            /\ records[identity].request.kind = "put"
+            /\ externalState[identity] = "idle"
+
+QuotaReservationBound ==
+    \A pool \in Pools :
+        usedBytes[pool] + reservedBytes[pool] <= Limit(pool)
+
+ObjectStateCovered ==
+    \A pool \in Pools :
+        ObjectBytes(objects, PoolKeys(pool)) <=
+            usedBytes[pool] + reservedBytes[pool]
+
+ObjectQuotaBound ==
+    \A pool \in Pools :
+        ObjectBytes(objects, PoolKeys(pool)) <= Limit(pool)
+
+AccountingAtQuiescence ==
+    \A pool \in Pools :
+        (~\E identity \in Identities :
+            Active(identity) /\
+            PoolOf(records[identity].request.key) = pool) =>
+            usedBytes[pool] = ObjectBytes(objects, PoolKeys(pool))
+
+ObservationConsistent ==
+    observation # NoObservation =>
+        LET identity == Identity(observation.request)
+            record == records[identity]
+        IN /\ record # NoRecord
+           /\ CASE observation.disposition = "first" ->
+                    /\ record.request = observation.request
+                    /\ observation.result = StateResult(record.state)
+              [] observation.disposition = "replay" ->
+                    /\ record.request = observation.request
+                    /\ observation.result = StateResult(record.state)
+              [] observation.disposition = "conflict" ->
+                    /\ RequestFingerprint(record.request) #
+                       RequestFingerprint(observation.request)
+                    /\ observation.result = "conflict"
+
+RecordsNeverForgottenAction ==
+    \A identity \in Identities :
+        records[identity] # NoRecord => records'[identity] # NoRecord
+
+RequestBindingsStableAction ==
+    \A identity \in Identities :
+        records[identity] # NoRecord =>
+            records'[identity].request = records[identity].request
+
+TerminalRecordsStableAction ==
+    \A identity \in Identities :
+        Terminal(identity) => records'[identity] = records[identity]
+
+PostgresFailureFenceAction ==
+    (~postgresUp /\ ~postgresUp') =>
+        UNCHANGED <<usedBytes, reservedBytes, keyOwner, records>>
+
+ObjectStoreFailureFenceAction ==
+    (~objectStoreUp /\ ~objectStoreUp') =>
+        UNCHANGED <<objects, externalState>>
+
+SettledEffectsDoNotMutateAction ==
+    \A identity \in Identities :
+        Active(identity) /\ externalState[identity] = "settled" =>
+            LET key == records[identity].request.key
+            IN objects'[key] = objects[key]
+
+RecordsNeverForgotten == [][RecordsNeverForgottenAction]_vars
+RequestBindingsStable == [][RequestBindingsStableAction]_vars
+TerminalRecordsStable == [][TerminalRecordsStableAction]_vars
+PostgresFailureFencesDurableState == [][PostgresFailureFenceAction]_vars
+ObjectStoreFailureFencesObjects == [][ObjectStoreFailureFenceAction]_vars
+SettledEffectsDoNotMutate == [][SettledEffectsDoNotMutateAction]_vars
+
+UnfinishedEventuallyTerminal ==
+    \A identity \in Identities :
+        Active(identity) ~> Terminal(identity)
+
+LostUploadEventuallyTerminal ==
+    \A identity \in Identities :
+        (Active(identity) /\
+         records[identity].request.kind = "put" /\
+         ~uploadBodyAvailable[identity] /\
+         externalState[identity] = "idle")
+        ~> Terminal(identity)
+
+=============================================================================

docs/protocol/NuthatchContentIdentity.cfg

@@ -0,0 +1,48 @@
+SPECIFICATION Spec
+
+CHECK_DEADLOCK TRUE
+
+CONSTANTS
+    Users = {u1}
+    OperationIds = {op1}
+    EnabledKinds = {"put"}
+    PersonalKeyNames = {personalKey}
+    SharedKeys = {sharedKey}
+    Contents = {content1, content2}
+    MaxObjectSize = 1
+    PersonalLimit = 1
+    SharedLimit = 1
+    MaxPostgresLosses = 0
+    MaxObjectStoreLosses = 0
+    SharedPool = SharedPool
+    NoObject = NoObject
+    NoRecord = NoRecord
+    NoOwner = NoOwner
+    NoObservation = NoObservation
+
+INVARIANTS
+    TypeOK
+    RecordIdentityConsistent
+    PersonalRequestsUseOwnPool
+    PersonalPoolsDisjoint
+    FingerprintBindsRequest
+    ReservationsConsistent
+    LocksConsistent
+    ActiveObjectConsistent
+    ExternalStateConsistent
+    UploadBodyConsistent
+    QuotaReservationBound
+    ObjectStateCovered
+    ObjectQuotaBound
+    AccountingAtQuiescence
+    ObservationConsistent
+
+PROPERTIES
+    RecordsNeverForgotten
+    RequestBindingsStable
+    TerminalRecordsStable
+    PostgresFailureFencesDurableState
+    ObjectStoreFailureFencesObjects
+    SettledEffectsDoNotMutate
+    UnfinishedEventuallyTerminal
+    LostUploadEventuallyTerminal

docs/protocol/NuthatchKinds.cfg

@@ -0,0 +1,48 @@
+SPECIFICATION Spec
+
+CHECK_DEADLOCK TRUE
+
+CONSTANTS
+    Users = {u1}
+    OperationIds = {op1}
+    EnabledKinds = {"put", "delete"}
+    PersonalKeyNames = {personalKey}
+    SharedKeys = {sharedKey}
+    Contents = {content1}
+    MaxObjectSize = 1
+    PersonalLimit = 1
+    SharedLimit = 1
+    MaxPostgresLosses = 0
+    MaxObjectStoreLosses = 0
+    SharedPool = SharedPool
+    NoObject = NoObject
+    NoRecord = NoRecord
+    NoOwner = NoOwner
+    NoObservation = NoObservation
+
+INVARIANTS
+    TypeOK
+    RecordIdentityConsistent
+    PersonalRequestsUseOwnPool
+    PersonalPoolsDisjoint
+    FingerprintBindsRequest
+    ReservationsConsistent
+    LocksConsistent
+    ActiveObjectConsistent
+    ExternalStateConsistent
+    UploadBodyConsistent
+    QuotaReservationBound
+    ObjectStateCovered
+    ObjectQuotaBound
+    AccountingAtQuiescence
+    ObservationConsistent
+
+PROPERTIES
+    RecordsNeverForgotten
+    RequestBindingsStable
+    TerminalRecordsStable
+    PostgresFailureFencesDurableState
+    ObjectStoreFailureFencesObjects
+    SettledEffectsDoNotMutate
+    UnfinishedEventuallyTerminal
+    LostUploadEventuallyTerminal

docs/protocol/NuthatchQuota.cfg

@@ -0,0 +1,48 @@
+SPECIFICATION Spec
+
+CHECK_DEADLOCK TRUE
+
+CONSTANTS
+    Users = {u1}
+    OperationIds = {op1, op2}
+    EnabledKinds = {"put"}
+    PersonalKeyNames = {personalKey1, personalKey2, personalKey3}
+    SharedKeys = {sharedKey}
+    Contents = {content1}
+    MaxObjectSize = 1
+    PersonalLimit = 2
+    SharedLimit = 1
+    MaxPostgresLosses = 0
+    MaxObjectStoreLosses = 0
+    SharedPool = SharedPool
+    NoObject = NoObject
+    NoRecord = NoRecord
+    NoOwner = NoOwner
+    NoObservation = NoObservation
+
+INVARIANTS
+    TypeOK
+    RecordIdentityConsistent
+    PersonalRequestsUseOwnPool
+    PersonalPoolsDisjoint
+    FingerprintBindsRequest
+    ReservationsConsistent
+    LocksConsistent
+    ActiveObjectConsistent
+    ExternalStateConsistent
+    UploadBodyConsistent
+    QuotaReservationBound
+    ObjectStateCovered
+    ObjectQuotaBound
+    AccountingAtQuiescence
+    ObservationConsistent
+
+PROPERTIES
+    RecordsNeverForgotten
+    RequestBindingsStable
+    TerminalRecordsStable
+    PostgresFailureFencesDurableState
+    ObjectStoreFailureFencesObjects
+    SettledEffectsDoNotMutate
+    UnfinishedEventuallyTerminal
+    LostUploadEventuallyTerminal

docs/protocol/NuthatchSequential.cfg

@@ -0,0 +1,48 @@
+SPECIFICATION Spec
+
+CHECK_DEADLOCK TRUE
+
+CONSTANTS
+    Users = {u1}
+    OperationIds = {op1, op2}
+    EnabledKinds = {"put"}
+    PersonalKeyNames = {personalKey}
+    SharedKeys = {sharedKey}
+    Contents = {content1}
+    MaxObjectSize = 2
+    PersonalLimit = 3
+    SharedLimit = 3
+    MaxPostgresLosses = 0
+    MaxObjectStoreLosses = 0
+    SharedPool = SharedPool
+    NoObject = NoObject
+    NoRecord = NoRecord
+    NoOwner = NoOwner
+    NoObservation = NoObservation
+
+INVARIANTS
+    TypeOK
+    RecordIdentityConsistent
+    PersonalRequestsUseOwnPool
+    PersonalPoolsDisjoint
+    FingerprintBindsRequest
+    ReservationsConsistent
+    LocksConsistent
+    ActiveObjectConsistent
+    ExternalStateConsistent
+    UploadBodyConsistent
+    QuotaReservationBound
+    ObjectStateCovered
+    ObjectQuotaBound
+    AccountingAtQuiescence
+    ObservationConsistent
+
+PROPERTIES
+    RecordsNeverForgotten
+    RequestBindingsStable
+    TerminalRecordsStable
+    PostgresFailureFencesDurableState
+    ObjectStoreFailureFencesObjects
+    SettledEffectsDoNotMutate
+    UnfinishedEventuallyTerminal
+    LostUploadEventuallyTerminal

docs/spec/nuthatch.md

@@ -0,0 +1,501 @@
+# Nuthatch — user-facing S3 BFF
+
+## Status and terminology
+
+This document defines the version-one product behavior, architecture, trust
+boundaries, dependency roles, and externally observable requirements for
+Nuthatch. It does not define mutable HTTP schemas, concrete database schemas,
+implementation libraries, or deployment-specific numeric settings.
+
+The key words **MUST**, **MUST NOT**, **SHOULD**, **SHOULD NOT**, and **MAY** are
+to be interpreted as described by RFC 2119 and RFC 8174.
+
+Requirements tables and explicitly labeled rules are normative even when they
+do not use one of these key words.
+
+## Specification ownership
+
+This document is authoritative for:
+
+- user-visible storage behavior and access boundaries;
+- the identity and object-root model;
+- externally visible quota and mutation guarantees;
+- allocation of authority among Nuthatch, PostgreSQL, Redis, and the object
+  store; and
+- failure behavior at dependency boundaries.
+
+[`Nuthatch.tla`](../protocol/Nuthatch.tla) is the design of record for the
+abstract mutation protocol. It owns mutation states, transition ordering,
+reservations, exclusive same-key admission, duplicate requests, upload-body
+loss, crash recovery, object-store settlement assumptions, and the safety and
+liveness properties that establish the guarantees in this document. Concrete
+mutation code MUST refine that model.
+
+The TLA+ configurations are checked finite instances. Their finite users, keys,
+sizes, contents, operation ids, quotas, and fault counts are not production
+limits.
+
+HTTP routes and schemas, SQL layouts, libraries, object-store vendors, identity
+providers, cryptographic algorithms, and deployment topology are separate
+contracts or deployment choices. They may narrow behavior but MUST preserve
+this specification.
+
+### Core terms
+
+| Term | Definition |
+| ---- | ---------- |
+| Pool | One quota-accounting domain: a personal root or the shared root |
+| Mutation | One put, overwrite, or delete against one canonical object key |
+| Operation id | A caller-generated opaque identifier for one logical mutation |
+| Request fingerprint | A canonical identity of the mutation kind, target, and content where applicable |
+| Content identity | A collision-resistant digest binding the exact uploaded bytes; its algorithm and encoding belong to the API contract |
+| Successful mutation | A mutation whose object state and PostgreSQL accounting are durably finalized |
+| Unfinished mutation | A durably recorded mutation that has not yet reached its successful or unsuccessful final result |
+
+## 1. Problem statement
+
+| Item | Detail |
+| ---- | ------ |
+| Need | Authenticated end users must upload, download, browse, search, and delete files without operator privileges on the object store |
+| Problem with admin UIs | Typical S3 admin consoles are operator-privileged and must not be the user plane |
+| Scale | Fewer than ten users |
+
+| Must not | Reason |
+| -------- | ------ |
+| Anonymous / public share links | Out of trust model |
+| Desktop sync or desktop app | Not required |
+| Office / collaboration suite | Not required |
+| Browser-held object-store credentials | Key leak and over-privilege |
+| User access to the object-store admin console | Wrong control plane |
+
+---
+
+## 2. Proposed solution
+
+**Nuthatch** is a **backend-for-frontend (BFF)**: a web UI and API that call an S3-compatible API with a single service principal and enforce identity-derived key prefixes. End users never receive object-store credentials.
+
+Nuthatch is an **OIDC relying party** and performs its own authorization-code flow. An external auth proxy is not required on this path.
+
+| Decision | Choice |
+| -------- | ------ |
+| Product name | **Nuthatch** |
+| Service / executable name | `nuthatch` |
+| Auth | OIDC. Login UI is the identity provider’s hosted login |
+| Session | Signed JWT cookie (HttpOnly, Secure, host-only). No server session store |
+| CSRF | Required for cookie-authenticated state-changing requests |
+| Durable state | **PostgreSQL**: quota limits, usage, and protocol-required mutation metadata |
+| Shared ephemeral state | **Redis-compatible** store: list/metadata cache and bandwidth throttle |
+| Object store | **S3-compatible** API, one bucket, one service principal |
+| Key layout | Personal `users/{user_id}/` derived from verified (`iss`, `sub`). Shared `shared/` |
+| Audit | Structured JSON on stdout |
+| Admin product surface | None. Object-store admin tooling remains operators-only |
+| Implementation language | Not fixed by this design |
+
+| Principle | Meaning |
+| --------- | ------- |
+| Stateless controller | No in-process authoritative or cross-request product state. Any replica may serve any request |
+| Shared ephemeral coordination | Cache and throttle live in Redis so multi-replica works without sticky sessions |
+| Service object principal | One IAM-style user. The app enforces per-user and shared prefixes |
+| No browser to object store | All product-path object bytes stream through Nuthatch |
+| Objects for end users only via BFF | Operators may inspect objects separately but MUST NOT mutate Nuthatch-owned prefixes while product service is enabled |
+| Mutation integrity | Configured quota limits are never exceeded. A successful mutation includes durable quota accounting, and retrying one logical mutation within the advertised idempotency window is idempotent |
+
+### 2.1 Runtime dependencies
+
+| Dependency | Role | Interface |
+| ---------- | ---- | --------- |
+| OIDC identity provider | Authenticate users | Standard OIDC (authorization code + PKCE) |
+| PostgreSQL | Quota limits and usage | SQL |
+| Redis-compatible store | List/metadata cache, bandwidth throttle | Redis protocol |
+| S3-compatible object store | Object bytes | Ordinary object put, get, head, delete, and list operations |
+
+Nuthatch MUST NOT require object versioning, conditional mutation, event
+delivery, server-side copy, or transactional object-store extensions for
+correctness.
+
+No other runtime dependencies are required for the product (no message bus, no search engine, no durable server-side session store, no auth proxy). Short-lived OIDC transaction state may be held in an integrity-protected, HttpOnly browser cookie.
+
+Secrets (OIDC client secret, object-store keys, JWT signing material, Redis password if any) are supplied by the deployment environment.
+
+---
+
+## 3. Architecture
+
+### 3.1 Context
+
+```mermaid
+flowchart TB
+  User[End user browser] -->|HTTPS| App[Nuthatch]
+  App -->|OIDC| IdP[OIDC identity provider]
+  App -->|quota| PG[(PostgreSQL)]
+  App -->|cache and throttle| Redis[(Redis compatible)]
+  App -->|S3 API| S3[S3 compatible store]
+  App -->|stdout| Logs[Log sink]
+```
+
+### 3.2 Request path
+
+Every authenticated request validates the Nuthatch session JWT and binds the verified OIDC identity. The server derives the active storage root; the browser never supplies a complete S3 key. Bandwidth accounting applies to **upload and download** body bytes (see §3.8). Further steps depend on the operation.
+
+```mermaid
+flowchart TD
+  start[Request] --> jwt{Valid JWT cookie}
+  jwt -->|no| oidc[OIDC code flow]
+  oidc --> mint[Mint JWT cookie]
+  mint --> ready
+  jwt -->|yes| ready{Operation}
+  ready -->|browse| listPath[List under active root]
+  ready -->|search| searchPath[Search under active root]
+  ready -->|download| dlPath[Download under allowed key]
+  ready -->|upload| upPath[Upload under allowed key]
+  ready -->|delete| delPath[Delete key or prefix page]
+  listPath --> listCache[List cache then S3 ListObjects]
+  listCache --> page[HTML or JSON]
+  searchPath --> searchS3[S3 list under root then filter keys]
+  searchS3 --> page
+  dlPath --> dlThrottle[Count download bytes toward throttle]
+  dlThrottle --> streamOut[Byte stream response]
+  upPath --> csrfUp[CSRF check]
+  csrfUp --> quota[Admit mutation under quota invariant]
+  quota --> upThrottle[Count upload bytes toward throttle]
+  upThrottle --> put[S3 put]
+  put --> usageUp[Finalize object and durable accounting]
+  usageUp --> inv[Invalidate list cache for parent prefixes]
+  inv --> page
+  delPath --> csrfDel[CSRF check]
+  csrfDel --> delKey[S3 delete one key]
+  delKey --> usageDn[Finalize per-key mutation outcome]
+  usageDn --> inv
+```
+
+### 3.3 Trust and data planes
+
+| Plane | Store | Contents |
+| ----- | ----- | -------- |
+| Identity | OIDC IdP | Users and OIDC client registration for Nuthatch |
+| Objects | S3-compatible store | Keys under `users/*` and `shared/*` |
+| Durable app | PostgreSQL | Quota limits, usage, and any durable mutation metadata required by the integrity protocol |
+| Shared ephemeral | Redis-compatible | List/metadata cache. Per-`user_id` bandwidth throttle |
+| Secrets | Deployment secret store | OIDC client secret, S3 access keys, JWT signing key, Redis auth if required |
+| Audit | stdout | Authn, authz, and mutations |
+
+| Store | Holds | Does not hold |
+| ----- | ----- | ------------- |
+| PostgreSQL | Quota limits, usage, protocol-required mutation metadata | Browser sessions, throttle, object bytes |
+| Redis-compatible | Cache, bandwidth throttle | Sessions, durable quota, object bytes |
+| JWT cookie | Session | — |
+| S3-compatible | Object bytes | Authn, quota policy |
+
+### 3.4 Object key layout and roots
+
+| Scope | Prefix | Access |
+| ----- | ------ | ------ |
+| Personal | `users/{user_id}/` | Owning verified OIDC identity only |
+| Shared | `shared/` | Any authenticated user may list, read, write, and delete any object |
+
+| Rule | Detail |
+| ---- | ------ |
+| Identity tuple | Ownership is based on the verified pair (`iss`, `sub`), not email |
+| User id | `user_id` is a deterministic, collision-resistant, URL-safe, opaque encoding of (`iss`, `sub`), defined by a versioned implementation contract and stable across deployments using the same identity configuration |
+| Active root | UI and API operate in one root at a time: personal or shared. The client selects only the root enum |
+| Shared trust | Shared is a **single trust domain** with no per-object ACLs. Any authenticated user can wipe shared content. Accepted at this scale |
+| Shared UX | The UI must clearly distinguish the shared root and identify destructive actions as affecting all users |
+| Folders | Virtual via S3 delimiter `/` |
+| Client path | The client supplies a relative logical path, never a complete S3 key or ownership prefix |
+| Path representation | A logical object path is an NFC-normalized sequence of non-empty Unicode segments separated by `/` |
+| Invalid paths | Reject invalid UTF-8, leading or trailing separators, empty segments, `.` or `..` segments, backslashes, Unicode control characters, and encoded forms that introduce any rejected value |
+| Decoding | Percent decoding occurs exactly once at the HTTP boundary. Decoded input is validated rather than rewritten into an alias |
+| Authorization check | After construction, the canonical key must remain beneath the server-derived active root |
+| Bucket | Single configured bucket (name is deployment configuration) |
+| Principal | Single configured access key id for the BFF (deployment configuration) |
+
+The exact `user_id` encoding is an implementation contract and must avoid collisions. Raw `sub` and email values are not inserted directly into object keys.
+
+### 3.5 S3 access model
+
+```mermaid
+flowchart LR
+  App[Nuthatch] -->|service credentials| Principal[S3 principal]
+  Principal --> Bucket[configured bucket]
+```
+
+| Layer | Control |
+| ----- | ------- |
+| Network | Users never call the object store directly for product traffic |
+| IAM / policy | Principal limited to the configured bucket (and optional prefix constraints) |
+| App | Verified (`iss`, `sub`) maps to an opaque personal root. Shared root is open to any authenticated user |
+| Transport | Upload and download are proxied through Nuthatch |
+
+### 3.6 OIDC, session (JWT), and CSRF
+
+| Item | Value |
+| ---- | ----- |
+| OIDC flow | Authorization code with PKCE using the identity provider’s hosted login |
+| Client type | Confidential OIDC client |
+| Transaction state | Short-lived integrity-protected, HttpOnly browser state for `state`, `nonce`, PKCE verifier, and a validated local return target. Not a durable server session |
+| Callback validation | Validate `state`, issuer, signature, audience/client id, nonce, code exchange result, token lifetime, and the exact configured redirect URI |
+| Token validation policy | Trusted issuer metadata and deployment policy determine accepted keys and algorithms; an untrusted token header alone does not |
+| Mechanism | App-signed Nuthatch session JWT after successful OIDC validation |
+| Cookie name | Deployment configuration |
+| Cookie flags | `HttpOnly`, `Secure`, host-only (no `Domain`), `Path=/` |
+| SameSite | `Lax` (compatible with OIDC top-level return) |
+| Lifetime | Bound to JWT `exp`. Duration is deployment configuration |
+| Claims | Identify Nuthatch as session issuer and audience and carry the verified upstream `oidc_issuer` and `oidc_subject`; `iat` and `exp` are required. Email or display name may be copied for UI display only |
+| Signing | Nuthatch signs and validates the session under an explicitly configured policy using app key material from the deployment secret store |
+| Authorization model | Authenticated user only. No role matrix beyond personal vs shared roots |
+| Logout | Clear the Nuthatch session cookie and transient login cookies. An authenticated logout is state-changing and follows the CSRF rule |
+| Revocation | JWT expiry only. No server-side session list |
+| Not used | Redis or PostgreSQL browser-session rows |
+
+| CSRF rule | Detail |
+| --------- | ------ |
+| Applies to | State-changing requests that use the session cookie: upload, delete, and any other mutating API |
+| Does not replace | OIDC transaction validation or normal authorization checks |
+| Mechanism | Synchronizer token or equivalent double-submit mechanism issued by Nuthatch and required on mutating requests |
+| Safe methods | `GET` and other safe methods for browse, search, and download do not require a CSRF token |
+| Failure | A mutating request without a valid CSRF token is rejected before object or quota mutation |
+
+### 3.7 Quota and mutation integrity
+
+| Pool | Scope | Accounting |
+| ---- | ----- | ---------- |
+| Personal | `users/{user_id}/` | One deployment-configured personal limit applied independently to each user, with per-user `used_bytes` in PostgreSQL |
+| Shared | `shared/` | One global limit and `used_bytes` in PostgreSQL |
+
+| Rule | Detail |
+| ---- | ------ |
+| Limit values | Deployment configuration. Not fixed by this design |
+| No overshoot | No object state produced by a Nuthatch mutation may place its pool above the configured limit |
+| Successful mutation | Put, overwrite, or delete succeeds only after the intended object state and its PostgreSQL accounting are durably finalized |
+| Unfinished mutation | A mutation that cannot be finalized MUST NOT report success. Conflicting mutations against its key fail closed until recovery resolves the unfinished operation |
+| Idempotency identity | Every mutation carries a client-generated operation id scoped to the verified user identity and operation kind |
+| Replay | Within the advertised idempotency window, exact replay of an unfinished mutation reports it as unfinished without starting a second effect. Once terminal, exact replay returns the recorded terminal result. Reuse with a different request fingerprint is rejected |
+| Request fingerprint | The fingerprint binds the operation kind, active root, canonical relative path, declared size, and content identity for puts. Its wire encoding and the idempotency-window duration belong to the API contract |
+| Same-key contention | At most one mutation may be admitted for a canonical object key. A new mutation for a locked key is rejected rather than queued; retry after resolution uses a new operation id |
+| Accounting | When no mutation in a pool is unfinished, `used_bytes` equals the sum of object sizes in that Nuthatch-owned pool |
+| Shared writes | Charge only the shared pool |
+| Personal writes | Charge only that user’s personal pool |
+| Downloads | Do not change usage |
+| Size known | Uploads must declare total size through the API contract before body acceptance |
+| Request settlement | An object-store request is settled only when it can no longer mutate its key. The reservation and key lock remain held until settlement |
+| Object observation | After settlement, Nuthatch identifies the stored bytes through their size and content identity using ordinary head or streaming get behavior |
+| Required object semantics | Put and delete settle within a configured maximum interval, replace one key atomically, and cannot take effect after settlement. After settlement, reads observe either the complete prior object or the complete requested object |
+| Unsupported store | A deployment that cannot establish the required object semantics is unsupported |
+| Upload interruption | Nuthatch does not retain or resume a lost request body. After request settlement, recovery finalizes success if the intended content is present; otherwise it records failure and releases the reservation and key lock |
+| Upload retry | The UI identifies the failed upload and allows the user to start a complete replacement upload with a new operation id |
+| Overwrite | Admission and final accounting use the size delta between the prior committed object and the replacement, not the replacement size alone |
+| Authority | Usage is app-maintained for mutations through Nuthatch |
+| Dependency loss | PostgreSQL or object-store loss fails new mutations closed. Unfinished mutations remain durably resolvable after transient dependency loss and process restart |
+| Object-store assumption | The object store supplies ordinary blob operations but no cross-system transaction, compare-and-swap, versioning, or event guarantee |
+| Deployment writer contract | Nuthatch is the exclusive writer to `users/` and `shared/` while product service is enabled. Out-of-band mutation violates the deployment contract and may invalidate accounting |
+
+Standing “list the bucket and rewrite counters” is not the primary correctness mechanism. A repair or verification job may exist as supplementary defense, but normal mutation success and crash recovery must not depend on a full-bucket reconciliation.
+
+### 3.8 Throttle (Redis-compatible)
+
+| Item | Value |
+| ---- | ----- |
+| Dimension | Bytes transferred per time unit per verified user identity |
+| Directions | **Upload and download** body bytes both count against one user bucket |
+| Algorithm | Shared token bucket with configurable sustained rate and burst allowance |
+| Store | Redis-compatible, shared across replicas |
+| When checked | Before accepting or sending further body bytes |
+| Mid-stream exhaustion | Stop transferring and fail the request. Bytes already moved still count |
+| Over limit before start | HTTP 429 |
+| Parallel streams | All replicas and simultaneous streams consume from the same logical user bucket |
+| Aborted transfers | Count bytes actually proxied before termination |
+| Range requests | Not supported in version one |
+| Redis unavailable | Throttle enforcement fails closed for new uploads and downloads. Browse and search may continue without cache |
+| Not used | Request-count limits, concurrent-upload caps, in-process counters |
+
+### 3.9 Browse, search, upload, download, and delete
+
+| Feature | Behavior |
+| ------- | -------- |
+| Browse | `ListObjectsV2` with delimiter `/`, configured page size, and continuation token under the active root |
+| Search | Case-insensitive substring on object keys under the active root only. List and filter in process; no content index |
+| Search bounds | Configurable maximum keys/pages scanned and request timeout. A bounded partial result must be identified as partial |
+| Upload | Stream request bodies to the object store. Require declared total size and enforce maximum object size before accepting the body. Interrupted bodies are not resumed |
+| Overwrite | Replacing an existing key is allowed and follows the mutation-integrity protocol and delta accounting in §3.7 |
+| Download | Stream from the object store through Nuthatch. Default to `Content-Disposition: attachment` with a safely encoded filename |
+| Download safety | Set `X-Content-Type-Options: nosniff`; do not render arbitrary stored HTML or SVG inline; apply a restrictive application Content Security Policy |
+| Delete object | Delete one canonical key through the mutation-integrity protocol, then invalidate affected list-cache entries |
+| Delete folder | After UI confirmation, enumerate keys under the prefix in configured-size pages and execute one idempotent mutation per key |
+| Folder delete consistency | Recursive deletion is not atomic or snapshot-isolated. Each observed key has an independent result, and keys created concurrently may remain |
+| Partial folder delete | Return per-key progress sufficient for a client to retry unfinished pages; do not report a key as deleted until its mutation succeeds |
+| Missing key | Deleting an already-absent key succeeds without changing usage |
+| Max object size | Configurable deployment limit |
+| Previews | None |
+| Versioning UI | None |
+
+Per-key recursive deletion is deliberate: it bounds each mutation and avoids one
+large deferred accounting adjustment. The version-one mutation protocol model
+covers individual keys. A durable server-side batch deletion protocol is
+outside version one.
+
+### 3.10 Caching (Redis-compatible)
+
+| Item | Value |
+| ---- | ----- |
+| What | Browse list and light metadata for a cache key of active root plus list parameters (prefix, delimiter, continuation, page size) |
+| Search | May reuse browse list entries for the same root listing parameters, then filter in process. Do not cache search-filter results as a separate long-lived view unless keyed by the query string as well |
+| Where | Redis-compatible store (same instance as throttle is allowed, distinct key namespace from throttle keys) |
+| TTL | Configurable |
+| Invalidate | After a put or delete mutation succeeds, invalidate list-cache keys for the active root whose listed prefix is the object’s parent prefix or an ancestor prefix under that root (including the root listing). TTL remains the backstop |
+| Redis unavailable | Cache fails open: bypass cache and read from S3. Cache failure alone does not make browse/search unavailable |
+| Required | Yes |
+
+### 3.11 Observability
+
+| Signal | Path |
+| ------ | ---- |
+| Audit | Structured JSON on stdout for authentication events, authorization failures, and object mutations |
+| Common fields | `ts`, `action`, `result`, and request or mutation correlation id |
+| Failure fields | Stable `reason` value when the result is not successful |
+| Identity fields | Stable opaque user id when a verified identity is available |
+| Object fields | `root` and `bytes` when applicable to the recorded action |
+| Object key policy | Full key, relative path, or keyed hash is configurable. Deployments must treat filenames as potentially sensitive |
+| Prohibited log data | Session JWTs, OIDC tokens/codes, CSRF tokens, object bodies, secrets, and raw authorization headers |
+| Read logging | Download and browse/search key detail may be reduced or hashed to limit privacy exposure |
+| Metrics | Not required by this design |
+
+### 3.12 Process and dependency failure model
+
+| Item | Value |
+| ---- | ----- |
+| App processes | Stateless with respect to browser sessions and in-process product authority. Multi-replica safe when PostgreSQL and Redis-compatible backends are shared |
+| Liveness | Reports whether the process can serve requests; does not fail solely because a dependency is unavailable |
+| Readiness | Reports whether the replica can safely serve its configured traffic class |
+| PostgreSQL unavailable | Reject mutations that require quota or protocol state. Read-only object operations may continue only if their authorization and audit requirements can still be met |
+| Redis unavailable | Bypass cache; fail closed for new uploads/downloads because bandwidth enforcement is unavailable |
+| S3 unavailable | Reject object operations with a dependency error; authentication and non-object endpoints may remain available |
+| Recovery | After transient dependency loss or process restart, unfinished mutations are recoverable without sticky sessions or leader election |
+| Leader election | None required by this design |
+
+
+### 3.13 Mutation protocol boundary
+
+This document defines the externally visible mutation guarantees and the
+authority assigned to PostgreSQL and the object store. `Nuthatch.tla` defines
+the abstract state machine that implements those guarantees.
+
+The protocol model defines:
+
+- admissible states and transitions for put, overwrite, delete, retry, and recovery;
+- quota reservation and release semantics;
+- exclusive same-key admission and contention rejection;
+- duplicate operation-id and request-fingerprint behavior;
+- crash and dependency-failure recovery;
+- the relationship between object state and PostgreSQL accounting; and
+- safety invariants sufficient to establish the no-overshoot, idempotency, and
+  accounting requirements in §3.7.
+
+The model may assume that transient dependency failures eventually recover and
+that recovery processing continues. Permanent dependency loss is outside its
+liveness boundary and causes Nuthatch to remain failed closed.
+
+The version-one model covers individual object keys. Recursive prefix deletion
+is a sequence of per-key operations and is not part of that model. A concrete
+implementation is conformant only when its mutation behavior refines the model.
+
+---
+
+## 4. Requirements
+
+### 4.1 Functional
+
+| ID | Requirement |
+| -- | ----------- |
+| F1 | Serve a browser UI and API over HTTPS |
+| F2 | Authenticate with OIDC using the IdP-hosted login as Nuthatch’s own OIDC client |
+| F3 | Issue and validate a JWT session cookie after OIDC |
+| F4 | Protect state-changing cookie-authenticated requests with CSRF tokens |
+| F5 | Personal root `users/{user_id}/` derived from verified (`iss`, `sub`) |
+| F6 | Shared root `shared/` for any authenticated user (single trust domain) |
+| F7 | Client-selectable active root (personal or shared) |
+| F8 | Paginated browse under the active root |
+| F9 | Drag-and-drop and form upload under the active root; identify failed uploads and allow complete user-initiated retry without server-side body resumption |
+| F10 | Download under the active root |
+| F11 | Delete an object or recursively enumerate a prefix under the active root using paged, independently idempotent per-key mutations |
+| F12 | Search by key substring under the active root |
+| F13 | Enforce configurable personal and shared storage quotas in PostgreSQL under the mutation-integrity invariants in §3.7 |
+| F14 | Enforce a configurable shared token-bucket byte throttle per user via Redis-compatible store for **upload and download** |
+| F15 | List and metadata cache in Redis-compatible store with configurable TTL |
+| F16 | Structured audit logs on stdout |
+| F17 | Require operation ids for put, overwrite, and delete; within the advertised idempotency window, replay exact requests without starting a second effect, preserve terminal results, and reject conflicting reuse; reject new mutations while their canonical key is locked |
+| F18 | Bound search work by configurable scan and timeout limits |
+| F19 | Use safe attachment-oriented download headers and prohibit inline rendering of active stored content |
+
+### 4.2 Non-functional
+
+| ID | Requirement |
+| -- | ----------- |
+| N1 | Stateless app processes. No sticky sessions. No in-process cache or throttle authority |
+| N2 | No object-store credentials in the browser |
+| N3 | No object-store admin console dependency for user workflows |
+| N4 | Stream bodies. Do not buffer whole objects in memory |
+| N5 | Design is language-agnostic. Language is chosen at implementation start |
+| N6 | Preserve mutation integrity under concurrent replicas and recover unfinished mutations after process restart or transient dependency loss |
+| N7 | Cache failure degrades to direct S3 access; throttle failure does not silently disable bandwidth enforcement |
+
+### 4.3 Security
+
+| ID | Requirement |
+| -- | ----------- |
+| S1 | Confidential OIDC client using authorization code with PKCE. Secrets from the deployment secret store |
+| S2 | JWT signed with an app key. Cookie: Secure, HttpOnly, host-only, `Path=/`, SameSite=Lax |
+| S3 | CSRF token required on mutating cookie-authenticated requests |
+| S4 | Construct keys server-side from a root enum and canonical relative path; reject ambiguous, absolute, traversal, and cross-root forms |
+| S5 | User A cannot access another identity’s `users/{user_id}/` root |
+| S6 | Shared root requires authentication |
+| S7 | No anonymous access to object data. Unauthenticated callers may only hit login, callback, logout, health, and static entrypoints explicitly intended for them |
+| S8 | Validate OIDC `state`, `nonce`, issuer, audience, signature, redirect URI, and token lifetime |
+| S9 | Do not log tokens, secrets, CSRF values, object bodies, or raw authorization headers |
+
+---
+
+## 5. Scope
+
+### 5.1 In scope
+
+| Area |
+| ---- |
+| OIDC (Nuthatch as RP), transient login transaction state, and JWT session cookie |
+| CSRF on mutating cookie-authenticated requests |
+| Active root selection (personal or shared) |
+| Paginated browse, upload, download |
+| Search by key substring under active root |
+| Delete object and recursive prefix with per-key usage updates |
+| PostgreSQL quotas and durable mutation metadata required by §3.7 and the protocol boundary in §3.13 |
+| Redis-compatible shared token-bucket throttle for upload and download bytes |
+| Redis-compatible list and metadata cache (configurable TTL) |
+| S3-compatible object storage via a service principal |
+| stdout audit logs |
+
+### 5.2 Out of scope
+
+| Item |
+| ---- |
+| Anonymous or time-limited public links |
+| Desktop client or sync |
+| Office or WOPI |
+| Object versioning UI |
+| In-browser previews |
+| Range downloads |
+| Server-side upload resumption |
+| WebDAV or SFTP |
+| Per-user object-store IAM users |
+| Browser presigned object-store URLs |
+| App-local durable disk for product data |
+| In-process throttle or cache as authority |
+| Redis or PostgreSQL as session store |
+| Standing bucket-list reconciler as the primary mutation-correctness mechanism |
+| Request-rate or concurrency throttle dimensions |
+| Fixed quota or throttle numeric defaults in this document |
+| Per-object ACLs on the shared root |
+| External auth proxy in front of Nuthatch |
+| Virus scan or DLP |
+| Teams or orgs beyond the single shared prefix |
+| Cluster ingress, DNS, network policy, and GitOps wiring |
+| Mutation state machine, transition ordering, and proof obligations (owned by the protocol model) |
+| Test harness and environment design |