Implement replay-safe authorized public HTTP API

Cargo.lock

@@ -3263,13 +3263,16 @@ version = "0.0.0"
 dependencies = [
  "actix-tls",
  "actix-web",
+ "base64",
  "getrandom 0.3.4",
+ "jsonwebtoken",
  "rcgen",
  "remora-admission",
  "remora-attempt",
  "remora-contracts",
  "remora-coordination",
  "remora-credentials",
+ "remora-deletion",
  "remora-domain",
  "remora-durable",
  "remora-identity",

Dockerfile.test

@@ -314,6 +314,13 @@ COPY crates crates
 COPY --from=kind-worker-api-verified /tmp/remora-api-tests.tar.zst /tests/remora-api-tests.tar.zst
 CMD ["bash", "-ec", "test -s /run/remora-fixture/profile.env && test -n \"$DATABASE_URL\" && test -n \"$REMORA_COORDINATION_URL\" && export REMORA_WORKER_API_REQUIRE=1 && cargo-nextest nextest run --archive-file /tests/remora-api-tests.tar.zst --workspace-remap /workspace --test-threads=1 --no-fail-fast"]
 
+FROM rust-tools AS kind-public-api-runner
+WORKDIR /workspace
+COPY Cargo.toml ./
+COPY crates crates
+COPY --from=kind-worker-api-verified /tmp/remora-api-tests.tar.zst /tests/remora-api-tests.tar.zst
+CMD ["bash", "-ec", "test -s /run/remora-fixture/profile.env && test -n \"$DATABASE_URL\" && test -n \"$REMORA_COORDINATION_URL\" && test -n \"$REMORA_OIDC_ISSUER\" && test -n \"$REMORA_TARGET_ENDPOINT\" && test -n \"$REMORA_VAULT_ADDR\" && case $REMORA_COORDINATION_PROVIDER in redis|dragonfly) ;; *) exit 1 ;; esac && case $REMORA_OIDC_PROVIDER in keycloak|zitadel) ;; *) exit 1 ;; esac && case $REMORA_TARGET_PROVIDER in garage|versitygw) ;; *) exit 1 ;; esac && export REMORA_PUBLIC_API_REQUIRE=1 && cargo-nextest nextest run --archive-file /tests/remora-api-tests.tar.zst --workspace-remap /workspace --test-threads=1 --no-fail-fast -E 'binary(public_api)'"]
+
 FROM rust-tools AS kind-worker-build
 WORKDIR /workspace
 COPY Cargo.toml Cargo.lock rust-toolchain.toml ./

Makefile

@@ -1,4 +1,4 @@
-.PHONY: check-dependencies check-fixture-lib check-kind-sample check-durable check-coordination check-identity check-credentials check-admission check-attempt check-attempt-mutations check-recovery check-worker-api check-worker check-stream check-target check-result check-deletion check-outbound check-http-source check-object-source check-hf-source check-fast check-full check-fast-host check-full-host rust-check web-install web-check web-test-fast web-test-full
+.PHONY: check-dependencies check-fixture-lib check-kind-sample check-durable check-coordination check-identity check-credentials check-admission check-attempt check-attempt-mutations check-recovery check-worker-api check-public-api check-worker check-stream check-target check-result check-deletion check-outbound check-http-source check-object-source check-hf-source check-fast check-full check-fast-host check-full-host rust-check web-install web-check web-test-fast web-test-full
 
 CONTAINER_ENGINE ?= nerdctl
 CONTAINER_RUN_FLAGS ?= --memory=16g --memory-swap=24g --cpus=4
@@ -83,6 +83,9 @@ check-recovery:
 check-worker-api:
 	./tests/kind/run-worker-api-job.sh
 
+check-public-api:
+	./tests/kind/run-public-api-job.sh
+
 check-worker:
 	./tests/kind/run-worker-job.sh
 

crates/remora-admission/src/service.rs

@@ -4,7 +4,8 @@ use remora_credentials::{
 };
 use remora_domain::{OpaqueId, Target, fingerprint, ports::MutationScope};
 use remora_durable::{
-    DurableError, MutationDisposition, MutationStatus, enqueue, execute_mutation, with_transaction,
+    DurableError, MutationDisposition, MutationStatus, PgTx, enqueue, execute_mutation,
+    with_transaction,
 };
 use remora_identity::{
     DefaultTarget, Principal, TargetOverride, may_request_execution_grant_revocation,
@@ -99,9 +100,30 @@ impl AdmissionService {
         if !may_request_execution_grant_revocation(principal) {
             return Err(AdmissionError::Unauthorized);
         }
+        let principal = principal.clone();
         let grant_id = grant_id.clone();
         with_transaction(&self.pool, move |tx| {
-            Box::pin(async move {
+            Box::pin(
+                async move { Self::revoke_execution_grant_in_tx(tx, &principal, &grant_id).await },
+            )
+        })
+        .await
+        .map_err(|error| match &error {
+            DurableError::Effect(message) if message == "grant_not_found" => {
+                AdmissionError::GrantNotFound
+            }
+            _ => AdmissionError::Durable(error),
+        })
+    }
+
+    pub async fn revoke_execution_grant_in_tx(
+        tx: &mut PgTx<'_>,
+        principal: &Principal,
+        grant_id: &OpaqueId,
+    ) -> Result<(), DurableError> {
+        if !may_request_execution_grant_revocation(principal) {
+            return Err(DurableError::Effect("grant_unauthorized".into()));
+        }
         let row = sqlx::query_as::<_, (String, Option<String>)>(
             "SELECT status, credential_id
              FROM execution_grants
@@ -148,15 +170,6 @@ impl AdmissionService {
         )
         .await?;
         Ok(())
-            })
-        })
-        .await
-        .map_err(|error| match &error {
-            DurableError::Effect(message) if message == "grant_not_found" => {
-                AdmissionError::GrantNotFound
-            }
-            _ => AdmissionError::Durable(error),
-        })
     }
 }
 

crates/remora-api/Cargo.toml

@@ -9,9 +9,14 @@ version.workspace = true
 actix-web = { version = "=4.14.0", features = ["rustls-0_23"] }
 actix-tls = "=3.5.0"
 getrandom = "0.3.4"
+remora-admission = { path = "../remora-admission" }
 remora-attempt = { path = "../remora-attempt" }
 remora-contracts = { path = "../remora-contracts" }
+remora-credentials = { path = "../remora-credentials" }
+remora-deletion = { path = "../remora-deletion" }
 remora-domain.workspace = true
+remora-durable.workspace = true
+remora-identity = { path = "../remora-identity" }
 remora-result = { path = "../remora-result" }
 remora-target = { path = "../remora-target" }
 serde.workspace = true
@@ -22,12 +27,10 @@ thiserror.workspace = true
 time.workspace = true
 
 [dev-dependencies]
+base64 = "=0.22.1"
+jsonwebtoken = "=9.3.1"
 rcgen = "=0.14.8"
-remora-admission = { path = "../remora-admission" }
 remora-coordination = { path = "../remora-coordination" }
-remora-credentials = { path = "../remora-credentials" }
+reqwest = { version = "=0.13.4", default-features = false, features = ["form", "json", "rustls"] }
-remora-durable.workspace = true
-remora-identity = { path = "../remora-identity" }
-reqwest = { version = "=0.13.4", default-features = false, features = ["json", "rustls"] }
 rustls = "=0.23.42"
 tokio.workspace = true

crates/remora-api/src/lib.rs

@@ -1,5 +1,7 @@
+mod public;
 mod worker;
 
+pub use public::{PublicApiConfig, PublicApiServices, PublicApiState, configure_public_api};
 pub use worker::{
     AuthenticatedWorker, WorkerApiState, authenticate_rustls_worker, configure_worker_api,
 };

crates/remora-attempt/src/lib.rs

@@ -12,7 +12,7 @@ pub use migrate::migrate;
 pub use mutation_types::{
     AttemptMutationOutcome, CheckpointMultipart, CheckpointProjection, CheckpointRecord,
     CommittedManifestRecord, CommittedResultRecord, DurableDescriptor, FailureRecord,
-    JobMutationProjection, MutationRejection, ProgressRecord,
+    JobMutationProjection, MutationRejection, ProgressRecord, ProgressSnapshot,
 };
 pub use recovery_types::{
     CancelOutcome, CleanupClaim, CleanupCompletion, CleanupIdentity, CooperativeStopOutcome,

crates/remora-attempt/src/mutation_types.rs

@@ -59,6 +59,14 @@ pub struct ProgressRecord {
     pub total_members: Option<u64>,
 }
 
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct ProgressSnapshot {
+    pub attempt_id: OpaqueId,
+    pub fence: u64,
+    pub progress: ProgressRecord,
+    pub observed_at: time::OffsetDateTime,
+}
+
 #[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
 #[serde(rename_all = "snake_case")]
 pub enum MutationRejection {

crates/remora-attempt/src/recovery.rs

@@ -5,7 +5,8 @@ use remora_domain::{
     FailureClass, FailureDisposition, JobPhase, OpaqueId, fingerprint, ports::MutationScope,
 };
 use remora_durable::{
-    DurableError, MutationDisposition, MutationStatus, enqueue, execute_mutation, with_transaction,
+    DurableError, MutationDisposition, MutationStatus, PgTx, enqueue, execute_mutation,
+    with_transaction,
 };
 use remora_target::{attempt_key_attempt_id, attempt_staging_prefix, validate_attempt_key};
 use serde::Serialize;
@@ -24,7 +25,16 @@ impl AttemptService {
     pub async fn request_cancel(&self, job_id: &OpaqueId) -> Result<CancelOutcome, AttemptError> {
         with_transaction(&self.pool, |tx| {
             let job_id = job_id.clone();
-            Box::pin(async move {
+            Box::pin(async move { Self::request_cancel_in_tx(tx, &job_id).await })
+        })
+        .await
+        .map_err(Into::into)
+    }
+
+    pub async fn request_cancel_in_tx(
+        tx: &mut PgTx<'_>,
+        job_id: &OpaqueId,
+    ) -> Result<CancelOutcome, DurableError> {
         let Some(row) = sqlx::query_as::<_, CancelRow>(
             "SELECT phase, cancel_requested FROM jobs WHERE job_id = $1 FOR UPDATE",
         )
@@ -68,10 +78,6 @@ impl AttemptService {
         }
         enqueue(tx, "job.updated", json!({ "job_id": job_id })).await?;
         Ok(CancelOutcome::Requested)
-            })
-        })
-        .await
-        .map_err(Into::into)
     }
 
     pub async fn cooperative_stop(

crates/remora-attempt/src/service.rs

@@ -16,8 +16,8 @@ use time::OffsetDateTime;
 use crate::{
     ActivationMutationOutcome, ActivationOutcome, AttemptAuthority, AttemptError,
     AttemptFaultInjector, AttemptFaultPoint, AttemptState, CheckpointProjection, ClaimOutcome,
-    ClaimRejection, ExecutionEnvelopeRecord, LeaseProjection, RecoveryOutcome,
+    ClaimRejection, ExecutionEnvelopeRecord, LeaseProjection, ProgressRecord, ProgressSnapshot,
-    RenewalMutationOutcome, RenewalOutcome,
+    RecoveryOutcome, RenewalMutationOutcome, RenewalOutcome,
 };
 
 struct NoFaults;
@@ -87,6 +87,27 @@ impl AttemptService {
         })
     }
 
+    pub async fn progress_snapshot(
+        &self,
+        job_id: &OpaqueId,
+    ) -> Result<Option<ProgressSnapshot>, AttemptError> {
+        let Some(hint) = self.coordination.get_progress(job_id.as_str()).await? else {
+            return Ok(None);
+        };
+        Ok(Some(ProgressSnapshot {
+            attempt_id: OpaqueId::parse(hint.attempt_id)
+                .map_err(|error| AttemptError::Invalid(error.to_string()))?,
+            fence: hint.fence,
+            progress: ProgressRecord {
+                completed_bytes: hint.bytes,
+                total_bytes: hint.total_bytes,
+                completed_members: hint.completed_members,
+                total_members: hint.total_members,
+            },
+            observed_at: OffsetDateTime::now_utc(),
+        }))
+    }
+
     pub async fn claim(
         &self,
         job_id: &OpaqueId,

crates/remora-credentials/src/lib.rs

@@ -18,7 +18,9 @@ mod vault;
 pub use authz::{CredentialCaller, may_manage, require_manage};
 pub use error::CredentialError;
 pub use migrate::{migrate, migrator};
-pub use registry::CredentialRegistry;
+pub use registry::{
+    CredentialRegistry, PreparedCredentialCreate, PreparedCredentialRotate, RotateTxOutcome,
+};
 pub use resolve::GrantBoundResolver;
 pub use types::{
     CreateCredential, CredentialGrantVersion, SecretMaterial, SourceCredential,

crates/remora-credentials/src/registry.rs

@@ -24,6 +24,30 @@ pub struct CredentialRegistry {
     reconcile_min_age: time::Duration,
 }
 
+pub struct PreparedCredentialCreate {
+    intent_id: OpaqueId,
+    credential_id: OpaqueId,
+    version_id: OpaqueId,
+    owner: OpaqueId,
+    label: String,
+    kind: SourceCredentialKind,
+    vault_kv_version: u64,
+    secret_mount: String,
+    credential_path: String,
+    secret_path: String,
+    created_at: OffsetDateTime,
+}
+
+pub struct PreparedCredentialRotate {
+    intent_id: OpaqueId,
+    credential_id: OpaqueId,
+    version_id: OpaqueId,
+    vault_kv_version: u64,
+    secret_mount: String,
+    secret_path: String,
+    created_at: OffsetDateTime,
+}
+
 impl CredentialRegistry {
     #[must_use]
     pub fn new(pool: PgPool, vault: VaultClient) -> Self {
@@ -46,6 +70,38 @@ impl CredentialRegistry {
         owner: &OpaqueId,
         request: CreateCredential,
     ) -> Result<SourceCredential, CredentialError> {
+        let prepared = self.prepare_create(owner, request).await?;
+        let credential_id = prepared.credential_id.clone();
+        let intent_id = prepared.intent_id.clone();
+        let now = prepared.created_at;
+        let result = with_transaction(&self.pool, move |tx| {
+            Box::pin(async move { Self::commit_prepared_create_in_tx(tx, prepared).await })
+        })
+        .await;
+        match result {
+            Ok(credential) => Ok(credential),
+            Err(error) => {
+                if let Ok(Some(credential)) = self.get_internal(&credential_id).await {
+                    let _ = mark_intent_status_if(
+                        &self.pool,
+                        &intent_id,
+                        IntentStatus::VaultWritten,
+                        IntentStatus::DurableCommitted,
+                        now,
+                    )
+                    .await;
+                    return Ok(credential);
+                }
+                Err(error.into())
+            }
+        }
+    }
+
+    pub async fn prepare_create(
+        &self,
+        owner: &OpaqueId,
+        request: CreateCredential,
+    ) -> Result<PreparedCredentialCreate, CredentialError> {
         validate_label(&request.label)?;
         request.material.validate()?;
         let kind = request.material.kind();
@@ -89,44 +145,53 @@ impl CredentialRegistry {
                 return Err(error);
             }
         };
-        // Durable KV version must be recorded before any later step; retry so a
-        // transient DB blip does not leave a Vault secret without vault_written.
         record_vault_written_with_retry(&self.pool, &intent_id, vault_kv_version, now).await?;
-
+        Ok(PreparedCredentialCreate {
-        match self
+            intent_id,
-            .materialize_create_intent(
+            credential_id,
-                &intent_id,
+            version_id,
-                &credential_id,
+            owner: owner.clone(),
-                &version_id,
+            label: request.label,
-                owner,
-                &request.label,
             kind,
             vault_kv_version,
-                &secret_mount,
+            secret_mount,
-                &credential_path,
+            credential_path,
-                &secret_path,
+            secret_path,
-                now,
+            created_at: now,
-            )
+        })
-            .await
+    }
+
+    pub async fn commit_prepared_create_in_tx(
+        tx: &mut PgTx<'_>,
+        prepared: PreparedCredentialCreate,
+    ) -> Result<SourceCredential, DurableError> {
+        advisory_xact_lock(tx, credential_advisory_key(&prepared.credential_id)).await?;
+        if !claim_intent_durable_committed_in_tx(tx, &prepared.intent_id, prepared.created_at)
+            .await?
         {
-            Ok(credential) => Ok(credential),
+            return Err(DurableError::Effect(
-            Err(error) => {
+                "create intent was claimed by reconciliation".into(),
-                // Never destroy Vault on materialize failure: the durable commit
+            ));
-                // may have succeeded with a lost response. Recover if rows exist.
+        }
-                if let Ok(Some(credential)) = self.get_internal(&credential_id).await {
+        insert_credential_with_version(
-                    let _ = mark_intent_status_if(
+            tx,
-                        &self.pool,
+            InsertCredential {
-                        &intent_id,
+                credential_id: &prepared.credential_id,
-                        IntentStatus::VaultWritten,
+                owner: &prepared.owner,
-                        IntentStatus::DurableCommitted,
+                label: &prepared.label,
-                        now,
+                kind: prepared.kind,
+                version_id: &prepared.version_id,
+                vault_kv_version: prepared.vault_kv_version,
+                secret_mount: &prepared.secret_mount,
+                credential_path: &prepared.credential_path,
+                secret_path: &prepared.secret_path,
+                now: prepared.created_at,
+            },
         )
-                    .await;
+        .await?;
-                    return Ok(credential);
+        get_internal_in_tx(tx, &prepared.credential_id)
-                }
+            .await?
-                Err(error)
+            .ok_or_else(|| DurableError::Effect("created credential is missing".into()))
-            }
-        }
     }
 
     pub async fn rotate(
@@ -135,6 +200,66 @@ impl CredentialRegistry {
         credential_id: &OpaqueId,
         material: SecretMaterial,
     ) -> Result<SourceCredential, CredentialError> {
+        let prepared = self.prepare_rotate(caller, credential_id, material).await?;
+        let intent_id = prepared.intent_id.clone();
+        let version_id = prepared.version_id.clone();
+        let secret_mount = prepared.secret_mount.clone();
+        let secret_path = prepared.secret_path.clone();
+        let vault_kv_version = prepared.vault_kv_version;
+        let now = prepared.created_at;
+        let result = with_transaction(&self.pool, move |tx| {
+            Box::pin(async move { Self::commit_prepared_rotate_in_tx(tx, prepared).await })
+        })
+        .await;
+
+        match result {
+            Ok(RotateTxOutcome::Applied(credential)) => Ok(credential),
+            Ok(RotateTxOutcome::Revoked) => {
+                if self
+                    .vault
+                    .destroy_version(&secret_mount, &secret_path, vault_kv_version)
+                    .await
+                    .is_ok()
+                {
+                    let _ = mark_intent_status_if(
+                        &self.pool,
+                        &intent_id,
+                        IntentStatus::Abandoning,
+                        IntentStatus::Abandoned,
+                        now,
+                    )
+                    .await;
+                }
+                Err(CredentialError::Revoked)
+            }
+            Ok(RotateTxOutcome::IntentClaimed) => Err(CredentialError::Config(
+                "rotate intent was claimed by reconciliation".into(),
+            )),
+            Err(error) => {
+                if version_row_exists(&self.pool, &version_id).await? {
+                    let _ = mark_intent_status_if(
+                        &self.pool,
+                        &intent_id,
+                        IntentStatus::VaultWritten,
+                        IntentStatus::DurableCommitted,
+                        now,
+                    )
+                    .await;
+                    if let Ok(Some(credential)) = self.get_internal(credential_id).await {
+                        return Ok(credential);
+                    }
+                }
+                Err(error.into())
+            }
+        }
+    }
+
+    pub async fn prepare_rotate(
+        &self,
+        caller: &CredentialCaller,
+        credential_id: &OpaqueId,
+        material: SecretMaterial,
+    ) -> Result<PreparedCredentialRotate, CredentialError> {
         material.validate()?;
         let existing = self
             .get(caller, credential_id)
@@ -181,30 +306,41 @@ impl CredentialRegistry {
             .put_version(&secret_mount, &secret_path, &material)
             .await?;
         record_vault_written_with_retry(&self.pool, &intent_id, vault_kv_version, now).await?;
+        Ok(PreparedCredentialRotate {
+            intent_id,
+            credential_id: credential_id.clone(),
+            version_id,
+            vault_kv_version,
+            secret_mount,
+            secret_path,
+            created_at: now,
+        })
+    }
 
-        let result = with_transaction(&self.pool, {
+    pub async fn commit_prepared_rotate_in_tx(
-            let credential_id = credential_id.clone();
+        tx: &mut PgTx<'_>,
-            let version_id = version_id.clone();
+        prepared: PreparedCredentialRotate,
-            let intent_id = intent_id.clone();
+    ) -> Result<RotateTxOutcome, DurableError> {
-            let secret_mount = secret_mount.clone();
+        lock_credential(tx, &prepared.credential_id).await?;
-            let secret_path = secret_path.clone();
+        let status: String =
-            move |tx| {
+            sqlx::query_scalar("SELECT status FROM source_credentials WHERE credential_id = $1")
-                Box::pin(async move {
+                .bind(prepared.credential_id.as_str())
-                    lock_credential(tx, &credential_id).await?;
-                    let status: String = sqlx::query_scalar(
-                        "SELECT status FROM source_credentials WHERE credential_id = $1",
-                    )
-                    .bind(credential_id.as_str())
                 .fetch_optional(tx.as_mut())
                 .await?
                 .ok_or_else(|| DurableError::Effect("credential missing under lock".into()))?;
         if status != "active" {
-                        claim_intent_abandoning_in_tx(tx, &intent_id, vault_kv_version, now)
+            claim_intent_abandoning_in_tx(
+                tx,
+                &prepared.intent_id,
+                prepared.vault_kv_version,
+                prepared.created_at,
+            )
             .await?;
             return Ok(RotateTxOutcome::Revoked);
         }
-                    // Refuse to materialize if reconciler already claimed the intent.
+        if !claim_intent_durable_committed_in_tx(tx, &prepared.intent_id, prepared.created_at)
-                    if !claim_intent_durable_committed_in_tx(tx, &intent_id, now).await? {
+            .await?
+        {
             return Ok(RotateTxOutcome::IntentClaimed);
         }
         sqlx::query(
@@ -213,12 +349,12 @@ impl CredentialRegistry {
                 secret_mount, secret_path, created_at, destroyed_at
              ) VALUES ($1, $2, $3, $4, $5, $6, NULL)",
         )
-                    .bind(version_id.as_str())
+        .bind(prepared.version_id.as_str())
-                    .bind(credential_id.as_str())
+        .bind(prepared.credential_id.as_str())
-                    .bind(vault_kv_version as i64)
+        .bind(prepared.vault_kv_version as i64)
-                    .bind(&secret_mount)
+        .bind(&prepared.secret_mount)
-                    .bind(&secret_path)
+        .bind(&prepared.secret_path)
-                    .bind(now)
+        .bind(prepared.created_at)
         .execute(tx.as_mut())
         .await?;
         sqlx::query(
@@ -226,68 +362,15 @@ impl CredentialRegistry {
              SET active_version_id = $2, updated_at = $3
              WHERE credential_id = $1",
         )
-                    .bind(credential_id.as_str())
+        .bind(prepared.credential_id.as_str())
-                    .bind(version_id.as_str())
+        .bind(prepared.version_id.as_str())
-                    .bind(now)
+        .bind(prepared.created_at)
         .execute(tx.as_mut())
         .await?;
-                    Ok(RotateTxOutcome::Applied)
+        let credential = get_internal_in_tx(tx, &prepared.credential_id)
-                })
-            }
-        })
-        .await;
-
-        match result {
-            Ok(RotateTxOutcome::Applied) => self
-                .get_internal(credential_id)
             .await?
-                .ok_or_else(|| CredentialError::Config("rotated credential is missing".into())),
+            .ok_or_else(|| DurableError::Effect("rotated credential is missing".into()))?;
-            Ok(RotateTxOutcome::Revoked) => {
+        Ok(RotateTxOutcome::Applied(credential))
-                match self
-                    .vault
-                    .destroy_version(&secret_mount, &secret_path, vault_kv_version)
-                    .await
-                {
-                    Ok(()) => {
-                        let _ = mark_intent_status_if(
-                            &self.pool,
-                            &intent_id,
-                            IntentStatus::Abandoning,
-                            IntentStatus::Abandoned,
-                            now,
-                        )
-                        .await;
-                    }
-                    Err(_) => {
-                        // Leave abandoning so reconciliation retries cleanup.
-                    }
-                }
-                Err(CredentialError::Revoked)
-            }
-            Ok(RotateTxOutcome::IntentClaimed) => {
-                // Reconciler owns the intent; do not destroy or materialize.
-                Err(CredentialError::Config(
-                    "rotate intent was claimed by reconciliation".into(),
-                ))
-            }
-            Err(error) => {
-                // Ambiguous commit: do not destroy. Recover if the version row landed.
-                if version_row_exists(&self.pool, &version_id).await? {
-                    let _ = mark_intent_status_if(
-                        &self.pool,
-                        &intent_id,
-                        IntentStatus::VaultWritten,
-                        IntentStatus::DurableCommitted,
-                        now,
-                    )
-                    .await;
-                    if let Ok(Some(credential)) = self.get_internal(credential_id).await {
-                        return Ok(credential);
-                    }
-                }
-                Err(error.into())
-            }
-        }
     }
 
     pub async fn revoke(
@@ -295,19 +378,29 @@ impl CredentialRegistry {
         caller: &CredentialCaller,
         credential_id: &OpaqueId,
     ) -> Result<SourceCredential, CredentialError> {
-        let existing = self
+        let caller = caller.clone();
-            .get(caller, credential_id)
+        let credential_id = credential_id.clone();
+        with_transaction(&self.pool, {
+            move |tx| Box::pin(async move { Self::revoke_in_tx(tx, &caller, &credential_id).await })
+        })
+        .await
+        .map_err(Into::into)
+    }
+
+    pub async fn revoke_in_tx(
+        tx: &mut PgTx<'_>,
+        caller: &CredentialCaller,
+        credential_id: &OpaqueId,
+    ) -> Result<SourceCredential, DurableError> {
+        lock_credential(tx, credential_id).await?;
+        let existing = get_internal_in_tx(tx, credential_id)
             .await?
-            .ok_or(CredentialError::NotFound)?;
+            .ok_or_else(|| DurableError::Effect(CredentialError::NotFound.to_string()))?;
+        require_manage(caller, Some(&existing)).map_err(DurableError::effect)?;
         if existing.status == SourceCredentialStatus::Revoked {
             return Ok(existing);
         }
         let now = OffsetDateTime::now_utc();
-        with_transaction(&self.pool, {
-            let credential_id = credential_id.clone();
-            move |tx| {
-                Box::pin(async move {
-                    lock_credential(tx, &credential_id).await?;
         sqlx::query(
             "UPDATE source_credentials
              SET status = 'revoked',
@@ -319,14 +412,9 @@ impl CredentialRegistry {
         .bind(now)
         .execute(tx.as_mut())
         .await?;
-                    Ok(())
+        get_internal_in_tx(tx, credential_id)
-                })
-            }
-        })
-        .await?;
-        self.get_internal(credential_id)
             .await?
-            .ok_or_else(|| CredentialError::Config("revoked credential is missing".into()))
+            .ok_or_else(|| DurableError::Effect("revoked credential is missing".into()))
     }
 
     pub async fn get(
@@ -473,7 +561,7 @@ impl CredentialRegistry {
         })
     }
 
-    /// Pin under an existing transaction for atomic composition (e.g. WP-14 admission).
+    /// Pin under an existing transaction for atomic composition with admission.
     ///
     /// Takes the same credential advisory lock as revoke. Caller must already
     /// authorize the owner/administrator relationship.
@@ -557,8 +645,8 @@ impl CredentialRegistry {
 
     /// Bind a pin to an execution grant for worker resolution.
     ///
-    /// WP-14 calls this when accepting a job. Resolution always re-reads this
+    /// Resolution always re-reads this durable binding, so revocation fails
-    /// durable binding; revocation fails closed even if workers retain the grant id.
+    /// closed even if workers retain the grant id.
     pub async fn bind_execution_grant(
         &self,
         grant_id: &OpaqueId,
@@ -950,66 +1038,6 @@ impl CredentialRegistry {
         row.map(VersionCoordRow::into_coord).transpose()
     }
 
-    #[allow(clippy::too_many_arguments)]
-    async fn materialize_create_intent(
-        &self,
-        intent_id: &OpaqueId,
-        credential_id: &OpaqueId,
-        version_id: &OpaqueId,
-        owner: &OpaqueId,
-        label: &str,
-        kind: SourceCredentialKind,
-        vault_kv_version: u64,
-        secret_mount: &str,
-        credential_path: &str,
-        secret_path: &str,
-        now: OffsetDateTime,
-    ) -> Result<SourceCredential, CredentialError> {
-        with_transaction(&self.pool, {
-            let intent_id = intent_id.clone();
-            let credential_id = credential_id.clone();
-            let version_id = version_id.clone();
-            let owner = owner.clone();
-            let label = label.to_owned();
-            let secret_mount = secret_mount.to_owned();
-            let credential_path = credential_path.to_owned();
-            let secret_path = secret_path.to_owned();
-            move |tx| {
-                Box::pin(async move {
-                    // Serialize with reconcile claim on this credential id.
-                    advisory_xact_lock(tx, credential_advisory_key(&credential_id)).await?;
-                    if !claim_intent_durable_committed_in_tx(tx, &intent_id, now).await? {
-                        return Err(DurableError::Effect(
-                            "create intent was claimed by reconciliation".into(),
-                        ));
-                    }
-                    insert_credential_with_version(
-                        tx,
-                        InsertCredential {
-                            credential_id: &credential_id,
-                            owner: &owner,
-                            label: &label,
-                            kind,
-                            version_id: &version_id,
-                            vault_kv_version,
-                            secret_mount: &secret_mount,
-                            credential_path: &credential_path,
-                            secret_path: &secret_path,
-                            now,
-                        },
-                    )
-                    .await
-                    .map_err(DurableError::from)?;
-                    Ok(())
-                })
-            }
-        })
-        .await?;
-        self.get_internal(credential_id)
-            .await?
-            .ok_or_else(|| CredentialError::Config("created credential is missing".into()))
-    }
-
     async fn get_internal(
         &self,
         credential_id: &OpaqueId,
@@ -1048,6 +1076,27 @@ impl CredentialRegistry {
     }
 }
 
+async fn get_internal_in_tx(
+    tx: &mut PgTx<'_>,
+    credential_id: &OpaqueId,
+) -> Result<Option<SourceCredential>, DurableError> {
+    let row = sqlx::query_as::<_, CredentialRow>(
+        "SELECT c.credential_id, c.owner_id, c.label, c.kind, c.status,
+                c.created_at, c.updated_at,
+                v.version_id AS active_version_id, v.created_at AS active_version_created_at
+         FROM source_credentials c
+         LEFT JOIN source_credential_versions v
+           ON v.version_id = c.active_version_id
+         WHERE c.credential_id = $1",
+    )
+    .bind(credential_id.as_str())
+    .fetch_optional(tx.as_mut())
+    .await?;
+    row.map(CredentialRow::into_record)
+        .transpose()
+        .map_err(DurableError::effect)
+}
+
 const EFFECT_REVOKED: &str = "remora.credentials.revoked";
 const EFFECT_NO_ACTIVE: &str = "remora.credentials.no_active_version";
 const EFFECT_MISSING: &str = "remora.credentials.missing";
@@ -1209,9 +1258,9 @@ async fn pin_current_version_in_tx(
     })
 }
 
-#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+#[derive(Clone, Debug, Eq, PartialEq)]
-enum RotateTxOutcome {
+pub enum RotateTxOutcome {
-    Applied,
+    Applied(SourceCredential),
     Revoked,
     IntentClaimed,
 }

crates/remora-deletion/src/registry.rs

@@ -36,7 +36,7 @@ impl TargetRegistry {
             .remove(&(target_store.to_owned(), bucket.to_owned()))
     }
 
-    pub(crate) fn resolve(&self, target_store: &str, bucket: &str) -> Option<TargetStore> {
+    pub fn resolve(&self, target_store: &str, bucket: &str) -> Option<TargetStore> {
         self.stores
             .read()
             .expect("target registry poisoned")

crates/remora-deletion/src/service.rs

@@ -2,7 +2,7 @@ use std::time::Duration;
 
 use futures_util::TryStreamExt;
 use remora_domain::{BucketName, OpaqueId, Target, TargetPrefix, TargetStoreId};
-use remora_durable::enqueue;
+use remora_durable::{PgTx, enqueue};
 use remora_identity::{
     AuthorizedResource, ManagementOperation, Principal, ResourceVisibility, resource_visibility,
 };
@@ -52,14 +52,23 @@ impl DeletionService {
         job_id: &OpaqueId,
     ) -> Result<LifecycleMutation, DeletionError> {
         let mut tx = self.pool.begin().await?;
-        let row = lock_job(&mut tx, job_id).await?;
+        let outcome = Self::soft_delete_in_tx(&mut tx, principal, job_id).await?;
+        tx.commit().await?;
+        Ok(outcome)
+    }
+
+    pub async fn soft_delete_in_tx(
+        tx: &mut PgTx<'_>,
+        principal: &Principal,
+        job_id: &OpaqueId,
+    ) -> Result<LifecycleMutation, DeletionError> {
+        let row = lock_job(tx, job_id).await?;
         authorize(principal, &row, ManagementOperation::SoftDelete)?;
         require_terminal(&row.phase)?;
         if row.deletion_operation_id.is_some() {
             return Err(DeletionError::HardDeleteStarted);
         }
         if row.lifecycle == "soft_deleted" {
-            tx.commit().await?;
             return Ok(LifecycleMutation::Unchanged);
         }
         let now = OffsetDateTime::now_utc();
@@ -72,8 +81,7 @@ impl DeletionService {
         .bind(now)
         .execute(tx.as_mut())
         .await?;
-        enqueue(&mut tx, "job.updated", json!({ "job_id": job_id })).await?;
+        enqueue(tx, "job.updated", json!({ "job_id": job_id })).await?;
-        tx.commit().await?;
         Ok(LifecycleMutation::Changed)
     }
 
@@ -83,14 +91,23 @@ impl DeletionService {
         job_id: &OpaqueId,
     ) -> Result<LifecycleMutation, DeletionError> {
         let mut tx = self.pool.begin().await?;
-        let row = lock_job(&mut tx, job_id).await?;
+        let outcome = Self::restore_in_tx(&mut tx, principal, job_id).await?;
+        tx.commit().await?;
+        Ok(outcome)
+    }
+
+    pub async fn restore_in_tx(
+        tx: &mut PgTx<'_>,
+        principal: &Principal,
+        job_id: &OpaqueId,
+    ) -> Result<LifecycleMutation, DeletionError> {
+        let row = lock_job(tx, job_id).await?;
         authorize(principal, &row, ManagementOperation::Restore)?;
         require_terminal(&row.phase)?;
         if row.deletion_operation_id.is_some() {
             return Err(DeletionError::HardDeleteStarted);
         }
         if row.lifecycle == "visible" {
-            tx.commit().await?;
             return Ok(LifecycleMutation::Unchanged);
         }
         let now = OffsetDateTime::now_utc();
@@ -103,8 +120,7 @@ impl DeletionService {
         .bind(now)
         .execute(tx.as_mut())
         .await?;
-        enqueue(&mut tx, "job.updated", json!({ "job_id": job_id })).await?;
+        enqueue(tx, "job.updated", json!({ "job_id": job_id })).await?;
-        tx.commit().await?;
         Ok(LifecycleMutation::Changed)
     }
 
@@ -113,21 +129,30 @@ impl DeletionService {
         principal: &Principal,
         job_id: &OpaqueId,
     ) -> Result<HardDeleteOutcome, DeletionError> {
-        let operation_id = new_id('d')?;
         let mut tx = self.pool.begin().await?;
-        let row = lock_job(&mut tx, job_id).await?;
+        let outcome = Self::hard_delete_in_tx(&mut tx, principal, job_id).await?;
+        tx.commit().await?;
+        Ok(outcome)
+    }
+
+    pub async fn hard_delete_in_tx(
+        tx: &mut PgTx<'_>,
+        principal: &Principal,
+        job_id: &OpaqueId,
+    ) -> Result<HardDeleteOutcome, DeletionError> {
+        let operation_id = new_id('d')?;
+        let row = lock_job(tx, job_id).await?;
         authorize(principal, &row, ManagementOperation::HardDelete)?;
         require_terminal(&row.phase)?;
         if let Some(existing) = row.deletion_operation_id {
-            let operation = load_operation_in_tx(&mut tx, &existing).await?;
+            let operation = load_operation_in_tx(tx, &existing).await?;
-            tx.commit().await?;
             return Ok(HardDeleteOutcome {
                 mutation: LifecycleMutation::Unchanged,
                 operation,
             });
         }
-        validate_committed_inventory(&mut tx, &row).await?;
+        validate_committed_inventory(tx, &row).await?;
-        validate_deletion_identities(&mut tx, &row).await?;
+        validate_deletion_identities(tx, &row).await?;
         let now = OffsetDateTime::now_utc();
         sqlx::query(
             "INSERT INTO deletion_operations (
@@ -150,16 +175,15 @@ impl DeletionService {
         .bind(now)
         .execute(tx.as_mut())
         .await?;
-        materialize_items(&mut tx, job_id, &operation_id, now).await?;
+        materialize_items(tx, job_id, &operation_id, now).await?;
-        finalize_if_empty(&mut tx, job_id, &operation_id, now).await?;
+        finalize_if_empty(tx, job_id, &operation_id, now).await?;
         enqueue(
-            &mut tx,
+            tx,
             "job.updated",
             json!({ "job_id": job_id, "deletion_operation_id": operation_id }),
         )
         .await?;
-        let operation = load_operation_in_tx(&mut tx, operation_id.as_str()).await?;
+        let operation = load_operation_in_tx(tx, operation_id.as_str()).await?;
-        tx.commit().await?;
         Ok(HardDeleteOutcome {
             mutation: LifecycleMutation::Changed,
             operation,

crates/remora-durable/src/ledger.rs

@@ -48,6 +48,51 @@ pub struct MutationOutcome {
     pub response: Value,
 }
 
+pub async fn observe_mutation(
+    pool: &PgPool,
+    scope: MutationScope<'_>,
+    kind: &str,
+    eligibility_claim: &[u8],
+    request_fingerprint: RequestFingerprint,
+) -> Result<Option<MutationOutcome>, DurableError> {
+    let existing = sqlx::query_as::<_, (String, Vec<u8>, Vec<u8>, String, Value)>(
+        "SELECT kind, eligibility_claim, request_fingerprint, response_status, response_body
+         FROM mutation_records
+         WHERE scope_subject = $1
+           AND scope_operation = $2
+           AND operation_key = $3",
+    )
+    .bind(scope.subject.as_str())
+    .bind(scope.operation)
+    .bind(scope.key)
+    .fetch_optional(pool)
+    .await?;
+    existing
+        .map(
+            |(
+                stored_kind,
+                stored_eligibility_claim,
+                stored_request_fingerprint,
+                response_status,
+                response_body,
+            )| {
+                observe_existing(
+                    kind,
+                    eligibility_claim,
+                    request_fingerprint.as_bytes(),
+                    StoredRecord {
+                        kind: stored_kind,
+                        eligibility_claim: stored_eligibility_claim,
+                        request_fingerprint: stored_request_fingerprint,
+                        response_status,
+                        response_body,
+                    },
+                )
+            },
+        )
+        .transpose()
+}
+
 #[derive(Clone, Debug)]
 struct StoredRecord {
     kind: String,

crates/remora-durable/src/lib.rs

@@ -19,6 +19,7 @@ pub use cursor::{CursorCodec, KeysetCursor, KeysetPosition, first_page_cursor, n
 pub use error::DurableError;
 pub use ledger::{
     MutationDisposition, MutationOutcome, MutationStatus, execute_mutation, execute_mutation_in_tx,
+    observe_mutation,
 };
 pub use migrate::{migrate, migrator};
 pub use outbox::{

tests/kind/public-api-job.yaml.in

@@ -0,0 +1,40 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: @@JOB_NAME@@
+  namespace: remora-testbed
+spec:
+  activeDeadlineSeconds: 900
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 300
+  template:
+    spec:
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      restartPolicy: Never
+      dnsPolicy: None
+      dnsConfig:
+        nameservers: [10.96.0.53]
+        searches: [remora-testbed.svc.cluster.local, svc.cluster.local, cluster.local]
+        options:
+          - {name: ndots, value: "5"}
+      containers:
+        - name: tests
+          image: @@TEST_IMAGE@@
+          imagePullPolicy: Never
+          envFrom:
+            - configMapRef: {name: @@PROFILE_NAME@@}
+          env:
+            - name: REMORA_PUBLIC_API_REQUIRE
+              value: "1"
+          volumeMounts:
+            - {name: fixture-profile, mountPath: /run/remora-fixture, readOnly: true}
+          resources:
+            requests: {cpu: 250m, memory: 512Mi}
+            limits: {cpu: "2", memory: 2Gi}
+      volumes:
+        - name: fixture-profile
+          configMap:
+            name: @@PROFILE_NAME@@-file
+            items:
+              - {key: profile.env, path: profile.env}

tests/kind/run-public-api-job.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
+source "$ROOT/tests/kind/lib.sh"
+
+USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
+SUCCEEDED=false
+OWNS_FIXTURE=false
+
+cleanup() {
+  local status=$?
+  trap - EXIT INT TERM
+  if [[ "${REMORA_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
+    printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
+    printf 'retained context: %s\n' "$CONTEXT"
+    exit "$status"
+  fi
+  if [[ "$OWNS_FIXTURE" = true ]]; then
+    destroy_fixture || status=1
+  fi
+  if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
+    printf 'user kubeconfig changed unexpectedly\n' >&2
+    status=1
+  fi
+  exit "$status"
+}
+
+trap cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+acquire_fixture_lock
+REMORA_REAL_NERDCTL="$(command -v nerdctl)"
+export REMORA_REAL_NERDCTL
+"$REMORA_REAL_NERDCTL" --namespace default build \
+  --file "$ROOT/Dockerfile.test" \
+  --target kind-public-api-runner \
+  --tag remora-kind-public-api:local \
+  "$ROOT"
+OWNS_FIXTURE=true
+setup_fixture durable coordination oidc target secret
+load_local_image remora-kind-public-api:local
+run_job_matrix \
+  public-api \
+  "$TESTBED/public-api-job.yaml.in" \
+  remora-kind-public-api:local \
+  durable coordination oidc target secret
+SUCCEEDED=true
+printf 'public API package test Jobs passed\n'