Harden durable replay and lock recovery

crates/remora-durable/src/ledger.rs

@@ -39,6 +39,8 @@ impl MutationStatus {
     }
 }
 
+/// On conflict, `response` is `Null`; higher layers must map `disposition`
+/// instead of serializing it as a stored API response.
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub struct MutationOutcome {
     pub disposition: MutationDisposition,
@@ -165,6 +167,9 @@ struct MutationInsert<'a> {
 
 fn identity_lock_key(scope: MutationScope<'_>) -> i64 {
     let mut hasher = Sha256::new();
+    hasher.update(b"remora.advisory.mutation.v1");
+    hasher.update(tx::ADVISORY_CLASS_MUTATION.to_be_bytes());
+    hasher.update([0xff]);
     hasher.update(scope.subject.as_str().as_bytes());
     hasher.update([0xff]);
     hasher.update(scope.operation.as_bytes());

crates/remora-durable/src/lib.rs

@@ -26,4 +26,7 @@ pub use outbox::{
     enqueue_after, mark_projected, mark_projected_pool, release_with_error, unprojected_count,
 };
 pub use pool::{PoolConfig, connect};
-pub use tx::{BoxFuture, PgTx, advisory_xact_lock, set_lock_timeout, with_transaction};
+pub use tx::{
+    ADVISORY_CLASS_APP, ADVISORY_CLASS_MUTATION, BoxFuture, PgTx, advisory_key, advisory_xact_lock,
+    set_lock_timeout, with_transaction,
+};

crates/remora-durable/src/outbox.rs

@@ -102,6 +102,11 @@ pub async fn claim_batch_in_tx(
     limit: i64,
     lease: time::Duration,
 ) -> Result<Vec<OutboxMessage>, DurableError> {
+    if limit <= 0 {
+        return Err(DurableError::Config(
+            "outbox claim limit must be positive".into(),
+        ));
+    }
     let lease_secs = lease.as_seconds_f64().max(0.0);
     let topic = match filter {
         OutboxClaimFilter::Any => None,
@@ -237,9 +242,8 @@ pub async fn mark_projected_pool(pool: &PgPool, claim: &OutboxClaim) -> Result<(
     .await
 }
 
-/// Release a claimed message for later retry. Fenced to the claim token so a
-/// stale claimant cannot clear a reclaimed lease. Retry availability uses the
-/// database clock.
+/// Fenced release is idempotent for ambiguous commits; missing messages and
+/// foreign claims fail distinctly, and retry timing uses the database clock.
 pub async fn release_with_error(
     tx: &mut PgTx<'_>,
     claim: &OutboxClaim,
@@ -265,12 +269,38 @@ pub async fn release_with_error(
     .bind(error)
     .execute(tx.as_mut())
     .await?;
-    if result.rows_affected() != 1 {
-        return Err(DurableError::OutboxClaimMismatch {
+    if result.rows_affected() == 1 {
+        return Ok(());
+    }
+
+    let row: Option<(bool, Option<String>, i64, Option<String>)> = sqlx::query_as(
+        "SELECT
+            projected_at IS NOT NULL,
+            claimed_by,
+            claim_generation,
+            last_error
+         FROM outbox_messages
+         WHERE id = $1",
+    )
+    .bind(claim.message_id)
+    .fetch_optional(tx.as_mut())
+    .await?;
+
+    match row {
+        None => Err(DurableError::OutboxMissing {
             message_id: claim.message_id,
-        });
+        }),
+        // Already released under this claim generation: unclaimed, unprojected,
+        // same generation, same last_error as this release attempt.
+        Some((false, None, generation, last_error))
+            if generation == claim.generation && last_error.as_deref() == Some(error) =>
+        {
+            Ok(())
+        }
+        Some(_) => Err(DurableError::OutboxClaimMismatch {
+            message_id: claim.message_id,
+        }),
     }
-    Ok(())
 }
 
 pub async fn unprojected_count(

crates/remora-durable/src/tx.rs

@@ -1,6 +1,7 @@
 use std::future::Future;
 use std::pin::Pin;
 
+use sha2::{Digest, Sha256};
 use sqlx::{PgPool, Postgres, Transaction};
 
 use crate::error::DurableError;
@@ -33,6 +34,29 @@ pub async fn set_lock_timeout(tx: &mut PgTx<'_>) -> Result<(), DurableError> {
     Ok(())
 }
 
+/// Advisory lock class reserved for mutation ledger identity locks.
+/// Feature packages must not use this class.
+pub const ADVISORY_CLASS_MUTATION: i32 = 1;
+
+/// Advisory lock class for application and feature-package locks.
+/// Prefer this (or higher package-owned classes) over the mutation class.
+pub const ADVISORY_CLASS_APP: i32 = 2;
+
+/// Mixes the class into a 64-bit key to separate lock domains without reducing
+/// the PostgreSQL `bigint` identity space.
+#[must_use]
+pub fn advisory_key(class: i32, key: i32) -> i64 {
+    let mut hasher = Sha256::new();
+    hasher.update(b"remora.advisory.v1");
+    hasher.update(class.to_be_bytes());
+    hasher.update(key.to_be_bytes());
+    let digest = hasher.finalize();
+    i64::from_be_bytes([
+        digest[0], digest[1], digest[2], digest[3], digest[4], digest[5], digest[6], digest[7],
+    ])
+}
+
+/// Transaction-scoped advisory lock on a domain-separated 64-bit key.
 pub async fn advisory_xact_lock(tx: &mut PgTx<'_>, key: i64) -> Result<(), DurableError> {
     sqlx::query("SELECT pg_advisory_xact_lock($1)")
         .bind(key)