Ship production deployment and runtime observability

crates/remora-attempt/src/recovery.rs

@@ -97,7 +97,6 @@ impl AttemptService {
                 .coordination
                 .release(authority.job_id.as_str(), &lease_owner(authority))
                 .await;
-            self.release_local(&authority.job_id);
         }
         Ok(outcome)
     }
@@ -154,7 +153,6 @@ impl AttemptService {
                 .coordination
                 .release(authority.job_id.as_str(), &lease_owner(authority))
                 .await;
-            self.release_local(&authority.job_id);
         }
         Ok(decision)
     }
@@ -649,7 +647,6 @@ impl AttemptService {
         };
         let mut report = ReconcileReport::default();
         if row.phase == "retry_wait" {
-            self.release_local_if_stopped(job_id).await?;
             if row.retry_not_before.is_none() {
                 let due =
                     OffsetDateTime::now_utc() + time_duration(config.retry_delay, "retry_delay")?;
@@ -731,7 +728,6 @@ impl AttemptService {
             }
         }
         if parse_phase(&row.phase)?.is_terminal() {
-            self.release_local_if_stopped(job_id).await?;
             let expired = self.coordination.expire_lease(job_id.as_str()).await?;
             self.coordination.clean_terminal(job_id.as_str()).await?;
             if expired.code.is_success() {
@@ -774,9 +770,6 @@ impl AttemptService {
         })
         .await?;
         let _ = self.coordination.expire_lease(job_id.as_str()).await?;
-        if reaped.release_local {
-            self.release_local(job_id);
-        }
         Ok(reaped.changed)
     }
 
@@ -861,22 +854,6 @@ impl AttemptService {
         .await?;
         Ok(members.rows_affected() + manifests.rows_affected())
     }
-
-    async fn release_local_if_stopped(&self, job_id: &OpaqueId) -> Result<(), AttemptError> {
-        let stopped = sqlx::query_scalar::<_, bool>(
-            "SELECT NOT EXISTS (
-                SELECT 1 FROM attempts
-                WHERE job_id = $1 AND state = 'executing' AND worker_down_at IS NULL
-             )",
-        )
-        .bind(job_id.as_str())
-        .fetch_one(&self.pool)
-        .await?;
-        if stopped {
-            self.release_local(job_id);
-        }
-        Ok(())
-    }
 }
 
 async fn cooperative_stop_transaction(
@@ -1242,10 +1219,7 @@ async fn reap_lost_attempt(
         .await?;
         enqueue(tx, "job.updated", json!({ "job_id": job_id })).await?;
     }
-    Ok(ReapEffect {
-        changed: true,
-        release_local: row.state == "authorized" || row.worker_down_at.is_some(),
-    })
+    Ok(ReapEffect { changed: true })
 }
 
 async fn validate_staging_key(
@@ -1393,7 +1367,6 @@ struct LockedReapRow {
 #[derive(Default)]
 struct ReapEffect {
     changed: bool,
-    release_local: bool,
 }
 
 #[derive(FromRow)]

crates/remora-attempt/src/service.rs

@@ -2,7 +2,7 @@ use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, Ordering};
 
 use remora_coordination::{
-    ActivateOptions, CoordinationStore, LeaseOwner, LocalPermitGate, MAX_SAFE_INTEGER, OpCode,
+    ActivateOptions, CoordinationStore, LeaseOwner, MAX_SAFE_INTEGER, OpCode,
 };
 use remora_domain::{OpaqueId, fingerprint, ports::MutationScope};
 use remora_durable::{
@@ -29,15 +29,11 @@ impl AttemptFaultInjector for NoFaults {
 #[derive(Clone, Debug)]
 pub struct AttemptConfig {
     pub max_attempts: u64,
-    pub local_limit: u64,
 }
 
 impl Default for AttemptConfig {
     fn default() -> Self {
-        Self {
-            max_attempts: 3,
-            local_limit: 4,
-        }
+        Self { max_attempts: 3 }
     }
 }
 
@@ -45,7 +41,6 @@ impl Default for AttemptConfig {
 pub struct AttemptService {
     pub(crate) pool: PgPool,
     pub(crate) coordination: CoordinationStore,
-    local_permits: LocalPermitGate,
     pub(crate) operation_subject: OpaqueId,
     pub(crate) faults: Arc<dyn AttemptFaultInjector>,
     pub(crate) config: AttemptConfig,
@@ -71,15 +66,9 @@ impl AttemptService {
                 "max_attempts must fit the coordination safe-integer range".into(),
             ));
         }
-        if config.local_limit == 0 {
-            return Err(AttemptError::Invalid(
-                "local_limit must be greater than zero".into(),
-            ));
-        }
         Ok(Self {
             pool,
             coordination,
-            local_permits: LocalPermitGate::new(config.local_limit),
             operation_subject: OpaqueId::parse("remora_attempt_service")
                 .map_err(|error| AttemptError::Invalid(error.to_string()))?,
             faults,
@@ -114,12 +103,6 @@ impl AttemptService {
         worker_incarnation: &OpaqueId,
         operation_id: &OpaqueId,
     ) -> Result<ClaimOutcome, AttemptError> {
-        if self.local_permits.acquire(job_id.as_str()).is_err() {
-            return Ok(ClaimOutcome::Rejected {
-                reason: ClaimRejection::LocalCapacity,
-                disposition: None,
-            });
-        }
         let request = ClaimRequest {
             job_id,
             worker_incarnation,
@@ -177,22 +160,16 @@ impl AttemptService {
                 if reserved.load(Ordering::Acquire) {
                     self.compensate_reservation(job_id, worker_incarnation)
                         .await;
-                } else {
-                    self.local_permits.release(job_id.as_str());
                 }
                 return Err(error.into());
             }
         };
         self.faults.hit(AttemptFaultPoint::ClaimAfterDurable);
         if outcome.disposition == MutationDisposition::Conflict {
-            self.local_permits.release(job_id.as_str());
             return Err(AttemptError::OperationConflict);
         }
         let decision: ClaimOutcome = serde_json::from_value(outcome.response)
             .map_err(|error| AttemptError::Invalid(error.to_string()))?;
-        if matches!(decision, ClaimOutcome::Rejected { .. }) {
-            self.local_permits.release(job_id.as_str());
-        }
         Ok(decision.with_disposition(outcome.disposition))
     }
 
@@ -564,21 +541,11 @@ impl AttemptService {
             }))
     }
 
-    pub fn release_local(&self, job_id: &OpaqueId) -> bool {
-        self.local_permits.release_all(job_id.as_str())
-    }
-
-    #[must_use]
-    pub fn local_used(&self) -> u64 {
-        self.local_permits.used()
-    }
-
     async fn compensate_reservation(&self, job_id: &OpaqueId, worker: &OpaqueId) {
         let _ = self
             .coordination
             .free_reserve(job_id.as_str(), Some(worker.as_str()))
             .await;
-        self.local_permits.release(job_id.as_str());
     }
 
     pub(crate) async fn release_exact(&self, authority: &AttemptAuthority) {

crates/remora-attempt/src/types.rs

@@ -58,7 +58,6 @@ pub struct AttemptAuthority {
 pub enum ClaimRejection {
     JobUnavailable,
     AttemptsExhausted,
-    LocalCapacity,
     GlobalCapacity,
     ReservationHeld,
 }