ms2k/remora
SHA256
Watch
0
Code Releases Activity

Ship production deployment and runtime observability

Browse source
This commit is contained in:
ms2k 2026-07-18 04:22:14 +09:00
commit 775dd61963
6 changed files with 3712 additions and 175 deletions
Download patch file Download diff file Expand all files Collapse all files

29
crates/remora-attempt/src/recovery.rs
View file

@ -97,7 +97,6 @@ impl AttemptService {
.coordination
.release(authority.job_id.as_str(), &lease_owner(authority))
.await;
self.release_local(&authority.job_id);
}
Ok(outcome)
}
@ -154,7 +153,6 @@ impl AttemptService {
.coordination
.release(authority.job_id.as_str(), &lease_owner(authority))
.await;
self.release_local(&authority.job_id);
}
Ok(decision)
}
@ -649,7 +647,6 @@ impl AttemptService {
};
let mut report = ReconcileReport::default();
if row.phase == "retry_wait" {
self.release_local_if_stopped(job_id).await?;
if row.retry_not_before.is_none() {
let due =
OffsetDateTime::now_utc() + time_duration(config.retry_delay, "retry_delay")?;
@ -731,7 +728,6 @@ impl AttemptService {
}
}
if parse_phase(&row.phase)?.is_terminal() {
self.release_local_if_stopped(job_id).await?;
let expired = self.coordination.expire_lease(job_id.as_str()).await?;
self.coordination.clean_terminal(job_id.as_str()).await?;
if expired.code.is_success() {
@ -774,9 +770,6 @@ impl AttemptService {
})
.await?;
let _ = self.coordination.expire_lease(job_id.as_str()).await?;
if reaped.release_local {
self.release_local(job_id);
}
Ok(reaped.changed)
}
@ -861,22 +854,6 @@ impl AttemptService {
.await?;
Ok(members.rows_affected() + manifests.rows_affected())
}
async fn release_local_if_stopped(&self, job_id: &OpaqueId) -> Result<(), AttemptError> {
let stopped = sqlx::query_scalar::<_, bool>(
"SELECT NOT EXISTS (
SELECT 1 FROM attempts
WHERE job_id = $1 AND state = 'executing' AND worker_down_at IS NULL
)",
)
.bind(job_id.as_str())
.fetch_one(&self.pool)
.await?;
if stopped {
self.release_local(job_id);
}
Ok(())
}
}
async fn cooperative_stop_transaction(
@ -1242,10 +1219,7 @@ async fn reap_lost_attempt(
.await?;
enqueue(tx, "job.updated", json!({ "job_id": job_id })).await?;
}
Ok(ReapEffect {
changed: true,
release_local: row.state == "authorized" || row.worker_down_at.is_some(),
})
Ok(ReapEffect { changed: true })
}
async fn validate_staging_key(
@ -1393,7 +1367,6 @@ struct LockedReapRow {
#[derive(Default)]
struct ReapEffect {
changed: bool,
release_local: bool,
}
#[derive(FromRow)]

37
crates/remora-attempt/src/service.rs
View file

@ -2,7 +2,7 @@ use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use remora_coordination::{
ActivateOptions, CoordinationStore, LeaseOwner, LocalPermitGate, MAX_SAFE_INTEGER, OpCode,
ActivateOptions, CoordinationStore, LeaseOwner, MAX_SAFE_INTEGER, OpCode,
};
use remora_domain::{OpaqueId, fingerprint, ports::MutationScope};
use remora_durable::{
@ -29,15 +29,11 @@ impl AttemptFaultInjector for NoFaults {
#[derive(Clone, Debug)]
pub struct AttemptConfig {
pub max_attempts: u64,
pub local_limit: u64,
}
impl Default for AttemptConfig {
fn default() -> Self {
Self {
max_attempts: 3,
local_limit: 4,
}
Self { max_attempts: 3 }
}
}
@ -45,7 +41,6 @@ impl Default for AttemptConfig {
pub struct AttemptService {
pub(crate) pool: PgPool,
pub(crate) coordination: CoordinationStore,
local_permits: LocalPermitGate,
pub(crate) operation_subject: OpaqueId,
pub(crate) faults: Arc<dyn AttemptFaultInjector>,
pub(crate) config: AttemptConfig,
@ -71,15 +66,9 @@ impl AttemptService {
"max_attempts must fit the coordination safe-integer range".into(),
));
}
if config.local_limit == 0 {
return Err(AttemptError::Invalid(
"local_limit must be greater than zero".into(),
));
}
Ok(Self {
pool,
coordination,
local_permits: LocalPermitGate::new(config.local_limit),
operation_subject: OpaqueId::parse("remora_attempt_service")
.map_err(|error| AttemptError::Invalid(error.to_string()))?,
faults,
@ -114,12 +103,6 @@ impl AttemptService {
worker_incarnation: &OpaqueId,
operation_id: &OpaqueId,
) -> Result<ClaimOutcome, AttemptError> {
if self.local_permits.acquire(job_id.as_str()).is_err() {
return Ok(ClaimOutcome::Rejected {
reason: ClaimRejection::LocalCapacity,
disposition: None,
});
}
let request = ClaimRequest {
job_id,
worker_incarnation,
@ -177,22 +160,16 @@ impl AttemptService {
if reserved.load(Ordering::Acquire) {
self.compensate_reservation(job_id, worker_incarnation)
.await;
} else {
self.local_permits.release(job_id.as_str());
}
return Err(error.into());
}
};
self.faults.hit(AttemptFaultPoint::ClaimAfterDurable);
if outcome.disposition == MutationDisposition::Conflict {
self.local_permits.release(job_id.as_str());
return Err(AttemptError::OperationConflict);
}
let decision: ClaimOutcome = serde_json::from_value(outcome.response)
.map_err(|error| AttemptError::Invalid(error.to_string()))?;
if matches!(decision, ClaimOutcome::Rejected { .. }) {
self.local_permits.release(job_id.as_str());
}
Ok(decision.with_disposition(outcome.disposition))
}
@ -564,21 +541,11 @@ impl AttemptService {
}))
}
pub fn release_local(&self, job_id: &OpaqueId) -> bool {
self.local_permits.release_all(job_id.as_str())
}
#[must_use]
pub fn local_used(&self) -> u64 {
self.local_permits.used()
}
async fn compensate_reservation(&self, job_id: &OpaqueId, worker: &OpaqueId) {
let _ = self
.coordination
.free_reserve(job_id.as_str(), Some(worker.as_str()))
.await;
self.local_permits.release(job_id.as_str());
}
pub(crate) async fn release_exact(&self, authority: &AttemptAuthority) {

1
crates/remora-attempt/src/types.rs
View file

@ -58,7 +58,6 @@ pub struct AttemptAuthority {
pub enum ClaimRejection {
JobUnavailable,
AttemptsExhausted,
LocalCapacity,
GlobalCapacity,
ReservationHeld,
}

62
crates/remora-attempt/tests/authority.rs
View file

@ -61,31 +61,20 @@ impl Harness {
Some(Self { pool, coordination })
}
fn service(&self, max_attempts: u64, local_limit: u64) -> AttemptService {
fn service(&self, max_attempts: u64) -> AttemptService {
AttemptService::new(
self.pool.clone(),
self.coordination.clone(),
AttemptConfig {
max_attempts,
local_limit,
},
AttemptConfig { max_attempts },
)
.expect("attempt service")
}
fn service_with_fault(
&self,
point: AttemptFaultPoint,
max_attempts: u64,
local_limit: u64,
) -> AttemptService {
fn service_with_fault(&self, point: AttemptFaultPoint, max_attempts: u64) -> AttemptService {
AttemptService::new_with_fault_injector(
self.pool.clone(),
self.coordination.clone(),
AttemptConfig {
max_attempts,
local_limit,
},
AttemptConfig { max_attempts },
Arc::new(PanicOnce::new(point)),
)
.expect("fault-injected attempt service")
@ -154,7 +143,7 @@ async fn claim_replay_conflict_and_fence_allocation_are_durable() {
let job = harness.job().await;
let worker = unique_id("worker");
let operation = unique_id("claimop");
let service = harness.service(2, 2);
let service = harness.service(2);
let first = claimed(
service
@ -189,7 +178,6 @@ async fn claim_replay_conflict_and_fence_allocation_are_durable() {
service.recover_inactive(&authority).await.unwrap(),
RecoveryOutcome::Reaped
);
service.release_local(&job);
harness.coordination.enqueue(job.as_str()).await.unwrap();
let second = claimed(
service
@ -201,7 +189,6 @@ async fn claim_replay_conflict_and_fence_allocation_are_durable() {
sleep(Duration::from_millis(2200)).await;
let second_authority = service.authority(&second.2).await.unwrap().unwrap();
service.recover_inactive(&second_authority).await.unwrap();
service.release_local(&job);
harness.coordination.enqueue(job.as_str()).await.unwrap();
let exhausted = service
.claim(&job, &worker, &unique_id("claimop"))
@ -217,7 +204,7 @@ async fn activation_replay_requires_the_complete_live_authority_tuple() {
};
let job = harness.job().await;
let worker = unique_id("worker");
let service = harness.service(3, 2);
let service = harness.service(3);
let claim_operation = unique_id("claimop");
let (_, _, attempt) = claimed(
service
@ -240,10 +227,7 @@ async fn activation_replay_requires_the_complete_live_authority_tuple() {
let unavailable = AttemptService::new(
unavailable_pool,
harness.coordination.clone(),
AttemptConfig {
max_attempts: 3,
local_limit: 2,
},
AttemptConfig { max_attempts: 3 },
)
.unwrap();
assert!(
@ -338,7 +322,7 @@ async fn activation_and_inactive_recovery_serialize_on_durable_authority() {
};
let job = harness.job().await;
let worker = unique_id("worker");
let service = harness.service(3, 2);
let service = harness.service(3);
let (_, _, attempt) = claimed(
service
.claim(&job, &worker, &unique_id("claimop"))
@ -380,11 +364,11 @@ async fn activation_and_inactive_recovery_serialize_on_durable_authority() {
}
#[tokio::test]
async fn local_and_global_permits_bound_claims_and_survive_lease_expiry() {
async fn global_permits_bound_claims_and_survive_lease_expiry() {
let Some(harness) = Harness::new("permits", Duration::from_millis(500), 1).await else {
return;
};
let service = harness.service(3, 1);
let service = harness.service(3);
let worker = unique_id("worker");
let first_job = harness.job().await;
let (_, _, attempt) = claimed(
@ -399,8 +383,6 @@ async fn local_and_global_permits_bound_claims_and_survive_lease_expiry() {
.await
.unwrap();
sleep(Duration::from_millis(600)).await;
assert_eq!(service.local_used(), 1);
let second_job = harness.job().await;
assert!(matches!(
service
@ -409,13 +391,11 @@ async fn local_and_global_permits_bound_claims_and_survive_lease_expiry() {
.unwrap(),
ClaimOutcome::Rejected { .. }
));
service.release_local(&first_job);
harness
.coordination
.expire_lease(first_job.as_str())
.await
.unwrap();
assert_eq!(service.local_used(), 0);
}
#[tokio::test]
@ -425,7 +405,7 @@ async fn precommit_lease_is_compensated_when_durable_authority_was_reaped() {
};
let job = harness.job().await;
let worker = unique_id("worker");
let service = harness.service(3, 2);
let service = harness.service(3);
let (_, _, attempt) = claimed(
service
.claim(&job, &worker, &unique_id("claimop"))
@ -479,11 +459,11 @@ async fn crash_injection_covers_every_cross_store_and_lost_response_boundary() {
return;
};
let worker = unique_id("worker");
let normal = harness.service(32, 32);
let normal = harness.service(32);
let before_reserve_job = harness.job().await;
crash_claim(
harness.service_with_fault(AttemptFaultPoint::ClaimBeforeReserve, 32, 32),
harness.service_with_fault(AttemptFaultPoint::ClaimBeforeReserve, 32),
before_reserve_job.clone(),
worker.clone(),
unique_id("claimop"),
@ -500,7 +480,7 @@ async fn crash_injection_covers_every_cross_store_and_lost_response_boundary() {
let after_reserve_job = harness.job().await;
crash_claim(
harness.service_with_fault(AttemptFaultPoint::ClaimAfterReserve, 32, 32),
harness.service_with_fault(AttemptFaultPoint::ClaimAfterReserve, 32),
after_reserve_job.clone(),
worker.clone(),
unique_id("claimop"),
@ -523,7 +503,7 @@ async fn crash_injection_covers_every_cross_store_and_lost_response_boundary() {
let lost_claim_job = harness.job().await;
let lost_claim_operation = unique_id("claimop");
crash_claim(
harness.service_with_fault(AttemptFaultPoint::ClaimAfterDurable, 32, 32),
harness.service_with_fault(AttemptFaultPoint::ClaimAfterDurable, 32),
lost_claim_job.clone(),
worker.clone(),
lost_claim_operation.clone(),
@ -544,7 +524,7 @@ async fn crash_injection_covers_every_cross_store_and_lost_response_boundary() {
let before_lease_operation = unique_id("activateop");
crash_activate(
harness.service_with_fault(AttemptFaultPoint::ActivationBeforeLease, 32, 32),
harness.service_with_fault(AttemptFaultPoint::ActivationBeforeLease, 32),
lost_claim_authority.clone(),
before_lease_operation.clone(),
)
@ -579,7 +559,7 @@ async fn crash_injection_covers_every_cross_store_and_lost_response_boundary() {
.unwrap();
let after_lease_operation = unique_id("activateop");
crash_activate(
harness.service_with_fault(AttemptFaultPoint::ActivationAfterLease, 32, 32),
harness.service_with_fault(AttemptFaultPoint::ActivationAfterLease, 32),
after_lease_authority.clone(),
after_lease_operation.clone(),
)
@ -614,7 +594,7 @@ async fn crash_injection_covers_every_cross_store_and_lost_response_boundary() {
.unwrap();
let lost_activation_operation = unique_id("activateop");
crash_activate(
harness.service_with_fault(AttemptFaultPoint::ActivationAfterDurable, 32, 32),
harness.service_with_fault(AttemptFaultPoint::ActivationAfterDurable, 32),
lost_activation_authority.clone(),
lost_activation_operation.clone(),
)
@ -633,12 +613,12 @@ async fn crash_injection_covers_every_cross_store_and_lost_response_boundary() {
);
crash_renew(
harness.service_with_fault(AttemptFaultPoint::RenewalBeforeLease, 32, 32),
harness.service_with_fault(AttemptFaultPoint::RenewalBeforeLease, 32),
executing.clone(),
)
.await;
crash_renew(
harness.service_with_fault(AttemptFaultPoint::RenewalAfterLease, 32, 32),
harness.service_with_fault(AttemptFaultPoint::RenewalAfterLease, 32),
executing.clone(),
)
.await;
@ -666,7 +646,7 @@ async fn crash_injection_covers_every_cross_store_and_lost_response_boundary() {
sleep(Duration::from_millis(600)).await;
for (point, authority) in recovery_cases {
let used_before = harness.coordination.global_used().await.unwrap();
crash_recovery(harness.service_with_fault(point, 32, 32), authority.clone()).await;
crash_recovery(harness.service_with_fault(point, 32), authority.clone()).await;
let expected = if point == AttemptFaultPoint::RecoveryAfterDurable {
RecoveryOutcome::AlreadyInactive
} else {

6
crates/remora-attempt/tests/mutations.rs
View file

@ -67,10 +67,7 @@ impl Harness {
AttemptService::new(
self.pool.clone(),
self.coordination.clone(),
AttemptConfig {
max_attempts,
local_limit: 8,
},
AttemptConfig { max_attempts },
)
.expect("attempt service")
}
@ -637,7 +634,6 @@ async fn failure_disposition_retries_with_budget_and_becomes_terminal_when_exhau
)
.await
.unwrap();
service.release_local(&job);
harness.coordination.enqueue(job.as_str()).await.unwrap();
let second = harness.executing(&service, &job).await;
assert_eq!(second.fence, 2);

25
crates/remora-attempt/tests/recovery.rs
View file

@ -163,10 +163,7 @@ impl Harness {
AttemptService::new(
self.pool.clone(),
self.coordination.clone(),
AttemptConfig {
max_attempts,
local_limit: 16,
},
AttemptConfig { max_attempts },
)
.expect("attempt service")
}
@ -304,10 +301,7 @@ async fn activation_and_recovery_share_authority_serialization() {
let service = AttemptService::new_with_fault_injector(
harness.pool.clone(),
harness.coordination.clone(),
AttemptConfig {
max_attempts: 3,
local_limit: 16,
},
AttemptConfig { max_attempts: 3 },
activation_gate.clone(),
)
.unwrap();
@ -371,10 +365,7 @@ async fn activation_and_recovery_share_authority_serialization() {
let service = AttemptService::new_with_fault_injector(
harness.pool.clone(),
harness.coordination.clone(),
AttemptConfig {
max_attempts: 3,
local_limit: 16,
},
AttemptConfig { max_attempts: 3 },
recovery_gate.clone(),
)
.unwrap();
@ -485,10 +476,7 @@ async fn coordination_partition_rolls_back_and_restart_converges() {
let service = AttemptService::new(
harness.pool.clone(),
store.clone(),
AttemptConfig {
max_attempts: 3,
local_limit: 16,
},
AttemptConfig { max_attempts: 3 },
)
.unwrap();
let job = harness.job().await;
@ -529,10 +517,7 @@ async fn coordination_partition_rolls_back_and_restart_converges() {
let restarted = AttemptService::new(
harness.pool.clone(),
resumed_store.clone(),
AttemptConfig {
max_attempts: 3,
local_limit: 16,
},
AttemptConfig { max_attempts: 3 },
)
.unwrap();
assert_eq!(