ms2k/remora
SHA256
Watch
0
Code Releases Activity

Ship production deployment and runtime observability

Browse source
This commit is contained in:
ms2k 2026-07-18 04:22:14 +09:00
commit 775dd61963
3 changed files with 3712 additions and 175 deletions
Download patch file Download diff file Expand all files Collapse all files

62
crates/remora-attempt/tests/authority.rs
View file

@ -61,31 +61,20 @@ impl Harness {
Some(Self { pool, coordination })
}
fn service(&self, max_attempts: u64, local_limit: u64) -> AttemptService {
fn service(&self, max_attempts: u64) -> AttemptService {
AttemptService::new(
self.pool.clone(),
self.coordination.clone(),
AttemptConfig {
max_attempts,
local_limit,
},
AttemptConfig { max_attempts },
)
.expect("attempt service")
}
fn service_with_fault(
&self,
point: AttemptFaultPoint,
max_attempts: u64,
local_limit: u64,
) -> AttemptService {
fn service_with_fault(&self, point: AttemptFaultPoint, max_attempts: u64) -> AttemptService {
AttemptService::new_with_fault_injector(
self.pool.clone(),
self.coordination.clone(),
AttemptConfig {
max_attempts,
local_limit,
},
AttemptConfig { max_attempts },
Arc::new(PanicOnce::new(point)),
)
.expect("fault-injected attempt service")
@ -154,7 +143,7 @@ async fn claim_replay_conflict_and_fence_allocation_are_durable() {
let job = harness.job().await;
let worker = unique_id("worker");
let operation = unique_id("claimop");
let service = harness.service(2, 2);
let service = harness.service(2);
let first = claimed(
service
@ -189,7 +178,6 @@ async fn claim_replay_conflict_and_fence_allocation_are_durable() {
service.recover_inactive(&authority).await.unwrap(),
RecoveryOutcome::Reaped
);
service.release_local(&job);
harness.coordination.enqueue(job.as_str()).await.unwrap();
let second = claimed(
service
@ -201,7 +189,6 @@ async fn claim_replay_conflict_and_fence_allocation_are_durable() {
sleep(Duration::from_millis(2200)).await;
let second_authority = service.authority(&second.2).await.unwrap().unwrap();
service.recover_inactive(&second_authority).await.unwrap();
service.release_local(&job);
harness.coordination.enqueue(job.as_str()).await.unwrap();
let exhausted = service
.claim(&job, &worker, &unique_id("claimop"))
@ -217,7 +204,7 @@ async fn activation_replay_requires_the_complete_live_authority_tuple() {
};
let job = harness.job().await;
let worker = unique_id("worker");
let service = harness.service(3, 2);
let service = harness.service(3);
let claim_operation = unique_id("claimop");
let (_, _, attempt) = claimed(
service
@ -240,10 +227,7 @@ async fn activation_replay_requires_the_complete_live_authority_tuple() {
let unavailable = AttemptService::new(
unavailable_pool,
harness.coordination.clone(),
AttemptConfig {
max_attempts: 3,
local_limit: 2,
},
AttemptConfig { max_attempts: 3 },
)
.unwrap();
assert!(
@ -338,7 +322,7 @@ async fn activation_and_inactive_recovery_serialize_on_durable_authority() {
};
let job = harness.job().await;
let worker = unique_id("worker");
let service = harness.service(3, 2);
let service = harness.service(3);
let (_, _, attempt) = claimed(
service
.claim(&job, &worker, &unique_id("claimop"))
@ -380,11 +364,11 @@ async fn activation_and_inactive_recovery_serialize_on_durable_authority() {
}
#[tokio::test]
async fn local_and_global_permits_bound_claims_and_survive_lease_expiry() {
async fn global_permits_bound_claims_and_survive_lease_expiry() {
let Some(harness) = Harness::new("permits", Duration::from_millis(500), 1).await else {
return;
};
let service = harness.service(3, 1);
let service = harness.service(3);
let worker = unique_id("worker");
let first_job = harness.job().await;
let (_, _, attempt) = claimed(
@ -399,8 +383,6 @@ async fn local_and_global_permits_bound_claims_and_survive_lease_expiry() {
.await
.unwrap();
sleep(Duration::from_millis(600)).await;
assert_eq!(service.local_used(), 1);
let second_job = harness.job().await;
assert!(matches!(
service
@ -409,13 +391,11 @@ async fn local_and_global_permits_bound_claims_and_survive_lease_expiry() {
.unwrap(),
ClaimOutcome::Rejected { .. }
));
service.release_local(&first_job);
harness
.coordination
.expire_lease(first_job.as_str())
.await
.unwrap();
assert_eq!(service.local_used(), 0);
}
#[tokio::test]
@ -425,7 +405,7 @@ async fn precommit_lease_is_compensated_when_durable_authority_was_reaped() {
};
let job = harness.job().await;
let worker = unique_id("worker");
let service = harness.service(3, 2);
let service = harness.service(3);
let (_, _, attempt) = claimed(
service
.claim(&job, &worker, &unique_id("claimop"))
@ -479,11 +459,11 @@ async fn crash_injection_covers_every_cross_store_and_lost_response_boundary() {
return;
};
let worker = unique_id("worker");
let normal = harness.service(32, 32);
let normal = harness.service(32);
let before_reserve_job = harness.job().await;
crash_claim(
harness.service_with_fault(AttemptFaultPoint::ClaimBeforeReserve, 32, 32),
harness.service_with_fault(AttemptFaultPoint::ClaimBeforeReserve, 32),
before_reserve_job.clone(),
worker.clone(),
unique_id("claimop"),
@ -500,7 +480,7 @@ async fn crash_injection_covers_every_cross_store_and_lost_response_boundary() {
let after_reserve_job = harness.job().await;
crash_claim(
harness.service_with_fault(AttemptFaultPoint::ClaimAfterReserve, 32, 32),
harness.service_with_fault(AttemptFaultPoint::ClaimAfterReserve, 32),
after_reserve_job.clone(),
worker.clone(),
unique_id("claimop"),
@ -523,7 +503,7 @@ async fn crash_injection_covers_every_cross_store_and_lost_response_boundary() {
let lost_claim_job = harness.job().await;
let lost_claim_operation = unique_id("claimop");
crash_claim(
harness.service_with_fault(AttemptFaultPoint::ClaimAfterDurable, 32, 32),
harness.service_with_fault(AttemptFaultPoint::ClaimAfterDurable, 32),
lost_claim_job.clone(),
worker.clone(),
lost_claim_operation.clone(),
@ -544,7 +524,7 @@ async fn crash_injection_covers_every_cross_store_and_lost_response_boundary() {
let before_lease_operation = unique_id("activateop");
crash_activate(
harness.service_with_fault(AttemptFaultPoint::ActivationBeforeLease, 32, 32),
harness.service_with_fault(AttemptFaultPoint::ActivationBeforeLease, 32),
lost_claim_authority.clone(),
before_lease_operation.clone(),
)
@ -579,7 +559,7 @@ async fn crash_injection_covers_every_cross_store_and_lost_response_boundary() {
.unwrap();
let after_lease_operation = unique_id("activateop");
crash_activate(
harness.service_with_fault(AttemptFaultPoint::ActivationAfterLease, 32, 32),
harness.service_with_fault(AttemptFaultPoint::ActivationAfterLease, 32),
after_lease_authority.clone(),
after_lease_operation.clone(),
)
@ -614,7 +594,7 @@ async fn crash_injection_covers_every_cross_store_and_lost_response_boundary() {
.unwrap();
let lost_activation_operation = unique_id("activateop");
crash_activate(
harness.service_with_fault(AttemptFaultPoint::ActivationAfterDurable, 32, 32),
harness.service_with_fault(AttemptFaultPoint::ActivationAfterDurable, 32),
lost_activation_authority.clone(),
lost_activation_operation.clone(),
)
@ -633,12 +613,12 @@ async fn crash_injection_covers_every_cross_store_and_lost_response_boundary() {
);
crash_renew(
harness.service_with_fault(AttemptFaultPoint::RenewalBeforeLease, 32, 32),
harness.service_with_fault(AttemptFaultPoint::RenewalBeforeLease, 32),
executing.clone(),
)
.await;
crash_renew(
harness.service_with_fault(AttemptFaultPoint::RenewalAfterLease, 32, 32),
harness.service_with_fault(AttemptFaultPoint::RenewalAfterLease, 32),
executing.clone(),
)
.await;
@ -666,7 +646,7 @@ async fn crash_injection_covers_every_cross_store_and_lost_response_boundary() {
sleep(Duration::from_millis(600)).await;
for (point, authority) in recovery_cases {
let used_before = harness.coordination.global_used().await.unwrap();
crash_recovery(harness.service_with_fault(point, 32, 32), authority.clone()).await;
crash_recovery(harness.service_with_fault(point, 32), authority.clone()).await;
let expected = if point == AttemptFaultPoint::RecoveryAfterDurable {
RecoveryOutcome::AlreadyInactive
} else {

6
crates/remora-attempt/tests/mutations.rs
View file

@ -67,10 +67,7 @@ impl Harness {
AttemptService::new(
self.pool.clone(),
self.coordination.clone(),
AttemptConfig {
max_attempts,
local_limit: 8,
},
AttemptConfig { max_attempts },
)
.expect("attempt service")
}
@ -637,7 +634,6 @@ async fn failure_disposition_retries_with_budget_and_becomes_terminal_when_exhau
)
.await
.unwrap();
service.release_local(&job);
harness.coordination.enqueue(job.as_str()).await.unwrap();
let second = harness.executing(&service, &job).await;
assert_eq!(second.fence, 2);

25
crates/remora-attempt/tests/recovery.rs
View file

@ -163,10 +163,7 @@ impl Harness {
AttemptService::new(
self.pool.clone(),
self.coordination.clone(),
AttemptConfig {
max_attempts,
local_limit: 16,
},
AttemptConfig { max_attempts },
)
.expect("attempt service")
}
@ -304,10 +301,7 @@ async fn activation_and_recovery_share_authority_serialization() {
let service = AttemptService::new_with_fault_injector(
harness.pool.clone(),
harness.coordination.clone(),
AttemptConfig {
max_attempts: 3,
local_limit: 16,
},
AttemptConfig { max_attempts: 3 },
activation_gate.clone(),
)
.unwrap();
@ -371,10 +365,7 @@ async fn activation_and_recovery_share_authority_serialization() {
let service = AttemptService::new_with_fault_injector(
harness.pool.clone(),
harness.coordination.clone(),
AttemptConfig {
max_attempts: 3,
local_limit: 16,
},
AttemptConfig { max_attempts: 3 },
recovery_gate.clone(),
)
.unwrap();
@ -485,10 +476,7 @@ async fn coordination_partition_rolls_back_and_restart_converges() {
let service = AttemptService::new(
harness.pool.clone(),
store.clone(),
AttemptConfig {
max_attempts: 3,
local_limit: 16,
},
AttemptConfig { max_attempts: 3 },
)
.unwrap();
let job = harness.job().await;
@ -529,10 +517,7 @@ async fn coordination_partition_rolls_back_and_restart_converges() {
let restarted = AttemptService::new(
harness.pool.clone(),
resumed_store.clone(),
AttemptConfig {
max_attempts: 3,
local_limit: 16,
},
AttemptConfig { max_attempts: 3 },
)
.unwrap();
assert_eq!(