1026 lines
32 KiB
Rust
1026 lines
32 KiB
Rust
|
use std::net::SocketAddr;
|
||
|
|
use std::sync::{Arc, Barrier, Mutex};
|
||
|
|
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||
|
|
|
||
|
|
use remora_admission::{
|
||
|
|
AdmissionConfig, AdmissionDecision, AdmissionService, migrate as admission_migrate,
|
||
|
|
};
|
||
|
|
use remora_attempt::{
|
||
|
|
ActivationOutcome, AttemptConfig, AttemptService, CancelOutcome, CleanupCompletion,
|
||
|
|
CleanupIdentity, CooperativeStopOutcome, RecoveryConfig, StagingMultipart, StagingObject,
|
||
|
|
migrate as attempt_migrate,
|
||
|
|
};
|
||
|
|
use remora_contracts::{HttpSourceSubmissionDto, JobSubmissionDto, SourceSubmissionDto};
|
||
|
|
use remora_coordination::{CoordinationStore, StoreConfig};
|
||
|
|
use remora_credentials::migrate as credentials_migrate;
|
||
|
|
use remora_domain::{
|
||
|
|
BucketName, OpaqueId, RuntimeFailureClass, StorageScope, TargetPrefix, TargetStoreId,
|
||
|
|
};
|
||
|
|
use remora_durable::{PoolConfig, connect, migrate as durable_migrate};
|
||
|
|
use remora_identity::{DefaultTarget, Principal};
|
||
|
|
use remora_result::{PublicationInput, migrate as result_migrate};
|
||
|
|
use remora_target::{
|
||
|
|
CreateObjectRequest, MultipartIdentity, ObjectIdentity, StoredObject, TargetStore,
|
||
|
|
TargetStoreConfig, attempt_manifest_key, attempt_object_key,
|
||
|
|
};
|
||
|
|
use sqlx::PgPool;
|
||
|
|
use tokio::io::copy_bidirectional;
|
||
|
|
use tokio::net::{TcpListener, TcpStream};
|
||
|
|
use tokio::task::JoinHandle;
|
||
|
|
|
||
|
|
struct Harness {
|
||
|
|
pool: PgPool,
|
||
|
|
coordination: CoordinationStore,
|
||
|
|
target: TargetStore,
|
||
|
|
target_config: TargetStoreConfig,
|
||
|
|
}
|
||
|
|
|
||
|
|
struct CoordinationProxy {
|
||
|
|
address: SocketAddr,
|
||
|
|
accept_task: JoinHandle<()>,
|
||
|
|
connections: Arc<Mutex<Vec<JoinHandle<()>>>>,
|
||
|
|
}
|
||
|
|
|
||
|
|
struct GateFault {
|
||
|
|
point: remora_attempt::AttemptFaultPoint,
|
||
|
|
entered: Barrier,
|
||
|
|
release: Barrier,
|
||
|
|
}
|
||
|
|
|
||
|
|
impl GateFault {
|
||
|
|
fn new(point: remora_attempt::AttemptFaultPoint) -> Self {
|
||
|
|
Self {
|
||
|
|
point,
|
||
|
|
entered: Barrier::new(2),
|
||
|
|
release: Barrier::new(2),
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
impl remora_attempt::AttemptFaultInjector for GateFault {
|
||
|
|
fn hit(&self, point: remora_attempt::AttemptFaultPoint) {
|
||
|
|
if point == self.point {
|
||
|
|
self.entered.wait();
|
||
|
|
self.release.wait();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
impl CoordinationProxy {
|
||
|
|
async fn start(upstream: String, address: Option<SocketAddr>) -> Self {
|
||
|
|
let listener = TcpListener::bind(address.unwrap_or_else(|| "127.0.0.1:0".parse().unwrap()))
|
||
|
|
.await
|
||
|
|
.unwrap();
|
||
|
|
let address = listener.local_addr().unwrap();
|
||
|
|
let connections = Arc::new(Mutex::new(Vec::new()));
|
||
|
|
let active = connections.clone();
|
||
|
|
let accept_task = tokio::spawn(async move {
|
||
|
|
while let Ok((mut downstream, _)) = listener.accept().await {
|
||
|
|
let Ok(mut upstream_stream) = TcpStream::connect(&upstream).await else {
|
||
|
|
continue;
|
||
|
|
};
|
||
|
|
let task = tokio::spawn(async move {
|
||
|
|
let _ = copy_bidirectional(&mut downstream, &mut upstream_stream).await;
|
||
|
|
});
|
||
|
|
active.lock().unwrap().push(task);
|
||
|
|
}
|
||
|
|
});
|
||
|
|
Self {
|
||
|
|
address,
|
||
|
|
accept_task,
|
||
|
|
connections,
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
async fn partition(self) -> SocketAddr {
|
||
|
|
self.accept_task.abort();
|
||
|
|
let _ = self.accept_task.await;
|
||
|
|
for task in self.connections.lock().unwrap().drain(..) {
|
||
|
|
task.abort();
|
||
|
|
}
|
||
|
|
tokio::task::yield_now().await;
|
||
|
|
self.address
|
||
|
|
}
|
||
|
|
|
||
|
|
fn url(&self) -> String {
|
||
|
|
format!("redis://{}/", self.address)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
impl Harness {
|
||
|
|
async fn new(name: &str) -> Option<Self> {
|
||
|
|
let database_url = required_env("DATABASE_URL")?;
|
||
|
|
let coordination_url = required_env("REMORA_COORDINATION_URL")?;
|
||
|
|
let target_config = match TargetStoreConfig::from_env() {
|
||
|
|
Ok(config) => config,
|
||
|
|
Err(error) if required() => panic!("target fixture is required: {error}"),
|
||
|
|
Err(_) => return None,
|
||
|
|
};
|
||
|
|
let target = TargetStore::connect(target_config.clone()).expect("connect target");
|
||
|
|
let pool = connect(
|
||
|
|
&database_url,
|
||
|
|
PoolConfig {
|
||
|
|
max_connections: 16,
|
||
|
|
acquire_timeout: Duration::from_secs(10),
|
||
|
|
idle_timeout: Some(Duration::from_secs(30)),
|
||
|
|
},
|
||
|
|
)
|
||
|
|
.await
|
||
|
|
.expect("connect postgres");
|
||
|
|
durable_migrate(&pool).await.expect("durable migrate");
|
||
|
|
credentials_migrate(&pool)
|
||
|
|
.await
|
||
|
|
.expect("credentials migrate");
|
||
|
|
admission_migrate(&pool).await.expect("admission migrate");
|
||
|
|
attempt_migrate(&pool).await.expect("attempt migrate");
|
||
|
|
result_migrate(&pool).await.expect("result migrate");
|
||
|
|
sqlx::query("TRUNCATE TABLE jobs CASCADE")
|
||
|
|
.execute(&pool)
|
||
|
|
.await
|
||
|
|
.expect("reset recovery test state");
|
||
|
|
let coordination = CoordinationStore::connect(
|
||
|
|
&coordination_url,
|
||
|
|
format!("{}wp17:{name}:{}:", coordination_namespace(), nonce()),
|
||
|
|
StoreConfig {
|
||
|
|
global_limit: 32,
|
||
|
|
lease_ttl: Duration::from_secs(10),
|
||
|
|
reserve_ttl: Duration::from_secs(10),
|
||
|
|
progress_ttl: Duration::from_secs(10),
|
||
|
|
cancel_ttl: None,
|
||
|
|
},
|
||
|
|
)
|
||
|
|
.await
|
||
|
|
.expect("connect coordination store");
|
||
|
|
Some(Self {
|
||
|
|
pool,
|
||
|
|
coordination,
|
||
|
|
target,
|
||
|
|
target_config,
|
||
|
|
})
|
||
|
|
}
|
||
|
|
|
||
|
|
fn service(&self, max_attempts: u64) -> AttemptService {
|
||
|
|
AttemptService::new(
|
||
|
|
self.pool.clone(),
|
||
|
|
self.coordination.clone(),
|
||
|
|
AttemptConfig {
|
||
|
|
max_attempts,
|
||
|
|
local_limit: 16,
|
||
|
|
},
|
||
|
|
)
|
||
|
|
.expect("attempt service")
|
||
|
|
}
|
||
|
|
|
||
|
|
async fn job(&self) -> OpaqueId {
|
||
|
|
let store = TargetStoreId::parse(&self.target_config.target_store_id).unwrap();
|
||
|
|
let bucket = BucketName::parse(&self.target_config.bucket).unwrap();
|
||
|
|
let prefix = TargetPrefix::parse("users/recovery", ["_remora"]).unwrap();
|
||
|
|
let owner = unique_id("owner");
|
||
|
|
let principal = Principal {
|
||
|
|
id: owner,
|
||
|
|
issuer: "https://issuer.example".into(),
|
||
|
|
subject: unique_id("subject").to_string(),
|
||
|
|
administrator: false,
|
||
|
|
scopes: [StorageScope {
|
||
|
|
store: store.clone(),
|
||
|
|
bucket: bucket.clone(),
|
||
|
|
prefix: prefix.clone(),
|
||
|
|
}]
|
||
|
|
.into_iter()
|
||
|
|
.collect(),
|
||
|
|
};
|
||
|
|
let admission = AdmissionService::new(
|
||
|
|
self.pool.clone(),
|
||
|
|
AdmissionConfig {
|
||
|
|
default_target: DefaultTarget {
|
||
|
|
store,
|
||
|
|
bucket,
|
||
|
|
prefix,
|
||
|
|
},
|
||
|
|
reserved_target_segments: vec!["_remora".into()],
|
||
|
|
policy_version: "recovery-tests-v1".into(),
|
||
|
|
},
|
||
|
|
)
|
||
|
|
.unwrap();
|
||
|
|
let outcome = admission
|
||
|
|
.submit(
|
||
|
|
&principal,
|
||
|
|
unique_id("submit").as_str(),
|
||
|
|
JobSubmissionDto {
|
||
|
|
source: SourceSubmissionDto::Http(HttpSourceSubmissionDto {
|
||
|
|
url: "https://example.invalid/recovery".into(),
|
||
|
|
result_path: "artifact.bin".into(),
|
||
|
|
credential_id: None,
|
||
|
|
}),
|
||
|
|
target: None,
|
||
|
|
label: Some("recovery-test".into()),
|
||
|
|
},
|
||
|
|
)
|
||
|
|
.await
|
||
|
|
.expect("submit job");
|
||
|
|
let AdmissionDecision::Accepted(job) = outcome.decision else {
|
||
|
|
panic!("job rejected")
|
||
|
|
};
|
||
|
|
job.job_id
|
||
|
|
}
|
||
|
|
|
||
|
|
async fn executing(
|
||
|
|
&self,
|
||
|
|
service: &AttemptService,
|
||
|
|
job: &OpaqueId,
|
||
|
|
) -> remora_attempt::AttemptAuthority {
|
||
|
|
self.coordination
|
||
|
|
.enqueue(job.as_str())
|
||
|
|
.await
|
||
|
|
.expect("enqueue");
|
||
|
|
let claim = service
|
||
|
|
.claim(job, &unique_id("worker"), &unique_id("claimop"))
|
||
|
|
.await
|
||
|
|
.expect("claim");
|
||
|
|
let remora_attempt::ClaimOutcome::Claimed { authority, .. } = claim else {
|
||
|
|
panic!("claim rejected")
|
||
|
|
};
|
||
|
|
assert!(matches!(
|
||
|
|
service
|
||
|
|
.activate(&authority, &unique_id("activateop"))
|
||
|
|
.await
|
||
|
|
.expect("activate"),
|
||
|
|
ActivationOutcome::Activated(_)
|
||
|
|
));
|
||
|
|
service
|
||
|
|
.authority(&authority.attempt_id)
|
||
|
|
.await
|
||
|
|
.unwrap()
|
||
|
|
.unwrap()
|
||
|
|
}
|
||
|
|
|
||
|
|
async fn seal_result(
|
||
|
|
&self,
|
||
|
|
authority: &remora_attempt::AttemptAuthority,
|
||
|
|
manifest: StoredObject,
|
||
|
|
) -> PublicationInput {
|
||
|
|
let manifest_id = unique_id("manifest");
|
||
|
|
sqlx::query(
|
||
|
|
"INSERT INTO result_inventories (
|
||
|
|
attempt_id, state, manifest_schema_version, manifest_id, manifest_store,
|
||
|
|
manifest_bucket, manifest_key, manifest_version_id, manifest_sha256,
|
||
|
|
manifest_size, member_count, total_bytes
|
||
|
|
) VALUES ($1, 'sealed', 1, $2, $3, $4, $5, $6, $7, $8, 0, 0)",
|
||
|
|
)
|
||
|
|
.bind(authority.attempt_id.as_str())
|
||
|
|
.bind(manifest_id.as_str())
|
||
|
|
.bind(&manifest.identity.target_store_id)
|
||
|
|
.bind(&manifest.identity.bucket)
|
||
|
|
.bind(&manifest.identity.key)
|
||
|
|
.bind(&manifest.identity.version_id)
|
||
|
|
.bind(manifest.content_sha256.as_str())
|
||
|
|
.bind(i64::try_from(manifest.size).unwrap())
|
||
|
|
.execute(&self.pool)
|
||
|
|
.await
|
||
|
|
.unwrap();
|
||
|
|
PublicationInput {
|
||
|
|
attempt_id: authority.attempt_id.clone(),
|
||
|
|
manifest_id,
|
||
|
|
schema_version: 1,
|
||
|
|
manifest,
|
||
|
|
member_count: 0,
|
||
|
|
total_bytes: 0,
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||
|
|
async fn activation_and_recovery_share_authority_serialization() {
|
||
|
|
let Some(harness) = Harness::new("activation-race").await else {
|
||
|
|
return;
|
||
|
|
};
|
||
|
|
let activation_gate = Arc::new(GateFault::new(
|
||
|
|
remora_attempt::AttemptFaultPoint::ActivationAfterLease,
|
||
|
|
));
|
||
|
|
let service = AttemptService::new_with_fault_injector(
|
||
|
|
harness.pool.clone(),
|
||
|
|
harness.coordination.clone(),
|
||
|
|
AttemptConfig {
|
||
|
|
max_attempts: 3,
|
||
|
|
local_limit: 16,
|
||
|
|
},
|
||
|
|
activation_gate.clone(),
|
||
|
|
)
|
||
|
|
.unwrap();
|
||
|
|
let activation_job = harness.job().await;
|
||
|
|
harness
|
||
|
|
.coordination
|
||
|
|
.enqueue(activation_job.as_str())
|
||
|
|
.await
|
||
|
|
.unwrap();
|
||
|
|
let activation_claim = service
|
||
|
|
.claim(&activation_job, &unique_id("worker"), &unique_id("claimop"))
|
||
|
|
.await
|
||
|
|
.unwrap();
|
||
|
|
let remora_attempt::ClaimOutcome::Claimed {
|
||
|
|
authority: activation_authority,
|
||
|
|
..
|
||
|
|
} = activation_claim
|
||
|
|
else {
|
||
|
|
panic!("claim rejected")
|
||
|
|
};
|
||
|
|
let activating = service.clone();
|
||
|
|
let activation_input = activation_authority.clone();
|
||
|
|
let activation = tokio::spawn(async move {
|
||
|
|
activating
|
||
|
|
.activate(&activation_input, &unique_id("activateop"))
|
||
|
|
.await
|
||
|
|
.unwrap()
|
||
|
|
});
|
||
|
|
activation_gate.entered.wait();
|
||
|
|
let recovering = service.clone();
|
||
|
|
let recovery =
|
||
|
|
tokio::spawn(async move { recovering.reconcile_once(&recovery_config()).await.unwrap() });
|
||
|
|
tokio::task::yield_now().await;
|
||
|
|
activation_gate.release.wait();
|
||
|
|
assert!(matches!(
|
||
|
|
activation.await.unwrap(),
|
||
|
|
ActivationOutcome::Activated(_)
|
||
|
|
));
|
||
|
|
assert_eq!(recovery.await.unwrap().attempts_reaped, 0);
|
||
|
|
assert_eq!(
|
||
|
|
service
|
||
|
|
.authority(&activation_authority.attempt_id)
|
||
|
|
.await
|
||
|
|
.unwrap()
|
||
|
|
.unwrap()
|
||
|
|
.state,
|
||
|
|
remora_attempt::AttemptState::Executing
|
||
|
|
);
|
||
|
|
service
|
||
|
|
.publish_failure(
|
||
|
|
&activation_authority,
|
||
|
|
&unique_id("failure"),
|
||
|
|
RuntimeFailureClass::Authorization,
|
||
|
|
)
|
||
|
|
.await
|
||
|
|
.unwrap();
|
||
|
|
|
||
|
|
let recovery_gate = Arc::new(GateFault::new(
|
||
|
|
remora_attempt::AttemptFaultPoint::RecoveryBeforeRevalidation,
|
||
|
|
));
|
||
|
|
let service = AttemptService::new_with_fault_injector(
|
||
|
|
harness.pool.clone(),
|
||
|
|
harness.coordination.clone(),
|
||
|
|
AttemptConfig {
|
||
|
|
max_attempts: 3,
|
||
|
|
local_limit: 16,
|
||
|
|
},
|
||
|
|
recovery_gate.clone(),
|
||
|
|
)
|
||
|
|
.unwrap();
|
||
|
|
let recovery_job = harness.job().await;
|
||
|
|
harness
|
||
|
|
.coordination
|
||
|
|
.enqueue(recovery_job.as_str())
|
||
|
|
.await
|
||
|
|
.unwrap();
|
||
|
|
let worker = unique_id("worker");
|
||
|
|
let recovery_claim = service
|
||
|
|
.claim(&recovery_job, &worker, &unique_id("claimop"))
|
||
|
|
.await
|
||
|
|
.unwrap();
|
||
|
|
let remora_attempt::ClaimOutcome::Claimed {
|
||
|
|
authority: recovery_authority,
|
||
|
|
..
|
||
|
|
} = recovery_claim
|
||
|
|
else {
|
||
|
|
panic!("claim rejected")
|
||
|
|
};
|
||
|
|
assert_eq!(
|
||
|
|
service.cooperative_stop(&recovery_authority).await.unwrap(),
|
||
|
|
CooperativeStopOutcome::StopNotRequired
|
||
|
|
);
|
||
|
|
harness
|
||
|
|
.coordination
|
||
|
|
.free_reserve(recovery_job.as_str(), Some(worker.as_str()))
|
||
|
|
.await
|
||
|
|
.unwrap();
|
||
|
|
let recovering = service.clone();
|
||
|
|
let recovery =
|
||
|
|
tokio::spawn(async move { recovering.reconcile_once(&recovery_config()).await.unwrap() });
|
||
|
|
recovery_gate.entered.wait();
|
||
|
|
let activating = service.clone();
|
||
|
|
let activation_input = recovery_authority.clone();
|
||
|
|
let activation = tokio::spawn(async move {
|
||
|
|
activating
|
||
|
|
.activate(&activation_input, &unique_id("activateop"))
|
||
|
|
.await
|
||
|
|
.unwrap()
|
||
|
|
});
|
||
|
|
tokio::task::yield_now().await;
|
||
|
|
recovery_gate.release.wait();
|
||
|
|
assert_eq!(recovery.await.unwrap().attempts_reaped, 1);
|
||
|
|
assert_eq!(
|
||
|
|
activation.await.unwrap(),
|
||
|
|
ActivationOutcome::PreconditionFailed
|
||
|
|
);
|
||
|
|
assert!(
|
||
|
|
service
|
||
|
|
.authority(&recovery_authority.attempt_id)
|
||
|
|
.await
|
||
|
|
.unwrap()
|
||
|
|
.is_none()
|
||
|
|
);
|
||
|
|
}
|
||
|
|
|
||
|
|
#[tokio::test]
|
||
|
|
async fn batched_reconciliation_rotates_across_eligible_jobs() {
|
||
|
|
let Some(harness) = Harness::new("fairness").await else {
|
||
|
|
return;
|
||
|
|
};
|
||
|
|
let service = harness.service(3);
|
||
|
|
let first = harness.job().await;
|
||
|
|
let second = harness.job().await;
|
||
|
|
sqlx::query(
|
||
|
|
"UPDATE jobs SET recovery_checked_at = $1
|
||
|
|
WHERE job_id <> $2 AND job_id <> $3",
|
||
|
|
)
|
||
|
|
.bind(time::OffsetDateTime::now_utc())
|
||
|
|
.bind(first.as_str())
|
||
|
|
.bind(second.as_str())
|
||
|
|
.execute(&harness.pool)
|
||
|
|
.await
|
||
|
|
.unwrap();
|
||
|
|
let config = RecoveryConfig {
|
||
|
|
batch_size: 1,
|
||
|
|
..recovery_config()
|
||
|
|
};
|
||
|
|
assert_eq!(service.reconcile_once(&config).await.unwrap().enqueued, 1);
|
||
|
|
assert_eq!(service.reconcile_once(&config).await.unwrap().enqueued, 1);
|
||
|
|
let mut projected = [
|
||
|
|
harness.coordination.dequeue().await.unwrap().unwrap(),
|
||
|
|
harness.coordination.dequeue().await.unwrap().unwrap(),
|
||
|
|
];
|
||
|
|
projected.sort();
|
||
|
|
let mut expected = [first.to_string(), second.to_string()];
|
||
|
|
expected.sort();
|
||
|
|
assert_eq!(projected, expected);
|
||
|
|
}
|
||
|
|
|
||
|
|
#[tokio::test]
|
||
|
|
async fn coordination_partition_rolls_back_and_restart_converges() {
|
||
|
|
let Some(harness) = Harness::new("partition").await else {
|
||
|
|
return;
|
||
|
|
};
|
||
|
|
let upstream = coordination_upstream();
|
||
|
|
let proxy = CoordinationProxy::start(upstream.clone(), None).await;
|
||
|
|
let proxy_url = proxy.url();
|
||
|
|
let store = CoordinationStore::connect(
|
||
|
|
&proxy_url,
|
||
|
|
format!("{}wp17:partition:{}:", coordination_namespace(), nonce()),
|
||
|
|
StoreConfig::default(),
|
||
|
|
)
|
||
|
|
.await
|
||
|
|
.unwrap();
|
||
|
|
let service = AttemptService::new(
|
||
|
|
harness.pool.clone(),
|
||
|
|
store.clone(),
|
||
|
|
AttemptConfig {
|
||
|
|
max_attempts: 3,
|
||
|
|
local_limit: 16,
|
||
|
|
},
|
||
|
|
)
|
||
|
|
.unwrap();
|
||
|
|
let job = harness.job().await;
|
||
|
|
assert_eq!(
|
||
|
|
service
|
||
|
|
.reconcile_once(&recovery_config())
|
||
|
|
.await
|
||
|
|
.unwrap()
|
||
|
|
.enqueued,
|
||
|
|
1
|
||
|
|
);
|
||
|
|
store.drop_job_state(&[job.as_str()]).await.unwrap();
|
||
|
|
let address = proxy.partition().await;
|
||
|
|
let interrupted = tokio::time::timeout(
|
||
|
|
Duration::from_millis(250),
|
||
|
|
service.reconcile_once(&recovery_config()),
|
||
|
|
)
|
||
|
|
.await;
|
||
|
|
assert!(interrupted.is_err() || interrupted.unwrap().is_err());
|
||
|
|
assert_eq!(
|
||
|
|
service
|
||
|
|
.mutation_projection(&job)
|
||
|
|
.await
|
||
|
|
.unwrap()
|
||
|
|
.unwrap()
|
||
|
|
.phase,
|
||
|
|
remora_domain::JobPhase::Queued
|
||
|
|
);
|
||
|
|
|
||
|
|
let resumed_proxy = CoordinationProxy::start(upstream, Some(address)).await;
|
||
|
|
let resumed_store = CoordinationStore::connect(
|
||
|
|
&resumed_proxy.url(),
|
||
|
|
format!("{}wp17:partition:{}:", coordination_namespace(), nonce()),
|
||
|
|
StoreConfig::default(),
|
||
|
|
)
|
||
|
|
.await
|
||
|
|
.unwrap();
|
||
|
|
let restarted = AttemptService::new(
|
||
|
|
harness.pool.clone(),
|
||
|
|
resumed_store.clone(),
|
||
|
|
AttemptConfig {
|
||
|
|
max_attempts: 3,
|
||
|
|
local_limit: 16,
|
||
|
|
},
|
||
|
|
)
|
||
|
|
.unwrap();
|
||
|
|
assert_eq!(
|
||
|
|
restarted
|
||
|
|
.reconcile_once(&recovery_config())
|
||
|
|
.await
|
||
|
|
.unwrap()
|
||
|
|
.enqueued,
|
||
|
|
1
|
||
|
|
);
|
||
|
|
assert_eq!(
|
||
|
|
resumed_store.dequeue().await.unwrap(),
|
||
|
|
Some(job.to_string())
|
||
|
|
);
|
||
|
|
resumed_proxy.partition().await;
|
||
|
|
}
|
||
|
|
|
||
|
|
#[tokio::test]
|
||
|
|
async fn coordination_wipes_reap_lost_attempts_and_reconstruct_dispatch() {
|
||
|
|
let Some(harness) = Harness::new("wipe").await else {
|
||
|
|
return;
|
||
|
|
};
|
||
|
|
let service = harness.service(3);
|
||
|
|
let config = recovery_config();
|
||
|
|
let queued = harness.job().await;
|
||
|
|
let first = service.reconcile_once(&config).await.unwrap();
|
||
|
|
assert_eq!(first.enqueued, 1);
|
||
|
|
assert!(harness.coordination.dequeue().await.unwrap().is_some());
|
||
|
|
harness
|
||
|
|
.coordination
|
||
|
|
.drop_job_state(&[queued.as_str()])
|
||
|
|
.await
|
||
|
|
.unwrap();
|
||
|
|
let restarted = harness.service(3);
|
||
|
|
let second = restarted.reconcile_once(&config).await.unwrap();
|
||
|
|
assert_eq!(second.enqueued, 1);
|
||
|
|
harness
|
||
|
|
.coordination
|
||
|
|
.drop_job_state(&[queued.as_str()])
|
||
|
|
.await
|
||
|
|
.unwrap();
|
||
|
|
assert_eq!(restarted.reconcile_once(&config).await.unwrap().enqueued, 1);
|
||
|
|
|
||
|
|
let authority = harness.executing(&service, &queued).await;
|
||
|
|
service
|
||
|
|
.report_progress(
|
||
|
|
&authority,
|
||
|
|
&unique_id("progress"),
|
||
|
|
remora_attempt::ProgressRecord {
|
||
|
|
completed_bytes: 64,
|
||
|
|
total_bytes: Some(128),
|
||
|
|
completed_members: 0,
|
||
|
|
total_members: Some(1),
|
||
|
|
},
|
||
|
|
)
|
||
|
|
.await
|
||
|
|
.unwrap();
|
||
|
|
harness
|
||
|
|
.coordination
|
||
|
|
.drop_job_state(&[queued.as_str()])
|
||
|
|
.await
|
||
|
|
.unwrap();
|
||
|
|
assert_eq!(
|
||
|
|
service
|
||
|
|
.reconcile_once(&config)
|
||
|
|
.await
|
||
|
|
.unwrap()
|
||
|
|
.attempts_reaped,
|
||
|
|
1
|
||
|
|
);
|
||
|
|
let scheduled = service.reconcile_once(&config).await.unwrap();
|
||
|
|
assert_eq!(scheduled.retries_scheduled, 1);
|
||
|
|
assert_eq!(scheduled.enqueued, 1);
|
||
|
|
assert!(
|
||
|
|
harness
|
||
|
|
.coordination
|
||
|
|
.get_progress(queued.as_str())
|
||
|
|
.await
|
||
|
|
.unwrap()
|
||
|
|
.is_none()
|
||
|
|
);
|
||
|
|
|
||
|
|
let worker = unique_id("worker");
|
||
|
|
let claim = service
|
||
|
|
.claim(&queued, &worker, &unique_id("claimop"))
|
||
|
|
.await
|
||
|
|
.unwrap();
|
||
|
|
let remora_attempt::ClaimOutcome::Claimed { authority, .. } = claim else {
|
||
|
|
panic!("retry claim rejected")
|
||
|
|
};
|
||
|
|
harness
|
||
|
|
.coordination
|
||
|
|
.free_reserve(queued.as_str(), Some(worker.as_str()))
|
||
|
|
.await
|
||
|
|
.unwrap();
|
||
|
|
assert_eq!(
|
||
|
|
service
|
||
|
|
.reconcile_once(&config)
|
||
|
|
.await
|
||
|
|
.unwrap()
|
||
|
|
.attempts_reaped,
|
||
|
|
1
|
||
|
|
);
|
||
|
|
assert_eq!(
|
||
|
|
sqlx::query_scalar::<_, String>("SELECT state FROM attempts WHERE attempt_id = $1")
|
||
|
|
.bind(authority.attempt_id.as_str())
|
||
|
|
.fetch_one(&harness.pool)
|
||
|
|
.await
|
||
|
|
.unwrap(),
|
||
|
|
"reaped"
|
||
|
|
);
|
||
|
|
}
|
||
|
|
|
||
|
|
#[tokio::test]
|
||
|
|
async fn cancellation_is_terminal_and_cooperative_stop_revokes_execution() {
|
||
|
|
let Some(harness) = Harness::new("cancel").await else {
|
||
|
|
return;
|
||
|
|
};
|
||
|
|
let service = harness.service(3);
|
||
|
|
let config = recovery_config();
|
||
|
|
let queued = harness.job().await;
|
||
|
|
assert_eq!(
|
||
|
|
service.request_cancel(&queued).await.unwrap(),
|
||
|
|
CancelOutcome::Requested
|
||
|
|
);
|
||
|
|
assert_eq!(
|
||
|
|
service.request_cancel(&queued).await.unwrap(),
|
||
|
|
CancelOutcome::AlreadyTerminal
|
||
|
|
);
|
||
|
|
assert_eq!(
|
||
|
|
service
|
||
|
|
.mutation_projection(&queued)
|
||
|
|
.await
|
||
|
|
.unwrap()
|
||
|
|
.unwrap()
|
||
|
|
.phase,
|
||
|
|
remora_domain::JobPhase::Cancelled
|
||
|
|
);
|
||
|
|
assert_eq!(
|
||
|
|
service
|
||
|
|
.reconcile_once(&config)
|
||
|
|
.await
|
||
|
|
.unwrap()
|
||
|
|
.terminal_coordination_cleaned,
|
||
|
|
1
|
||
|
|
);
|
||
|
|
|
||
|
|
let running = harness.job().await;
|
||
|
|
let authority = harness.executing(&service, &running).await;
|
||
|
|
assert_eq!(
|
||
|
|
service.request_cancel(&running).await.unwrap(),
|
||
|
|
CancelOutcome::Requested
|
||
|
|
);
|
||
|
|
let report = service.reconcile_once(&config).await.unwrap();
|
||
|
|
assert_eq!(report.cancellations_published, 1);
|
||
|
|
assert_eq!(report.cancellations_finalized, 1);
|
||
|
|
assert_eq!(report.terminal_coordination_cleaned, 0);
|
||
|
|
assert_eq!(harness.coordination.global_used().await.unwrap(), 1);
|
||
|
|
assert_eq!(
|
||
|
|
service.cooperative_stop(&authority).await.unwrap(),
|
||
|
|
CooperativeStopOutcome::Stopped
|
||
|
|
);
|
||
|
|
assert_eq!(
|
||
|
|
sqlx::query_scalar::<_, String>("SELECT state FROM attempts WHERE attempt_id = $1")
|
||
|
|
.bind(authority.attempt_id.as_str())
|
||
|
|
.fetch_one(&harness.pool)
|
||
|
|
.await
|
||
|
|
.unwrap(),
|
||
|
|
"stopped"
|
||
|
|
);
|
||
|
|
assert_eq!(
|
||
|
|
service
|
||
|
|
.reconcile_once(&config)
|
||
|
|
.await
|
||
|
|
.unwrap()
|
||
|
|
.terminal_coordination_cleaned,
|
||
|
|
1
|
||
|
|
);
|
||
|
|
assert_eq!(harness.coordination.global_used().await.unwrap(), 0);
|
||
|
|
assert_eq!(
|
||
|
|
service
|
||
|
|
.publish_failure(
|
||
|
|
&authority,
|
||
|
|
&unique_id("latefail"),
|
||
|
|
RuntimeFailureClass::Internal,
|
||
|
|
)
|
||
|
|
.await
|
||
|
|
.unwrap(),
|
||
|
|
remora_attempt::AttemptMutationOutcome::Rejected {
|
||
|
|
reason: remora_attempt::MutationRejection::AuthorityLost,
|
||
|
|
disposition: remora_durable::MutationDisposition::First,
|
||
|
|
}
|
||
|
|
);
|
||
|
|
}
|
||
|
|
|
||
|
|
#[tokio::test]
|
||
|
|
async fn worker_loss_reaps_live_authority_without_treating_coordination_as_durable() {
|
||
|
|
let Some(harness) = Harness::new("worker-down").await else {
|
||
|
|
return;
|
||
|
|
};
|
||
|
|
let service = harness.service(1);
|
||
|
|
let job = harness.job().await;
|
||
|
|
let authority = harness.executing(&service, &job).await;
|
||
|
|
assert_eq!(
|
||
|
|
service
|
||
|
|
.mark_worker_down(&authority.worker_incarnation)
|
||
|
|
.await
|
||
|
|
.unwrap(),
|
||
|
|
1
|
||
|
|
);
|
||
|
|
let report = service.reconcile_once(&recovery_config()).await.unwrap();
|
||
|
|
assert_eq!(report.attempts_reaped, 1);
|
||
|
|
assert_eq!(
|
||
|
|
service
|
||
|
|
.mutation_projection(&job)
|
||
|
|
.await
|
||
|
|
.unwrap()
|
||
|
|
.unwrap()
|
||
|
|
.phase,
|
||
|
|
remora_domain::JobPhase::Failed
|
||
|
|
);
|
||
|
|
assert_eq!(
|
||
|
|
service.cooperative_stop(&authority).await.unwrap(),
|
||
|
|
CooperativeStopOutcome::Stopped
|
||
|
|
);
|
||
|
|
}
|
||
|
|
|
||
|
|
#[tokio::test]
|
||
|
|
async fn cleanup_uses_exact_durable_identities_retries_and_excludes_the_winner() {
|
||
|
|
let Some(harness) = Harness::new("cleanup").await else {
|
||
|
|
return;
|
||
|
|
};
|
||
|
|
let service = harness.service(3);
|
||
|
|
|
||
|
|
let loser_job = harness.job().await;
|
||
|
|
let loser = harness.executing(&service, &loser_job).await;
|
||
|
|
let loser_key = attempt_object_key(&loser.attempt_id, &unique_id("object"), None);
|
||
|
|
let loser_object = harness
|
||
|
|
.target
|
||
|
|
.put_create_only(
|
||
|
|
&CreateObjectRequest::parse(loser_key.clone()).unwrap(),
|
||
|
|
b"loser-stage".as_slice(),
|
||
|
|
)
|
||
|
|
.await
|
||
|
|
.unwrap();
|
||
|
|
service
|
||
|
|
.register_staging_object(
|
||
|
|
&loser.attempt_id,
|
||
|
|
&StagingObject {
|
||
|
|
target_store: loser_object.identity.target_store_id.clone(),
|
||
|
|
bucket: loser_object.identity.bucket.clone(),
|
||
|
|
key: loser_object.identity.key.clone(),
|
||
|
|
version_id: loser_object.identity.version_id.clone(),
|
||
|
|
},
|
||
|
|
)
|
||
|
|
.await
|
||
|
|
.unwrap();
|
||
|
|
assert!(
|
||
|
|
service
|
||
|
|
.register_staging_object(
|
||
|
|
&loser.attempt_id,
|
||
|
|
&StagingObject {
|
||
|
|
target_store: "foreign-store".into(),
|
||
|
|
bucket: loser_object.identity.bucket.clone(),
|
||
|
|
key: loser_object.identity.key.clone(),
|
||
|
|
version_id: loser_object.identity.version_id.clone(),
|
||
|
|
},
|
||
|
|
)
|
||
|
|
.await
|
||
|
|
.is_err()
|
||
|
|
);
|
||
|
|
let multipart_key = attempt_object_key(&loser.attempt_id, &unique_id("multipart"), None);
|
||
|
|
let multipart = harness
|
||
|
|
.target
|
||
|
|
.begin_multipart(&CreateObjectRequest::parse(multipart_key.clone()).unwrap())
|
||
|
|
.await
|
||
|
|
.unwrap();
|
||
|
|
let multipart_identity = multipart.identity().clone();
|
||
|
|
service
|
||
|
|
.register_staging_multipart(
|
||
|
|
&loser.attempt_id,
|
||
|
|
&StagingMultipart {
|
||
|
|
target_store: multipart_identity.target_store_id.clone(),
|
||
|
|
bucket: multipart_identity.bucket.clone(),
|
||
|
|
key: multipart_identity.key.clone(),
|
||
|
|
upload_id: multipart_identity.upload_id.clone(),
|
||
|
|
},
|
||
|
|
)
|
||
|
|
.await
|
||
|
|
.unwrap();
|
||
|
|
service
|
||
|
|
.publish_failure(
|
||
|
|
&loser,
|
||
|
|
&unique_id("failure"),
|
||
|
|
RuntimeFailureClass::Authorization,
|
||
|
|
)
|
||
|
|
.await
|
||
|
|
.unwrap();
|
||
|
|
|
||
|
|
let winner_job = harness.job().await;
|
||
|
|
let winner = harness.executing(&service, &winner_job).await;
|
||
|
|
let winner_key = attempt_manifest_key(&winner.attempt_id, &unique_id("manifest"));
|
||
|
|
let winner_object = harness
|
||
|
|
.target
|
||
|
|
.put_create_only(
|
||
|
|
&CreateObjectRequest::parse(winner_key).unwrap(),
|
||
|
|
b"winner-manifest".as_slice(),
|
||
|
|
)
|
||
|
|
.await
|
||
|
|
.unwrap();
|
||
|
|
service
|
||
|
|
.register_staging_object(
|
||
|
|
&winner.attempt_id,
|
||
|
|
&StagingObject {
|
||
|
|
target_store: winner_object.identity.target_store_id.clone(),
|
||
|
|
bucket: winner_object.identity.bucket.clone(),
|
||
|
|
key: winner_object.identity.key.clone(),
|
||
|
|
version_id: winner_object.identity.version_id.clone(),
|
||
|
|
},
|
||
|
|
)
|
||
|
|
.await
|
||
|
|
.unwrap();
|
||
|
|
let publication = harness.seal_result(&winner, winner_object.clone()).await;
|
||
|
|
service
|
||
|
|
.publish_success(&winner, &unique_id("success"), publication)
|
||
|
|
.await
|
||
|
|
.unwrap();
|
||
|
|
|
||
|
|
let config = recovery_config();
|
||
|
|
let first = service.claim_cleanup(&config).await.unwrap().unwrap();
|
||
|
|
let failing_target = TargetStore::connect(
|
||
|
|
TargetStoreConfig::new_exclusive(
|
||
|
|
harness.target_config.target_store_id.clone(),
|
||
|
|
"http://127.0.0.1:9",
|
||
|
|
harness.target_config.bucket.clone(),
|
||
|
|
harness.target_config.region.clone(),
|
||
|
|
harness.target_config.access_key_id.clone(),
|
||
|
|
harness.target_config.secret_access_key.clone(),
|
||
|
|
"",
|
||
|
|
)
|
||
|
|
.unwrap(),
|
||
|
|
)
|
||
|
|
.unwrap();
|
||
|
|
assert!(execute_cleanup(&failing_target, &first).await.is_err());
|
||
|
|
assert_eq!(
|
||
|
|
service
|
||
|
|
.retry_cleanup(&first, "target_unavailable")
|
||
|
|
.await
|
||
|
|
.unwrap(),
|
||
|
|
CleanupCompletion::RetryScheduled
|
||
|
|
);
|
||
|
|
let short_claims = RecoveryConfig {
|
||
|
|
cleanup_claim_ttl: Duration::from_millis(1),
|
||
|
|
..config.clone()
|
||
|
|
};
|
||
|
|
let abandoned = service.claim_cleanup(&short_claims).await.unwrap().unwrap();
|
||
|
|
tokio::time::sleep(Duration::from_millis(10)).await;
|
||
|
|
let restarted = harness.service(3);
|
||
|
|
let reclaimed = restarted.claim_cleanup(&config).await.unwrap().unwrap();
|
||
|
|
assert_eq!(reclaimed.cleanup_id, abandoned.cleanup_id);
|
||
|
|
assert_ne!(reclaimed.claim_token, abandoned.claim_token);
|
||
|
|
assert_eq!(
|
||
|
|
service.complete_cleanup(&abandoned).await.unwrap(),
|
||
|
|
CleanupCompletion::ClaimLost
|
||
|
|
);
|
||
|
|
execute_cleanup(&harness.target, &reclaimed).await.unwrap();
|
||
|
|
assert_eq!(
|
||
|
|
restarted.complete_cleanup(&reclaimed).await.unwrap(),
|
||
|
|
CleanupCompletion::Completed
|
||
|
|
);
|
||
|
|
let mut completed = 1;
|
||
|
|
while let Some(claim) = restarted.claim_cleanup(&config).await.unwrap() {
|
||
|
|
assert_eq!(claim.attempt_id, loser.attempt_id);
|
||
|
|
execute_cleanup(&harness.target, &claim).await.unwrap();
|
||
|
|
assert_eq!(
|
||
|
|
restarted.complete_cleanup(&claim).await.unwrap(),
|
||
|
|
CleanupCompletion::Completed
|
||
|
|
);
|
||
|
|
assert_eq!(
|
||
|
|
restarted.complete_cleanup(&claim).await.unwrap(),
|
||
|
|
CleanupCompletion::ClaimLost
|
||
|
|
);
|
||
|
|
completed += 1;
|
||
|
|
}
|
||
|
|
assert_eq!(completed, 2);
|
||
|
|
assert!(
|
||
|
|
harness
|
||
|
|
.target
|
||
|
|
.head_exact(&loser_object.identity)
|
||
|
|
.await
|
||
|
|
.is_err()
|
||
|
|
);
|
||
|
|
assert!(
|
||
|
|
harness
|
||
|
|
.target
|
||
|
|
.head_exact(&winner_object.identity)
|
||
|
|
.await
|
||
|
|
.is_ok()
|
||
|
|
);
|
||
|
|
assert!(
|
||
|
|
service
|
||
|
|
.register_staging_object(
|
||
|
|
&loser.attempt_id,
|
||
|
|
&StagingObject {
|
||
|
|
target_store: harness.target_config.target_store_id.clone(),
|
||
|
|
bucket: harness.target_config.bucket.clone(),
|
||
|
|
key: attempt_object_key(&winner.attempt_id, &unique_id("wrong"), None),
|
||
|
|
version_id: None,
|
||
|
|
},
|
||
|
|
)
|
||
|
|
.await
|
||
|
|
.is_err()
|
||
|
|
);
|
||
|
|
}
|
||
|
|
|
||
|
|
async fn execute_cleanup(
|
||
|
|
target: &TargetStore,
|
||
|
|
claim: &remora_attempt::CleanupClaim,
|
||
|
|
) -> Result<(), remora_target::TargetError> {
|
||
|
|
match &claim.identity {
|
||
|
|
CleanupIdentity::Object(object) => {
|
||
|
|
target
|
||
|
|
.delete_exact(&ObjectIdentity {
|
||
|
|
target_store_id: object.target_store.clone(),
|
||
|
|
bucket: object.bucket.clone(),
|
||
|
|
key: object.key.clone(),
|
||
|
|
version_id: object.version_id.clone(),
|
||
|
|
})
|
||
|
|
.await
|
||
|
|
}
|
||
|
|
CleanupIdentity::Multipart(upload) => {
|
||
|
|
let (session, _) = target
|
||
|
|
.resume_multipart(
|
||
|
|
MultipartIdentity {
|
||
|
|
target_store_id: upload.target_store.clone(),
|
||
|
|
bucket: upload.bucket.clone(),
|
||
|
|
key: upload.key.clone(),
|
||
|
|
upload_id: upload.upload_id.clone(),
|
||
|
|
},
|
||
|
|
None,
|
||
|
|
)
|
||
|
|
.await?;
|
||
|
|
target.abort_multipart(session).await
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
fn recovery_config() -> RecoveryConfig {
|
||
|
|
RecoveryConfig {
|
||
|
|
retry_delay: Duration::ZERO,
|
||
|
|
cleanup_claim_ttl: Duration::from_secs(10),
|
||
|
|
batch_size: 128,
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
fn required() -> bool {
|
||
|
|
std::env::var("REMORA_RECOVERY_REQUIRE").is_ok()
|
||
|
|
}
|
||
|
|
|
||
|
|
fn required_env(name: &str) -> Option<String> {
|
||
|
|
match std::env::var(name) {
|
||
|
|
Ok(value) if !value.is_empty() => Some(value),
|
||
|
|
Ok(_) | Err(_) if required() => panic!("{name} is required"),
|
||
|
|
Ok(_) | Err(_) => None,
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
fn unique_id(prefix: &str) -> OpaqueId {
|
||
|
|
let nanos = SystemTime::now()
|
||
|
|
.duration_since(UNIX_EPOCH)
|
||
|
|
.expect("time")
|
||
|
|
.as_nanos();
|
||
|
|
OpaqueId::parse(format!("{prefix}{nanos:x}")).unwrap()
|
||
|
|
}
|
||
|
|
|
||
|
|
fn nonce() -> u128 {
|
||
|
|
SystemTime::now()
|
||
|
|
.duration_since(UNIX_EPOCH)
|
||
|
|
.expect("time")
|
||
|
|
.as_nanos()
|
||
|
|
}
|
||
|
|
|
||
|
|
fn coordination_namespace() -> String {
|
||
|
|
std::env::var("REMORA_COORDINATION_NAMESPACE").unwrap_or_else(|_| "remora:test:".into())
|
||
|
|
}
|
||
|
|
|
||
|
|
fn coordination_upstream() -> String {
|
||
|
|
std::env::var("REMORA_COORDINATION_URL")
|
||
|
|
.unwrap()
|
||
|
|
.trim_start_matches("redis://")
|
||
|
|
.split('/')
|
||
|
|
.next()
|
||
|
|
.unwrap()
|
||
|
|
.to_owned()
|
||
|
|
}
|