Implement crash-safe streaming worker runtime
This commit is contained in:
parent
16cba3c317
commit
cdeef3cc1a
1 changed files with 7448 additions and 194 deletions
|
|
@ -8,7 +8,7 @@ use remora_admission::{
|
|||
use remora_attempt::{
|
||||
ActivationOutcome, AttemptConfig, AttemptService, CancelOutcome, CleanupCompletion,
|
||||
CleanupIdentity, CooperativeStopOutcome, RecoveryConfig, StagingMultipart, StagingObject,
|
||||
migrate as attempt_migrate,
|
||||
StagingWriteIntent, migrate as attempt_migrate,
|
||||
};
|
||||
use remora_contracts::{HttpSourceSubmissionDto, JobSubmissionDto, SourceSubmissionDto};
|
||||
use remora_coordination::{CoordinationStore, StoreConfig};
|
||||
|
|
@ -21,7 +21,7 @@ use remora_identity::{DefaultTarget, Principal};
|
|||
use remora_result::{PublicationInput, migrate as result_migrate};
|
||||
use remora_target::{
|
||||
CreateObjectRequest, MultipartIdentity, ObjectIdentity, StoredObject, TargetStore,
|
||||
TargetStoreConfig, attempt_manifest_key, attempt_object_key,
|
||||
TargetStoreConfig, attempt_manifest_key_in, attempt_object_key_in,
|
||||
};
|
||||
use sqlx::PgPool;
|
||||
use tokio::io::copy_bidirectional;
|
||||
|
|
@ -264,8 +264,12 @@ impl Harness {
|
|||
"INSERT INTO result_inventories (
|
||||
attempt_id, state, manifest_schema_version, manifest_id, manifest_store,
|
||||
manifest_bucket, manifest_key, manifest_version_id, manifest_sha256,
|
||||
manifest_size, member_count, total_bytes
|
||||
) VALUES ($1, 'sealed', 1, $2, $3, $4, $5, $6, $7, $8, 0, 0)",
|
||||
manifest_size, member_count, total_bytes,
|
||||
staging_origin_attempt_id, target_prefix
|
||||
) VALUES (
|
||||
$1, 'sealed', 1, $2, $3, $4, $5, $6, $7, $8, 0, 0,
|
||||
$1, 'users/recovery'
|
||||
)",
|
||||
)
|
||||
.bind(authority.attempt_id.as_str())
|
||||
.bind(manifest_id.as_str())
|
||||
|
|
@ -765,7 +769,12 @@ async fn cleanup_uses_exact_durable_identities_retries_and_excludes_the_winner()
|
|||
|
||||
let loser_job = harness.job().await;
|
||||
let loser = harness.executing(&service, &loser_job).await;
|
||||
let loser_key = attempt_object_key(&loser.attempt_id, &unique_id("object"), None);
|
||||
let loser_key = attempt_object_key_in(
|
||||
"users/recovery",
|
||||
&loser.attempt_id,
|
||||
&unique_id("object"),
|
||||
None,
|
||||
);
|
||||
let loser_object = harness
|
||||
.target
|
||||
.put_create_only(
|
||||
|
|
@ -800,7 +809,12 @@ async fn cleanup_uses_exact_durable_identities_retries_and_excludes_the_winner()
|
|||
.await
|
||||
.is_err()
|
||||
);
|
||||
let multipart_key = attempt_object_key(&loser.attempt_id, &unique_id("multipart"), None);
|
||||
let multipart_key = attempt_object_key_in(
|
||||
"users/recovery",
|
||||
&loser.attempt_id,
|
||||
&unique_id("multipart"),
|
||||
None,
|
||||
);
|
||||
let multipart = harness
|
||||
.target
|
||||
.begin_multipart(&CreateObjectRequest::parse(multipart_key.clone()).unwrap())
|
||||
|
|
@ -830,7 +844,8 @@ async fn cleanup_uses_exact_durable_identities_retries_and_excludes_the_winner()
|
|||
|
||||
let winner_job = harness.job().await;
|
||||
let winner = harness.executing(&service, &winner_job).await;
|
||||
let winner_key = attempt_manifest_key(&winner.attempt_id, &unique_id("manifest"));
|
||||
let winner_key =
|
||||
attempt_manifest_key_in("users/recovery", &winner.attempt_id, &unique_id("manifest"));
|
||||
let winner_object = harness
|
||||
.target
|
||||
.put_create_only(
|
||||
|
|
@ -935,7 +950,12 @@ async fn cleanup_uses_exact_durable_identities_retries_and_excludes_the_winner()
|
|||
&StagingObject {
|
||||
target_store: harness.target_config.target_store_id.clone(),
|
||||
bucket: harness.target_config.bucket.clone(),
|
||||
key: attempt_object_key(&winner.attempt_id, &unique_id("wrong"), None),
|
||||
key: attempt_object_key_in(
|
||||
"users/recovery",
|
||||
&winner.attempt_id,
|
||||
&unique_id("wrong"),
|
||||
None,
|
||||
),
|
||||
version_id: None,
|
||||
},
|
||||
)
|
||||
|
|
@ -944,11 +964,201 @@ async fn cleanup_uses_exact_durable_identities_retries_and_excludes_the_winner()
|
|||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cleanup_reconciles_accepted_writes_from_durable_key_intents() {
|
||||
let Some(harness) = Harness::new("write-intent").await else {
|
||||
return;
|
||||
};
|
||||
let service = harness.service(1);
|
||||
let config = recovery_config();
|
||||
|
||||
let multipart_job = harness.job().await;
|
||||
let multipart_attempt = harness.executing(&service, &multipart_job).await;
|
||||
let multipart_key = attempt_object_key_in(
|
||||
"users/recovery",
|
||||
&multipart_attempt.attempt_id,
|
||||
&unique_id("multipart"),
|
||||
None,
|
||||
);
|
||||
service
|
||||
.register_staging_write_intent_operation(
|
||||
&multipart_attempt,
|
||||
&unique_id("intent"),
|
||||
StagingWriteIntent {
|
||||
target_store: harness.target_config.target_store_id.clone(),
|
||||
bucket: harness.target_config.bucket.clone(),
|
||||
key: multipart_key.clone(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let accepted_multipart = harness
|
||||
.target
|
||||
.begin_multipart(&CreateObjectRequest::parse(multipart_key.clone()).unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
let accepted_upload_id = accepted_multipart.identity().upload_id.clone();
|
||||
service
|
||||
.publish_failure(
|
||||
&multipart_attempt,
|
||||
&unique_id("failure"),
|
||||
RuntimeFailureClass::Authorization,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let intent_claim = service.claim_cleanup(&config).await.unwrap().unwrap();
|
||||
assert!(matches!(
|
||||
intent_claim.identity,
|
||||
CleanupIdentity::WriteIntent(_)
|
||||
));
|
||||
let discovered = harness
|
||||
.target
|
||||
.discover_staging_write(&multipart_key)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(discovered.objects.is_empty());
|
||||
assert_eq!(discovered.multiparts.len(), 1);
|
||||
assert_eq!(discovered.multiparts[0].upload_id, accepted_upload_id);
|
||||
assert_eq!(
|
||||
service
|
||||
.reconcile_write_intent(
|
||||
&intent_claim,
|
||||
Vec::new(),
|
||||
discovered
|
||||
.multiparts
|
||||
.into_iter()
|
||||
.map(|identity| StagingMultipart {
|
||||
target_store: identity.target_store_id,
|
||||
bucket: identity.bucket,
|
||||
key: identity.key,
|
||||
upload_id: identity.upload_id,
|
||||
})
|
||||
.collect(),
|
||||
)
|
||||
.await
|
||||
.unwrap(),
|
||||
CleanupCompletion::Completed
|
||||
);
|
||||
let multipart_claim = service.claim_cleanup(&config).await.unwrap().unwrap();
|
||||
assert!(matches!(
|
||||
multipart_claim.identity,
|
||||
CleanupIdentity::Multipart(_)
|
||||
));
|
||||
execute_cleanup(&harness.target, &multipart_claim)
|
||||
.await
|
||||
.unwrap();
|
||||
service.complete_cleanup(&multipart_claim).await.unwrap();
|
||||
assert!(
|
||||
harness
|
||||
.target
|
||||
.discover_staging_write(&multipart_key)
|
||||
.await
|
||||
.unwrap()
|
||||
.multiparts
|
||||
.is_empty()
|
||||
);
|
||||
|
||||
let object_job = harness.job().await;
|
||||
let object_attempt = harness.executing(&service, &object_job).await;
|
||||
let object_key = attempt_object_key_in(
|
||||
"users/recovery",
|
||||
&object_attempt.attempt_id,
|
||||
&unique_id("object"),
|
||||
None,
|
||||
);
|
||||
service
|
||||
.register_staging_write_intent_operation(
|
||||
&object_attempt,
|
||||
&unique_id("intent"),
|
||||
StagingWriteIntent {
|
||||
target_store: harness.target_config.target_store_id.clone(),
|
||||
bucket: harness.target_config.bucket.clone(),
|
||||
key: object_key.clone(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let accepted_object = harness
|
||||
.target
|
||||
.put_create_only(
|
||||
&CreateObjectRequest::parse(object_key.clone()).unwrap(),
|
||||
b"accepted-before-ledger".as_slice(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
service
|
||||
.publish_failure(
|
||||
&object_attempt,
|
||||
&unique_id("failure"),
|
||||
RuntimeFailureClass::Authorization,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let intent_claim = service.claim_cleanup(&config).await.unwrap().unwrap();
|
||||
let discovered = harness
|
||||
.target
|
||||
.discover_staging_write(&object_key)
|
||||
.await
|
||||
.unwrap();
|
||||
let [discovered_object] = discovered.objects.as_slice() else {
|
||||
panic!("expected one accepted object");
|
||||
};
|
||||
assert_eq!(
|
||||
discovered_object.target_store_id,
|
||||
accepted_object.identity.target_store_id
|
||||
);
|
||||
assert_eq!(discovered_object.bucket, accepted_object.identity.bucket);
|
||||
assert_eq!(discovered_object.key, accepted_object.identity.key);
|
||||
if let (Some(discovered_version), Some(accepted_version)) = (
|
||||
discovered_object.version_id.as_deref(),
|
||||
accepted_object.identity.version_id.as_deref(),
|
||||
) {
|
||||
assert_eq!(discovered_version, accepted_version);
|
||||
}
|
||||
assert!(discovered.multiparts.is_empty());
|
||||
assert_eq!(
|
||||
service
|
||||
.reconcile_write_intent(
|
||||
&intent_claim,
|
||||
discovered
|
||||
.objects
|
||||
.into_iter()
|
||||
.map(|identity| StagingObject {
|
||||
target_store: identity.target_store_id,
|
||||
bucket: identity.bucket,
|
||||
key: identity.key,
|
||||
version_id: identity.version_id,
|
||||
})
|
||||
.collect(),
|
||||
Vec::new(),
|
||||
)
|
||||
.await
|
||||
.unwrap(),
|
||||
CleanupCompletion::Completed
|
||||
);
|
||||
let object_claim = service.claim_cleanup(&config).await.unwrap().unwrap();
|
||||
assert!(matches!(object_claim.identity, CleanupIdentity::Object(_)));
|
||||
execute_cleanup(&harness.target, &object_claim)
|
||||
.await
|
||||
.unwrap();
|
||||
service.complete_cleanup(&object_claim).await.unwrap();
|
||||
assert!(
|
||||
harness
|
||||
.target
|
||||
.head_exact(&accepted_object.identity)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
async fn execute_cleanup(
|
||||
target: &TargetStore,
|
||||
claim: &remora_attempt::CleanupClaim,
|
||||
) -> Result<(), remora_target::TargetError> {
|
||||
match &claim.identity {
|
||||
CleanupIdentity::WriteIntent(_) => Err(remora_target::TargetError::internal(
|
||||
"write intents must be reconciled before exact cleanup",
|
||||
)),
|
||||
CleanupIdentity::Object(object) => {
|
||||
target
|
||||
.delete_exact(&ObjectIdentity {
|
||||
|
|
|
|||
Loading…
Reference in a new issue