Implement crash-safe streaming worker runtime
This commit is contained in:
parent
16cba3c317
commit
cdeef3cc1a
2 changed files with 7448 additions and 194 deletions
|
|
@ -18,7 +18,7 @@ use remora_domain::{
|
|||
use remora_durable::{MutationDisposition, PoolConfig, connect, migrate as durable_migrate};
|
||||
use remora_identity::{DefaultTarget, Principal};
|
||||
use remora_result::{PublicationInput, migrate as result_migrate};
|
||||
use remora_target::{ContentSha256, ObjectIdentity, StoredObject, attempt_manifest_key};
|
||||
use remora_target::{ContentSha256, ObjectIdentity, StoredObject, attempt_manifest_key_in};
|
||||
use sqlx::PgPool;
|
||||
|
||||
struct Harness {
|
||||
|
|
@ -168,7 +168,7 @@ impl Harness {
|
|||
) -> PublicationInput {
|
||||
let manifest_id = unique_id("manifest");
|
||||
let sha256 = format!("sha256:{}", "11".repeat(32));
|
||||
let key = attempt_manifest_key(attempt_id, &manifest_id);
|
||||
let key = attempt_manifest_key_in("users/alice", attempt_id, &manifest_id);
|
||||
let input = PublicationInput {
|
||||
attempt_id: attempt_id.clone(),
|
||||
manifest_id: manifest_id.clone(),
|
||||
|
|
@ -190,9 +190,10 @@ impl Harness {
|
|||
"INSERT INTO result_inventories (
|
||||
attempt_id, state, manifest_schema_version, manifest_id, manifest_store,
|
||||
manifest_bucket, manifest_key, manifest_version_id, manifest_sha256,
|
||||
manifest_size, member_count, total_bytes
|
||||
manifest_size, member_count, total_bytes,
|
||||
staging_origin_attempt_id, target_prefix
|
||||
) VALUES ($1, 'sealed', 1, $2, 'primary', 'remora-results', $3,
|
||||
'version-1', $4, 128, $5, $6)",
|
||||
'version-1', $4, 128, $5, $6, $1, 'users/alice')",
|
||||
)
|
||||
.bind(attempt_id.as_str())
|
||||
.bind(manifest_id.as_str())
|
||||
|
|
@ -283,6 +284,7 @@ async fn resolved_source_checkpoint_and_progress_are_fenced_and_replayable() {
|
|||
resolved_source_id: resolved.id.clone(),
|
||||
validated_bytes: 0,
|
||||
validated_members: 0,
|
||||
staging_multipart: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
|
|
@ -295,6 +297,7 @@ async fn resolved_source_checkpoint_and_progress_are_fenced_and_replayable() {
|
|||
resolved_source_id: resolved.id.clone(),
|
||||
validated_bytes: 1024,
|
||||
validated_members: 1,
|
||||
staging_multipart: None,
|
||||
};
|
||||
let mut wrong_source = checkpoint.clone();
|
||||
wrong_source.resolved_source_id = unique_id("resolved");
|
||||
|
|
@ -539,6 +542,7 @@ async fn failure_disposition_retries_with_budget_and_becomes_terminal_when_exhau
|
|||
resolved_source_id: retry_source.id.clone(),
|
||||
validated_bytes: 512,
|
||||
validated_members: 1,
|
||||
staging_multipart: None,
|
||||
};
|
||||
assert_rejected(
|
||||
service
|
||||
|
|
@ -555,6 +559,40 @@ async fn failure_disposition_retries_with_budget_and_becomes_terminal_when_exhau
|
|||
.persist_checkpoint(&first, &unique_id("checkpointop"), retry_checkpoint)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO result_inventories (
|
||||
attempt_id, staging_origin_attempt_id, target_prefix
|
||||
) VALUES ($1, $1, 'users/alice')",
|
||||
)
|
||||
.bind(first.attempt_id.as_str())
|
||||
.execute(&harness.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO result_inventory_members (
|
||||
attempt_id, member_id, path, size, object_store, object_bucket,
|
||||
object_key, object_sha256
|
||||
) VALUES ($1, $2, 'retry.bin', 512, 'target', 'bucket', $3, $4)",
|
||||
)
|
||||
.bind(first.attempt_id.as_str())
|
||||
.bind(unique_id("member").as_str())
|
||||
.bind(format!(
|
||||
"users/alice/_remora/a/{}/o/retry/retry.bin",
|
||||
first.attempt_id.as_str()
|
||||
))
|
||||
.bind(format!("sha256:{}", "0".repeat(64)))
|
||||
.execute(&harness.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"UPDATE result_inventories
|
||||
SET member_count = 1, total_bytes = 512
|
||||
WHERE attempt_id = $1",
|
||||
)
|
||||
.bind(first.attempt_id.as_str())
|
||||
.execute(&harness.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
service
|
||||
.publish_failure(
|
||||
|
|
@ -610,6 +648,20 @@ async fn failure_disposition_retries_with_budget_and_becomes_terminal_when_exhau
|
|||
.await
|
||||
.unwrap();
|
||||
assert_eq!(initial_progress, 512);
|
||||
let staging_origin: String =
|
||||
sqlx::query_scalar("SELECT staging_origin_attempt_id FROM attempts WHERE attempt_id = $1")
|
||||
.bind(second.attempt_id.as_str())
|
||||
.fetch_one(&harness.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(staging_origin, first.attempt_id.as_str());
|
||||
let inventory_owner: String =
|
||||
sqlx::query_scalar("SELECT attempt_id FROM result_inventories WHERE attempt_id = $1")
|
||||
.bind(second.attempt_id.as_str())
|
||||
.fetch_one(&harness.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(inventory_owner, second.attempt_id.as_str());
|
||||
service
|
||||
.publish_failure(
|
||||
&second,
|
||||
|
|
@ -760,7 +812,7 @@ async fn publication_from_database(pool: &PgPool, attempt_id: &OpaqueId) -> Publ
|
|||
identity: ObjectIdentity {
|
||||
target_store_id: row.1,
|
||||
bucket: row.2,
|
||||
key: attempt_manifest_key(attempt_id, &manifest_id),
|
||||
key: attempt_manifest_key_in("users/alice", attempt_id, &manifest_id),
|
||||
version_id: row.3,
|
||||
},
|
||||
content_sha256: ContentSha256::parse(row.4).unwrap(),
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ use remora_admission::{
|
|||
use remora_attempt::{
|
||||
ActivationOutcome, AttemptConfig, AttemptService, CancelOutcome, CleanupCompletion,
|
||||
CleanupIdentity, CooperativeStopOutcome, RecoveryConfig, StagingMultipart, StagingObject,
|
||||
migrate as attempt_migrate,
|
||||
StagingWriteIntent, migrate as attempt_migrate,
|
||||
};
|
||||
use remora_contracts::{HttpSourceSubmissionDto, JobSubmissionDto, SourceSubmissionDto};
|
||||
use remora_coordination::{CoordinationStore, StoreConfig};
|
||||
|
|
@ -21,7 +21,7 @@ use remora_identity::{DefaultTarget, Principal};
|
|||
use remora_result::{PublicationInput, migrate as result_migrate};
|
||||
use remora_target::{
|
||||
CreateObjectRequest, MultipartIdentity, ObjectIdentity, StoredObject, TargetStore,
|
||||
TargetStoreConfig, attempt_manifest_key, attempt_object_key,
|
||||
TargetStoreConfig, attempt_manifest_key_in, attempt_object_key_in,
|
||||
};
|
||||
use sqlx::PgPool;
|
||||
use tokio::io::copy_bidirectional;
|
||||
|
|
@ -264,8 +264,12 @@ impl Harness {
|
|||
"INSERT INTO result_inventories (
|
||||
attempt_id, state, manifest_schema_version, manifest_id, manifest_store,
|
||||
manifest_bucket, manifest_key, manifest_version_id, manifest_sha256,
|
||||
manifest_size, member_count, total_bytes
|
||||
) VALUES ($1, 'sealed', 1, $2, $3, $4, $5, $6, $7, $8, 0, 0)",
|
||||
manifest_size, member_count, total_bytes,
|
||||
staging_origin_attempt_id, target_prefix
|
||||
) VALUES (
|
||||
$1, 'sealed', 1, $2, $3, $4, $5, $6, $7, $8, 0, 0,
|
||||
$1, 'users/recovery'
|
||||
)",
|
||||
)
|
||||
.bind(authority.attempt_id.as_str())
|
||||
.bind(manifest_id.as_str())
|
||||
|
|
@ -765,7 +769,12 @@ async fn cleanup_uses_exact_durable_identities_retries_and_excludes_the_winner()
|
|||
|
||||
let loser_job = harness.job().await;
|
||||
let loser = harness.executing(&service, &loser_job).await;
|
||||
let loser_key = attempt_object_key(&loser.attempt_id, &unique_id("object"), None);
|
||||
let loser_key = attempt_object_key_in(
|
||||
"users/recovery",
|
||||
&loser.attempt_id,
|
||||
&unique_id("object"),
|
||||
None,
|
||||
);
|
||||
let loser_object = harness
|
||||
.target
|
||||
.put_create_only(
|
||||
|
|
@ -800,7 +809,12 @@ async fn cleanup_uses_exact_durable_identities_retries_and_excludes_the_winner()
|
|||
.await
|
||||
.is_err()
|
||||
);
|
||||
let multipart_key = attempt_object_key(&loser.attempt_id, &unique_id("multipart"), None);
|
||||
let multipart_key = attempt_object_key_in(
|
||||
"users/recovery",
|
||||
&loser.attempt_id,
|
||||
&unique_id("multipart"),
|
||||
None,
|
||||
);
|
||||
let multipart = harness
|
||||
.target
|
||||
.begin_multipart(&CreateObjectRequest::parse(multipart_key.clone()).unwrap())
|
||||
|
|
@ -830,7 +844,8 @@ async fn cleanup_uses_exact_durable_identities_retries_and_excludes_the_winner()
|
|||
|
||||
let winner_job = harness.job().await;
|
||||
let winner = harness.executing(&service, &winner_job).await;
|
||||
let winner_key = attempt_manifest_key(&winner.attempt_id, &unique_id("manifest"));
|
||||
let winner_key =
|
||||
attempt_manifest_key_in("users/recovery", &winner.attempt_id, &unique_id("manifest"));
|
||||
let winner_object = harness
|
||||
.target
|
||||
.put_create_only(
|
||||
|
|
@ -935,7 +950,12 @@ async fn cleanup_uses_exact_durable_identities_retries_and_excludes_the_winner()
|
|||
&StagingObject {
|
||||
target_store: harness.target_config.target_store_id.clone(),
|
||||
bucket: harness.target_config.bucket.clone(),
|
||||
key: attempt_object_key(&winner.attempt_id, &unique_id("wrong"), None),
|
||||
key: attempt_object_key_in(
|
||||
"users/recovery",
|
||||
&winner.attempt_id,
|
||||
&unique_id("wrong"),
|
||||
None,
|
||||
),
|
||||
version_id: None,
|
||||
},
|
||||
)
|
||||
|
|
@ -944,11 +964,201 @@ async fn cleanup_uses_exact_durable_identities_retries_and_excludes_the_winner()
|
|||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cleanup_reconciles_accepted_writes_from_durable_key_intents() {
|
||||
let Some(harness) = Harness::new("write-intent").await else {
|
||||
return;
|
||||
};
|
||||
let service = harness.service(1);
|
||||
let config = recovery_config();
|
||||
|
||||
let multipart_job = harness.job().await;
|
||||
let multipart_attempt = harness.executing(&service, &multipart_job).await;
|
||||
let multipart_key = attempt_object_key_in(
|
||||
"users/recovery",
|
||||
&multipart_attempt.attempt_id,
|
||||
&unique_id("multipart"),
|
||||
None,
|
||||
);
|
||||
service
|
||||
.register_staging_write_intent_operation(
|
||||
&multipart_attempt,
|
||||
&unique_id("intent"),
|
||||
StagingWriteIntent {
|
||||
target_store: harness.target_config.target_store_id.clone(),
|
||||
bucket: harness.target_config.bucket.clone(),
|
||||
key: multipart_key.clone(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let accepted_multipart = harness
|
||||
.target
|
||||
.begin_multipart(&CreateObjectRequest::parse(multipart_key.clone()).unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
let accepted_upload_id = accepted_multipart.identity().upload_id.clone();
|
||||
service
|
||||
.publish_failure(
|
||||
&multipart_attempt,
|
||||
&unique_id("failure"),
|
||||
RuntimeFailureClass::Authorization,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let intent_claim = service.claim_cleanup(&config).await.unwrap().unwrap();
|
||||
assert!(matches!(
|
||||
intent_claim.identity,
|
||||
CleanupIdentity::WriteIntent(_)
|
||||
));
|
||||
let discovered = harness
|
||||
.target
|
||||
.discover_staging_write(&multipart_key)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(discovered.objects.is_empty());
|
||||
assert_eq!(discovered.multiparts.len(), 1);
|
||||
assert_eq!(discovered.multiparts[0].upload_id, accepted_upload_id);
|
||||
assert_eq!(
|
||||
service
|
||||
.reconcile_write_intent(
|
||||
&intent_claim,
|
||||
Vec::new(),
|
||||
discovered
|
||||
.multiparts
|
||||
.into_iter()
|
||||
.map(|identity| StagingMultipart {
|
||||
target_store: identity.target_store_id,
|
||||
bucket: identity.bucket,
|
||||
key: identity.key,
|
||||
upload_id: identity.upload_id,
|
||||
})
|
||||
.collect(),
|
||||
)
|
||||
.await
|
||||
.unwrap(),
|
||||
CleanupCompletion::Completed
|
||||
);
|
||||
let multipart_claim = service.claim_cleanup(&config).await.unwrap().unwrap();
|
||||
assert!(matches!(
|
||||
multipart_claim.identity,
|
||||
CleanupIdentity::Multipart(_)
|
||||
));
|
||||
execute_cleanup(&harness.target, &multipart_claim)
|
||||
.await
|
||||
.unwrap();
|
||||
service.complete_cleanup(&multipart_claim).await.unwrap();
|
||||
assert!(
|
||||
harness
|
||||
.target
|
||||
.discover_staging_write(&multipart_key)
|
||||
.await
|
||||
.unwrap()
|
||||
.multiparts
|
||||
.is_empty()
|
||||
);
|
||||
|
||||
let object_job = harness.job().await;
|
||||
let object_attempt = harness.executing(&service, &object_job).await;
|
||||
let object_key = attempt_object_key_in(
|
||||
"users/recovery",
|
||||
&object_attempt.attempt_id,
|
||||
&unique_id("object"),
|
||||
None,
|
||||
);
|
||||
service
|
||||
.register_staging_write_intent_operation(
|
||||
&object_attempt,
|
||||
&unique_id("intent"),
|
||||
StagingWriteIntent {
|
||||
target_store: harness.target_config.target_store_id.clone(),
|
||||
bucket: harness.target_config.bucket.clone(),
|
||||
key: object_key.clone(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let accepted_object = harness
|
||||
.target
|
||||
.put_create_only(
|
||||
&CreateObjectRequest::parse(object_key.clone()).unwrap(),
|
||||
b"accepted-before-ledger".as_slice(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
service
|
||||
.publish_failure(
|
||||
&object_attempt,
|
||||
&unique_id("failure"),
|
||||
RuntimeFailureClass::Authorization,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let intent_claim = service.claim_cleanup(&config).await.unwrap().unwrap();
|
||||
let discovered = harness
|
||||
.target
|
||||
.discover_staging_write(&object_key)
|
||||
.await
|
||||
.unwrap();
|
||||
let [discovered_object] = discovered.objects.as_slice() else {
|
||||
panic!("expected one accepted object");
|
||||
};
|
||||
assert_eq!(
|
||||
discovered_object.target_store_id,
|
||||
accepted_object.identity.target_store_id
|
||||
);
|
||||
assert_eq!(discovered_object.bucket, accepted_object.identity.bucket);
|
||||
assert_eq!(discovered_object.key, accepted_object.identity.key);
|
||||
if let (Some(discovered_version), Some(accepted_version)) = (
|
||||
discovered_object.version_id.as_deref(),
|
||||
accepted_object.identity.version_id.as_deref(),
|
||||
) {
|
||||
assert_eq!(discovered_version, accepted_version);
|
||||
}
|
||||
assert!(discovered.multiparts.is_empty());
|
||||
assert_eq!(
|
||||
service
|
||||
.reconcile_write_intent(
|
||||
&intent_claim,
|
||||
discovered
|
||||
.objects
|
||||
.into_iter()
|
||||
.map(|identity| StagingObject {
|
||||
target_store: identity.target_store_id,
|
||||
bucket: identity.bucket,
|
||||
key: identity.key,
|
||||
version_id: identity.version_id,
|
||||
})
|
||||
.collect(),
|
||||
Vec::new(),
|
||||
)
|
||||
.await
|
||||
.unwrap(),
|
||||
CleanupCompletion::Completed
|
||||
);
|
||||
let object_claim = service.claim_cleanup(&config).await.unwrap().unwrap();
|
||||
assert!(matches!(object_claim.identity, CleanupIdentity::Object(_)));
|
||||
execute_cleanup(&harness.target, &object_claim)
|
||||
.await
|
||||
.unwrap();
|
||||
service.complete_cleanup(&object_claim).await.unwrap();
|
||||
assert!(
|
||||
harness
|
||||
.target
|
||||
.head_exact(&accepted_object.identity)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
async fn execute_cleanup(
|
||||
target: &TargetStore,
|
||||
claim: &remora_attempt::CleanupClaim,
|
||||
) -> Result<(), remora_target::TargetError> {
|
||||
match &claim.identity {
|
||||
CleanupIdentity::WriteIntent(_) => Err(remora_target::TargetError::internal(
|
||||
"write intents must be reconciled before exact cleanup",
|
||||
)),
|
||||
CleanupIdentity::Object(object) => {
|
||||
target
|
||||
.delete_exact(&ObjectIdentity {
|
||||
|
|
|
|||
Loading…
Reference in a new issue