Implement crash-safe streaming worker runtime
This commit is contained in:
parent
16cba3c317
commit
cdeef3cc1a
1 changed files with 7448 additions and 194 deletions
|
|
@ -15,11 +15,13 @@ use remora_attempt::{AttemptConfig, AttemptService, RecoveryConfig, migrate as a
|
|||
use remora_contracts::{
|
||||
ApiErrorDto, AttemptActivateRequestDto, AttemptCheckpointRequestDto, AttemptFailureOutcomeDto,
|
||||
AttemptOutcomeDto, AttemptOutcomeRequestDto, AttemptProgressRequestDto, AttemptRenewRequestDto,
|
||||
AttemptResolvedSourceRequestDto, AttemptRuntimeFailureDto, CheckpointDescriptorKindDto,
|
||||
CheckpointDescriptorRefDto, ClaimRequestDto, ClaimResponseDto, HttpSourceSubmissionDto,
|
||||
JobPhaseProjectionDto, JobSubmissionDto, PositiveIntegerDto, ProgressValueDto,
|
||||
ResolvedSourceDescriptorKindDto, ResolvedSourceDescriptorRefDto, ResultDescriptorDto,
|
||||
ResultKindDto, SafeIntegerDto, SourceSubmissionDto,
|
||||
AttemptResolvedSourceRequestDto, AttemptRuntimeFailureDto, AttemptStagingMultipartRequestDto,
|
||||
AttemptStagingObjectRequestDto, AttemptStagingWriteIntentRequestDto, AttemptStopRequestDto,
|
||||
CheckpointDescriptorKindDto, CheckpointDescriptorRefDto, ClaimRequestDto, ClaimResponseDto,
|
||||
HttpSourceSubmissionDto, JobPhaseProjectionDto, JobSubmissionDto, PositiveIntegerDto,
|
||||
ProgressValueDto, ResolvedSourceDescriptorKindDto, ResolvedSourceDescriptorRefDto,
|
||||
ResultDescriptorDto, ResultKindDto, SafeIntegerDto, SourceSubmissionDto,
|
||||
StagingMultipartIdentityDto, StagingObjectIdentityDto, StagingWriteIntentDto,
|
||||
};
|
||||
use remora_coordination::{CoordinationStore, StoreConfig};
|
||||
use remora_credentials::migrate as credentials_migrate;
|
||||
|
|
@ -29,7 +31,7 @@ use remora_domain::{
|
|||
use remora_durable::{PoolConfig, connect, migrate as durable_migrate};
|
||||
use remora_identity::{DefaultTarget, Principal};
|
||||
use remora_result::migrate as result_migrate;
|
||||
use remora_target::attempt_manifest_key;
|
||||
use remora_target::attempt_manifest_key_in;
|
||||
use reqwest::{Certificate as ClientCertificate, Client, Identity};
|
||||
use rustls::{
|
||||
RootCertStore, ServerConfig,
|
||||
|
|
@ -153,14 +155,15 @@ impl Harness {
|
|||
async fn seal_result(&self, attempt_id: &OpaqueId) -> ResultDescriptorDto {
|
||||
let manifest_id = unique_id("manifest");
|
||||
let checksum = format!("sha256:{}", "11".repeat(32));
|
||||
let key = attempt_manifest_key(attempt_id, &manifest_id);
|
||||
let key = attempt_manifest_key_in("users/alice", attempt_id, &manifest_id);
|
||||
sqlx::query(
|
||||
"INSERT INTO result_inventories (
|
||||
attempt_id, state, manifest_schema_version, manifest_id, manifest_store,
|
||||
manifest_bucket, manifest_key, manifest_version_id, manifest_sha256,
|
||||
manifest_size, member_count, total_bytes
|
||||
manifest_size, member_count, total_bytes,
|
||||
staging_origin_attempt_id, target_prefix
|
||||
) VALUES ($1, 'sealed', 1, $2, 'primary', 'remora-results', $3,
|
||||
'version-1', $4, 128, 1, 4096)",
|
||||
'version-1', $4, 128, 1, 4096, $1, 'users/alice')",
|
||||
)
|
||||
.bind(attempt_id.as_str())
|
||||
.bind(manifest_id.as_str())
|
||||
|
|
@ -336,6 +339,92 @@ async fn worker_operations_are_authenticated_fenced_replayable_and_redacted() {
|
|||
assert_eq!(response.status(), 409);
|
||||
}
|
||||
|
||||
let staging_key = format!(
|
||||
"{}o/{}",
|
||||
claimed.execution.staging_target.prefix,
|
||||
unique_id("object")
|
||||
);
|
||||
let staging_intent = AttemptStagingWriteIntentRequestDto {
|
||||
operation_id: unique_id("operation"),
|
||||
fence: claimed.authority.fence,
|
||||
worker_incarnation: worker.clone(),
|
||||
intent: StagingWriteIntentDto {
|
||||
store: claimed.execution.target.store.clone(),
|
||||
bucket: claimed.execution.target.bucket.clone(),
|
||||
key: staging_key.clone(),
|
||||
},
|
||||
};
|
||||
for _ in 0..2 {
|
||||
assert_eq!(
|
||||
client
|
||||
.post(format!("{attempt_url}/staging/write-intent"))
|
||||
.json(&staging_intent)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
204
|
||||
);
|
||||
}
|
||||
let staging_multipart = AttemptStagingMultipartRequestDto {
|
||||
operation_id: unique_id("operation"),
|
||||
fence: claimed.authority.fence,
|
||||
worker_incarnation: worker.clone(),
|
||||
multipart: StagingMultipartIdentityDto {
|
||||
store: claimed.execution.target.store.clone(),
|
||||
bucket: claimed.execution.target.bucket.clone(),
|
||||
key: staging_key.clone(),
|
||||
upload_id: "upload-identity".into(),
|
||||
},
|
||||
};
|
||||
assert_eq!(
|
||||
client
|
||||
.post(format!("{attempt_url}/staging/multipart"))
|
||||
.json(&staging_multipart)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
204
|
||||
);
|
||||
let staging_operation = unique_id("operation");
|
||||
let staging_object = AttemptStagingObjectRequestDto {
|
||||
operation_id: staging_operation.clone(),
|
||||
fence: claimed.authority.fence,
|
||||
worker_incarnation: worker.clone(),
|
||||
object: StagingObjectIdentityDto {
|
||||
store: claimed.execution.target.store.clone(),
|
||||
bucket: claimed.execution.target.bucket.clone(),
|
||||
key: staging_key.clone(),
|
||||
version_id: None,
|
||||
},
|
||||
completed_upload_id: Some("upload-identity".into()),
|
||||
};
|
||||
for _ in 0..2 {
|
||||
assert_eq!(
|
||||
client
|
||||
.post(format!("{attempt_url}/staging/object"))
|
||||
.json(&staging_object)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
204
|
||||
);
|
||||
}
|
||||
let mut changed_staging = staging_object;
|
||||
changed_staging.object.key.push_str("-changed");
|
||||
assert_eq!(
|
||||
client
|
||||
.post(format!("{attempt_url}/staging/object"))
|
||||
.json(&changed_staging)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
409
|
||||
);
|
||||
|
||||
let resolved_id = unique_id("resolved");
|
||||
let resolved = AttemptResolvedSourceRequestDto {
|
||||
operation_id: unique_id("operation"),
|
||||
|
|
@ -375,6 +464,7 @@ async fn worker_operations_are_authenticated_fenced_replayable_and_redacted() {
|
|||
resolved_source_id: resolved_id,
|
||||
validated_bytes: safe(1024),
|
||||
validated_members: safe(1),
|
||||
staging_multipart: None,
|
||||
};
|
||||
assert_eq!(
|
||||
client
|
||||
|
|
@ -470,7 +560,7 @@ async fn worker_operations_are_authenticated_fenced_replayable_and_redacted() {
|
|||
.json(&AttemptProgressRequestDto {
|
||||
operation_id: renewal_operation,
|
||||
fence: claimed.authority.fence,
|
||||
worker_incarnation: worker,
|
||||
worker_incarnation: worker.clone(),
|
||||
progress: ProgressValueDto {
|
||||
completed_bytes: safe(2048),
|
||||
total_bytes: Some(safe(4096)),
|
||||
|
|
@ -488,6 +578,23 @@ async fn worker_operations_are_authenticated_fenced_replayable_and_redacted() {
|
|||
);
|
||||
let error: ApiErrorDto = cross_kind.json().await.unwrap();
|
||||
assert_eq!(error.message, "precondition failed");
|
||||
let stop = AttemptStopRequestDto {
|
||||
operation_id: unique_id("operation"),
|
||||
fence: claimed.authority.fence,
|
||||
worker_incarnation: worker,
|
||||
};
|
||||
for _ in 0..2 {
|
||||
assert_eq!(
|
||||
client
|
||||
.post(format!("{attempt_url}/stop"))
|
||||
.json(&stop)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
204
|
||||
);
|
||||
}
|
||||
server.stop(true).await;
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue