Implement crash-safe streaming worker runtime
This commit is contained in:
parent
16cba3c317
commit
cdeef3cc1a
46 changed files with 7448 additions and 194 deletions
36
Cargo.lock
generated
36
Cargo.lock
generated
|
|
@ -3522,6 +3522,42 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "remora-worker"
|
||||
version = "0.0.0"
|
||||
dependencies = [
|
||||
"actix-tls",
|
||||
"actix-web",
|
||||
"base64",
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"getrandom 0.3.4",
|
||||
"http 1.4.2",
|
||||
"rcgen",
|
||||
"remora-admission",
|
||||
"remora-api",
|
||||
"remora-attempt",
|
||||
"remora-contracts",
|
||||
"remora-coordination",
|
||||
"remora-credentials",
|
||||
"remora-domain",
|
||||
"remora-durable",
|
||||
"remora-hf-source",
|
||||
"remora-http-source",
|
||||
"remora-identity",
|
||||
"remora-object-source",
|
||||
"remora-outbound",
|
||||
"remora-result",
|
||||
"remora-stream",
|
||||
"remora-target",
|
||||
"reqwest 0.13.4",
|
||||
"rustls",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_json_canonicalizer",
|
||||
"sha2 0.11.0",
|
||||
"sqlx",
|
||||
"thiserror",
|
||||
"time",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "reqsign-aws-v4"
|
||||
|
|
|
|||
|
|
@ -285,6 +285,35 @@ COPY crates crates
|
|||
COPY --from=kind-worker-api-verified /tmp/remora-api-tests.tar.zst /tests/remora-api-tests.tar.zst
|
||||
CMD ["bash", "-ec", "test -s /run/remora-fixture/profile.env && test -n \"$DATABASE_URL\" && test -n \"$REMORA_COORDINATION_URL\" && export REMORA_WORKER_API_REQUIRE=1 && cargo-nextest nextest run --archive-file /tests/remora-api-tests.tar.zst --workspace-remap /workspace --test-threads=1 --no-fail-fast"]
|
||||
|
||||
FROM rust-tools AS kind-worker-build
|
||||
WORKDIR /workspace
|
||||
COPY Cargo.toml Cargo.lock rust-toolchain.toml ./
|
||||
COPY .config .config
|
||||
COPY contracts contracts
|
||||
COPY crates crates
|
||||
COPY docs/api docs/api
|
||||
RUN cargo nextest archive \
|
||||
--package remora-worker \
|
||||
--build-jobs 4 \
|
||||
--locked \
|
||||
--archive-file /tmp/remora-worker-tests.tar.zst
|
||||
|
||||
FROM kind-worker-build AS kind-worker-verified
|
||||
RUN cargo clippy \
|
||||
--package remora-worker \
|
||||
--all-targets \
|
||||
--all-features \
|
||||
--locked \
|
||||
-- -D warnings \
|
||||
&& cargo test --package remora-worker --doc --locked
|
||||
|
||||
FROM rust-tools AS kind-worker-runner
|
||||
WORKDIR /workspace
|
||||
COPY Cargo.toml ./
|
||||
COPY crates crates
|
||||
COPY --from=kind-worker-verified /tmp/remora-worker-tests.tar.zst /tests/remora-worker-tests.tar.zst
|
||||
CMD ["bash", "-ec", "test -s /run/remora-fixture/profile.env && test -n \"$DATABASE_URL\" && test -n \"$REMORA_COORDINATION_URL\" && test -n \"$REMORA_TARGET_ENDPOINT\" && test -n \"$REMORA_VAULT_ADDR\" && case $REMORA_COORDINATION_PROVIDER in redis|dragonfly) ;; *) exit 1 ;; esac && case $REMORA_TARGET_PROVIDER in garage|versitygw) ;; *) exit 1 ;; esac && export REMORA_WORKER_REQUIRE=1 && cargo-nextest nextest run --archive-file /tests/remora-worker-tests.tar.zst --workspace-remap /workspace --test-threads=1 --no-fail-fast"]
|
||||
|
||||
FROM rust-tools AS kind-http-source-build
|
||||
WORKDIR /workspace
|
||||
COPY Cargo.toml Cargo.lock rust-toolchain.toml ./
|
||||
|
|
|
|||
5
Makefile
5
Makefile
|
|
@ -1,4 +1,4 @@
|
|||
.PHONY: check-dependencies check-fixture-lib check-kind-sample check-durable check-coordination check-identity check-credentials check-admission check-attempt check-attempt-mutations check-recovery check-worker-api check-stream check-target check-result check-outbound check-http-source check-object-source check-hf-source check-fast check-full check-fast-host check-full-host rust-check web-install web-check web-test-fast web-test-full
|
||||
.PHONY: check-dependencies check-fixture-lib check-kind-sample check-durable check-coordination check-identity check-credentials check-admission check-attempt check-attempt-mutations check-recovery check-worker-api check-worker check-stream check-target check-result check-outbound check-http-source check-object-source check-hf-source check-fast check-full check-fast-host check-full-host rust-check web-install web-check web-test-fast web-test-full
|
||||
|
||||
CONTAINER_ENGINE ?= nerdctl
|
||||
CONTAINER_RUN_FLAGS ?= --memory=16g --memory-swap=24g --cpus=4
|
||||
|
|
@ -83,6 +83,9 @@ check-recovery:
|
|||
check-worker-api:
|
||||
./tests/kind/run-worker-api-job.sh
|
||||
|
||||
check-worker:
|
||||
./tests/kind/run-worker-job.sh
|
||||
|
||||
check-stream:
|
||||
cargo fmt --all --check
|
||||
cargo clippy --package remora-stream --all-targets --all-features -- -D warnings
|
||||
|
|
|
|||
|
|
@ -8,22 +8,25 @@ use actix_web::{
|
|||
};
|
||||
use remora_attempt::{
|
||||
ActivationMutationOutcome, AttemptAuthority, AttemptError, AttemptMutationOutcome,
|
||||
AttemptService, CheckpointProjection, CheckpointRecord, ClaimOutcome, DurableDescriptor,
|
||||
LeaseProjection, MutationRejection, OutcomePublication, ProgressRecord, RenewalMutationOutcome,
|
||||
AttemptService, CheckpointMultipart, CheckpointProjection, CheckpointRecord, ClaimOutcome,
|
||||
DurableDescriptor, LeaseProjection, MutationRejection, OutcomePublication, ProgressRecord,
|
||||
RenewalMutationOutcome, StagingMultipart, StagingObject, StagingWriteIntent,
|
||||
};
|
||||
use remora_contracts::{
|
||||
ActivationResponseDto, ApiErrorCodeDto, ApiErrorDto, AttemptActivateRequestDto,
|
||||
AttemptAuthorityDto, AttemptCheckpointRequestDto, AttemptOutcomeDto, AttemptOutcomeRequestDto,
|
||||
AttemptProgressRequestDto, AttemptRenewRequestDto, AttemptResolvedSourceRequestDto,
|
||||
CheckpointDescriptorKindDto, CheckpointDescriptorRefDto, ClaimRequestDto, ClaimResponseDto,
|
||||
CredentialGrantVersionDto, DateTimeDto, DurableDescriptorKindDto, DurableDescriptorRefDto,
|
||||
ExecutionEnvelopeDto, JobPhaseProjectionDto, LeaseProjectionDto, NullableDto,
|
||||
PositiveIntegerDto, ResolvedSourceDescriptorKindDto, ResolvedSourceDescriptorRefDto,
|
||||
SafeIntegerDto, SourceProjectionDto, StagingTargetDto, TargetDto,
|
||||
AttemptStagingMultipartRequestDto, AttemptStagingObjectRequestDto,
|
||||
AttemptStagingWriteIntentRequestDto, AttemptStopRequestDto, CheckpointDescriptorKindDto,
|
||||
CheckpointDescriptorRefDto, ClaimRequestDto, ClaimResponseDto, CredentialGrantVersionDto,
|
||||
DateTimeDto, DurableDescriptorKindDto, DurableDescriptorRefDto, ExecutionEnvelopeDto,
|
||||
JobPhaseProjectionDto, LeaseProjectionDto, NullableDto, PositiveIntegerDto,
|
||||
ResolvedSourceDescriptorKindDto, ResolvedSourceDescriptorRefDto, SafeIntegerDto,
|
||||
SourceProjectionDto, StagingTargetDto, TargetDto,
|
||||
};
|
||||
use remora_domain::OpaqueId;
|
||||
use remora_result::PublicationInput;
|
||||
use remora_target::{ContentSha256, ObjectIdentity, RESERVED_NAMESPACE_SEGMENT, StoredObject};
|
||||
use remora_target::{ContentSha256, ObjectIdentity, StoredObject, attempt_staging_prefix};
|
||||
use sha2::{Digest, Sha256};
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
|
||||
|
|
@ -96,6 +99,19 @@ pub fn configure_worker_api(config: &mut web::ServiceConfig) {
|
|||
.route("/jobs/{job_id}/claim", web::post().to(claim))
|
||||
.route("/attempts/{attempt_id}/renew", web::post().to(renew))
|
||||
.route("/attempts/{attempt_id}/activate", web::post().to(activate))
|
||||
.route("/attempts/{attempt_id}/stop", web::post().to(stop))
|
||||
.route(
|
||||
"/attempts/{attempt_id}/staging/write-intent",
|
||||
web::post().to(register_staging_write_intent),
|
||||
)
|
||||
.route(
|
||||
"/attempts/{attempt_id}/staging/object",
|
||||
web::post().to(register_staging_object),
|
||||
)
|
||||
.route(
|
||||
"/attempts/{attempt_id}/staging/multipart",
|
||||
web::post().to(register_staging_multipart),
|
||||
)
|
||||
.route(
|
||||
"/attempts/{attempt_id}/resolved-source",
|
||||
web::post().to(pin_resolved_source),
|
||||
|
|
@ -115,6 +131,137 @@ pub fn configure_worker_api(config: &mut web::ServiceConfig) {
|
|||
);
|
||||
}
|
||||
|
||||
async fn register_staging_object(
|
||||
_: AuthenticatedWorker,
|
||||
state: web::Data<WorkerApiState>,
|
||||
path: web::Path<String>,
|
||||
request: web::Json<AttemptStagingObjectRequestDto>,
|
||||
) -> Result<HttpResponse, WorkerApiError> {
|
||||
let authority = request_authority(
|
||||
&state,
|
||||
path.into_inner(),
|
||||
request.fence.get(),
|
||||
&request.worker_incarnation,
|
||||
)
|
||||
.await?;
|
||||
let accepted = state
|
||||
.attempts
|
||||
.register_staging_object_operation(
|
||||
&authority,
|
||||
&request.operation_id,
|
||||
StagingObject {
|
||||
target_store: request.object.store.clone(),
|
||||
bucket: request.object.bucket.clone(),
|
||||
key: request.object.key.clone(),
|
||||
version_id: request.object.version_id.clone(),
|
||||
},
|
||||
request.completed_upload_id.clone(),
|
||||
)
|
||||
.await
|
||||
.map_err(WorkerApiError::from_attempt)?;
|
||||
if accepted {
|
||||
Ok(HttpResponse::NoContent().finish())
|
||||
} else {
|
||||
Err(WorkerApiError::precondition_failed())
|
||||
}
|
||||
}
|
||||
|
||||
async fn register_staging_write_intent(
|
||||
_: AuthenticatedWorker,
|
||||
state: web::Data<WorkerApiState>,
|
||||
path: web::Path<String>,
|
||||
request: web::Json<AttemptStagingWriteIntentRequestDto>,
|
||||
) -> Result<HttpResponse, WorkerApiError> {
|
||||
let authority = request_authority(
|
||||
&state,
|
||||
path.into_inner(),
|
||||
request.fence.get(),
|
||||
&request.worker_incarnation,
|
||||
)
|
||||
.await?;
|
||||
let accepted = state
|
||||
.attempts
|
||||
.register_staging_write_intent_operation(
|
||||
&authority,
|
||||
&request.operation_id,
|
||||
StagingWriteIntent {
|
||||
target_store: request.intent.store.clone(),
|
||||
bucket: request.intent.bucket.clone(),
|
||||
key: request.intent.key.clone(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(WorkerApiError::from_attempt)?;
|
||||
if accepted {
|
||||
Ok(HttpResponse::NoContent().finish())
|
||||
} else {
|
||||
Err(WorkerApiError::precondition_failed())
|
||||
}
|
||||
}
|
||||
|
||||
async fn register_staging_multipart(
|
||||
_: AuthenticatedWorker,
|
||||
state: web::Data<WorkerApiState>,
|
||||
path: web::Path<String>,
|
||||
request: web::Json<AttemptStagingMultipartRequestDto>,
|
||||
) -> Result<HttpResponse, WorkerApiError> {
|
||||
let authority = request_authority(
|
||||
&state,
|
||||
path.into_inner(),
|
||||
request.fence.get(),
|
||||
&request.worker_incarnation,
|
||||
)
|
||||
.await?;
|
||||
let accepted = state
|
||||
.attempts
|
||||
.register_staging_multipart_operation(
|
||||
&authority,
|
||||
&request.operation_id,
|
||||
StagingMultipart {
|
||||
target_store: request.multipart.store.clone(),
|
||||
bucket: request.multipart.bucket.clone(),
|
||||
key: request.multipart.key.clone(),
|
||||
upload_id: request.multipart.upload_id.clone(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(WorkerApiError::from_attempt)?;
|
||||
if accepted {
|
||||
Ok(HttpResponse::NoContent().finish())
|
||||
} else {
|
||||
Err(WorkerApiError::precondition_failed())
|
||||
}
|
||||
}
|
||||
|
||||
async fn stop(
|
||||
_: AuthenticatedWorker,
|
||||
state: web::Data<WorkerApiState>,
|
||||
path: web::Path<String>,
|
||||
request: web::Json<AttemptStopRequestDto>,
|
||||
) -> Result<HttpResponse, WorkerApiError> {
|
||||
let authority = request_authority(
|
||||
&state,
|
||||
path.into_inner(),
|
||||
request.fence.get(),
|
||||
&request.worker_incarnation,
|
||||
)
|
||||
.await?;
|
||||
match state
|
||||
.attempts
|
||||
.cooperative_stop_operation(&authority, &request.operation_id)
|
||||
.await
|
||||
.map_err(WorkerApiError::from_attempt)?
|
||||
{
|
||||
remora_attempt::CooperativeStopOutcome::Stopped
|
||||
| remora_attempt::CooperativeStopOutcome::StopNotRequired => {
|
||||
Ok(HttpResponse::NoContent().finish())
|
||||
}
|
||||
remora_attempt::CooperativeStopOutcome::AuthorityMismatch => {
|
||||
Err(WorkerApiError::precondition_failed())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn claim(
|
||||
_: AuthenticatedWorker,
|
||||
state: web::Data<WorkerApiState>,
|
||||
|
|
@ -245,6 +392,14 @@ async fn persist_checkpoint(
|
|||
resolved_source_id: request.resolved_source_id.clone(),
|
||||
validated_bytes: request.validated_bytes.get(),
|
||||
validated_members: request.validated_members.get(),
|
||||
staging_multipart: request.staging_multipart.as_ref().map(|identity| {
|
||||
CheckpointMultipart {
|
||||
target_store: identity.store.clone(),
|
||||
bucket: identity.bucket.clone(),
|
||||
key: identity.key.clone(),
|
||||
upload_id: identity.upload_id.clone(),
|
||||
}
|
||||
}),
|
||||
},
|
||||
)
|
||||
.await
|
||||
|
|
@ -387,6 +542,7 @@ async fn load_execution_envelope(
|
|||
let source: SourceProjectionDto =
|
||||
serde_json::from_value(row.source).map_err(|_| WorkerApiError::unavailable())?;
|
||||
let target_prefix = row.target_prefix.join("/");
|
||||
let staging_prefix = attempt_staging_prefix(&target_prefix, &row.staging_origin_attempt_id);
|
||||
let resolved_source_ref = row.resolved_source.map(|descriptor| {
|
||||
NullableDto(Some(descriptor_ref(
|
||||
descriptor,
|
||||
|
|
@ -425,10 +581,7 @@ async fn load_execution_envelope(
|
|||
staging_target: StagingTargetDto {
|
||||
store: row.target_store,
|
||||
bucket: row.target_bucket,
|
||||
prefix: format!(
|
||||
"{RESERVED_NAMESPACE_SEGMENT}/a/{}/",
|
||||
authority.attempt_id.as_str()
|
||||
),
|
||||
prefix: staging_prefix,
|
||||
},
|
||||
execution_grant_ref: row.execution_grant_id,
|
||||
checkpoint_ref: checkpoint_ref.unwrap_or(NullableDto(None)),
|
||||
|
|
|
|||
|
|
@ -15,11 +15,13 @@ use remora_attempt::{AttemptConfig, AttemptService, RecoveryConfig, migrate as a
|
|||
use remora_contracts::{
|
||||
ApiErrorDto, AttemptActivateRequestDto, AttemptCheckpointRequestDto, AttemptFailureOutcomeDto,
|
||||
AttemptOutcomeDto, AttemptOutcomeRequestDto, AttemptProgressRequestDto, AttemptRenewRequestDto,
|
||||
AttemptResolvedSourceRequestDto, AttemptRuntimeFailureDto, CheckpointDescriptorKindDto,
|
||||
CheckpointDescriptorRefDto, ClaimRequestDto, ClaimResponseDto, HttpSourceSubmissionDto,
|
||||
JobPhaseProjectionDto, JobSubmissionDto, PositiveIntegerDto, ProgressValueDto,
|
||||
ResolvedSourceDescriptorKindDto, ResolvedSourceDescriptorRefDto, ResultDescriptorDto,
|
||||
ResultKindDto, SafeIntegerDto, SourceSubmissionDto,
|
||||
AttemptResolvedSourceRequestDto, AttemptRuntimeFailureDto, AttemptStagingMultipartRequestDto,
|
||||
AttemptStagingObjectRequestDto, AttemptStagingWriteIntentRequestDto, AttemptStopRequestDto,
|
||||
CheckpointDescriptorKindDto, CheckpointDescriptorRefDto, ClaimRequestDto, ClaimResponseDto,
|
||||
HttpSourceSubmissionDto, JobPhaseProjectionDto, JobSubmissionDto, PositiveIntegerDto,
|
||||
ProgressValueDto, ResolvedSourceDescriptorKindDto, ResolvedSourceDescriptorRefDto,
|
||||
ResultDescriptorDto, ResultKindDto, SafeIntegerDto, SourceSubmissionDto,
|
||||
StagingMultipartIdentityDto, StagingObjectIdentityDto, StagingWriteIntentDto,
|
||||
};
|
||||
use remora_coordination::{CoordinationStore, StoreConfig};
|
||||
use remora_credentials::migrate as credentials_migrate;
|
||||
|
|
@ -29,7 +31,7 @@ use remora_domain::{
|
|||
use remora_durable::{PoolConfig, connect, migrate as durable_migrate};
|
||||
use remora_identity::{DefaultTarget, Principal};
|
||||
use remora_result::migrate as result_migrate;
|
||||
use remora_target::attempt_manifest_key;
|
||||
use remora_target::attempt_manifest_key_in;
|
||||
use reqwest::{Certificate as ClientCertificate, Client, Identity};
|
||||
use rustls::{
|
||||
RootCertStore, ServerConfig,
|
||||
|
|
@ -153,14 +155,15 @@ impl Harness {
|
|||
async fn seal_result(&self, attempt_id: &OpaqueId) -> ResultDescriptorDto {
|
||||
let manifest_id = unique_id("manifest");
|
||||
let checksum = format!("sha256:{}", "11".repeat(32));
|
||||
let key = attempt_manifest_key(attempt_id, &manifest_id);
|
||||
let key = attempt_manifest_key_in("users/alice", attempt_id, &manifest_id);
|
||||
sqlx::query(
|
||||
"INSERT INTO result_inventories (
|
||||
attempt_id, state, manifest_schema_version, manifest_id, manifest_store,
|
||||
manifest_bucket, manifest_key, manifest_version_id, manifest_sha256,
|
||||
manifest_size, member_count, total_bytes
|
||||
manifest_size, member_count, total_bytes,
|
||||
staging_origin_attempt_id, target_prefix
|
||||
) VALUES ($1, 'sealed', 1, $2, 'primary', 'remora-results', $3,
|
||||
'version-1', $4, 128, 1, 4096)",
|
||||
'version-1', $4, 128, 1, 4096, $1, 'users/alice')",
|
||||
)
|
||||
.bind(attempt_id.as_str())
|
||||
.bind(manifest_id.as_str())
|
||||
|
|
@ -336,6 +339,92 @@ async fn worker_operations_are_authenticated_fenced_replayable_and_redacted() {
|
|||
assert_eq!(response.status(), 409);
|
||||
}
|
||||
|
||||
let staging_key = format!(
|
||||
"{}o/{}",
|
||||
claimed.execution.staging_target.prefix,
|
||||
unique_id("object")
|
||||
);
|
||||
let staging_intent = AttemptStagingWriteIntentRequestDto {
|
||||
operation_id: unique_id("operation"),
|
||||
fence: claimed.authority.fence,
|
||||
worker_incarnation: worker.clone(),
|
||||
intent: StagingWriteIntentDto {
|
||||
store: claimed.execution.target.store.clone(),
|
||||
bucket: claimed.execution.target.bucket.clone(),
|
||||
key: staging_key.clone(),
|
||||
},
|
||||
};
|
||||
for _ in 0..2 {
|
||||
assert_eq!(
|
||||
client
|
||||
.post(format!("{attempt_url}/staging/write-intent"))
|
||||
.json(&staging_intent)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
204
|
||||
);
|
||||
}
|
||||
let staging_multipart = AttemptStagingMultipartRequestDto {
|
||||
operation_id: unique_id("operation"),
|
||||
fence: claimed.authority.fence,
|
||||
worker_incarnation: worker.clone(),
|
||||
multipart: StagingMultipartIdentityDto {
|
||||
store: claimed.execution.target.store.clone(),
|
||||
bucket: claimed.execution.target.bucket.clone(),
|
||||
key: staging_key.clone(),
|
||||
upload_id: "upload-identity".into(),
|
||||
},
|
||||
};
|
||||
assert_eq!(
|
||||
client
|
||||
.post(format!("{attempt_url}/staging/multipart"))
|
||||
.json(&staging_multipart)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
204
|
||||
);
|
||||
let staging_operation = unique_id("operation");
|
||||
let staging_object = AttemptStagingObjectRequestDto {
|
||||
operation_id: staging_operation.clone(),
|
||||
fence: claimed.authority.fence,
|
||||
worker_incarnation: worker.clone(),
|
||||
object: StagingObjectIdentityDto {
|
||||
store: claimed.execution.target.store.clone(),
|
||||
bucket: claimed.execution.target.bucket.clone(),
|
||||
key: staging_key.clone(),
|
||||
version_id: None,
|
||||
},
|
||||
completed_upload_id: Some("upload-identity".into()),
|
||||
};
|
||||
for _ in 0..2 {
|
||||
assert_eq!(
|
||||
client
|
||||
.post(format!("{attempt_url}/staging/object"))
|
||||
.json(&staging_object)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
204
|
||||
);
|
||||
}
|
||||
let mut changed_staging = staging_object;
|
||||
changed_staging.object.key.push_str("-changed");
|
||||
assert_eq!(
|
||||
client
|
||||
.post(format!("{attempt_url}/staging/object"))
|
||||
.json(&changed_staging)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
409
|
||||
);
|
||||
|
||||
let resolved_id = unique_id("resolved");
|
||||
let resolved = AttemptResolvedSourceRequestDto {
|
||||
operation_id: unique_id("operation"),
|
||||
|
|
@ -375,6 +464,7 @@ async fn worker_operations_are_authenticated_fenced_replayable_and_redacted() {
|
|||
resolved_source_id: resolved_id,
|
||||
validated_bytes: safe(1024),
|
||||
validated_members: safe(1),
|
||||
staging_multipart: None,
|
||||
};
|
||||
assert_eq!(
|
||||
client
|
||||
|
|
@ -470,7 +560,7 @@ async fn worker_operations_are_authenticated_fenced_replayable_and_redacted() {
|
|||
.json(&AttemptProgressRequestDto {
|
||||
operation_id: renewal_operation,
|
||||
fence: claimed.authority.fence,
|
||||
worker_incarnation: worker,
|
||||
worker_incarnation: worker.clone(),
|
||||
progress: ProgressValueDto {
|
||||
completed_bytes: safe(2048),
|
||||
total_bytes: Some(safe(4096)),
|
||||
|
|
@ -488,6 +578,23 @@ async fn worker_operations_are_authenticated_fenced_replayable_and_redacted() {
|
|||
);
|
||||
let error: ApiErrorDto = cross_kind.json().await.unwrap();
|
||||
assert_eq!(error.message, "precondition failed");
|
||||
let stop = AttemptStopRequestDto {
|
||||
operation_id: unique_id("operation"),
|
||||
fence: claimed.authority.fence,
|
||||
worker_incarnation: worker,
|
||||
};
|
||||
for _ in 0..2 {
|
||||
assert_eq!(
|
||||
client
|
||||
.post(format!("{attempt_url}/stop"))
|
||||
.json(&stop)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
204
|
||||
);
|
||||
}
|
||||
server.stop(true).await;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,39 @@
|
|||
ALTER TABLE attempts
|
||||
ADD COLUMN staging_origin_attempt_id TEXT;
|
||||
|
||||
UPDATE attempts
|
||||
SET staging_origin_attempt_id = attempt_id;
|
||||
|
||||
ALTER TABLE attempts
|
||||
ALTER COLUMN staging_origin_attempt_id SET NOT NULL,
|
||||
ADD CONSTRAINT attempts_staging_origin_fk
|
||||
FOREIGN KEY (staging_origin_attempt_id)
|
||||
REFERENCES attempts (attempt_id)
|
||||
DEFERRABLE INITIALLY DEFERRED,
|
||||
ADD CONSTRAINT attempts_id_job_unique UNIQUE (attempt_id, job_id);
|
||||
|
||||
ALTER TABLE jobs
|
||||
ADD COLUMN checkpoint_owner_attempt_id TEXT;
|
||||
|
||||
UPDATE jobs j
|
||||
SET checkpoint_owner_attempt_id = (
|
||||
SELECT a.attempt_id
|
||||
FROM attempts a
|
||||
WHERE a.job_id = j.job_id
|
||||
ORDER BY a.fence DESC
|
||||
LIMIT 1
|
||||
)
|
||||
WHERE j.checkpoint IS NOT NULL;
|
||||
|
||||
ALTER TABLE jobs
|
||||
ADD CONSTRAINT jobs_checkpoint_owner_fk
|
||||
FOREIGN KEY (checkpoint_owner_attempt_id, job_id)
|
||||
REFERENCES attempts (attempt_id, job_id)
|
||||
DEFERRABLE INITIALLY DEFERRED,
|
||||
ADD CONSTRAINT jobs_checkpoint_owner_check CHECK (
|
||||
(checkpoint IS NULL) = (checkpoint_owner_attempt_id IS NULL)
|
||||
);
|
||||
|
||||
CREATE INDEX jobs_checkpoint_owner_idx
|
||||
ON jobs (checkpoint_owner_attempt_id)
|
||||
WHERE checkpoint_owner_attempt_id IS NOT NULL;
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
ALTER TABLE attempt_staging_cleanup
|
||||
DROP CONSTRAINT attempt_staging_cleanup_kind_check,
|
||||
DROP CONSTRAINT attempt_staging_cleanup_identity_shape_check,
|
||||
ADD CONSTRAINT attempt_staging_cleanup_kind_check
|
||||
CHECK (kind IN ('write_intent', 'object', 'multipart')),
|
||||
ADD CONSTRAINT attempt_staging_cleanup_identity_shape_check CHECK (
|
||||
(kind IN ('write_intent', 'object') AND upload_id IS NULL)
|
||||
OR (kind = 'multipart' AND version_id IS NULL AND upload_id IS NOT NULL)
|
||||
);
|
||||
|
|
@ -10,13 +10,13 @@ mod types;
|
|||
pub use error::AttemptError;
|
||||
pub use migrate::migrate;
|
||||
pub use mutation_types::{
|
||||
AttemptMutationOutcome, CheckpointProjection, CheckpointRecord, CommittedManifestRecord,
|
||||
CommittedResultRecord, DurableDescriptor, FailureRecord, JobMutationProjection,
|
||||
MutationRejection, ProgressRecord,
|
||||
AttemptMutationOutcome, CheckpointMultipart, CheckpointProjection, CheckpointRecord,
|
||||
CommittedManifestRecord, CommittedResultRecord, DurableDescriptor, FailureRecord,
|
||||
JobMutationProjection, MutationRejection, ProgressRecord,
|
||||
};
|
||||
pub use recovery_types::{
|
||||
CancelOutcome, CleanupClaim, CleanupCompletion, CleanupIdentity, CooperativeStopOutcome,
|
||||
ReconcileReport, RecoveryConfig, StagingMultipart, StagingObject,
|
||||
ReconcileReport, RecoveryConfig, StagingMultipart, StagingObject, StagingWriteIntent,
|
||||
};
|
||||
pub use service::{AttemptConfig, AttemptService};
|
||||
pub use types::{
|
||||
|
|
|
|||
|
|
@ -18,6 +18,16 @@ pub struct CheckpointRecord {
|
|||
pub resolved_source_id: OpaqueId,
|
||||
pub validated_bytes: u64,
|
||||
pub validated_members: u64,
|
||||
#[serde(default)]
|
||||
pub staging_multipart: Option<CheckpointMultipart>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
pub struct CheckpointMultipart {
|
||||
pub target_store: String,
|
||||
pub bucket: String,
|
||||
pub key: String,
|
||||
pub upload_id: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
|
|
|
|||
|
|
@ -73,7 +73,11 @@ impl AttemptService {
|
|||
let outcome = self
|
||||
.publish_success(authority, operation_id, result)
|
||||
.await?;
|
||||
self.publication_outcome(operation_id, outcome).await
|
||||
let publication = self.publication_outcome(operation_id, outcome).await?;
|
||||
if matches!(publication, OutcomePublication::Accepted { .. }) {
|
||||
self.release_exact(authority).await;
|
||||
}
|
||||
Ok(publication)
|
||||
}
|
||||
|
||||
pub async fn persist_checkpoint(
|
||||
|
|
@ -118,13 +122,16 @@ impl AttemptService {
|
|||
}
|
||||
let now = OffsetDateTime::now_utc();
|
||||
sqlx::query(
|
||||
"UPDATE jobs SET checkpoint = $2, revision = revision + 1,
|
||||
updated_at = $3 WHERE job_id = $1",
|
||||
"UPDATE jobs
|
||||
SET checkpoint = $2, checkpoint_owner_attempt_id = $3,
|
||||
revision = revision + 1, updated_at = $4
|
||||
WHERE job_id = $1",
|
||||
)
|
||||
.bind(authority.job_id.as_str())
|
||||
.bind(Json(&CheckpointProjection::Reusable {
|
||||
checkpoint: checkpoint.clone(),
|
||||
}))
|
||||
.bind(authority.attempt_id.as_str())
|
||||
.bind(now)
|
||||
.execute(tx.as_mut())
|
||||
.await?;
|
||||
|
|
@ -143,7 +150,11 @@ impl AttemptService {
|
|||
class: RuntimeFailureClass,
|
||||
) -> Result<OutcomePublication, AttemptError> {
|
||||
let outcome = self.publish_failure(authority, operation_id, class).await?;
|
||||
self.publication_outcome(operation_id, outcome).await
|
||||
let publication = self.publication_outcome(operation_id, outcome).await?;
|
||||
if matches!(publication, OutcomePublication::Accepted { .. }) {
|
||||
self.release_exact(authority).await;
|
||||
}
|
||||
Ok(publication)
|
||||
}
|
||||
|
||||
pub async fn report_progress(
|
||||
|
|
@ -227,12 +238,14 @@ impl AttemptService {
|
|||
};
|
||||
let changed = sqlx::query(
|
||||
"UPDATE jobs SET phase = 'succeeded', authority_attempt_id = NULL,
|
||||
checkpoint = $2, committed_result = $3, terminal_at = $4,
|
||||
revision = revision + 1, updated_at = $4
|
||||
checkpoint = $2, checkpoint_owner_attempt_id = $3,
|
||||
committed_result = $4, terminal_at = $5,
|
||||
revision = revision + 1, updated_at = $5
|
||||
WHERE job_id = $1 AND phase = 'running' AND cancel_requested = FALSE",
|
||||
)
|
||||
.bind(authority.job_id.as_str())
|
||||
.bind(Json(&complete_checkpoint))
|
||||
.bind(authority.attempt_id.as_str())
|
||||
.bind(stored)
|
||||
.bind(now)
|
||||
.execute(tx.as_mut())
|
||||
|
|
@ -544,11 +557,26 @@ fn validate_checkpoint(checkpoint: &CheckpointRecord) -> Result<(), AttemptError
|
|||
if (checkpoint.validated_bytes == 0 && checkpoint.validated_members == 0)
|
||||
|| checkpoint.validated_bytes > MAX_SAFE_INTEGER
|
||||
|| checkpoint.validated_members > MAX_SAFE_INTEGER
|
||||
|| (checkpoint.validated_members == 0) != checkpoint.staging_multipart.is_some()
|
||||
{
|
||||
return Err(AttemptError::Invalid(
|
||||
"checkpoint exceeds the safe-integer range".into(),
|
||||
));
|
||||
}
|
||||
if checkpoint
|
||||
.staging_multipart
|
||||
.as_ref()
|
||||
.is_some_and(|identity| {
|
||||
identity.target_store.is_empty()
|
||||
|| identity.bucket.is_empty()
|
||||
|| identity.key.is_empty()
|
||||
|| identity.upload_id.is_empty()
|
||||
})
|
||||
{
|
||||
return Err(AttemptError::Invalid(
|
||||
"checkpoint multipart identity is invalid".into(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,14 @@
|
|||
use std::time::Duration;
|
||||
|
||||
use remora_coordination::{LeaseOwner, OpCode};
|
||||
use remora_domain::{FailureClass, FailureDisposition, JobPhase, OpaqueId};
|
||||
use remora_durable::{DurableError, enqueue, with_transaction};
|
||||
use remora_target::validate_attempt_key;
|
||||
use remora_domain::{
|
||||
FailureClass, FailureDisposition, JobPhase, OpaqueId, fingerprint, ports::MutationScope,
|
||||
};
|
||||
use remora_durable::{
|
||||
DurableError, MutationDisposition, MutationStatus, enqueue, execute_mutation, with_transaction,
|
||||
};
|
||||
use remora_target::{attempt_key_attempt_id, attempt_staging_prefix, validate_attempt_key};
|
||||
use serde::Serialize;
|
||||
use serde_json::json;
|
||||
use sqlx::{FromRow, Postgres, Transaction, types::Json};
|
||||
use time::OffsetDateTime;
|
||||
|
|
@ -12,7 +17,7 @@ use crate::service::new_id;
|
|||
use crate::{
|
||||
AttemptAuthority, AttemptError, AttemptService, CancelOutcome, CleanupClaim, CleanupCompletion,
|
||||
CleanupIdentity, CooperativeStopOutcome, FailureRecord, ReconcileReport, RecoveryConfig,
|
||||
StagingMultipart, StagingObject,
|
||||
StagingMultipart, StagingObject, StagingWriteIntent,
|
||||
};
|
||||
|
||||
impl AttemptService {
|
||||
|
|
@ -73,57 +78,79 @@ impl AttemptService {
|
|||
&self,
|
||||
authority: &AttemptAuthority,
|
||||
) -> Result<CooperativeStopOutcome, AttemptError> {
|
||||
let row = sqlx::query_as::<_, StopRow>(
|
||||
"SELECT a.job_id, a.fence, a.worker_incarnation, a.state,
|
||||
j.authority_attempt_id, j.cancel_requested
|
||||
FROM attempts a JOIN jobs j ON j.job_id = a.job_id
|
||||
WHERE a.attempt_id = $1",
|
||||
)
|
||||
.bind(authority.attempt_id.as_str())
|
||||
.fetch_optional(&self.pool)
|
||||
let authority_for_effect = authority.clone();
|
||||
let coordination = self.coordination.clone();
|
||||
let outcome = with_transaction(&self.pool, move |tx| {
|
||||
Box::pin(async move {
|
||||
cooperative_stop_transaction(tx, &coordination, &authority_for_effect).await
|
||||
})
|
||||
})
|
||||
.await?;
|
||||
let Some(row) = row else {
|
||||
return Ok(CooperativeStopOutcome::AuthorityMismatch);
|
||||
};
|
||||
if row.job_id != authority.job_id.as_str()
|
||||
|| u64::try_from(row.fence).ok() != Some(authority.fence)
|
||||
|| row.worker_incarnation != authority.worker_incarnation.as_str()
|
||||
{
|
||||
return Ok(CooperativeStopOutcome::AuthorityMismatch);
|
||||
if outcome == CooperativeStopOutcome::Stopped {
|
||||
let _ = self
|
||||
.coordination
|
||||
.release(authority.job_id.as_str(), &lease_owner(authority))
|
||||
.await;
|
||||
self.release_local(&authority.job_id);
|
||||
}
|
||||
if row.state == "stopped" {
|
||||
return Ok(CooperativeStopOutcome::Stopped);
|
||||
}
|
||||
if row.state != "executing" {
|
||||
return Ok(CooperativeStopOutcome::StopNotRequired);
|
||||
}
|
||||
let current = row.authority_attempt_id.as_deref() == Some(authority.attempt_id.as_str());
|
||||
let live = self
|
||||
.coordination
|
||||
.get_live_lease(authority.job_id.as_str())
|
||||
.await?
|
||||
.is_some_and(|lease| lease.record.owner() == lease_owner(authority));
|
||||
if current && live && !row.cancel_requested {
|
||||
return Ok(CooperativeStopOutcome::StopNotRequired);
|
||||
}
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let changed = sqlx::query(
|
||||
"UPDATE attempts SET state = 'stopped', stopped_at = $2
|
||||
WHERE attempt_id = $1 AND state IN ('authorized', 'executing')",
|
||||
Ok(outcome)
|
||||
}
|
||||
|
||||
pub async fn cooperative_stop_operation(
|
||||
&self,
|
||||
authority: &AttemptAuthority,
|
||||
operation_id: &OpaqueId,
|
||||
) -> Result<CooperativeStopOutcome, AttemptError> {
|
||||
let request_fingerprint = fingerprint(&StopRequest {
|
||||
job_id: &authority.job_id,
|
||||
attempt_id: &authority.attempt_id,
|
||||
fence: authority.fence,
|
||||
worker_incarnation: &authority.worker_incarnation,
|
||||
})
|
||||
.map_err(|error| AttemptError::Invalid(error.to_string()))?;
|
||||
let authority_for_effect = authority.clone();
|
||||
let coordination = self.coordination.clone();
|
||||
let outcome = execute_mutation(
|
||||
&self.pool,
|
||||
MutationScope {
|
||||
subject: &self.operation_subject,
|
||||
operation: "attempt.mutation",
|
||||
key: operation_id.as_str(),
|
||||
},
|
||||
"cooperative_stop",
|
||||
&[],
|
||||
request_fingerprint,
|
||||
move |tx| {
|
||||
Box::pin(async move {
|
||||
let decision =
|
||||
cooperative_stop_transaction(tx, &coordination, &authority_for_effect)
|
||||
.await?;
|
||||
let status = if decision == CooperativeStopOutcome::AuthorityMismatch {
|
||||
MutationStatus::Rejected
|
||||
} else {
|
||||
MutationStatus::Accepted
|
||||
};
|
||||
Ok((
|
||||
status,
|
||||
serde_json::to_value(decision).map_err(DurableError::effect)?,
|
||||
))
|
||||
})
|
||||
},
|
||||
)
|
||||
.bind(authority.attempt_id.as_str())
|
||||
.bind(now)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
if changed.rows_affected() == 0 {
|
||||
return Ok(CooperativeStopOutcome::StopNotRequired);
|
||||
if outcome.disposition == MutationDisposition::Conflict {
|
||||
return Err(AttemptError::OperationConflict);
|
||||
}
|
||||
let _ = self
|
||||
.coordination
|
||||
.release(authority.job_id.as_str(), &lease_owner(authority))
|
||||
.await;
|
||||
self.release_local(&authority.job_id);
|
||||
Ok(CooperativeStopOutcome::Stopped)
|
||||
let decision = serde_json::from_value(outcome.response)
|
||||
.map_err(|error| AttemptError::Invalid(error.to_string()))?;
|
||||
if decision == CooperativeStopOutcome::Stopped {
|
||||
let _ = self
|
||||
.coordination
|
||||
.release(authority.job_id.as_str(), &lease_owner(authority))
|
||||
.await;
|
||||
self.release_local(&authority.job_id);
|
||||
}
|
||||
Ok(decision)
|
||||
}
|
||||
|
||||
pub async fn mark_worker_down(
|
||||
|
|
@ -146,7 +173,7 @@ impl AttemptService {
|
|||
attempt_id: &OpaqueId,
|
||||
object: &StagingObject,
|
||||
) -> Result<(), AttemptError> {
|
||||
validate_staging_key(attempt_id, &object.key)?;
|
||||
validate_staging_key(&self.pool, attempt_id, &object.key).await?;
|
||||
insert_cleanup_identity(
|
||||
&self.pool,
|
||||
attempt_id,
|
||||
|
|
@ -162,12 +189,105 @@ impl AttemptService {
|
|||
.await
|
||||
}
|
||||
|
||||
pub async fn register_staging_write_intent_operation(
|
||||
&self,
|
||||
authority: &AttemptAuthority,
|
||||
operation_id: &OpaqueId,
|
||||
intent: StagingWriteIntent,
|
||||
) -> Result<bool, AttemptError> {
|
||||
self.register_staging_operation(
|
||||
authority,
|
||||
operation_id,
|
||||
"register_staging_write_intent",
|
||||
intent.clone(),
|
||||
move |tx, attempt_id| {
|
||||
Box::pin(async move {
|
||||
validate_staging_key_transaction(tx, &attempt_id, &intent.key).await?;
|
||||
insert_cleanup_identity_transaction(
|
||||
tx,
|
||||
&attempt_id,
|
||||
NewCleanupIdentity {
|
||||
kind: "write_intent",
|
||||
target_store: &intent.target_store,
|
||||
bucket: &intent.bucket,
|
||||
key: &intent.key,
|
||||
version_id: None,
|
||||
upload_id: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
})
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn register_staging_object_operation(
|
||||
&self,
|
||||
authority: &AttemptAuthority,
|
||||
operation_id: &OpaqueId,
|
||||
object: StagingObject,
|
||||
completed_upload_id: Option<String>,
|
||||
) -> Result<bool, AttemptError> {
|
||||
self.register_staging_operation(
|
||||
authority,
|
||||
operation_id,
|
||||
"register_staging_object",
|
||||
StagingObjectRegistration {
|
||||
object: object.clone(),
|
||||
completed_upload_id: completed_upload_id.clone(),
|
||||
},
|
||||
move |tx, attempt_id| {
|
||||
Box::pin(async move {
|
||||
validate_staging_key_transaction(tx, &attempt_id, &object.key).await?;
|
||||
insert_cleanup_identity_transaction(
|
||||
tx,
|
||||
&attempt_id,
|
||||
NewCleanupIdentity {
|
||||
kind: "object",
|
||||
target_store: &object.target_store,
|
||||
bucket: &object.bucket,
|
||||
key: &object.key,
|
||||
version_id: object.version_id.as_deref(),
|
||||
upload_id: None,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
complete_pending_identity(
|
||||
tx,
|
||||
&attempt_id,
|
||||
"write_intent",
|
||||
&object.target_store,
|
||||
&object.bucket,
|
||||
&object.key,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
if let Some(upload_id) = completed_upload_id {
|
||||
complete_pending_identity(
|
||||
tx,
|
||||
&attempt_id,
|
||||
"multipart",
|
||||
&object.target_store,
|
||||
&object.bucket,
|
||||
&object.key,
|
||||
Some(&upload_id),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn register_staging_multipart(
|
||||
&self,
|
||||
attempt_id: &OpaqueId,
|
||||
multipart: &StagingMultipart,
|
||||
) -> Result<(), AttemptError> {
|
||||
validate_staging_key(attempt_id, &multipart.key)?;
|
||||
validate_staging_key(&self.pool, attempt_id, &multipart.key).await?;
|
||||
if multipart.upload_id.is_empty() {
|
||||
return Err(AttemptError::Invalid(
|
||||
"multipart upload identity cannot be empty".into(),
|
||||
|
|
@ -188,6 +308,107 @@ impl AttemptService {
|
|||
.await
|
||||
}
|
||||
|
||||
pub async fn register_staging_multipart_operation(
|
||||
&self,
|
||||
authority: &AttemptAuthority,
|
||||
operation_id: &OpaqueId,
|
||||
multipart: StagingMultipart,
|
||||
) -> Result<bool, AttemptError> {
|
||||
if multipart.upload_id.is_empty() {
|
||||
return Err(AttemptError::Invalid(
|
||||
"multipart upload identity cannot be empty".into(),
|
||||
));
|
||||
}
|
||||
self.register_staging_operation(
|
||||
authority,
|
||||
operation_id,
|
||||
"register_staging_multipart",
|
||||
multipart.clone(),
|
||||
move |tx, attempt_id| {
|
||||
Box::pin(async move {
|
||||
validate_staging_key_transaction(tx, &attempt_id, &multipart.key).await?;
|
||||
insert_cleanup_identity_transaction(
|
||||
tx,
|
||||
&attempt_id,
|
||||
NewCleanupIdentity {
|
||||
kind: "multipart",
|
||||
target_store: &multipart.target_store,
|
||||
bucket: &multipart.bucket,
|
||||
key: &multipart.key,
|
||||
version_id: None,
|
||||
upload_id: Some(&multipart.upload_id),
|
||||
},
|
||||
)
|
||||
.await
|
||||
})
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn register_staging_operation<T, F>(
|
||||
&self,
|
||||
authority: &AttemptAuthority,
|
||||
operation_id: &OpaqueId,
|
||||
kind: &'static str,
|
||||
request: T,
|
||||
effect: F,
|
||||
) -> Result<bool, AttemptError>
|
||||
where
|
||||
T: Serialize,
|
||||
F: for<'c> FnOnce(
|
||||
&'c mut remora_durable::PgTx<'_>,
|
||||
OpaqueId,
|
||||
) -> remora_durable::BoxFuture<'c, Result<(), DurableError>>
|
||||
+ Send
|
||||
+ 'static,
|
||||
{
|
||||
let request_fingerprint = fingerprint(&RegistrationRequest {
|
||||
job_id: &authority.job_id,
|
||||
attempt_id: &authority.attempt_id,
|
||||
fence: authority.fence,
|
||||
worker_incarnation: &authority.worker_incarnation,
|
||||
value: request,
|
||||
})
|
||||
.map_err(|error| AttemptError::Invalid(error.to_string()))?;
|
||||
let authority = authority.clone();
|
||||
let outcome = execute_mutation(
|
||||
&self.pool,
|
||||
MutationScope {
|
||||
subject: &self.operation_subject,
|
||||
operation: "attempt.mutation",
|
||||
key: operation_id.as_str(),
|
||||
},
|
||||
kind,
|
||||
&[],
|
||||
request_fingerprint,
|
||||
move |tx| {
|
||||
Box::pin(async move {
|
||||
let eligible = lock_registration_authority(tx, &authority).await?;
|
||||
if eligible {
|
||||
effect(tx, authority.attempt_id).await?;
|
||||
}
|
||||
Ok((
|
||||
if eligible {
|
||||
MutationStatus::Accepted
|
||||
} else {
|
||||
MutationStatus::Rejected
|
||||
},
|
||||
serde_json::Value::Bool(eligible),
|
||||
))
|
||||
})
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
if outcome.disposition == MutationDisposition::Conflict {
|
||||
return Err(AttemptError::OperationConflict);
|
||||
}
|
||||
outcome
|
||||
.response
|
||||
.as_bool()
|
||||
.ok_or_else(|| AttemptError::Invalid("invalid staging operation result".into()))
|
||||
}
|
||||
|
||||
pub async fn reconcile_once(
|
||||
&self,
|
||||
config: &RecoveryConfig,
|
||||
|
|
@ -235,6 +456,8 @@ impl AttemptService {
|
|||
WHERE (c.state = 'pending'
|
||||
OR (c.state = 'claimed' AND c.claimed_until <= $1))
|
||||
AND j.authority_attempt_id IS DISTINCT FROM a.attempt_id
|
||||
AND (j.phase IN ('succeeded', 'failed', 'cancelled')
|
||||
OR j.checkpoint_owner_attempt_id IS DISTINCT FROM a.attempt_id)
|
||||
AND COALESCE(j.committed_result ->> 'attempt_id', '') <> a.attempt_id
|
||||
AND (a.state IN ('stopped', 'reaped') OR a.worker_down_at IS NOT NULL)
|
||||
ORDER BY c.cleanup_id
|
||||
|
|
@ -279,6 +502,104 @@ impl AttemptService {
|
|||
})
|
||||
}
|
||||
|
||||
pub async fn reconcile_write_intent(
|
||||
&self,
|
||||
claim: &CleanupClaim,
|
||||
objects: Vec<StagingObject>,
|
||||
multiparts: Vec<StagingMultipart>,
|
||||
) -> Result<CleanupCompletion, AttemptError> {
|
||||
let CleanupIdentity::WriteIntent(intent) = &claim.identity else {
|
||||
return Err(AttemptError::Invalid(
|
||||
"cleanup claim is not a staging write intent".into(),
|
||||
));
|
||||
};
|
||||
for object in &objects {
|
||||
if object.target_store != intent.target_store
|
||||
|| object.bucket != intent.bucket
|
||||
|| object.key != intent.key
|
||||
{
|
||||
return Err(AttemptError::Invalid(
|
||||
"discovered object does not match the write intent".into(),
|
||||
));
|
||||
}
|
||||
}
|
||||
for multipart in &multiparts {
|
||||
if multipart.target_store != intent.target_store
|
||||
|| multipart.bucket != intent.bucket
|
||||
|| multipart.key != intent.key
|
||||
|| multipart.upload_id.is_empty()
|
||||
{
|
||||
return Err(AttemptError::Invalid(
|
||||
"discovered multipart does not match the write intent".into(),
|
||||
));
|
||||
}
|
||||
}
|
||||
let cleanup_id = i64::try_from(claim.cleanup_id).map_err(invalid_number)?;
|
||||
let claim_token = claim.claim_token.clone();
|
||||
let attempt_id = claim.attempt_id.clone();
|
||||
with_transaction(&self.pool, move |tx| {
|
||||
Box::pin(async move {
|
||||
let retained = sqlx::query_scalar::<_, i64>(
|
||||
"SELECT cleanup_id FROM attempt_staging_cleanup
|
||||
WHERE cleanup_id = $1 AND attempt_id = $2 AND kind = 'write_intent'
|
||||
AND state = 'claimed' AND claim_token = $3
|
||||
FOR UPDATE",
|
||||
)
|
||||
.bind(cleanup_id)
|
||||
.bind(attempt_id.as_str())
|
||||
.bind(claim_token.as_str())
|
||||
.fetch_optional(tx.as_mut())
|
||||
.await?;
|
||||
if retained.is_none() {
|
||||
return Ok(CleanupCompletion::ClaimLost);
|
||||
}
|
||||
for object in &objects {
|
||||
insert_cleanup_identity_transaction(
|
||||
tx,
|
||||
&attempt_id,
|
||||
NewCleanupIdentity {
|
||||
kind: "object",
|
||||
target_store: &object.target_store,
|
||||
bucket: &object.bucket,
|
||||
key: &object.key,
|
||||
version_id: object.version_id.as_deref(),
|
||||
upload_id: None,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
for multipart in &multiparts {
|
||||
insert_cleanup_identity_transaction(
|
||||
tx,
|
||||
&attempt_id,
|
||||
NewCleanupIdentity {
|
||||
kind: "multipart",
|
||||
target_store: &multipart.target_store,
|
||||
bucket: &multipart.bucket,
|
||||
key: &multipart.key,
|
||||
version_id: None,
|
||||
upload_id: Some(&multipart.upload_id),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
sqlx::query(
|
||||
"UPDATE attempt_staging_cleanup
|
||||
SET state = 'complete', claim_token = NULL, claimed_until = NULL,
|
||||
completed_at = $2, last_error = NULL
|
||||
WHERE cleanup_id = $1",
|
||||
)
|
||||
.bind(cleanup_id)
|
||||
.bind(OffsetDateTime::now_utc())
|
||||
.execute(tx.as_mut())
|
||||
.await?;
|
||||
Ok(CleanupCompletion::Completed)
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
pub async fn retry_cleanup(
|
||||
&self,
|
||||
claim: &CleanupClaim,
|
||||
|
|
@ -499,6 +820,8 @@ impl AttemptService {
|
|||
JOIN attempts a ON a.attempt_id = m.attempt_id
|
||||
JOIN jobs j ON j.job_id = a.job_id
|
||||
WHERE j.authority_attempt_id IS DISTINCT FROM a.attempt_id
|
||||
AND (j.phase IN ('succeeded', 'failed', 'cancelled')
|
||||
OR j.checkpoint_owner_attempt_id IS DISTINCT FROM a.attempt_id)
|
||||
AND COALESCE(j.committed_result ->> 'attempt_id', '') <> a.attempt_id
|
||||
AND (a.state IN ('stopped', 'reaped') OR a.worker_down_at IS NOT NULL)
|
||||
ON CONFLICT DO NOTHING",
|
||||
|
|
@ -517,6 +840,8 @@ impl AttemptService {
|
|||
JOIN jobs j ON j.job_id = a.job_id
|
||||
WHERE r.state = 'sealed'
|
||||
AND j.authority_attempt_id IS DISTINCT FROM a.attempt_id
|
||||
AND (j.phase IN ('succeeded', 'failed', 'cancelled')
|
||||
OR j.checkpoint_owner_attempt_id IS DISTINCT FROM a.attempt_id)
|
||||
AND COALESCE(j.committed_result ->> 'attempt_id', '') <> a.attempt_id
|
||||
AND (a.state IN ('stopped', 'reaped') OR a.worker_down_at IS NOT NULL)
|
||||
ON CONFLICT DO NOTHING",
|
||||
|
|
@ -544,6 +869,199 @@ impl AttemptService {
|
|||
}
|
||||
}
|
||||
|
||||
async fn cooperative_stop_transaction(
|
||||
tx: &mut remora_durable::PgTx<'_>,
|
||||
coordination: &remora_coordination::CoordinationStore,
|
||||
authority: &AttemptAuthority,
|
||||
) -> Result<CooperativeStopOutcome, DurableError> {
|
||||
let row = sqlx::query_as::<_, StopRow>(
|
||||
"SELECT a.job_id, a.fence, a.worker_incarnation, a.state,
|
||||
j.authority_attempt_id, j.cancel_requested
|
||||
FROM attempts a JOIN jobs j ON j.job_id = a.job_id
|
||||
WHERE a.attempt_id = $1
|
||||
FOR UPDATE OF a, j",
|
||||
)
|
||||
.bind(authority.attempt_id.as_str())
|
||||
.fetch_optional(tx.as_mut())
|
||||
.await?;
|
||||
let Some(row) = row else {
|
||||
return Ok(CooperativeStopOutcome::AuthorityMismatch);
|
||||
};
|
||||
if row.job_id != authority.job_id.as_str()
|
||||
|| u64::try_from(row.fence).ok() != Some(authority.fence)
|
||||
|| row.worker_incarnation != authority.worker_incarnation.as_str()
|
||||
{
|
||||
return Ok(CooperativeStopOutcome::AuthorityMismatch);
|
||||
}
|
||||
if row.state == "stopped" {
|
||||
return Ok(CooperativeStopOutcome::Stopped);
|
||||
}
|
||||
if row.state != "executing" {
|
||||
return Ok(CooperativeStopOutcome::StopNotRequired);
|
||||
}
|
||||
let current = row.authority_attempt_id.as_deref() == Some(authority.attempt_id.as_str());
|
||||
let live = coordination
|
||||
.get_live_lease(authority.job_id.as_str())
|
||||
.await
|
||||
.map_err(DurableError::effect)?
|
||||
.is_some_and(|lease| lease.record.owner() == lease_owner(authority));
|
||||
if current && live && !row.cancel_requested {
|
||||
return Ok(CooperativeStopOutcome::StopNotRequired);
|
||||
}
|
||||
let changed = sqlx::query(
|
||||
"UPDATE attempts SET state = 'stopped', stopped_at = $2
|
||||
WHERE attempt_id = $1 AND state IN ('authorized', 'executing')",
|
||||
)
|
||||
.bind(authority.attempt_id.as_str())
|
||||
.bind(OffsetDateTime::now_utc())
|
||||
.execute(tx.as_mut())
|
||||
.await?;
|
||||
Ok(if changed.rows_affected() == 1 {
|
||||
CooperativeStopOutcome::Stopped
|
||||
} else {
|
||||
CooperativeStopOutcome::StopNotRequired
|
||||
})
|
||||
}
|
||||
|
||||
async fn lock_registration_authority(
|
||||
tx: &mut remora_durable::PgTx<'_>,
|
||||
authority: &AttemptAuthority,
|
||||
) -> Result<bool, DurableError> {
|
||||
let row = sqlx::query_as::<_, RegistrationAuthorityRow>(
|
||||
"SELECT a.job_id, a.fence, a.worker_incarnation, a.state,
|
||||
j.authority_attempt_id, j.cancel_requested
|
||||
FROM attempts a JOIN jobs j ON j.job_id = a.job_id
|
||||
WHERE a.attempt_id = $1
|
||||
FOR UPDATE OF a, j",
|
||||
)
|
||||
.bind(authority.attempt_id.as_str())
|
||||
.fetch_optional(tx.as_mut())
|
||||
.await?;
|
||||
Ok(row.is_some_and(|row| {
|
||||
row.job_id == authority.job_id.as_str()
|
||||
&& u64::try_from(row.fence).ok() == Some(authority.fence)
|
||||
&& row.worker_incarnation == authority.worker_incarnation.as_str()
|
||||
&& row.state == "executing"
|
||||
&& row.authority_attempt_id.as_deref() == Some(authority.attempt_id.as_str())
|
||||
&& !row.cancel_requested
|
||||
}))
|
||||
}
|
||||
|
||||
async fn validate_staging_key_transaction(
|
||||
tx: &mut remora_durable::PgTx<'_>,
|
||||
attempt_id: &OpaqueId,
|
||||
key: &str,
|
||||
) -> Result<(), DurableError> {
|
||||
validate_attempt_key(key).map_err(DurableError::effect)?;
|
||||
let (staging_origin, target_prefix) = sqlx::query_as::<_, (String, Json<Vec<String>>)>(
|
||||
"SELECT a.staging_origin_attempt_id, j.target_prefix
|
||||
FROM attempts a JOIN jobs j ON j.job_id = a.job_id
|
||||
WHERE a.attempt_id = $1",
|
||||
)
|
||||
.bind(attempt_id.as_str())
|
||||
.fetch_one(tx.as_mut())
|
||||
.await?;
|
||||
let staging_origin = OpaqueId::parse(staging_origin).map_err(DurableError::effect)?;
|
||||
if attempt_key_attempt_id(key).map_err(DurableError::effect)? != staging_origin
|
||||
|| !key.starts_with(&attempt_staging_prefix(
|
||||
&target_prefix.0.join("/"),
|
||||
&staging_origin,
|
||||
))
|
||||
{
|
||||
return Err(DurableError::effect(
|
||||
"staging key belongs to a different attempt",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn insert_cleanup_identity_transaction(
|
||||
tx: &mut remora_durable::PgTx<'_>,
|
||||
attempt_id: &OpaqueId,
|
||||
identity: NewCleanupIdentity<'_>,
|
||||
) -> Result<(), DurableError> {
|
||||
if identity.target_store.is_empty() || identity.bucket.is_empty() {
|
||||
return Err(DurableError::effect(
|
||||
"staging target store and bucket cannot be empty",
|
||||
));
|
||||
}
|
||||
let changed = sqlx::query(
|
||||
"INSERT INTO attempt_staging_cleanup (
|
||||
attempt_id, kind, target_store, bucket, object_key, version_id, upload_id, created_at
|
||||
)
|
||||
SELECT $1, $2, $3, $4, $5, $6, $7, $8
|
||||
WHERE EXISTS (
|
||||
SELECT 1 FROM attempts a JOIN jobs j ON j.job_id = a.job_id
|
||||
WHERE a.attempt_id = $1 AND j.target_store = $3 AND j.target_bucket = $4
|
||||
)
|
||||
ON CONFLICT DO NOTHING",
|
||||
)
|
||||
.bind(attempt_id.as_str())
|
||||
.bind(identity.kind)
|
||||
.bind(identity.target_store)
|
||||
.bind(identity.bucket)
|
||||
.bind(identity.key)
|
||||
.bind(identity.version_id)
|
||||
.bind(identity.upload_id)
|
||||
.bind(OffsetDateTime::now_utc())
|
||||
.execute(tx.as_mut())
|
||||
.await?;
|
||||
if changed.rows_affected() == 0 {
|
||||
let binding = sqlx::query_as::<_, (String, String)>(
|
||||
"SELECT j.target_store, j.target_bucket
|
||||
FROM attempts a JOIN jobs j ON j.job_id = a.job_id
|
||||
WHERE a.attempt_id = $1",
|
||||
)
|
||||
.bind(attempt_id.as_str())
|
||||
.fetch_optional(tx.as_mut())
|
||||
.await?;
|
||||
match binding {
|
||||
None => return Err(DurableError::effect("attempt does not exist")),
|
||||
Some((store, bucket))
|
||||
if store != identity.target_store || bucket != identity.bucket =>
|
||||
{
|
||||
return Err(DurableError::effect(
|
||||
"staging identity does not match the attempt target",
|
||||
));
|
||||
}
|
||||
Some(_) => {}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn complete_pending_identity(
|
||||
tx: &mut remora_durable::PgTx<'_>,
|
||||
attempt_id: &OpaqueId,
|
||||
kind: &str,
|
||||
target_store: &str,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
upload_id: Option<&str>,
|
||||
) -> Result<(), DurableError> {
|
||||
let changed = sqlx::query(
|
||||
"UPDATE attempt_staging_cleanup
|
||||
SET state = 'complete', completed_at = $7
|
||||
WHERE attempt_id = $1 AND kind = $2 AND target_store = $3 AND bucket = $4
|
||||
AND object_key = $5 AND upload_id IS NOT DISTINCT FROM $6 AND state = 'pending'",
|
||||
)
|
||||
.bind(attempt_id.as_str())
|
||||
.bind(kind)
|
||||
.bind(target_store)
|
||||
.bind(bucket)
|
||||
.bind(key)
|
||||
.bind(upload_id)
|
||||
.bind(OffsetDateTime::now_utc())
|
||||
.execute(tx.as_mut())
|
||||
.await?;
|
||||
if changed.rows_affected() != 1 {
|
||||
return Err(DurableError::effect(
|
||||
"staging write lifecycle identity is missing or unavailable",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn insert_cleanup_identity(
|
||||
pool: &sqlx::PgPool,
|
||||
attempt_id: &OpaqueId,
|
||||
|
|
@ -662,6 +1180,7 @@ async fn reap_lost_attempt(
|
|||
let inactive = match row.state.as_str() {
|
||||
"authorized" => reservation.is_none() && !exact_live,
|
||||
"executing" => row.worker_down_at.is_some() || !exact_live,
|
||||
"stopped" => !exact_live,
|
||||
_ => false,
|
||||
};
|
||||
if !inactive {
|
||||
|
|
@ -718,9 +1237,30 @@ async fn reap_lost_attempt(
|
|||
})
|
||||
}
|
||||
|
||||
fn validate_staging_key(attempt_id: &OpaqueId, key: &str) -> Result<(), AttemptError> {
|
||||
async fn validate_staging_key(
|
||||
pool: &sqlx::PgPool,
|
||||
attempt_id: &OpaqueId,
|
||||
key: &str,
|
||||
) -> Result<(), AttemptError> {
|
||||
validate_attempt_key(key).map_err(|error| AttemptError::Invalid(error.to_string()))?;
|
||||
if key.split('/').nth(2) != Some(attempt_id.as_str()) {
|
||||
let (staging_origin, target_prefix) = sqlx::query_as::<_, (String, Json<Vec<String>>)>(
|
||||
"SELECT a.staging_origin_attempt_id, j.target_prefix
|
||||
FROM attempts a JOIN jobs j ON j.job_id = a.job_id
|
||||
WHERE a.attempt_id = $1",
|
||||
)
|
||||
.bind(attempt_id.as_str())
|
||||
.fetch_optional(pool)
|
||||
.await?
|
||||
.ok_or_else(|| AttemptError::Invalid("attempt does not exist".into()))?;
|
||||
let staging_origin = OpaqueId::parse(staging_origin)
|
||||
.map_err(|error| AttemptError::Invalid(error.to_string()))?;
|
||||
if attempt_key_attempt_id(key).map_err(|error| AttemptError::Invalid(error.to_string()))?
|
||||
!= staging_origin
|
||||
|| !key.starts_with(&attempt_staging_prefix(
|
||||
&target_prefix.0.join("/"),
|
||||
&staging_origin,
|
||||
))
|
||||
{
|
||||
return Err(AttemptError::Invalid(
|
||||
"staging key belongs to a different attempt".into(),
|
||||
));
|
||||
|
|
@ -788,6 +1328,39 @@ struct StopRow {
|
|||
cancel_requested: bool,
|
||||
}
|
||||
|
||||
#[derive(FromRow)]
|
||||
struct RegistrationAuthorityRow {
|
||||
job_id: String,
|
||||
fence: i64,
|
||||
worker_incarnation: String,
|
||||
state: String,
|
||||
authority_attempt_id: Option<String>,
|
||||
cancel_requested: bool,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct StopRequest<'a> {
|
||||
job_id: &'a OpaqueId,
|
||||
attempt_id: &'a OpaqueId,
|
||||
fence: u64,
|
||||
worker_incarnation: &'a OpaqueId,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct RegistrationRequest<'a, T> {
|
||||
job_id: &'a OpaqueId,
|
||||
attempt_id: &'a OpaqueId,
|
||||
fence: u64,
|
||||
worker_incarnation: &'a OpaqueId,
|
||||
value: T,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct StagingObjectRegistration {
|
||||
object: StagingObject,
|
||||
completed_upload_id: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(FromRow)]
|
||||
struct RecoveryJobRow {
|
||||
phase: String,
|
||||
|
|
@ -827,6 +1400,11 @@ struct CleanupRow {
|
|||
impl CleanupRow {
|
||||
fn try_into_claim(self, claim_token: OpaqueId) -> Result<CleanupClaim, AttemptError> {
|
||||
let identity = match self.kind.as_str() {
|
||||
"write_intent" => CleanupIdentity::WriteIntent(StagingWriteIntent {
|
||||
target_store: self.target_store,
|
||||
bucket: self.bucket,
|
||||
key: self.object_key,
|
||||
}),
|
||||
"object" => CleanupIdentity::Object(StagingObject {
|
||||
target_store: self.target_store,
|
||||
bucket: self.bucket,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
use std::time::Duration;
|
||||
|
||||
use remora_domain::OpaqueId;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct RecoveryConfig {
|
||||
|
|
@ -27,7 +28,8 @@ pub enum CancelOutcome {
|
|||
NotFound,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum CooperativeStopOutcome {
|
||||
Stopped,
|
||||
StopNotRequired,
|
||||
|
|
@ -59,7 +61,7 @@ impl ReconcileReport {
|
|||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Serialize)]
|
||||
pub struct StagingObject {
|
||||
pub target_store: String,
|
||||
pub bucket: String,
|
||||
|
|
@ -67,7 +69,7 @@ pub struct StagingObject {
|
|||
pub version_id: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Serialize)]
|
||||
pub struct StagingMultipart {
|
||||
pub target_store: String,
|
||||
pub bucket: String,
|
||||
|
|
@ -75,8 +77,16 @@ pub struct StagingMultipart {
|
|||
pub upload_id: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Serialize)]
|
||||
pub struct StagingWriteIntent {
|
||||
pub target_store: String,
|
||||
pub bucket: String,
|
||||
pub key: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub enum CleanupIdentity {
|
||||
WriteIntent(StagingWriteIntent),
|
||||
Object(StagingObject),
|
||||
Multipart(StagingMultipart),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -559,7 +559,7 @@ impl AttemptService {
|
|||
self.local_permits.release(job_id.as_str());
|
||||
}
|
||||
|
||||
async fn release_exact(&self, authority: &AttemptAuthority) {
|
||||
pub(crate) async fn release_exact(&self, authority: &AttemptAuthority) {
|
||||
if let Ok(owner) = lease_owner(authority) {
|
||||
let _ = self
|
||||
.coordination
|
||||
|
|
@ -662,16 +662,25 @@ async fn authorize_attempt(
|
|||
Option<String>,
|
||||
Option<Json<crate::DurableDescriptor>>,
|
||||
Option<Json<CheckpointProjection>>,
|
||||
Option<String>,
|
||||
),
|
||||
>(
|
||||
"SELECT phase, dispatch_needed, current_fence, authority_attempt_id,
|
||||
resolved_source, checkpoint
|
||||
resolved_source, checkpoint, checkpoint_owner_attempt_id
|
||||
FROM jobs WHERE job_id = $1 FOR UPDATE",
|
||||
)
|
||||
.bind(input.job_id.as_str())
|
||||
.fetch_optional(tx.as_mut())
|
||||
.await?;
|
||||
let Some((phase, dispatch_needed, current_fence, authority, resolved_source, checkpoint)) = row
|
||||
let Some((
|
||||
phase,
|
||||
dispatch_needed,
|
||||
current_fence,
|
||||
authority,
|
||||
resolved_source,
|
||||
checkpoint,
|
||||
checkpoint_owner,
|
||||
)) = row
|
||||
else {
|
||||
return Ok(claim_rejected(ClaimRejection::JobUnavailable));
|
||||
};
|
||||
|
|
@ -682,6 +691,26 @@ async fn authorize_attempt(
|
|||
if current_fence >= input.max_attempts {
|
||||
return Ok(claim_rejected(ClaimRejection::AttemptsExhausted));
|
||||
}
|
||||
let reusable = resolved_source
|
||||
.as_ref()
|
||||
.zip(checkpoint.as_ref())
|
||||
.and_then(|(source, checkpoint)| checkpoint.0.reusable().map(|value| (source, value)))
|
||||
.filter(|(source, checkpoint)| checkpoint.resolved_source_id == source.0.id);
|
||||
if let (Some(_), Some(owner)) = (reusable, checkpoint_owner.as_deref()) {
|
||||
let stopped = sqlx::query_scalar::<_, bool>(
|
||||
"SELECT state = 'stopped'
|
||||
FROM attempts
|
||||
WHERE attempt_id = $1 AND job_id = $2",
|
||||
)
|
||||
.bind(owner)
|
||||
.bind(input.job_id.as_str())
|
||||
.fetch_optional(tx.as_mut())
|
||||
.await?
|
||||
.unwrap_or(false);
|
||||
if !stopped {
|
||||
return Ok(claim_rejected(ClaimRejection::JobUnavailable));
|
||||
}
|
||||
}
|
||||
input.faults.hit(AttemptFaultPoint::ClaimBeforeReserve);
|
||||
let reservation = input
|
||||
.coordination
|
||||
|
|
@ -697,26 +726,47 @@ async fn authorize_attempt(
|
|||
input.reserved.store(true, Ordering::Release);
|
||||
input.faults.hit(AttemptFaultPoint::ClaimAfterReserve);
|
||||
let fence = current_fence + 1;
|
||||
let initial_progress = match (resolved_source, checkpoint) {
|
||||
(Some(source), Some(checkpoint))
|
||||
if checkpoint
|
||||
.0
|
||||
.reusable()
|
||||
.is_some_and(|value| value.resolved_source_id == source.0.id) =>
|
||||
{
|
||||
checkpoint.0.reusable().unwrap().validated_bytes
|
||||
let checkpoint_stage = match (reusable, checkpoint_owner.as_deref()) {
|
||||
(Some((_, checkpoint)), Some(owner)) if checkpoint.validated_bytes > 0 => {
|
||||
load_checkpoint_stage(tx, input.job_id, owner, checkpoint).await?
|
||||
}
|
||||
_ => 0,
|
||||
_ => None,
|
||||
};
|
||||
if checkpoint.is_some() && checkpoint_stage.is_none() {
|
||||
sqlx::query(
|
||||
"UPDATE jobs
|
||||
SET checkpoint = NULL, checkpoint_owner_attempt_id = NULL,
|
||||
revision = revision + 1, updated_at = $2
|
||||
WHERE job_id = $1",
|
||||
)
|
||||
.bind(input.job_id.as_str())
|
||||
.bind(OffsetDateTime::now_utc())
|
||||
.execute(tx.as_mut())
|
||||
.await?;
|
||||
}
|
||||
let initial_progress = checkpoint_stage
|
||||
.as_ref()
|
||||
.map_or(0, |stage| stage.validated_bytes);
|
||||
let staging_origin = checkpoint_stage
|
||||
.as_ref()
|
||||
.map_or(input.attempt_id, |stage| &stage.staging_origin_attempt_id);
|
||||
let checkpoint_owner_after_claim = checkpoint_stage.as_ref().map(|_| input.attempt_id.as_str());
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let envelope =
|
||||
load_execution_envelope_record(tx, input.job_id, input.attempt_id, fence, input.worker)
|
||||
.await?;
|
||||
let envelope = load_execution_envelope_record(
|
||||
tx,
|
||||
input.job_id,
|
||||
input.attempt_id,
|
||||
fence,
|
||||
input.worker,
|
||||
staging_origin,
|
||||
)
|
||||
.await?;
|
||||
sqlx::query(
|
||||
"INSERT INTO attempts (
|
||||
attempt_id, job_id, fence, worker_incarnation, state,
|
||||
initial_progress, authorized_at, executing_at, reaped_at, execution_envelope
|
||||
) VALUES ($1, $2, $3, $4, 'authorized', $5, $6, NULL, NULL, $7)",
|
||||
initial_progress, authorized_at, executing_at, reaped_at, execution_envelope,
|
||||
staging_origin_attempt_id
|
||||
) VALUES ($1, $2, $3, $4, 'authorized', $5, $6, NULL, NULL, $7, $8)",
|
||||
)
|
||||
.bind(input.attempt_id.as_str())
|
||||
.bind(input.job_id.as_str())
|
||||
|
|
@ -725,18 +775,24 @@ async fn authorize_attempt(
|
|||
.bind(i64::try_from(initial_progress).map_err(DurableError::effect)?)
|
||||
.bind(now)
|
||||
.bind(Json(&envelope))
|
||||
.bind(staging_origin.as_str())
|
||||
.execute(tx.as_mut())
|
||||
.await?;
|
||||
if let Some(stage) = checkpoint_stage {
|
||||
transfer_checkpoint_stage(tx, &stage, input.attempt_id).await?;
|
||||
}
|
||||
sqlx::query(
|
||||
"UPDATE jobs
|
||||
SET phase = 'running', current_fence = $2, authority_attempt_id = $3,
|
||||
dispatch_needed = FALSE, revision = revision + 1, updated_at = $4
|
||||
checkpoint_owner_attempt_id = $5, dispatch_needed = FALSE,
|
||||
revision = revision + 1, updated_at = $4
|
||||
WHERE job_id = $1",
|
||||
)
|
||||
.bind(input.job_id.as_str())
|
||||
.bind(i64::try_from(fence).map_err(DurableError::effect)?)
|
||||
.bind(input.attempt_id.as_str())
|
||||
.bind(now)
|
||||
.bind(checkpoint_owner_after_claim)
|
||||
.execute(tx.as_mut())
|
||||
.await?;
|
||||
Ok(ClaimOutcome::Claimed {
|
||||
|
|
@ -751,12 +807,277 @@ async fn authorize_attempt(
|
|||
})
|
||||
}
|
||||
|
||||
struct CheckpointStage {
|
||||
owner_attempt_id: OpaqueId,
|
||||
staging_origin_attempt_id: OpaqueId,
|
||||
validated_bytes: u64,
|
||||
validated_members: u64,
|
||||
inventory_bytes: u64,
|
||||
staging_multipart: Option<crate::CheckpointMultipart>,
|
||||
}
|
||||
|
||||
async fn load_checkpoint_stage(
|
||||
tx: &mut remora_durable::PgTx<'_>,
|
||||
job_id: &OpaqueId,
|
||||
owner: &str,
|
||||
checkpoint: &crate::CheckpointRecord,
|
||||
) -> Result<Option<CheckpointStage>, DurableError> {
|
||||
let Some(staging_origin) = sqlx::query_scalar::<_, String>(
|
||||
"SELECT staging_origin_attempt_id
|
||||
FROM attempts
|
||||
WHERE attempt_id = $1 AND job_id = $2 AND state = 'stopped'",
|
||||
)
|
||||
.bind(owner)
|
||||
.bind(job_id.as_str())
|
||||
.fetch_optional(tx.as_mut())
|
||||
.await?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Some((state, member_count, inventory_origin, inventory_prefix, target_prefix)) =
|
||||
sqlx::query_as::<
|
||||
_,
|
||||
(
|
||||
String,
|
||||
i64,
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
Json<Vec<String>>,
|
||||
),
|
||||
>(
|
||||
"SELECT i.state, i.member_count, i.staging_origin_attempt_id,
|
||||
i.target_prefix, j.target_prefix
|
||||
FROM result_inventories i
|
||||
JOIN attempts a ON a.attempt_id = i.attempt_id
|
||||
JOIN jobs j ON j.job_id = a.job_id
|
||||
WHERE i.attempt_id = $1
|
||||
FOR UPDATE OF i",
|
||||
)
|
||||
.bind(owner)
|
||||
.fetch_optional(tx.as_mut())
|
||||
.await?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
if inventory_origin.as_deref() != Some(staging_origin.as_str())
|
||||
|| inventory_prefix.as_deref() != Some(target_prefix.0.join("/").as_str())
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
let validated_members =
|
||||
i64::try_from(checkpoint.validated_members).map_err(DurableError::effect)?;
|
||||
let validated_bytes =
|
||||
i64::try_from(checkpoint.validated_bytes).map_err(DurableError::effect)?;
|
||||
if state != "open" || member_count < validated_members {
|
||||
return Ok(None);
|
||||
}
|
||||
let (retained_members, retained_bytes) = sqlx::query_as::<_, (i64, i64)>(
|
||||
"SELECT COUNT(*), COALESCE(SUM(size), 0)::BIGINT
|
||||
FROM (
|
||||
SELECT size
|
||||
FROM result_inventory_members
|
||||
WHERE attempt_id = $1
|
||||
ORDER BY path COLLATE \"C\", member_id COLLATE \"C\"
|
||||
LIMIT $2
|
||||
) retained",
|
||||
)
|
||||
.bind(owner)
|
||||
.bind(validated_members)
|
||||
.fetch_one(tx.as_mut())
|
||||
.await?;
|
||||
if retained_members != validated_members || retained_bytes > validated_bytes {
|
||||
return Ok(None);
|
||||
}
|
||||
let partial = retained_bytes < validated_bytes;
|
||||
if partial != checkpoint.staging_multipart.is_some() {
|
||||
return Ok(None);
|
||||
}
|
||||
let cleanup_started = sqlx::query_scalar::<_, bool>(
|
||||
"WITH retained AS (
|
||||
SELECT object_store, object_bucket, object_key, object_version_id
|
||||
FROM result_inventory_members
|
||||
WHERE attempt_id = $1
|
||||
ORDER BY path COLLATE \"C\", member_id COLLATE \"C\"
|
||||
LIMIT $2
|
||||
)
|
||||
SELECT EXISTS (
|
||||
SELECT 1
|
||||
FROM attempt_staging_cleanup c
|
||||
JOIN retained r
|
||||
ON r.object_store = c.target_store
|
||||
AND r.object_bucket = c.bucket
|
||||
AND r.object_key = c.object_key
|
||||
AND r.object_version_id IS NOT DISTINCT FROM c.version_id
|
||||
WHERE c.attempt_id = $1
|
||||
AND c.kind = 'object'
|
||||
AND c.state <> 'pending'
|
||||
)",
|
||||
)
|
||||
.bind(owner)
|
||||
.bind(validated_members)
|
||||
.fetch_one(tx.as_mut())
|
||||
.await?;
|
||||
if cleanup_started {
|
||||
return Ok(None);
|
||||
}
|
||||
if let Some(multipart) = &checkpoint.staging_multipart {
|
||||
let available = sqlx::query_scalar::<_, bool>(
|
||||
"SELECT EXISTS (
|
||||
SELECT 1 FROM attempt_staging_cleanup
|
||||
WHERE attempt_id = $1 AND kind = 'multipart'
|
||||
AND target_store = $2 AND bucket = $3
|
||||
AND object_key = $4 AND upload_id = $5
|
||||
AND state = 'pending'
|
||||
)",
|
||||
)
|
||||
.bind(owner)
|
||||
.bind(&multipart.target_store)
|
||||
.bind(&multipart.bucket)
|
||||
.bind(&multipart.key)
|
||||
.bind(&multipart.upload_id)
|
||||
.fetch_one(tx.as_mut())
|
||||
.await?;
|
||||
if !available {
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
Ok(Some(CheckpointStage {
|
||||
owner_attempt_id: OpaqueId::parse(owner).map_err(DurableError::effect)?,
|
||||
staging_origin_attempt_id: OpaqueId::parse(staging_origin).map_err(DurableError::effect)?,
|
||||
validated_bytes: checkpoint.validated_bytes,
|
||||
validated_members: checkpoint.validated_members,
|
||||
inventory_bytes: u64::try_from(retained_bytes).map_err(DurableError::effect)?,
|
||||
staging_multipart: checkpoint.staging_multipart.clone(),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn transfer_checkpoint_stage(
|
||||
tx: &mut remora_durable::PgTx<'_>,
|
||||
stage: &CheckpointStage,
|
||||
replacement: &OpaqueId,
|
||||
) -> Result<(), DurableError> {
|
||||
let retained_members = i64::try_from(stage.validated_members).map_err(DurableError::effect)?;
|
||||
let retained_bytes = i64::try_from(stage.inventory_bytes).map_err(DurableError::effect)?;
|
||||
let now = OffsetDateTime::now_utc();
|
||||
sqlx::query(
|
||||
"WITH ranked AS (
|
||||
SELECT *, ROW_NUMBER() OVER (
|
||||
ORDER BY path COLLATE \"C\", member_id COLLATE \"C\"
|
||||
) AS ordinal
|
||||
FROM result_inventory_members
|
||||
WHERE attempt_id = $1
|
||||
)
|
||||
INSERT INTO attempt_staging_cleanup (
|
||||
attempt_id, kind, target_store, bucket, object_key, version_id, created_at
|
||||
)
|
||||
SELECT $1, 'object', object_store, object_bucket, object_key,
|
||||
object_version_id, $3
|
||||
FROM ranked
|
||||
WHERE ordinal > $2
|
||||
ON CONFLICT DO NOTHING",
|
||||
)
|
||||
.bind(stage.owner_attempt_id.as_str())
|
||||
.bind(retained_members)
|
||||
.bind(now)
|
||||
.execute(tx.as_mut())
|
||||
.await?;
|
||||
sqlx::query(
|
||||
"WITH retained AS (
|
||||
SELECT object_store, object_bucket, object_key, object_version_id
|
||||
FROM result_inventory_members
|
||||
WHERE attempt_id = $1
|
||||
ORDER BY path COLLATE \"C\", member_id COLLATE \"C\"
|
||||
LIMIT $3
|
||||
)
|
||||
UPDATE attempt_staging_cleanup c
|
||||
SET attempt_id = $2
|
||||
FROM retained r
|
||||
WHERE c.attempt_id = $1
|
||||
AND c.kind = 'object'
|
||||
AND c.state = 'pending'
|
||||
AND r.object_store = c.target_store
|
||||
AND r.object_bucket = c.bucket
|
||||
AND r.object_key = c.object_key
|
||||
AND r.object_version_id IS NOT DISTINCT FROM c.version_id",
|
||||
)
|
||||
.bind(stage.owner_attempt_id.as_str())
|
||||
.bind(replacement.as_str())
|
||||
.bind(retained_members)
|
||||
.execute(tx.as_mut())
|
||||
.await?;
|
||||
if let Some(multipart) = &stage.staging_multipart {
|
||||
let changed = sqlx::query(
|
||||
"UPDATE attempt_staging_cleanup
|
||||
SET attempt_id = $2
|
||||
WHERE attempt_id = $1 AND state = 'pending'
|
||||
AND (
|
||||
(kind = 'multipart' AND target_store = $3 AND bucket = $4
|
||||
AND object_key = $5 AND upload_id = $6)
|
||||
OR (kind = 'write_intent' AND target_store = $3 AND bucket = $4
|
||||
AND object_key = $5)
|
||||
)",
|
||||
)
|
||||
.bind(stage.owner_attempt_id.as_str())
|
||||
.bind(replacement.as_str())
|
||||
.bind(&multipart.target_store)
|
||||
.bind(&multipart.bucket)
|
||||
.bind(&multipart.key)
|
||||
.bind(&multipart.upload_id)
|
||||
.execute(tx.as_mut())
|
||||
.await?;
|
||||
if changed.rows_affected() == 0 {
|
||||
return Err(DurableError::effect(
|
||||
"checkpoint multipart ownership transfer failed",
|
||||
));
|
||||
}
|
||||
}
|
||||
sqlx::query(
|
||||
"WITH extras AS (
|
||||
SELECT attempt_id, member_id
|
||||
FROM (
|
||||
SELECT attempt_id, member_id, ROW_NUMBER() OVER (
|
||||
ORDER BY path COLLATE \"C\", member_id COLLATE \"C\"
|
||||
) AS ordinal
|
||||
FROM result_inventory_members
|
||||
WHERE attempt_id = $1
|
||||
) ranked
|
||||
WHERE ordinal > $2
|
||||
)
|
||||
DELETE FROM result_inventory_members m
|
||||
USING extras
|
||||
WHERE m.attempt_id = extras.attempt_id
|
||||
AND m.member_id = extras.member_id",
|
||||
)
|
||||
.bind(stage.owner_attempt_id.as_str())
|
||||
.bind(retained_members)
|
||||
.execute(tx.as_mut())
|
||||
.await?;
|
||||
let changed = sqlx::query(
|
||||
"UPDATE result_inventories
|
||||
SET attempt_id = $2, member_count = $3, total_bytes = $4
|
||||
WHERE attempt_id = $1 AND state = 'open'",
|
||||
)
|
||||
.bind(stage.owner_attempt_id.as_str())
|
||||
.bind(replacement.as_str())
|
||||
.bind(retained_members)
|
||||
.bind(retained_bytes)
|
||||
.execute(tx.as_mut())
|
||||
.await?;
|
||||
if changed.rows_affected() != 1 {
|
||||
return Err(DurableError::effect(
|
||||
"checkpoint staging inventory transfer failed",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn load_execution_envelope_record(
|
||||
tx: &mut remora_durable::PgTx<'_>,
|
||||
job_id: &OpaqueId,
|
||||
attempt_id: &OpaqueId,
|
||||
fence: u64,
|
||||
worker: &OpaqueId,
|
||||
staging_origin_attempt_id: &OpaqueId,
|
||||
) -> Result<ExecutionEnvelopeRecord, DurableError> {
|
||||
type Row = (
|
||||
Json<serde_json::Value>,
|
||||
|
|
@ -804,6 +1125,7 @@ async fn load_execution_envelope_record(
|
|||
target_store: row.2,
|
||||
target_bucket: row.3,
|
||||
target_prefix: row.4.0,
|
||||
staging_origin_attempt_id: staging_origin_attempt_id.clone(),
|
||||
execution_grant_id: grant_id,
|
||||
checkpoint: row.9.map(|value| value.0),
|
||||
})
|
||||
|
|
|
|||
|
|
@ -157,6 +157,7 @@ pub struct ExecutionEnvelopeRecord {
|
|||
pub target_store: String,
|
||||
pub target_bucket: String,
|
||||
pub target_prefix: Vec<String>,
|
||||
pub staging_origin_attempt_id: OpaqueId,
|
||||
pub execution_grant_id: OpaqueId,
|
||||
pub checkpoint: Option<crate::CheckpointProjection>,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ use remora_domain::{
|
|||
use remora_durable::{MutationDisposition, PoolConfig, connect, migrate as durable_migrate};
|
||||
use remora_identity::{DefaultTarget, Principal};
|
||||
use remora_result::{PublicationInput, migrate as result_migrate};
|
||||
use remora_target::{ContentSha256, ObjectIdentity, StoredObject, attempt_manifest_key};
|
||||
use remora_target::{ContentSha256, ObjectIdentity, StoredObject, attempt_manifest_key_in};
|
||||
use sqlx::PgPool;
|
||||
|
||||
struct Harness {
|
||||
|
|
@ -168,7 +168,7 @@ impl Harness {
|
|||
) -> PublicationInput {
|
||||
let manifest_id = unique_id("manifest");
|
||||
let sha256 = format!("sha256:{}", "11".repeat(32));
|
||||
let key = attempt_manifest_key(attempt_id, &manifest_id);
|
||||
let key = attempt_manifest_key_in("users/alice", attempt_id, &manifest_id);
|
||||
let input = PublicationInput {
|
||||
attempt_id: attempt_id.clone(),
|
||||
manifest_id: manifest_id.clone(),
|
||||
|
|
@ -190,9 +190,10 @@ impl Harness {
|
|||
"INSERT INTO result_inventories (
|
||||
attempt_id, state, manifest_schema_version, manifest_id, manifest_store,
|
||||
manifest_bucket, manifest_key, manifest_version_id, manifest_sha256,
|
||||
manifest_size, member_count, total_bytes
|
||||
manifest_size, member_count, total_bytes,
|
||||
staging_origin_attempt_id, target_prefix
|
||||
) VALUES ($1, 'sealed', 1, $2, 'primary', 'remora-results', $3,
|
||||
'version-1', $4, 128, $5, $6)",
|
||||
'version-1', $4, 128, $5, $6, $1, 'users/alice')",
|
||||
)
|
||||
.bind(attempt_id.as_str())
|
||||
.bind(manifest_id.as_str())
|
||||
|
|
@ -283,6 +284,7 @@ async fn resolved_source_checkpoint_and_progress_are_fenced_and_replayable() {
|
|||
resolved_source_id: resolved.id.clone(),
|
||||
validated_bytes: 0,
|
||||
validated_members: 0,
|
||||
staging_multipart: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
|
|
@ -295,6 +297,7 @@ async fn resolved_source_checkpoint_and_progress_are_fenced_and_replayable() {
|
|||
resolved_source_id: resolved.id.clone(),
|
||||
validated_bytes: 1024,
|
||||
validated_members: 1,
|
||||
staging_multipart: None,
|
||||
};
|
||||
let mut wrong_source = checkpoint.clone();
|
||||
wrong_source.resolved_source_id = unique_id("resolved");
|
||||
|
|
@ -539,6 +542,7 @@ async fn failure_disposition_retries_with_budget_and_becomes_terminal_when_exhau
|
|||
resolved_source_id: retry_source.id.clone(),
|
||||
validated_bytes: 512,
|
||||
validated_members: 1,
|
||||
staging_multipart: None,
|
||||
};
|
||||
assert_rejected(
|
||||
service
|
||||
|
|
@ -555,6 +559,40 @@ async fn failure_disposition_retries_with_budget_and_becomes_terminal_when_exhau
|
|||
.persist_checkpoint(&first, &unique_id("checkpointop"), retry_checkpoint)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO result_inventories (
|
||||
attempt_id, staging_origin_attempt_id, target_prefix
|
||||
) VALUES ($1, $1, 'users/alice')",
|
||||
)
|
||||
.bind(first.attempt_id.as_str())
|
||||
.execute(&harness.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO result_inventory_members (
|
||||
attempt_id, member_id, path, size, object_store, object_bucket,
|
||||
object_key, object_sha256
|
||||
) VALUES ($1, $2, 'retry.bin', 512, 'target', 'bucket', $3, $4)",
|
||||
)
|
||||
.bind(first.attempt_id.as_str())
|
||||
.bind(unique_id("member").as_str())
|
||||
.bind(format!(
|
||||
"users/alice/_remora/a/{}/o/retry/retry.bin",
|
||||
first.attempt_id.as_str()
|
||||
))
|
||||
.bind(format!("sha256:{}", "0".repeat(64)))
|
||||
.execute(&harness.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"UPDATE result_inventories
|
||||
SET member_count = 1, total_bytes = 512
|
||||
WHERE attempt_id = $1",
|
||||
)
|
||||
.bind(first.attempt_id.as_str())
|
||||
.execute(&harness.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
service
|
||||
.publish_failure(
|
||||
|
|
@ -610,6 +648,20 @@ async fn failure_disposition_retries_with_budget_and_becomes_terminal_when_exhau
|
|||
.await
|
||||
.unwrap();
|
||||
assert_eq!(initial_progress, 512);
|
||||
let staging_origin: String =
|
||||
sqlx::query_scalar("SELECT staging_origin_attempt_id FROM attempts WHERE attempt_id = $1")
|
||||
.bind(second.attempt_id.as_str())
|
||||
.fetch_one(&harness.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(staging_origin, first.attempt_id.as_str());
|
||||
let inventory_owner: String =
|
||||
sqlx::query_scalar("SELECT attempt_id FROM result_inventories WHERE attempt_id = $1")
|
||||
.bind(second.attempt_id.as_str())
|
||||
.fetch_one(&harness.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(inventory_owner, second.attempt_id.as_str());
|
||||
service
|
||||
.publish_failure(
|
||||
&second,
|
||||
|
|
@ -760,7 +812,7 @@ async fn publication_from_database(pool: &PgPool, attempt_id: &OpaqueId) -> Publ
|
|||
identity: ObjectIdentity {
|
||||
target_store_id: row.1,
|
||||
bucket: row.2,
|
||||
key: attempt_manifest_key(attempt_id, &manifest_id),
|
||||
key: attempt_manifest_key_in("users/alice", attempt_id, &manifest_id),
|
||||
version_id: row.3,
|
||||
},
|
||||
content_sha256: ContentSha256::parse(row.4).unwrap(),
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ use remora_admission::{
|
|||
use remora_attempt::{
|
||||
ActivationOutcome, AttemptConfig, AttemptService, CancelOutcome, CleanupCompletion,
|
||||
CleanupIdentity, CooperativeStopOutcome, RecoveryConfig, StagingMultipart, StagingObject,
|
||||
migrate as attempt_migrate,
|
||||
StagingWriteIntent, migrate as attempt_migrate,
|
||||
};
|
||||
use remora_contracts::{HttpSourceSubmissionDto, JobSubmissionDto, SourceSubmissionDto};
|
||||
use remora_coordination::{CoordinationStore, StoreConfig};
|
||||
|
|
@ -21,7 +21,7 @@ use remora_identity::{DefaultTarget, Principal};
|
|||
use remora_result::{PublicationInput, migrate as result_migrate};
|
||||
use remora_target::{
|
||||
CreateObjectRequest, MultipartIdentity, ObjectIdentity, StoredObject, TargetStore,
|
||||
TargetStoreConfig, attempt_manifest_key, attempt_object_key,
|
||||
TargetStoreConfig, attempt_manifest_key_in, attempt_object_key_in,
|
||||
};
|
||||
use sqlx::PgPool;
|
||||
use tokio::io::copy_bidirectional;
|
||||
|
|
@ -264,8 +264,12 @@ impl Harness {
|
|||
"INSERT INTO result_inventories (
|
||||
attempt_id, state, manifest_schema_version, manifest_id, manifest_store,
|
||||
manifest_bucket, manifest_key, manifest_version_id, manifest_sha256,
|
||||
manifest_size, member_count, total_bytes
|
||||
) VALUES ($1, 'sealed', 1, $2, $3, $4, $5, $6, $7, $8, 0, 0)",
|
||||
manifest_size, member_count, total_bytes,
|
||||
staging_origin_attempt_id, target_prefix
|
||||
) VALUES (
|
||||
$1, 'sealed', 1, $2, $3, $4, $5, $6, $7, $8, 0, 0,
|
||||
$1, 'users/recovery'
|
||||
)",
|
||||
)
|
||||
.bind(authority.attempt_id.as_str())
|
||||
.bind(manifest_id.as_str())
|
||||
|
|
@ -765,7 +769,12 @@ async fn cleanup_uses_exact_durable_identities_retries_and_excludes_the_winner()
|
|||
|
||||
let loser_job = harness.job().await;
|
||||
let loser = harness.executing(&service, &loser_job).await;
|
||||
let loser_key = attempt_object_key(&loser.attempt_id, &unique_id("object"), None);
|
||||
let loser_key = attempt_object_key_in(
|
||||
"users/recovery",
|
||||
&loser.attempt_id,
|
||||
&unique_id("object"),
|
||||
None,
|
||||
);
|
||||
let loser_object = harness
|
||||
.target
|
||||
.put_create_only(
|
||||
|
|
@ -800,7 +809,12 @@ async fn cleanup_uses_exact_durable_identities_retries_and_excludes_the_winner()
|
|||
.await
|
||||
.is_err()
|
||||
);
|
||||
let multipart_key = attempt_object_key(&loser.attempt_id, &unique_id("multipart"), None);
|
||||
let multipart_key = attempt_object_key_in(
|
||||
"users/recovery",
|
||||
&loser.attempt_id,
|
||||
&unique_id("multipart"),
|
||||
None,
|
||||
);
|
||||
let multipart = harness
|
||||
.target
|
||||
.begin_multipart(&CreateObjectRequest::parse(multipart_key.clone()).unwrap())
|
||||
|
|
@ -830,7 +844,8 @@ async fn cleanup_uses_exact_durable_identities_retries_and_excludes_the_winner()
|
|||
|
||||
let winner_job = harness.job().await;
|
||||
let winner = harness.executing(&service, &winner_job).await;
|
||||
let winner_key = attempt_manifest_key(&winner.attempt_id, &unique_id("manifest"));
|
||||
let winner_key =
|
||||
attempt_manifest_key_in("users/recovery", &winner.attempt_id, &unique_id("manifest"));
|
||||
let winner_object = harness
|
||||
.target
|
||||
.put_create_only(
|
||||
|
|
@ -935,7 +950,12 @@ async fn cleanup_uses_exact_durable_identities_retries_and_excludes_the_winner()
|
|||
&StagingObject {
|
||||
target_store: harness.target_config.target_store_id.clone(),
|
||||
bucket: harness.target_config.bucket.clone(),
|
||||
key: attempt_object_key(&winner.attempt_id, &unique_id("wrong"), None),
|
||||
key: attempt_object_key_in(
|
||||
"users/recovery",
|
||||
&winner.attempt_id,
|
||||
&unique_id("wrong"),
|
||||
None,
|
||||
),
|
||||
version_id: None,
|
||||
},
|
||||
)
|
||||
|
|
@ -944,11 +964,201 @@ async fn cleanup_uses_exact_durable_identities_retries_and_excludes_the_winner()
|
|||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cleanup_reconciles_accepted_writes_from_durable_key_intents() {
|
||||
let Some(harness) = Harness::new("write-intent").await else {
|
||||
return;
|
||||
};
|
||||
let service = harness.service(1);
|
||||
let config = recovery_config();
|
||||
|
||||
let multipart_job = harness.job().await;
|
||||
let multipart_attempt = harness.executing(&service, &multipart_job).await;
|
||||
let multipart_key = attempt_object_key_in(
|
||||
"users/recovery",
|
||||
&multipart_attempt.attempt_id,
|
||||
&unique_id("multipart"),
|
||||
None,
|
||||
);
|
||||
service
|
||||
.register_staging_write_intent_operation(
|
||||
&multipart_attempt,
|
||||
&unique_id("intent"),
|
||||
StagingWriteIntent {
|
||||
target_store: harness.target_config.target_store_id.clone(),
|
||||
bucket: harness.target_config.bucket.clone(),
|
||||
key: multipart_key.clone(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let accepted_multipart = harness
|
||||
.target
|
||||
.begin_multipart(&CreateObjectRequest::parse(multipart_key.clone()).unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
let accepted_upload_id = accepted_multipart.identity().upload_id.clone();
|
||||
service
|
||||
.publish_failure(
|
||||
&multipart_attempt,
|
||||
&unique_id("failure"),
|
||||
RuntimeFailureClass::Authorization,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let intent_claim = service.claim_cleanup(&config).await.unwrap().unwrap();
|
||||
assert!(matches!(
|
||||
intent_claim.identity,
|
||||
CleanupIdentity::WriteIntent(_)
|
||||
));
|
||||
let discovered = harness
|
||||
.target
|
||||
.discover_staging_write(&multipart_key)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(discovered.objects.is_empty());
|
||||
assert_eq!(discovered.multiparts.len(), 1);
|
||||
assert_eq!(discovered.multiparts[0].upload_id, accepted_upload_id);
|
||||
assert_eq!(
|
||||
service
|
||||
.reconcile_write_intent(
|
||||
&intent_claim,
|
||||
Vec::new(),
|
||||
discovered
|
||||
.multiparts
|
||||
.into_iter()
|
||||
.map(|identity| StagingMultipart {
|
||||
target_store: identity.target_store_id,
|
||||
bucket: identity.bucket,
|
||||
key: identity.key,
|
||||
upload_id: identity.upload_id,
|
||||
})
|
||||
.collect(),
|
||||
)
|
||||
.await
|
||||
.unwrap(),
|
||||
CleanupCompletion::Completed
|
||||
);
|
||||
let multipart_claim = service.claim_cleanup(&config).await.unwrap().unwrap();
|
||||
assert!(matches!(
|
||||
multipart_claim.identity,
|
||||
CleanupIdentity::Multipart(_)
|
||||
));
|
||||
execute_cleanup(&harness.target, &multipart_claim)
|
||||
.await
|
||||
.unwrap();
|
||||
service.complete_cleanup(&multipart_claim).await.unwrap();
|
||||
assert!(
|
||||
harness
|
||||
.target
|
||||
.discover_staging_write(&multipart_key)
|
||||
.await
|
||||
.unwrap()
|
||||
.multiparts
|
||||
.is_empty()
|
||||
);
|
||||
|
||||
let object_job = harness.job().await;
|
||||
let object_attempt = harness.executing(&service, &object_job).await;
|
||||
let object_key = attempt_object_key_in(
|
||||
"users/recovery",
|
||||
&object_attempt.attempt_id,
|
||||
&unique_id("object"),
|
||||
None,
|
||||
);
|
||||
service
|
||||
.register_staging_write_intent_operation(
|
||||
&object_attempt,
|
||||
&unique_id("intent"),
|
||||
StagingWriteIntent {
|
||||
target_store: harness.target_config.target_store_id.clone(),
|
||||
bucket: harness.target_config.bucket.clone(),
|
||||
key: object_key.clone(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let accepted_object = harness
|
||||
.target
|
||||
.put_create_only(
|
||||
&CreateObjectRequest::parse(object_key.clone()).unwrap(),
|
||||
b"accepted-before-ledger".as_slice(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
service
|
||||
.publish_failure(
|
||||
&object_attempt,
|
||||
&unique_id("failure"),
|
||||
RuntimeFailureClass::Authorization,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let intent_claim = service.claim_cleanup(&config).await.unwrap().unwrap();
|
||||
let discovered = harness
|
||||
.target
|
||||
.discover_staging_write(&object_key)
|
||||
.await
|
||||
.unwrap();
|
||||
let [discovered_object] = discovered.objects.as_slice() else {
|
||||
panic!("expected one accepted object");
|
||||
};
|
||||
assert_eq!(
|
||||
discovered_object.target_store_id,
|
||||
accepted_object.identity.target_store_id
|
||||
);
|
||||
assert_eq!(discovered_object.bucket, accepted_object.identity.bucket);
|
||||
assert_eq!(discovered_object.key, accepted_object.identity.key);
|
||||
if let (Some(discovered_version), Some(accepted_version)) = (
|
||||
discovered_object.version_id.as_deref(),
|
||||
accepted_object.identity.version_id.as_deref(),
|
||||
) {
|
||||
assert_eq!(discovered_version, accepted_version);
|
||||
}
|
||||
assert!(discovered.multiparts.is_empty());
|
||||
assert_eq!(
|
||||
service
|
||||
.reconcile_write_intent(
|
||||
&intent_claim,
|
||||
discovered
|
||||
.objects
|
||||
.into_iter()
|
||||
.map(|identity| StagingObject {
|
||||
target_store: identity.target_store_id,
|
||||
bucket: identity.bucket,
|
||||
key: identity.key,
|
||||
version_id: identity.version_id,
|
||||
})
|
||||
.collect(),
|
||||
Vec::new(),
|
||||
)
|
||||
.await
|
||||
.unwrap(),
|
||||
CleanupCompletion::Completed
|
||||
);
|
||||
let object_claim = service.claim_cleanup(&config).await.unwrap().unwrap();
|
||||
assert!(matches!(object_claim.identity, CleanupIdentity::Object(_)));
|
||||
execute_cleanup(&harness.target, &object_claim)
|
||||
.await
|
||||
.unwrap();
|
||||
service.complete_cleanup(&object_claim).await.unwrap();
|
||||
assert!(
|
||||
harness
|
||||
.target
|
||||
.head_exact(&accepted_object.identity)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
async fn execute_cleanup(
|
||||
target: &TargetStore,
|
||||
claim: &remora_attempt::CleanupClaim,
|
||||
) -> Result<(), remora_target::TargetError> {
|
||||
match &claim.identity {
|
||||
CleanupIdentity::WriteIntent(_) => Err(remora_target::TargetError::internal(
|
||||
"write intents must be reconciled before exact cleanup",
|
||||
)),
|
||||
CleanupIdentity::Object(object) => {
|
||||
target
|
||||
.delete_exact(&ObjectIdentity {
|
||||
|
|
|
|||
|
|
@ -71,6 +71,78 @@ pub struct AttemptActivateRequestDto {
|
|||
pub worker_incarnation: OpaqueId,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct AttemptStopRequestDto {
|
||||
pub operation_id: OpaqueId,
|
||||
pub fence: PositiveIntegerDto,
|
||||
pub worker_incarnation: OpaqueId,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct AttemptStagingObjectRequestDto {
|
||||
pub operation_id: OpaqueId,
|
||||
pub fence: PositiveIntegerDto,
|
||||
pub worker_incarnation: OpaqueId,
|
||||
pub object: StagingObjectIdentityDto,
|
||||
#[serde(
|
||||
default,
|
||||
deserialize_with = "crate::dto::optional_non_null",
|
||||
skip_serializing_if = "Option::is_none"
|
||||
)]
|
||||
pub completed_upload_id: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct StagingObjectIdentityDto {
|
||||
pub store: String,
|
||||
pub bucket: String,
|
||||
pub key: String,
|
||||
#[serde(
|
||||
default,
|
||||
deserialize_with = "crate::dto::optional_non_null",
|
||||
skip_serializing_if = "Option::is_none"
|
||||
)]
|
||||
pub version_id: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct AttemptStagingWriteIntentRequestDto {
|
||||
pub operation_id: OpaqueId,
|
||||
pub fence: PositiveIntegerDto,
|
||||
pub worker_incarnation: OpaqueId,
|
||||
pub intent: StagingWriteIntentDto,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct StagingWriteIntentDto {
|
||||
pub store: String,
|
||||
pub bucket: String,
|
||||
pub key: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct AttemptStagingMultipartRequestDto {
|
||||
pub operation_id: OpaqueId,
|
||||
pub fence: PositiveIntegerDto,
|
||||
pub worker_incarnation: OpaqueId,
|
||||
pub multipart: StagingMultipartIdentityDto,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct StagingMultipartIdentityDto {
|
||||
pub store: String,
|
||||
pub bucket: String,
|
||||
pub key: String,
|
||||
pub upload_id: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ResolvedSourceDescriptorRefDto {
|
||||
|
|
@ -160,6 +232,12 @@ pub struct AttemptCheckpointRequestDto {
|
|||
pub resolved_source_id: OpaqueId,
|
||||
pub validated_bytes: SafeIntegerDto,
|
||||
pub validated_members: SafeIntegerDto,
|
||||
#[serde(
|
||||
default,
|
||||
deserialize_with = "crate::dto::optional_non_null",
|
||||
skip_serializing_if = "Option::is_none"
|
||||
)]
|
||||
pub staging_multipart: Option<StagingMultipartIdentityDto>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
|
|
|
|||
|
|
@ -30,9 +30,11 @@ pub use events::{
|
|||
pub use internal::{
|
||||
ActivationResponseDto, AttemptActivateRequestDto, AttemptCheckpointRequestDto,
|
||||
AttemptMutationDto, AttemptProgressRequestDto, AttemptRenewRequestDto,
|
||||
AttemptResolvedSourceRequestDto, CheckpointDescriptorKindDto, CheckpointDescriptorRefDto,
|
||||
ClaimRequestDto, ClaimResponseDto, ExecutionEnvelopeDto, ResolvedSourceDescriptorKindDto,
|
||||
ResolvedSourceDescriptorRefDto,
|
||||
AttemptResolvedSourceRequestDto, AttemptStagingMultipartRequestDto,
|
||||
AttemptStagingObjectRequestDto, AttemptStagingWriteIntentRequestDto, AttemptStopRequestDto,
|
||||
CheckpointDescriptorKindDto, CheckpointDescriptorRefDto, ClaimRequestDto, ClaimResponseDto,
|
||||
ExecutionEnvelopeDto, ResolvedSourceDescriptorKindDto, ResolvedSourceDescriptorRefDto,
|
||||
StagingMultipartIdentityDto, StagingObjectIdentityDto, StagingWriteIntentDto,
|
||||
};
|
||||
pub use public::{
|
||||
ApiErrorCodeDto, ApiErrorDto, ApiErrorFieldDto, AttemptPageDto, AttemptRecordDto,
|
||||
|
|
|
|||
|
|
@ -3,8 +3,9 @@ use std::{collections::BTreeSet, fs};
|
|||
use remora_contracts::{
|
||||
ActivationResponseDto, AttemptActivateRequestDto, AttemptCheckpointRequestDto,
|
||||
AttemptOutcomeRequestDto, AttemptProgressRequestDto, AttemptRenewRequestDto,
|
||||
AttemptResolvedSourceRequestDto, ClaimRequestDto, ClaimResponseDto, JobPhaseProjectionDto,
|
||||
LeaseProjectionDto,
|
||||
AttemptResolvedSourceRequestDto, AttemptStagingMultipartRequestDto,
|
||||
AttemptStagingObjectRequestDto, AttemptStagingWriteIntentRequestDto, AttemptStopRequestDto,
|
||||
ClaimRequestDto, ClaimResponseDto, JobPhaseProjectionDto, LeaseProjectionDto,
|
||||
};
|
||||
use serde::{Serialize, de::DeserializeOwned};
|
||||
use serde_json::Value;
|
||||
|
|
@ -96,7 +97,7 @@ fn worker_routes_and_dtos_conform_to_the_mutable_openapi_contract() {
|
|||
"staging_target": {
|
||||
"store": "primary",
|
||||
"bucket": "remora-results",
|
||||
"prefix": "_remora/a/attempt_0001/"
|
||||
"prefix": "users/alice/_remora/a/attempt_0001/"
|
||||
},
|
||||
"execution_grant_ref": "grant_0001",
|
||||
"checkpoint_ref": null
|
||||
|
|
@ -135,6 +136,58 @@ fn worker_routes_and_dtos_conform_to_the_mutable_openapi_contract() {
|
|||
"AttemptActivateRequest",
|
||||
mutation.clone(),
|
||||
);
|
||||
assert_dto_and_schema::<AttemptStopRequestDto>(
|
||||
&component_schema,
|
||||
"AttemptStopRequest",
|
||||
serde_json::json!({
|
||||
"operation_id": "operation_0001",
|
||||
"fence": 1,
|
||||
"worker_incarnation": "worker_0001"
|
||||
}),
|
||||
);
|
||||
assert_dto_and_schema::<AttemptStagingObjectRequestDto>(
|
||||
&component_schema,
|
||||
"AttemptStagingObjectRequest",
|
||||
serde_json::json!({
|
||||
"operation_id": "operation_0001",
|
||||
"fence": 1,
|
||||
"worker_incarnation": "worker_0001",
|
||||
"object": {
|
||||
"store": "primary",
|
||||
"bucket": "remora-results",
|
||||
"key": "users/alice/_remora/a/attempt_0001/o/object_0001"
|
||||
}
|
||||
}),
|
||||
);
|
||||
assert_dto_and_schema::<AttemptStagingWriteIntentRequestDto>(
|
||||
&component_schema,
|
||||
"AttemptStagingWriteIntentRequest",
|
||||
serde_json::json!({
|
||||
"operation_id": "operation_0001",
|
||||
"fence": 1,
|
||||
"worker_incarnation": "worker_0001",
|
||||
"intent": {
|
||||
"store": "primary",
|
||||
"bucket": "remora-results",
|
||||
"key": "users/alice/_remora/a/attempt_0001/o/object_0001"
|
||||
}
|
||||
}),
|
||||
);
|
||||
assert_dto_and_schema::<AttemptStagingMultipartRequestDto>(
|
||||
&component_schema,
|
||||
"AttemptStagingMultipartRequest",
|
||||
serde_json::json!({
|
||||
"operation_id": "operation_0001",
|
||||
"fence": 1,
|
||||
"worker_incarnation": "worker_0001",
|
||||
"multipart": {
|
||||
"store": "primary",
|
||||
"bucket": "remora-results",
|
||||
"key": "users/alice/_remora/a/attempt_0001/o/object_0001",
|
||||
"upload_id": "upload_0001"
|
||||
}
|
||||
}),
|
||||
);
|
||||
assert_dto_and_schema::<ActivationResponseDto>(
|
||||
&component_schema,
|
||||
"ActivationResponse",
|
||||
|
|
@ -228,6 +281,30 @@ fn worker_routes_and_dtos_conform_to_the_mutable_openapi_contract() {
|
|||
"200",
|
||||
Some("ActivationResponse"),
|
||||
),
|
||||
(
|
||||
"/internal/v1/attempts/{attempt_id}/stop",
|
||||
"AttemptStopRequest",
|
||||
"204",
|
||||
None,
|
||||
),
|
||||
(
|
||||
"/internal/v1/attempts/{attempt_id}/staging/write-intent",
|
||||
"AttemptStagingWriteIntentRequest",
|
||||
"204",
|
||||
None,
|
||||
),
|
||||
(
|
||||
"/internal/v1/attempts/{attempt_id}/staging/object",
|
||||
"AttemptStagingObjectRequest",
|
||||
"204",
|
||||
None,
|
||||
),
|
||||
(
|
||||
"/internal/v1/attempts/{attempt_id}/staging/multipart",
|
||||
"AttemptStagingMultipartRequest",
|
||||
"204",
|
||||
None,
|
||||
),
|
||||
(
|
||||
"/internal/v1/attempts/{attempt_id}/resolved-source",
|
||||
"AttemptResolvedSourceRequest",
|
||||
|
|
|
|||
|
|
@ -40,6 +40,26 @@ impl<R: Resolver> HfSource<R> {
|
|||
}
|
||||
}
|
||||
|
||||
pub fn new_with_root_certificates(
|
||||
resolver: R,
|
||||
policy: OutboundPolicy,
|
||||
timing: RetryTiming,
|
||||
roots: impl IntoIterator<Item = Vec<u8>>,
|
||||
) -> Result<Self, HfSourceError> {
|
||||
let timing = timing.sanitized();
|
||||
let mut client =
|
||||
OutboundClient::builder(resolver, policy).connect_timeout(timing.connect_timeout);
|
||||
for root in roots {
|
||||
let root =
|
||||
reqwest::Certificate::from_der(&root).map_err(|_| HfSourceError::InvalidRequest)?;
|
||||
client = client.add_root_certificate(root);
|
||||
}
|
||||
Ok(Self {
|
||||
client: client.build(),
|
||||
timing,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn resolve(
|
||||
&self,
|
||||
request: HfSourceRequest,
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue