Implement crash-safe streaming worker runtime
This commit is contained in:
parent
16cba3c317
commit
cdeef3cc1a
1 changed files with 7448 additions and 194 deletions
|
|
@ -8,22 +8,25 @@ use actix_web::{
|
|||
};
|
||||
use remora_attempt::{
|
||||
ActivationMutationOutcome, AttemptAuthority, AttemptError, AttemptMutationOutcome,
|
||||
AttemptService, CheckpointProjection, CheckpointRecord, ClaimOutcome, DurableDescriptor,
|
||||
LeaseProjection, MutationRejection, OutcomePublication, ProgressRecord, RenewalMutationOutcome,
|
||||
AttemptService, CheckpointMultipart, CheckpointProjection, CheckpointRecord, ClaimOutcome,
|
||||
DurableDescriptor, LeaseProjection, MutationRejection, OutcomePublication, ProgressRecord,
|
||||
RenewalMutationOutcome, StagingMultipart, StagingObject, StagingWriteIntent,
|
||||
};
|
||||
use remora_contracts::{
|
||||
ActivationResponseDto, ApiErrorCodeDto, ApiErrorDto, AttemptActivateRequestDto,
|
||||
AttemptAuthorityDto, AttemptCheckpointRequestDto, AttemptOutcomeDto, AttemptOutcomeRequestDto,
|
||||
AttemptProgressRequestDto, AttemptRenewRequestDto, AttemptResolvedSourceRequestDto,
|
||||
CheckpointDescriptorKindDto, CheckpointDescriptorRefDto, ClaimRequestDto, ClaimResponseDto,
|
||||
CredentialGrantVersionDto, DateTimeDto, DurableDescriptorKindDto, DurableDescriptorRefDto,
|
||||
ExecutionEnvelopeDto, JobPhaseProjectionDto, LeaseProjectionDto, NullableDto,
|
||||
PositiveIntegerDto, ResolvedSourceDescriptorKindDto, ResolvedSourceDescriptorRefDto,
|
||||
SafeIntegerDto, SourceProjectionDto, StagingTargetDto, TargetDto,
|
||||
AttemptStagingMultipartRequestDto, AttemptStagingObjectRequestDto,
|
||||
AttemptStagingWriteIntentRequestDto, AttemptStopRequestDto, CheckpointDescriptorKindDto,
|
||||
CheckpointDescriptorRefDto, ClaimRequestDto, ClaimResponseDto, CredentialGrantVersionDto,
|
||||
DateTimeDto, DurableDescriptorKindDto, DurableDescriptorRefDto, ExecutionEnvelopeDto,
|
||||
JobPhaseProjectionDto, LeaseProjectionDto, NullableDto, PositiveIntegerDto,
|
||||
ResolvedSourceDescriptorKindDto, ResolvedSourceDescriptorRefDto, SafeIntegerDto,
|
||||
SourceProjectionDto, StagingTargetDto, TargetDto,
|
||||
};
|
||||
use remora_domain::OpaqueId;
|
||||
use remora_result::PublicationInput;
|
||||
use remora_target::{ContentSha256, ObjectIdentity, RESERVED_NAMESPACE_SEGMENT, StoredObject};
|
||||
use remora_target::{ContentSha256, ObjectIdentity, StoredObject, attempt_staging_prefix};
|
||||
use sha2::{Digest, Sha256};
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
|
||||
|
|
@ -96,6 +99,19 @@ pub fn configure_worker_api(config: &mut web::ServiceConfig) {
|
|||
.route("/jobs/{job_id}/claim", web::post().to(claim))
|
||||
.route("/attempts/{attempt_id}/renew", web::post().to(renew))
|
||||
.route("/attempts/{attempt_id}/activate", web::post().to(activate))
|
||||
.route("/attempts/{attempt_id}/stop", web::post().to(stop))
|
||||
.route(
|
||||
"/attempts/{attempt_id}/staging/write-intent",
|
||||
web::post().to(register_staging_write_intent),
|
||||
)
|
||||
.route(
|
||||
"/attempts/{attempt_id}/staging/object",
|
||||
web::post().to(register_staging_object),
|
||||
)
|
||||
.route(
|
||||
"/attempts/{attempt_id}/staging/multipart",
|
||||
web::post().to(register_staging_multipart),
|
||||
)
|
||||
.route(
|
||||
"/attempts/{attempt_id}/resolved-source",
|
||||
web::post().to(pin_resolved_source),
|
||||
|
|
@ -115,6 +131,137 @@ pub fn configure_worker_api(config: &mut web::ServiceConfig) {
|
|||
);
|
||||
}
|
||||
|
||||
async fn register_staging_object(
|
||||
_: AuthenticatedWorker,
|
||||
state: web::Data<WorkerApiState>,
|
||||
path: web::Path<String>,
|
||||
request: web::Json<AttemptStagingObjectRequestDto>,
|
||||
) -> Result<HttpResponse, WorkerApiError> {
|
||||
let authority = request_authority(
|
||||
&state,
|
||||
path.into_inner(),
|
||||
request.fence.get(),
|
||||
&request.worker_incarnation,
|
||||
)
|
||||
.await?;
|
||||
let accepted = state
|
||||
.attempts
|
||||
.register_staging_object_operation(
|
||||
&authority,
|
||||
&request.operation_id,
|
||||
StagingObject {
|
||||
target_store: request.object.store.clone(),
|
||||
bucket: request.object.bucket.clone(),
|
||||
key: request.object.key.clone(),
|
||||
version_id: request.object.version_id.clone(),
|
||||
},
|
||||
request.completed_upload_id.clone(),
|
||||
)
|
||||
.await
|
||||
.map_err(WorkerApiError::from_attempt)?;
|
||||
if accepted {
|
||||
Ok(HttpResponse::NoContent().finish())
|
||||
} else {
|
||||
Err(WorkerApiError::precondition_failed())
|
||||
}
|
||||
}
|
||||
|
||||
async fn register_staging_write_intent(
|
||||
_: AuthenticatedWorker,
|
||||
state: web::Data<WorkerApiState>,
|
||||
path: web::Path<String>,
|
||||
request: web::Json<AttemptStagingWriteIntentRequestDto>,
|
||||
) -> Result<HttpResponse, WorkerApiError> {
|
||||
let authority = request_authority(
|
||||
&state,
|
||||
path.into_inner(),
|
||||
request.fence.get(),
|
||||
&request.worker_incarnation,
|
||||
)
|
||||
.await?;
|
||||
let accepted = state
|
||||
.attempts
|
||||
.register_staging_write_intent_operation(
|
||||
&authority,
|
||||
&request.operation_id,
|
||||
StagingWriteIntent {
|
||||
target_store: request.intent.store.clone(),
|
||||
bucket: request.intent.bucket.clone(),
|
||||
key: request.intent.key.clone(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(WorkerApiError::from_attempt)?;
|
||||
if accepted {
|
||||
Ok(HttpResponse::NoContent().finish())
|
||||
} else {
|
||||
Err(WorkerApiError::precondition_failed())
|
||||
}
|
||||
}
|
||||
|
||||
async fn register_staging_multipart(
|
||||
_: AuthenticatedWorker,
|
||||
state: web::Data<WorkerApiState>,
|
||||
path: web::Path<String>,
|
||||
request: web::Json<AttemptStagingMultipartRequestDto>,
|
||||
) -> Result<HttpResponse, WorkerApiError> {
|
||||
let authority = request_authority(
|
||||
&state,
|
||||
path.into_inner(),
|
||||
request.fence.get(),
|
||||
&request.worker_incarnation,
|
||||
)
|
||||
.await?;
|
||||
let accepted = state
|
||||
.attempts
|
||||
.register_staging_multipart_operation(
|
||||
&authority,
|
||||
&request.operation_id,
|
||||
StagingMultipart {
|
||||
target_store: request.multipart.store.clone(),
|
||||
bucket: request.multipart.bucket.clone(),
|
||||
key: request.multipart.key.clone(),
|
||||
upload_id: request.multipart.upload_id.clone(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(WorkerApiError::from_attempt)?;
|
||||
if accepted {
|
||||
Ok(HttpResponse::NoContent().finish())
|
||||
} else {
|
||||
Err(WorkerApiError::precondition_failed())
|
||||
}
|
||||
}
|
||||
|
||||
async fn stop(
|
||||
_: AuthenticatedWorker,
|
||||
state: web::Data<WorkerApiState>,
|
||||
path: web::Path<String>,
|
||||
request: web::Json<AttemptStopRequestDto>,
|
||||
) -> Result<HttpResponse, WorkerApiError> {
|
||||
let authority = request_authority(
|
||||
&state,
|
||||
path.into_inner(),
|
||||
request.fence.get(),
|
||||
&request.worker_incarnation,
|
||||
)
|
||||
.await?;
|
||||
match state
|
||||
.attempts
|
||||
.cooperative_stop_operation(&authority, &request.operation_id)
|
||||
.await
|
||||
.map_err(WorkerApiError::from_attempt)?
|
||||
{
|
||||
remora_attempt::CooperativeStopOutcome::Stopped
|
||||
| remora_attempt::CooperativeStopOutcome::StopNotRequired => {
|
||||
Ok(HttpResponse::NoContent().finish())
|
||||
}
|
||||
remora_attempt::CooperativeStopOutcome::AuthorityMismatch => {
|
||||
Err(WorkerApiError::precondition_failed())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn claim(
|
||||
_: AuthenticatedWorker,
|
||||
state: web::Data<WorkerApiState>,
|
||||
|
|
@ -245,6 +392,14 @@ async fn persist_checkpoint(
|
|||
resolved_source_id: request.resolved_source_id.clone(),
|
||||
validated_bytes: request.validated_bytes.get(),
|
||||
validated_members: request.validated_members.get(),
|
||||
staging_multipart: request.staging_multipart.as_ref().map(|identity| {
|
||||
CheckpointMultipart {
|
||||
target_store: identity.store.clone(),
|
||||
bucket: identity.bucket.clone(),
|
||||
key: identity.key.clone(),
|
||||
upload_id: identity.upload_id.clone(),
|
||||
}
|
||||
}),
|
||||
},
|
||||
)
|
||||
.await
|
||||
|
|
@ -387,6 +542,7 @@ async fn load_execution_envelope(
|
|||
let source: SourceProjectionDto =
|
||||
serde_json::from_value(row.source).map_err(|_| WorkerApiError::unavailable())?;
|
||||
let target_prefix = row.target_prefix.join("/");
|
||||
let staging_prefix = attempt_staging_prefix(&target_prefix, &row.staging_origin_attempt_id);
|
||||
let resolved_source_ref = row.resolved_source.map(|descriptor| {
|
||||
NullableDto(Some(descriptor_ref(
|
||||
descriptor,
|
||||
|
|
@ -425,10 +581,7 @@ async fn load_execution_envelope(
|
|||
staging_target: StagingTargetDto {
|
||||
store: row.target_store,
|
||||
bucket: row.target_bucket,
|
||||
prefix: format!(
|
||||
"{RESERVED_NAMESPACE_SEGMENT}/a/{}/",
|
||||
authority.attempt_id.as_str()
|
||||
),
|
||||
prefix: staging_prefix,
|
||||
},
|
||||
execution_grant_ref: row.execution_grant_id,
|
||||
checkpoint_ref: checkpoint_ref.unwrap_or(NullableDto(None)),
|
||||
|
|
|
|||
Loading…
Reference in a new issue