Add authenticated internal worker API
This commit is contained in:
parent
9d96c4d563
commit
ebf818d23c
4 changed files with 2745 additions and 43 deletions
|
|
@ -4,3 +4,30 @@ edition.workspace = true
|
|||
publish = false
|
||||
rust-version.workspace = true
|
||||
version.workspace = true
|
||||
|
||||
[dependencies]
|
||||
actix-web = { version = "=4.14.0", features = ["rustls-0_23"] }
|
||||
actix-tls = "=3.5.0"
|
||||
getrandom = "0.3.4"
|
||||
remora-attempt = { path = "../remora-attempt" }
|
||||
remora-contracts = { path = "../remora-contracts" }
|
||||
remora-domain.workspace = true
|
||||
remora-result = { path = "../remora-result" }
|
||||
remora-target = { path = "../remora-target" }
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
sha2.workspace = true
|
||||
sqlx.workspace = true
|
||||
thiserror.workspace = true
|
||||
time.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
rcgen = "=0.14.8"
|
||||
remora-admission = { path = "../remora-admission" }
|
||||
remora-coordination = { path = "../remora-coordination" }
|
||||
remora-credentials = { path = "../remora-credentials" }
|
||||
remora-durable.workspace = true
|
||||
remora-identity = { path = "../remora-identity" }
|
||||
reqwest = { version = "=0.13.4", default-features = false, features = ["json", "rustls"] }
|
||||
rustls = "=0.23.42"
|
||||
tokio.workspace = true
|
||||
|
|
|
|||
|
|
@ -1,3 +1,9 @@
|
|||
mod worker;
|
||||
|
||||
pub use worker::{
|
||||
AuthenticatedWorker, WorkerApiState, authenticate_rustls_worker, configure_worker_api,
|
||||
};
|
||||
|
||||
pub const COMPONENT_NAME: &str = "remora-api";
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
|
|||
613
crates/remora-api/src/worker.rs
Normal file
613
crates/remora-api/src/worker.rs
Normal file
|
|
@ -0,0 +1,613 @@
|
|||
use std::{
|
||||
any::Any,
|
||||
future::{Ready, ready},
|
||||
};
|
||||
|
||||
use actix_web::{
|
||||
FromRequest, HttpRequest, HttpResponse, ResponseError, dev::Payload, http::StatusCode, web,
|
||||
};
|
||||
use remora_attempt::{
|
||||
ActivationMutationOutcome, AttemptAuthority, AttemptError, AttemptMutationOutcome,
|
||||
AttemptService, CheckpointProjection, CheckpointRecord, ClaimOutcome, DurableDescriptor,
|
||||
LeaseProjection, MutationRejection, OutcomePublication, ProgressRecord, RenewalMutationOutcome,
|
||||
};
|
||||
use remora_contracts::{
|
||||
ActivationResponseDto, ApiErrorCodeDto, ApiErrorDto, AttemptActivateRequestDto,
|
||||
AttemptAuthorityDto, AttemptCheckpointRequestDto, AttemptOutcomeDto, AttemptOutcomeRequestDto,
|
||||
AttemptProgressRequestDto, AttemptRenewRequestDto, AttemptResolvedSourceRequestDto,
|
||||
CheckpointDescriptorKindDto, CheckpointDescriptorRefDto, ClaimRequestDto, ClaimResponseDto,
|
||||
CredentialGrantVersionDto, DateTimeDto, DurableDescriptorKindDto, DurableDescriptorRefDto,
|
||||
ExecutionEnvelopeDto, JobPhaseProjectionDto, LeaseProjectionDto, NullableDto,
|
||||
PositiveIntegerDto, ResolvedSourceDescriptorKindDto, ResolvedSourceDescriptorRefDto,
|
||||
SafeIntegerDto, SourceProjectionDto, StagingTargetDto, TargetDto,
|
||||
};
|
||||
use remora_domain::OpaqueId;
|
||||
use remora_result::PublicationInput;
|
||||
use remora_target::{ContentSha256, ObjectIdentity, RESERVED_NAMESPACE_SEGMENT, StoredObject};
|
||||
use sha2::{Digest, Sha256};
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct AuthenticatedWorker {
|
||||
pub identity: OpaqueId,
|
||||
}
|
||||
|
||||
pub fn authenticate_rustls_worker(
|
||||
connection: &dyn Any,
|
||||
extensions: &mut actix_web::dev::Extensions,
|
||||
) {
|
||||
type Stream = actix_tls::accept::rustls_0_23::TlsStream<actix_web::rt::net::TcpStream>;
|
||||
let Some(stream) = connection.downcast_ref::<Stream>() else {
|
||||
return;
|
||||
};
|
||||
let Some(certificate) = stream
|
||||
.get_ref()
|
||||
.1
|
||||
.peer_certificates()
|
||||
.and_then(|chain| chain.first())
|
||||
else {
|
||||
return;
|
||||
};
|
||||
let digest = Sha256::digest(certificate.as_ref());
|
||||
let mut value = String::from("w");
|
||||
for byte in digest {
|
||||
use std::fmt::Write as _;
|
||||
write!(value, "{byte:02x}").expect("writing to String cannot fail");
|
||||
}
|
||||
extensions.insert(AuthenticatedWorker {
|
||||
identity: OpaqueId::parse(value).expect("certificate digest is a valid identity"),
|
||||
});
|
||||
}
|
||||
|
||||
impl FromRequest for AuthenticatedWorker {
|
||||
type Error = WorkerApiError;
|
||||
type Future = Ready<Result<Self, Self::Error>>;
|
||||
|
||||
fn from_request(request: &HttpRequest, _payload: &mut Payload) -> Self::Future {
|
||||
ready(
|
||||
request
|
||||
.conn_data::<Self>()
|
||||
.cloned()
|
||||
.ok_or_else(WorkerApiError::unauthenticated),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct WorkerApiState {
|
||||
attempts: AttemptService,
|
||||
}
|
||||
|
||||
impl WorkerApiState {
|
||||
#[must_use]
|
||||
pub fn new(attempts: AttemptService) -> Self {
|
||||
Self { attempts }
|
||||
}
|
||||
}
|
||||
|
||||
pub fn configure_worker_api(config: &mut web::ServiceConfig) {
|
||||
config.service(
|
||||
web::scope("/internal/v1")
|
||||
.app_data(
|
||||
web::JsonConfig::default()
|
||||
.limit(1024 * 1024)
|
||||
.error_handler(|_, _| WorkerApiError::invalid_request().into()),
|
||||
)
|
||||
.route("/jobs/{job_id}/claim", web::post().to(claim))
|
||||
.route("/attempts/{attempt_id}/renew", web::post().to(renew))
|
||||
.route("/attempts/{attempt_id}/activate", web::post().to(activate))
|
||||
.route(
|
||||
"/attempts/{attempt_id}/resolved-source",
|
||||
web::post().to(pin_resolved_source),
|
||||
)
|
||||
.route(
|
||||
"/attempts/{attempt_id}/checkpoint",
|
||||
web::post().to(persist_checkpoint),
|
||||
)
|
||||
.route(
|
||||
"/attempts/{attempt_id}/progress",
|
||||
web::post().to(report_progress),
|
||||
)
|
||||
.route(
|
||||
"/attempts/{attempt_id}/outcome",
|
||||
web::post().to(report_outcome),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
async fn claim(
|
||||
_: AuthenticatedWorker,
|
||||
state: web::Data<WorkerApiState>,
|
||||
path: web::Path<String>,
|
||||
request: web::Json<ClaimRequestDto>,
|
||||
) -> Result<HttpResponse, WorkerApiError> {
|
||||
let job_id = parse_id(path.into_inner())?;
|
||||
let outcome = state
|
||||
.attempts
|
||||
.claim(&job_id, &request.worker_incarnation, &request.operation_id)
|
||||
.await
|
||||
.map_err(WorkerApiError::from_attempt)?;
|
||||
let ClaimOutcome::Claimed { authority, .. } = outcome else {
|
||||
return Err(WorkerApiError::precondition_failed());
|
||||
};
|
||||
let execution = load_execution_envelope(&state.attempts, &authority).await?;
|
||||
Ok(HttpResponse::Ok().json(ClaimResponseDto {
|
||||
authority: authority_dto(&authority)?,
|
||||
execution,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn renew(
|
||||
_: AuthenticatedWorker,
|
||||
state: web::Data<WorkerApiState>,
|
||||
path: web::Path<String>,
|
||||
request: web::Json<AttemptRenewRequestDto>,
|
||||
) -> Result<HttpResponse, WorkerApiError> {
|
||||
let authority = request_authority(
|
||||
&state,
|
||||
path.into_inner(),
|
||||
request.fence.get(),
|
||||
&request.worker_incarnation,
|
||||
)
|
||||
.await?;
|
||||
match state
|
||||
.attempts
|
||||
.renew_operation(&authority, &request.operation_id)
|
||||
.await
|
||||
.map_err(WorkerApiError::from_attempt)?
|
||||
{
|
||||
RenewalMutationOutcome::Renewed { lease, .. } => {
|
||||
Ok(HttpResponse::Ok().json(lease_dto(lease)?))
|
||||
}
|
||||
RenewalMutationOutcome::AuthorityLost { .. } => Err(WorkerApiError::precondition_failed()),
|
||||
}
|
||||
}
|
||||
|
||||
async fn activate(
|
||||
_: AuthenticatedWorker,
|
||||
state: web::Data<WorkerApiState>,
|
||||
path: web::Path<String>,
|
||||
request: web::Json<AttemptActivateRequestDto>,
|
||||
) -> Result<HttpResponse, WorkerApiError> {
|
||||
let authority = request_authority(
|
||||
&state,
|
||||
path.into_inner(),
|
||||
request.fence.get(),
|
||||
&request.worker_incarnation,
|
||||
)
|
||||
.await?;
|
||||
let lease = match state
|
||||
.attempts
|
||||
.activate_operation(&authority, &request.operation_id)
|
||||
.await
|
||||
.map_err(WorkerApiError::from_attempt)?
|
||||
{
|
||||
ActivationMutationOutcome::Activated { lease, .. } => lease,
|
||||
ActivationMutationOutcome::PreconditionFailed => {
|
||||
return Err(WorkerApiError::precondition_failed());
|
||||
}
|
||||
};
|
||||
let execution = load_execution_envelope(&state.attempts, &authority).await?;
|
||||
Ok(HttpResponse::Ok().json(ActivationResponseDto {
|
||||
authority: authority_dto(&authority)?,
|
||||
execution,
|
||||
lease: lease_dto(lease)?,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn pin_resolved_source(
|
||||
_: AuthenticatedWorker,
|
||||
state: web::Data<WorkerApiState>,
|
||||
path: web::Path<String>,
|
||||
request: web::Json<AttemptResolvedSourceRequestDto>,
|
||||
) -> Result<HttpResponse, WorkerApiError> {
|
||||
let authority = request_authority(
|
||||
&state,
|
||||
path.into_inner(),
|
||||
request.fence.get(),
|
||||
&request.worker_incarnation,
|
||||
)
|
||||
.await?;
|
||||
accepted(
|
||||
state
|
||||
.attempts
|
||||
.pin_resolved_source(
|
||||
&authority,
|
||||
&request.operation_id,
|
||||
descriptor_from_resolved(&request.resolved_source_ref)?,
|
||||
)
|
||||
.await
|
||||
.map_err(WorkerApiError::from_attempt)?,
|
||||
)
|
||||
}
|
||||
|
||||
async fn persist_checkpoint(
|
||||
_: AuthenticatedWorker,
|
||||
state: web::Data<WorkerApiState>,
|
||||
path: web::Path<String>,
|
||||
request: web::Json<AttemptCheckpointRequestDto>,
|
||||
) -> Result<HttpResponse, WorkerApiError> {
|
||||
let authority = request_authority(
|
||||
&state,
|
||||
path.into_inner(),
|
||||
request.fence.get(),
|
||||
&request.worker_incarnation,
|
||||
)
|
||||
.await?;
|
||||
accepted(
|
||||
state
|
||||
.attempts
|
||||
.persist_checkpoint(
|
||||
&authority,
|
||||
&request.operation_id,
|
||||
CheckpointRecord {
|
||||
descriptor: descriptor_from_checkpoint(&request.checkpoint_ref)?,
|
||||
resolved_source_id: request.resolved_source_id.clone(),
|
||||
validated_bytes: request.validated_bytes.get(),
|
||||
validated_members: request.validated_members.get(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(WorkerApiError::from_attempt)?,
|
||||
)
|
||||
}
|
||||
|
||||
async fn report_progress(
|
||||
_: AuthenticatedWorker,
|
||||
state: web::Data<WorkerApiState>,
|
||||
path: web::Path<String>,
|
||||
request: web::Json<AttemptProgressRequestDto>,
|
||||
) -> Result<HttpResponse, WorkerApiError> {
|
||||
let authority = request_authority(
|
||||
&state,
|
||||
path.into_inner(),
|
||||
request.fence.get(),
|
||||
&request.worker_incarnation,
|
||||
)
|
||||
.await?;
|
||||
accepted(
|
||||
state
|
||||
.attempts
|
||||
.report_progress(
|
||||
&authority,
|
||||
&request.operation_id,
|
||||
ProgressRecord {
|
||||
completed_bytes: request.progress.completed_bytes.get(),
|
||||
total_bytes: request.progress.total_bytes.map(SafeIntegerDto::get),
|
||||
completed_members: request.progress.completed_members.get(),
|
||||
total_members: request.progress.total_members.map(SafeIntegerDto::get),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(WorkerApiError::from_attempt)?,
|
||||
)
|
||||
}
|
||||
|
||||
async fn report_outcome(
|
||||
_: AuthenticatedWorker,
|
||||
state: web::Data<WorkerApiState>,
|
||||
path: web::Path<String>,
|
||||
request: web::Json<AttemptOutcomeRequestDto>,
|
||||
) -> Result<HttpResponse, WorkerApiError> {
|
||||
let authority = request_authority(
|
||||
&state,
|
||||
path.into_inner(),
|
||||
request.fence.get(),
|
||||
&request.worker_incarnation,
|
||||
)
|
||||
.await?;
|
||||
let outcome = match &request.outcome {
|
||||
AttemptOutcomeDto::Success(success) => {
|
||||
let result = &success.result;
|
||||
state
|
||||
.attempts
|
||||
.publish_success_operation(
|
||||
&authority,
|
||||
&request.operation_id,
|
||||
PublicationInput {
|
||||
attempt_id: authority.attempt_id.clone(),
|
||||
manifest_id: result.manifest_id.clone(),
|
||||
schema_version: u32::try_from(result.schema_version.get())
|
||||
.map_err(|_| WorkerApiError::invalid_request())?,
|
||||
manifest: StoredObject {
|
||||
identity: ObjectIdentity {
|
||||
target_store_id: result.manifest.store.clone(),
|
||||
bucket: result.manifest.bucket.clone(),
|
||||
key: result.manifest.key.clone(),
|
||||
version_id: result.manifest.version_id.clone(),
|
||||
},
|
||||
content_sha256: ContentSha256::parse(&result.manifest.checksum)
|
||||
.map_err(|_| WorkerApiError::invalid_request())?,
|
||||
size: result.manifest.size.get(),
|
||||
},
|
||||
member_count: result.member_count.get(),
|
||||
total_bytes: result.total_bytes.get(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
AttemptOutcomeDto::Failure(failure) => {
|
||||
state
|
||||
.attempts
|
||||
.publish_failure_operation(&authority, &request.operation_id, failure.failure.class)
|
||||
.await
|
||||
}
|
||||
}
|
||||
.map_err(WorkerApiError::from_attempt)?;
|
||||
let phase = match outcome {
|
||||
OutcomePublication::Accepted { phase, .. } => phase,
|
||||
OutcomePublication::Rejected { reason, .. } => {
|
||||
return Err(WorkerApiError::from_rejection(reason));
|
||||
}
|
||||
};
|
||||
Ok(HttpResponse::Ok().json(JobPhaseProjectionDto { phase }))
|
||||
}
|
||||
|
||||
fn accepted(outcome: AttemptMutationOutcome) -> Result<HttpResponse, WorkerApiError> {
|
||||
match outcome {
|
||||
AttemptMutationOutcome::Accepted(_) => Ok(HttpResponse::NoContent().finish()),
|
||||
AttemptMutationOutcome::Rejected { reason, .. } => {
|
||||
Err(WorkerApiError::from_rejection(reason))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn request_authority(
|
||||
state: &WorkerApiState,
|
||||
attempt_id: String,
|
||||
fence: u64,
|
||||
worker: &OpaqueId,
|
||||
) -> Result<AttemptAuthority, WorkerApiError> {
|
||||
let attempt_id = parse_id(attempt_id)?;
|
||||
state
|
||||
.attempts
|
||||
.historical_authority(&attempt_id)
|
||||
.await
|
||||
.map_err(WorkerApiError::from_attempt)?
|
||||
.filter(|authority| authority.fence == fence && authority.worker_incarnation == *worker)
|
||||
.ok_or_else(WorkerApiError::precondition_failed)
|
||||
}
|
||||
|
||||
async fn load_execution_envelope(
|
||||
attempts: &AttemptService,
|
||||
authority: &AttemptAuthority,
|
||||
) -> Result<ExecutionEnvelopeDto, WorkerApiError> {
|
||||
let row = attempts
|
||||
.execution_envelope(&authority.attempt_id)
|
||||
.await
|
||||
.map_err(WorkerApiError::from_attempt)?
|
||||
.ok_or_else(WorkerApiError::precondition_failed)?;
|
||||
if row.job_id != authority.job_id
|
||||
|| row.attempt_id != authority.attempt_id
|
||||
|| row.fence != authority.fence
|
||||
|| row.worker_incarnation != authority.worker_incarnation
|
||||
{
|
||||
return Err(WorkerApiError::precondition_failed());
|
||||
}
|
||||
let source: SourceProjectionDto =
|
||||
serde_json::from_value(row.source).map_err(|_| WorkerApiError::unavailable())?;
|
||||
let target_prefix = row.target_prefix.join("/");
|
||||
let resolved_source_ref = row.resolved_source.map(|descriptor| {
|
||||
NullableDto(Some(descriptor_ref(
|
||||
descriptor,
|
||||
DurableDescriptorKindDto::ResolvedSource,
|
||||
)))
|
||||
});
|
||||
let checkpoint_ref = row.checkpoint.and_then(|checkpoint| match checkpoint {
|
||||
CheckpointProjection::Reusable { checkpoint } => Some(NullableDto(Some(descriptor_ref(
|
||||
checkpoint.descriptor,
|
||||
DurableDescriptorKindDto::Checkpoint,
|
||||
)))),
|
||||
CheckpointProjection::Complete { .. } => None,
|
||||
});
|
||||
Ok(ExecutionEnvelopeDto {
|
||||
id: row.id,
|
||||
job_id: authority.job_id.clone(),
|
||||
attempt_id: authority.attempt_id.clone(),
|
||||
fence: positive(authority.fence)?,
|
||||
worker_incarnation: authority.worker_incarnation.clone(),
|
||||
source,
|
||||
sensitive_source_ref: row.sensitive_source_ref,
|
||||
credential_version: NullableDto(match (row.credential_id, row.credential_version_id) {
|
||||
(Some(credential_id), Some(version_id)) => Some(CredentialGrantVersionDto {
|
||||
credential_id,
|
||||
version_id,
|
||||
}),
|
||||
(None, None) => None,
|
||||
_ => return Err(WorkerApiError::unavailable()),
|
||||
}),
|
||||
resolved_source_ref: resolved_source_ref.unwrap_or(NullableDto(None)),
|
||||
target: TargetDto {
|
||||
store: row.target_store.clone(),
|
||||
bucket: row.target_bucket.clone(),
|
||||
prefix: target_prefix,
|
||||
},
|
||||
staging_target: StagingTargetDto {
|
||||
store: row.target_store,
|
||||
bucket: row.target_bucket,
|
||||
prefix: format!(
|
||||
"{RESERVED_NAMESPACE_SEGMENT}/a/{}/",
|
||||
authority.attempt_id.as_str()
|
||||
),
|
||||
},
|
||||
execution_grant_ref: row.execution_grant_id,
|
||||
checkpoint_ref: checkpoint_ref.unwrap_or(NullableDto(None)),
|
||||
})
|
||||
}
|
||||
|
||||
fn descriptor_ref(
|
||||
descriptor: DurableDescriptor,
|
||||
kind: DurableDescriptorKindDto,
|
||||
) -> DurableDescriptorRefDto {
|
||||
DurableDescriptorRefDto {
|
||||
id: descriptor.id,
|
||||
kind,
|
||||
schema_version: positive(descriptor.schema_version.into()).expect("validated descriptor"),
|
||||
byte_length: safe(descriptor.byte_length).expect("validated descriptor"),
|
||||
item_count: descriptor
|
||||
.item_count
|
||||
.map(|value| safe(value).expect("validated descriptor")),
|
||||
checksum: descriptor.checksum,
|
||||
}
|
||||
}
|
||||
|
||||
fn descriptor_from_resolved(
|
||||
value: &ResolvedSourceDescriptorRefDto,
|
||||
) -> Result<DurableDescriptor, WorkerApiError> {
|
||||
let ResolvedSourceDescriptorKindDto::ResolvedSource = value.kind;
|
||||
Ok(DurableDescriptor {
|
||||
id: value.id.clone(),
|
||||
schema_version: u32::try_from(value.schema_version.get())
|
||||
.map_err(|_| WorkerApiError::invalid_request())?,
|
||||
byte_length: value.byte_length.get(),
|
||||
item_count: value.item_count.map(SafeIntegerDto::get),
|
||||
checksum: value.checksum.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
fn descriptor_from_checkpoint(
|
||||
value: &CheckpointDescriptorRefDto,
|
||||
) -> Result<DurableDescriptor, WorkerApiError> {
|
||||
let CheckpointDescriptorKindDto::Checkpoint = value.kind;
|
||||
Ok(DurableDescriptor {
|
||||
id: value.id.clone(),
|
||||
schema_version: u32::try_from(value.schema_version.get())
|
||||
.map_err(|_| WorkerApiError::invalid_request())?,
|
||||
byte_length: value.byte_length.get(),
|
||||
item_count: value.item_count.map(SafeIntegerDto::get),
|
||||
checksum: value.checksum.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
fn authority_dto(authority: &AttemptAuthority) -> Result<AttemptAuthorityDto, WorkerApiError> {
|
||||
Ok(AttemptAuthorityDto {
|
||||
attempt_id: authority.attempt_id.clone(),
|
||||
fence: positive(authority.fence)?,
|
||||
worker_incarnation: authority.worker_incarnation.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
fn lease_dto(lease: LeaseProjection) -> Result<LeaseProjectionDto, WorkerApiError> {
|
||||
let observed = timestamp(lease.observed_at_ms)?;
|
||||
let expires = timestamp(lease.expires_at_ms)?;
|
||||
let renew_at = observed + (expires - observed) / 2;
|
||||
Ok(LeaseProjectionDto {
|
||||
expires_at: datetime(expires)?,
|
||||
renew_after: datetime(renew_at)?,
|
||||
})
|
||||
}
|
||||
|
||||
fn timestamp(milliseconds: u64) -> Result<OffsetDateTime, WorkerApiError> {
|
||||
let nanoseconds = i128::from(milliseconds) * 1_000_000;
|
||||
OffsetDateTime::from_unix_timestamp_nanos(nanoseconds)
|
||||
.map_err(|_| WorkerApiError::unavailable())
|
||||
}
|
||||
|
||||
fn datetime(value: OffsetDateTime) -> Result<DateTimeDto, WorkerApiError> {
|
||||
let value = value
|
||||
.format(&Rfc3339)
|
||||
.map_err(|_| WorkerApiError::unavailable())?;
|
||||
DateTimeDto::parse(value).map_err(|_| WorkerApiError::unavailable())
|
||||
}
|
||||
|
||||
fn positive(value: u64) -> Result<PositiveIntegerDto, WorkerApiError> {
|
||||
PositiveIntegerDto::new(value).map_err(|_| WorkerApiError::unavailable())
|
||||
}
|
||||
|
||||
fn safe(value: u64) -> Result<SafeIntegerDto, WorkerApiError> {
|
||||
SafeIntegerDto::new(value).map_err(|_| WorkerApiError::unavailable())
|
||||
}
|
||||
|
||||
fn parse_id(value: impl Into<String>) -> Result<OpaqueId, WorkerApiError> {
|
||||
OpaqueId::parse(value).map_err(|_| WorkerApiError::invalid_request())
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("worker API request failed")]
|
||||
pub struct WorkerApiError {
|
||||
status: StatusCode,
|
||||
code: ApiErrorCodeDto,
|
||||
message: &'static str,
|
||||
}
|
||||
|
||||
impl WorkerApiError {
|
||||
fn invalid_request() -> Self {
|
||||
Self::new(
|
||||
StatusCode::BAD_REQUEST,
|
||||
ApiErrorCodeDto::InvalidRequest,
|
||||
"invalid request",
|
||||
)
|
||||
}
|
||||
|
||||
fn unauthenticated() -> Self {
|
||||
Self::new(
|
||||
StatusCode::UNAUTHORIZED,
|
||||
ApiErrorCodeDto::Unauthenticated,
|
||||
"authentication required",
|
||||
)
|
||||
}
|
||||
|
||||
fn precondition_failed() -> Self {
|
||||
Self::new(
|
||||
StatusCode::CONFLICT,
|
||||
ApiErrorCodeDto::PreconditionFailed,
|
||||
"precondition failed",
|
||||
)
|
||||
}
|
||||
|
||||
fn unavailable() -> Self {
|
||||
Self::new(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
ApiErrorCodeDto::Unavailable,
|
||||
"service unavailable",
|
||||
)
|
||||
}
|
||||
|
||||
fn new(status: StatusCode, code: ApiErrorCodeDto, message: &'static str) -> Self {
|
||||
Self {
|
||||
status,
|
||||
code,
|
||||
message,
|
||||
}
|
||||
}
|
||||
|
||||
fn from_attempt(error: AttemptError) -> Self {
|
||||
match error {
|
||||
AttemptError::Invalid(_) => Self::invalid_request(),
|
||||
AttemptError::OperationConflict => Self::precondition_failed(),
|
||||
AttemptError::Coordination(_) | AttemptError::Durable(_) => Self::unavailable(),
|
||||
}
|
||||
}
|
||||
|
||||
fn from_rejection(_reason: MutationRejection) -> Self {
|
||||
Self::precondition_failed()
|
||||
}
|
||||
}
|
||||
|
||||
impl ResponseError for WorkerApiError {
|
||||
fn status_code(&self) -> StatusCode {
|
||||
self.status
|
||||
}
|
||||
|
||||
fn error_response(&self) -> HttpResponse {
|
||||
HttpResponse::build(self.status)
|
||||
.content_type("application/problem+json")
|
||||
.json(ApiErrorDto {
|
||||
code: self.code,
|
||||
message: self.message.into(),
|
||||
request_id: request_id(),
|
||||
fields: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn request_id() -> OpaqueId {
|
||||
let mut bytes = [0_u8; 16];
|
||||
getrandom::fill(&mut bytes).expect("operating system randomness is required");
|
||||
let mut value = String::from("r");
|
||||
for byte in bytes {
|
||||
use std::fmt::Write as _;
|
||||
write!(value, "{byte:02x}").expect("writing to String cannot fail");
|
||||
}
|
||||
OpaqueId::parse(value).expect("generated request identifier is valid")
|
||||
}
|
||||
735
crates/remora-api/tests/worker_api.rs
Normal file
735
crates/remora-api/tests/worker_api.rs
Normal file
|
|
@ -0,0 +1,735 @@
|
|||
use std::{net::TcpListener, sync::Arc, time::Duration};
|
||||
|
||||
use actix_web::{App, HttpResponse, HttpServer, web};
|
||||
use rcgen::{
|
||||
BasicConstraints, Certificate, CertificateParams, ExtendedKeyUsagePurpose, IsCa, Issuer,
|
||||
KeyPair, KeyUsagePurpose,
|
||||
};
|
||||
use remora_admission::{
|
||||
AdmissionConfig, AdmissionDecision, AdmissionService, migrate as admission_migrate,
|
||||
};
|
||||
use remora_api::{
|
||||
AuthenticatedWorker, WorkerApiState, authenticate_rustls_worker, configure_worker_api,
|
||||
};
|
||||
use remora_attempt::{AttemptConfig, AttemptService, RecoveryConfig, migrate as attempt_migrate};
|
||||
use remora_contracts::{
|
||||
ApiErrorDto, AttemptActivateRequestDto, AttemptCheckpointRequestDto, AttemptFailureOutcomeDto,
|
||||
AttemptOutcomeDto, AttemptOutcomeRequestDto, AttemptProgressRequestDto, AttemptRenewRequestDto,
|
||||
AttemptResolvedSourceRequestDto, AttemptRuntimeFailureDto, CheckpointDescriptorKindDto,
|
||||
CheckpointDescriptorRefDto, ClaimRequestDto, ClaimResponseDto, HttpSourceSubmissionDto,
|
||||
JobPhaseProjectionDto, JobSubmissionDto, PositiveIntegerDto, ProgressValueDto,
|
||||
ResolvedSourceDescriptorKindDto, ResolvedSourceDescriptorRefDto, ResultDescriptorDto,
|
||||
ResultKindDto, SafeIntegerDto, SourceSubmissionDto,
|
||||
};
|
||||
use remora_coordination::{CoordinationStore, StoreConfig};
|
||||
use remora_credentials::migrate as credentials_migrate;
|
||||
use remora_domain::{
|
||||
BucketName, JobPhase, OpaqueId, RuntimeFailureClass, StorageScope, TargetPrefix, TargetStoreId,
|
||||
};
|
||||
use remora_durable::{PoolConfig, connect, migrate as durable_migrate};
|
||||
use remora_identity::{DefaultTarget, Principal};
|
||||
use remora_result::migrate as result_migrate;
|
||||
use remora_target::attempt_manifest_key;
|
||||
use reqwest::{Certificate as ClientCertificate, Client, Identity};
|
||||
use rustls::{
|
||||
RootCertStore, ServerConfig,
|
||||
pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer},
|
||||
server::WebPkiClientVerifier,
|
||||
};
|
||||
use sqlx::PgPool;
|
||||
|
||||
struct Harness {
|
||||
pool: PgPool,
|
||||
coordination: CoordinationStore,
|
||||
}
|
||||
|
||||
impl Harness {
|
||||
async fn new(name: &str) -> Option<Self> {
|
||||
let database_url = required_env("DATABASE_URL")?;
|
||||
let coordination_url = required_env("REMORA_COORDINATION_URL")?;
|
||||
let pool = connect(
|
||||
&database_url,
|
||||
PoolConfig {
|
||||
max_connections: 16,
|
||||
acquire_timeout: Duration::from_secs(10),
|
||||
idle_timeout: Some(Duration::from_secs(30)),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
durable_migrate(&pool).await.unwrap();
|
||||
credentials_migrate(&pool).await.unwrap();
|
||||
admission_migrate(&pool).await.unwrap();
|
||||
attempt_migrate(&pool).await.unwrap();
|
||||
result_migrate(&pool).await.unwrap();
|
||||
sqlx::query("TRUNCATE TABLE jobs CASCADE")
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let coordination = CoordinationStore::connect(
|
||||
&coordination_url,
|
||||
format!("{}wp18:{name}:{}:", coordination_namespace(), nonce()),
|
||||
StoreConfig {
|
||||
global_limit: 16,
|
||||
lease_ttl: Duration::from_secs(10),
|
||||
reserve_ttl: Duration::from_secs(10),
|
||||
progress_ttl: Duration::from_secs(10),
|
||||
cancel_ttl: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
Some(Self { pool, coordination })
|
||||
}
|
||||
|
||||
fn service(&self) -> AttemptService {
|
||||
AttemptService::new(
|
||||
self.pool.clone(),
|
||||
self.coordination.clone(),
|
||||
AttemptConfig {
|
||||
max_attempts: 3,
|
||||
local_limit: 8,
|
||||
},
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
async fn job(&self) -> OpaqueId {
|
||||
let store = TargetStoreId::parse("primary").unwrap();
|
||||
let bucket = BucketName::parse("remora-results").unwrap();
|
||||
let prefix = TargetPrefix::parse("users/alice", ["_remora"]).unwrap();
|
||||
let principal = Principal {
|
||||
id: unique_id("owner"),
|
||||
issuer: "https://issuer.example".into(),
|
||||
subject: "subject".into(),
|
||||
administrator: false,
|
||||
scopes: [StorageScope {
|
||||
store: store.clone(),
|
||||
bucket: bucket.clone(),
|
||||
prefix: prefix.clone(),
|
||||
}]
|
||||
.into_iter()
|
||||
.collect(),
|
||||
};
|
||||
let service = AdmissionService::new(
|
||||
self.pool.clone(),
|
||||
AdmissionConfig {
|
||||
default_target: DefaultTarget {
|
||||
store,
|
||||
bucket,
|
||||
prefix,
|
||||
},
|
||||
reserved_target_segments: vec!["_remora".into()],
|
||||
policy_version: "worker-api-v1".into(),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
let outcome = service
|
||||
.submit(
|
||||
&principal,
|
||||
unique_id("submit").as_str(),
|
||||
JobSubmissionDto {
|
||||
source: SourceSubmissionDto::Http(HttpSourceSubmissionDto {
|
||||
url: "https://example.invalid/artifact".into(),
|
||||
result_path: "artifact.bin".into(),
|
||||
credential_id: None,
|
||||
}),
|
||||
target: None,
|
||||
label: Some("worker-api-test".into()),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let AdmissionDecision::Accepted(job) = outcome.decision else {
|
||||
panic!("job rejected")
|
||||
};
|
||||
self.coordination
|
||||
.enqueue(job.job_id.as_str())
|
||||
.await
|
||||
.unwrap();
|
||||
job.job_id
|
||||
}
|
||||
|
||||
async fn seal_result(&self, attempt_id: &OpaqueId) -> ResultDescriptorDto {
|
||||
let manifest_id = unique_id("manifest");
|
||||
let checksum = format!("sha256:{}", "11".repeat(32));
|
||||
let key = attempt_manifest_key(attempt_id, &manifest_id);
|
||||
sqlx::query(
|
||||
"INSERT INTO result_inventories (
|
||||
attempt_id, state, manifest_schema_version, manifest_id, manifest_store,
|
||||
manifest_bucket, manifest_key, manifest_version_id, manifest_sha256,
|
||||
manifest_size, member_count, total_bytes
|
||||
) VALUES ($1, 'sealed', 1, $2, 'primary', 'remora-results', $3,
|
||||
'version-1', $4, 128, 1, 4096)",
|
||||
)
|
||||
.bind(attempt_id.as_str())
|
||||
.bind(manifest_id.as_str())
|
||||
.bind(&key)
|
||||
.bind(&checksum)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
ResultDescriptorDto {
|
||||
kind: ResultKindDto::Manifest,
|
||||
manifest_id,
|
||||
schema_version: positive(1),
|
||||
manifest: remora_contracts::ObjectIdentityDto {
|
||||
store: "primary".into(),
|
||||
bucket: "remora-results".into(),
|
||||
key,
|
||||
version_id: Some("version-1".into()),
|
||||
checksum,
|
||||
size: safe(128),
|
||||
},
|
||||
member_count: positive(1),
|
||||
total_bytes: safe(4096),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn worker_operations_are_authenticated_fenced_replayable_and_redacted() {
|
||||
let Some(harness) = Harness::new("flow").await else {
|
||||
return;
|
||||
};
|
||||
let state = WorkerApiState::new(harness.service());
|
||||
let (base, server, client) = start_authenticated_server(state).await;
|
||||
let job_id = harness.job().await;
|
||||
let worker = unique_id("worker");
|
||||
let claim_operation = unique_id("operation");
|
||||
let claim = ClaimRequestDto {
|
||||
operation_id: claim_operation.clone(),
|
||||
worker_incarnation: worker.clone(),
|
||||
};
|
||||
|
||||
let unauthenticated = actix_web::test::init_service(App::new().route(
|
||||
"/",
|
||||
web::post().to(|_: AuthenticatedWorker| async { HttpResponse::NoContent().finish() }),
|
||||
))
|
||||
.await;
|
||||
let response = actix_web::test::call_service(
|
||||
&unauthenticated,
|
||||
actix_web::test::TestRequest::post().uri("/").to_request(),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), 401);
|
||||
assert_eq!(
|
||||
response.headers().get("content-type").unwrap(),
|
||||
"application/problem+json"
|
||||
);
|
||||
|
||||
let claim_url = format!("{base}/internal/v1/jobs/{job_id}/claim");
|
||||
let response = client.post(&claim_url).json(&claim).send().await.unwrap();
|
||||
assert_eq!(response.status(), 200);
|
||||
let claimed: ClaimResponseDto = response.json().await.unwrap();
|
||||
assert_eq!(claimed.authority.worker_incarnation, worker);
|
||||
assert_eq!(claimed.execution.attempt_id, claimed.authority.attempt_id);
|
||||
assert_eq!(claimed.execution.job_id, job_id);
|
||||
assert!(
|
||||
claimed
|
||||
.execution
|
||||
.staging_target
|
||||
.prefix
|
||||
.contains(claimed.authority.attempt_id.as_str())
|
||||
);
|
||||
|
||||
let changed_claim = client
|
||||
.post(&claim_url)
|
||||
.json(&ClaimRequestDto {
|
||||
operation_id: claim_operation,
|
||||
worker_incarnation: unique_id("worker"),
|
||||
})
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(changed_claim.status(), 409);
|
||||
assert_eq!(
|
||||
changed_claim.headers().get("content-type").unwrap(),
|
||||
"application/problem+json"
|
||||
);
|
||||
|
||||
let attempt_url = format!(
|
||||
"{base}/internal/v1/attempts/{}",
|
||||
claimed.authority.attempt_id
|
||||
);
|
||||
let activation_operation = unique_id("operation");
|
||||
let activation = AttemptActivateRequestDto {
|
||||
operation_id: activation_operation,
|
||||
fence: claimed.authority.fence,
|
||||
worker_incarnation: worker.clone(),
|
||||
};
|
||||
let response = client
|
||||
.post(format!("{attempt_url}/activate"))
|
||||
.json(&activation)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 200);
|
||||
let activation_body: serde_json::Value = response.json().await.unwrap();
|
||||
let activated: remora_contracts::ActivationResponseDto =
|
||||
serde_json::from_value(activation_body.clone()).unwrap();
|
||||
assert!(activated.lease.renew_after.as_str() < activated.lease.expires_at.as_str());
|
||||
|
||||
let renewal_operation = unique_id("operation");
|
||||
let renewal = AttemptRenewRequestDto {
|
||||
operation_id: renewal_operation.clone(),
|
||||
fence: claimed.authority.fence,
|
||||
worker_incarnation: worker.clone(),
|
||||
};
|
||||
let first_renewal = client
|
||||
.post(format!("{attempt_url}/renew"))
|
||||
.json(&renewal)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(first_renewal.status(), 200);
|
||||
let first_lease: serde_json::Value = first_renewal.json().await.unwrap();
|
||||
let replayed_lease: serde_json::Value = client
|
||||
.post(format!("{attempt_url}/renew"))
|
||||
.json(&renewal)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.json()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(replayed_lease, first_lease);
|
||||
let replayed_activation: serde_json::Value = client
|
||||
.post(format!("{attempt_url}/activate"))
|
||||
.json(&activation)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.json()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(replayed_activation, activation_body);
|
||||
|
||||
let authority_mismatches = [
|
||||
(
|
||||
format!("{base}/internal/v1/attempts/{}/renew", unique_id("attempt")),
|
||||
claimed.authority.fence,
|
||||
worker.clone(),
|
||||
),
|
||||
(
|
||||
format!("{attempt_url}/renew"),
|
||||
positive(claimed.authority.fence.get() + 1),
|
||||
worker.clone(),
|
||||
),
|
||||
(
|
||||
format!("{attempt_url}/renew"),
|
||||
claimed.authority.fence,
|
||||
unique_id("worker"),
|
||||
),
|
||||
];
|
||||
for (url, fence, worker_incarnation) in authority_mismatches {
|
||||
let response = client
|
||||
.post(url)
|
||||
.json(&AttemptRenewRequestDto {
|
||||
operation_id: unique_id("operation"),
|
||||
fence,
|
||||
worker_incarnation,
|
||||
})
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 409);
|
||||
}
|
||||
|
||||
let resolved_id = unique_id("resolved");
|
||||
let resolved = AttemptResolvedSourceRequestDto {
|
||||
operation_id: unique_id("operation"),
|
||||
fence: claimed.authority.fence,
|
||||
worker_incarnation: worker.clone(),
|
||||
resolved_source_ref: ResolvedSourceDescriptorRefDto {
|
||||
id: resolved_id.clone(),
|
||||
kind: ResolvedSourceDescriptorKindDto::ResolvedSource,
|
||||
schema_version: positive(1),
|
||||
byte_length: safe(64),
|
||||
item_count: Some(safe(1)),
|
||||
checksum: format!("sha256:{}", "22".repeat(32)),
|
||||
},
|
||||
};
|
||||
assert_eq!(
|
||||
client
|
||||
.post(format!("{attempt_url}/resolved-source"))
|
||||
.json(&resolved)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
204
|
||||
);
|
||||
let checkpoint = AttemptCheckpointRequestDto {
|
||||
operation_id: unique_id("operation"),
|
||||
fence: claimed.authority.fence,
|
||||
worker_incarnation: worker.clone(),
|
||||
checkpoint_ref: CheckpointDescriptorRefDto {
|
||||
id: unique_id("checkpoint"),
|
||||
kind: CheckpointDescriptorKindDto::Checkpoint,
|
||||
schema_version: positive(1),
|
||||
byte_length: safe(32),
|
||||
item_count: Some(safe(1)),
|
||||
checksum: format!("sha256:{}", "33".repeat(32)),
|
||||
},
|
||||
resolved_source_id: resolved_id,
|
||||
validated_bytes: safe(1024),
|
||||
validated_members: safe(1),
|
||||
};
|
||||
assert_eq!(
|
||||
client
|
||||
.post(format!("{attempt_url}/checkpoint"))
|
||||
.json(&checkpoint)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
204
|
||||
);
|
||||
let progress = AttemptProgressRequestDto {
|
||||
operation_id: unique_id("operation"),
|
||||
fence: claimed.authority.fence,
|
||||
worker_incarnation: worker.clone(),
|
||||
progress: ProgressValueDto {
|
||||
completed_bytes: safe(2048),
|
||||
total_bytes: Some(safe(4096)),
|
||||
completed_members: safe(1),
|
||||
total_members: Some(safe(1)),
|
||||
},
|
||||
};
|
||||
assert_eq!(
|
||||
client
|
||||
.post(format!("{attempt_url}/progress"))
|
||||
.json(&progress)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
204
|
||||
);
|
||||
|
||||
let replayed_claim: ClaimResponseDto = client
|
||||
.post(&claim_url)
|
||||
.json(&claim)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.json()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
serde_json::to_value(replayed_claim).unwrap(),
|
||||
serde_json::to_value(&claimed).unwrap()
|
||||
);
|
||||
|
||||
let failure = AttemptOutcomeRequestDto {
|
||||
operation_id: unique_id("operation"),
|
||||
fence: claimed.authority.fence,
|
||||
worker_incarnation: worker.clone(),
|
||||
outcome: AttemptOutcomeDto::Failure(AttemptFailureOutcomeDto {
|
||||
failure: AttemptRuntimeFailureDto {
|
||||
class: RuntimeFailureClass::TransientSource,
|
||||
message: "secret-value-must-not-escape".into(),
|
||||
safe_details: None,
|
||||
},
|
||||
}),
|
||||
};
|
||||
let outcome_url = format!("{attempt_url}/outcome");
|
||||
let response = client
|
||||
.post(&outcome_url)
|
||||
.json(&failure)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 200);
|
||||
let bytes = response.bytes().await.unwrap();
|
||||
assert!(!String::from_utf8_lossy(&bytes).contains("secret-value-must-not-escape"));
|
||||
let projection: JobPhaseProjectionDto = serde_json::from_slice(&bytes).unwrap();
|
||||
assert_eq!(projection.phase, JobPhase::RetryWait);
|
||||
let report = harness
|
||||
.service()
|
||||
.reconcile_once(&RecoveryConfig {
|
||||
retry_delay: Duration::ZERO,
|
||||
cleanup_claim_ttl: Duration::from_secs(1),
|
||||
batch_size: 128,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(report.retries_scheduled, 1);
|
||||
let replay = client
|
||||
.post(&outcome_url)
|
||||
.json(&failure)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(replay.status(), 200);
|
||||
assert_eq!(replay.bytes().await.unwrap(), bytes);
|
||||
|
||||
let cross_kind = client
|
||||
.post(format!("{attempt_url}/progress"))
|
||||
.json(&AttemptProgressRequestDto {
|
||||
operation_id: renewal_operation,
|
||||
fence: claimed.authority.fence,
|
||||
worker_incarnation: worker,
|
||||
progress: ProgressValueDto {
|
||||
completed_bytes: safe(2048),
|
||||
total_bytes: Some(safe(4096)),
|
||||
completed_members: safe(1),
|
||||
total_members: Some(safe(1)),
|
||||
},
|
||||
})
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(cross_kind.status(), 409);
|
||||
assert_eq!(
|
||||
cross_kind.headers().get("content-type").unwrap(),
|
||||
"application/problem+json"
|
||||
);
|
||||
let error: ApiErrorDto = cross_kind.json().await.unwrap();
|
||||
assert_eq!(error.message, "precondition failed");
|
||||
server.stop(true).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn success_outcome_publishes_the_exact_manifest_and_replays() {
|
||||
let Some(harness) = Harness::new("success").await else {
|
||||
return;
|
||||
};
|
||||
let state = WorkerApiState::new(harness.service());
|
||||
let (base, server, client) = start_authenticated_server(state).await;
|
||||
let job_id = harness.job().await;
|
||||
let worker = unique_id("worker");
|
||||
let claim: ClaimResponseDto = client
|
||||
.post(format!("{base}/internal/v1/jobs/{job_id}/claim"))
|
||||
.json(&ClaimRequestDto {
|
||||
operation_id: unique_id("operation"),
|
||||
worker_incarnation: worker.clone(),
|
||||
})
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.json()
|
||||
.await
|
||||
.unwrap();
|
||||
let attempt_url = format!("{base}/internal/v1/attempts/{}", claim.authority.attempt_id);
|
||||
assert_eq!(
|
||||
client
|
||||
.post(format!("{attempt_url}/activate"))
|
||||
.json(&AttemptActivateRequestDto {
|
||||
operation_id: unique_id("operation"),
|
||||
fence: claim.authority.fence,
|
||||
worker_incarnation: worker.clone(),
|
||||
})
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
200
|
||||
);
|
||||
let result = harness.seal_result(&claim.authority.attempt_id).await;
|
||||
let operation = unique_id("operation");
|
||||
let outcome = AttemptOutcomeRequestDto {
|
||||
operation_id: operation,
|
||||
fence: claim.authority.fence,
|
||||
worker_incarnation: worker,
|
||||
outcome: AttemptOutcomeDto::Success(remora_contracts::AttemptSuccessOutcomeDto { result }),
|
||||
};
|
||||
let outcome_url = format!("{attempt_url}/outcome");
|
||||
for _ in 0..2 {
|
||||
let response = client
|
||||
.post(&outcome_url)
|
||||
.json(&outcome)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 200);
|
||||
assert_eq!(
|
||||
response
|
||||
.json::<JobPhaseProjectionDto>()
|
||||
.await
|
||||
.unwrap()
|
||||
.phase,
|
||||
JobPhase::Succeeded
|
||||
);
|
||||
}
|
||||
server.stop(true).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mutual_tls_requires_a_trusted_client_certificate() {
|
||||
let certificates = certificates();
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = HttpServer::new(|| {
|
||||
App::new().route(
|
||||
"/identity",
|
||||
web::get().to(|identity: AuthenticatedWorker| async move {
|
||||
HttpResponse::Ok().body(identity.identity.to_string())
|
||||
}),
|
||||
)
|
||||
})
|
||||
.on_connect(authenticate_rustls_worker)
|
||||
.listen_rustls_0_23(listener, certificates.server_config)
|
||||
.unwrap()
|
||||
.run();
|
||||
let handle = server.handle();
|
||||
tokio::spawn(server);
|
||||
let root = ClientCertificate::from_der(certificates.ca.der()).unwrap();
|
||||
let untrusted = Client::builder()
|
||||
.add_root_certificate(root.clone())
|
||||
.build()
|
||||
.unwrap();
|
||||
assert!(
|
||||
untrusted
|
||||
.get(format!("https://localhost:{}/identity", address.port()))
|
||||
.send()
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
let mut client_pem = certificates.client.pem();
|
||||
client_pem.push_str(&certificates.client_key.serialize_pem());
|
||||
let trusted = Client::builder()
|
||||
.add_root_certificate(root)
|
||||
.identity(Identity::from_pem(client_pem.as_bytes()).unwrap())
|
||||
.build()
|
||||
.unwrap();
|
||||
let response = trusted
|
||||
.get(format!("https://localhost:{}/identity", address.port()))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 200);
|
||||
let expected = certificate_identity(certificates.client.der().as_ref());
|
||||
assert_eq!(response.text().await.unwrap(), expected);
|
||||
handle.stop(true).await;
|
||||
}
|
||||
|
||||
async fn start_authenticated_server(
|
||||
state: WorkerApiState,
|
||||
) -> (String, actix_web::dev::ServerHandle, Client) {
|
||||
let certificates = certificates();
|
||||
let client = trusted_client(&certificates);
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = HttpServer::new(move || {
|
||||
App::new()
|
||||
.app_data(web::Data::new(state.clone()))
|
||||
.configure(configure_worker_api)
|
||||
})
|
||||
.on_connect(authenticate_rustls_worker)
|
||||
.listen_rustls_0_23(listener, certificates.server_config)
|
||||
.unwrap()
|
||||
.run();
|
||||
let handle = server.handle();
|
||||
tokio::spawn(server);
|
||||
(
|
||||
format!("https://localhost:{}", address.port()),
|
||||
handle,
|
||||
client,
|
||||
)
|
||||
}
|
||||
|
||||
fn trusted_client(certificates: &Certificates) -> Client {
|
||||
let root = ClientCertificate::from_der(certificates.ca.der()).unwrap();
|
||||
let mut client_pem = certificates.client.pem();
|
||||
client_pem.push_str(&certificates.client_key.serialize_pem());
|
||||
Client::builder()
|
||||
.add_root_certificate(root)
|
||||
.identity(Identity::from_pem(client_pem.as_bytes()).unwrap())
|
||||
.build()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn certificate_identity(certificate: &[u8]) -> String {
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
let mut value = String::from("w");
|
||||
for byte in Sha256::digest(certificate) {
|
||||
use std::fmt::Write as _;
|
||||
write!(value, "{byte:02x}").unwrap();
|
||||
}
|
||||
value
|
||||
}
|
||||
|
||||
struct Certificates {
|
||||
ca: Certificate,
|
||||
client: Certificate,
|
||||
client_key: KeyPair,
|
||||
server_config: ServerConfig,
|
||||
}
|
||||
|
||||
fn certificates() -> Certificates {
|
||||
let _ = rustls::crypto::ring::default_provider().install_default();
|
||||
let mut ca_params = CertificateParams::new(Vec::new()).unwrap();
|
||||
ca_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained);
|
||||
ca_params.key_usages = vec![
|
||||
KeyUsagePurpose::DigitalSignature,
|
||||
KeyUsagePurpose::KeyCertSign,
|
||||
KeyUsagePurpose::CrlSign,
|
||||
];
|
||||
let ca_key = KeyPair::generate().unwrap();
|
||||
let ca = ca_params.self_signed(&ca_key).unwrap();
|
||||
let issuer = Issuer::new(ca_params, ca_key);
|
||||
|
||||
let server_key = KeyPair::generate().unwrap();
|
||||
let mut server_params = CertificateParams::new(vec!["localhost".into()]).unwrap();
|
||||
server_params.key_usages = vec![KeyUsagePurpose::DigitalSignature];
|
||||
server_params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ServerAuth];
|
||||
let server = server_params.signed_by(&server_key, &issuer).unwrap();
|
||||
|
||||
let client_key = KeyPair::generate().unwrap();
|
||||
let mut client_params = CertificateParams::new(Vec::new()).unwrap();
|
||||
client_params.key_usages = vec![KeyUsagePurpose::DigitalSignature];
|
||||
client_params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ClientAuth];
|
||||
let client = client_params.signed_by(&client_key, &issuer).unwrap();
|
||||
|
||||
let mut roots = RootCertStore::empty();
|
||||
roots.add(ca.der().clone()).unwrap();
|
||||
let verifier = WebPkiClientVerifier::builder(Arc::new(roots))
|
||||
.build()
|
||||
.unwrap();
|
||||
let server_key_der = PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(server_key.serialize_der()));
|
||||
let server_config = ServerConfig::builder()
|
||||
.with_client_cert_verifier(verifier)
|
||||
.with_single_cert(vec![server.der().clone()], server_key_der)
|
||||
.unwrap();
|
||||
Certificates {
|
||||
ca,
|
||||
client,
|
||||
client_key,
|
||||
server_config,
|
||||
}
|
||||
}
|
||||
|
||||
fn required_env(name: &str) -> Option<String> {
|
||||
match std::env::var(name) {
|
||||
Ok(value) if !value.is_empty() => Some(value),
|
||||
_ if std::env::var_os("REMORA_WORKER_API_REQUIRE").is_some() => {
|
||||
panic!("{name} is required")
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn coordination_namespace() -> String {
|
||||
std::env::var("REMORA_COORDINATION_NAMESPACE").unwrap_or_else(|_| "remora:test:".into())
|
||||
}
|
||||
|
||||
fn nonce() -> u128 {
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
static COUNTER: AtomicU64 = AtomicU64::new(0);
|
||||
(u128::from(std::process::id()) << 64) | u128::from(COUNTER.fetch_add(1, Ordering::Relaxed))
|
||||
}
|
||||
|
||||
fn unique_id(prefix: &str) -> OpaqueId {
|
||||
OpaqueId::parse(format!("{prefix}{:032x}", nonce())).unwrap()
|
||||
}
|
||||
|
||||
fn positive(value: u64) -> PositiveIntegerDto {
|
||||
PositiveIntegerDto::new(value).unwrap()
|
||||
}
|
||||
|
||||
fn safe(value: u64) -> SafeIntegerDto {
|
||||
SafeIntegerDto::new(value).unwrap()
|
||||
}
|
||||
Loading…
Reference in a new issue