ms2k/remora
SHA256
Watch
0
Code Releases Activity

Add authenticated internal worker API

Browse source
This commit is contained in:
ms2k 2026-07-17 09:16:46 +09:00
commit ebf818d23c
13 changed files with 2745 additions and 43 deletions
Download patch file Download diff file Expand all files Collapse all files

27
crates/remora-api/Cargo.toml
View file

@ -4,3 +4,30 @@ edition.workspace = true
publish = false
rust-version.workspace = true
version.workspace = true
[dependencies]
actix-web = { version = "=4.14.0", features = ["rustls-0_23"] }
actix-tls = "=3.5.0"
getrandom = "0.3.4"
remora-attempt = { path = "../remora-attempt" }
remora-contracts = { path = "../remora-contracts" }
remora-domain.workspace = true
remora-result = { path = "../remora-result" }
remora-target = { path = "../remora-target" }
serde.workspace = true
serde_json.workspace = true
sha2.workspace = true
sqlx.workspace = true
thiserror.workspace = true
time.workspace = true
[dev-dependencies]
rcgen = "=0.14.8"
remora-admission = { path = "../remora-admission" }
remora-coordination = { path = "../remora-coordination" }
remora-credentials = { path = "../remora-credentials" }
remora-durable.workspace = true
remora-identity = { path = "../remora-identity" }
reqwest = { version = "=0.13.4", default-features = false, features = ["json", "rustls"] }
rustls = "=0.23.42"
tokio.workspace = true

6
crates/remora-api/src/lib.rs
View file

@ -1,3 +1,9 @@
mod worker;
pub use worker::{
AuthenticatedWorker, WorkerApiState, authenticate_rustls_worker, configure_worker_api,
};
pub const COMPONENT_NAME: &str = "remora-api";
#[cfg(test)]

613
crates/remora-api/src/worker.rs Normal file
View file

@ -0,0 +1,613 @@
use std::{
any::Any,
future::{Ready, ready},
};
use actix_web::{
FromRequest, HttpRequest, HttpResponse, ResponseError, dev::Payload, http::StatusCode, web,
};
use remora_attempt::{
ActivationMutationOutcome, AttemptAuthority, AttemptError, AttemptMutationOutcome,
AttemptService, CheckpointProjection, CheckpointRecord, ClaimOutcome, DurableDescriptor,
LeaseProjection, MutationRejection, OutcomePublication, ProgressRecord, RenewalMutationOutcome,
};
use remora_contracts::{
ActivationResponseDto, ApiErrorCodeDto, ApiErrorDto, AttemptActivateRequestDto,
AttemptAuthorityDto, AttemptCheckpointRequestDto, AttemptOutcomeDto, AttemptOutcomeRequestDto,
AttemptProgressRequestDto, AttemptRenewRequestDto, AttemptResolvedSourceRequestDto,
CheckpointDescriptorKindDto, CheckpointDescriptorRefDto, ClaimRequestDto, ClaimResponseDto,
CredentialGrantVersionDto, DateTimeDto, DurableDescriptorKindDto, DurableDescriptorRefDto,
ExecutionEnvelopeDto, JobPhaseProjectionDto, LeaseProjectionDto, NullableDto,
PositiveIntegerDto, ResolvedSourceDescriptorKindDto, ResolvedSourceDescriptorRefDto,
SafeIntegerDto, SourceProjectionDto, StagingTargetDto, TargetDto,
};
use remora_domain::OpaqueId;
use remora_result::PublicationInput;
use remora_target::{ContentSha256, ObjectIdentity, RESERVED_NAMESPACE_SEGMENT, StoredObject};
use sha2::{Digest, Sha256};
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct AuthenticatedWorker {
pub identity: OpaqueId,
}
pub fn authenticate_rustls_worker(
connection: &dyn Any,
extensions: &mut actix_web::dev::Extensions,
) {
type Stream = actix_tls::accept::rustls_0_23::TlsStream<actix_web::rt::net::TcpStream>;
let Some(stream) = connection.downcast_ref::<Stream>() else {
return;
};
let Some(certificate) = stream
.get_ref()
.1
.peer_certificates()
.and_then(|chain| chain.first())
else {
return;
};
let digest = Sha256::digest(certificate.as_ref());
let mut value = String::from("w");
for byte in digest {
use std::fmt::Write as _;
write!(value, "{byte:02x}").expect("writing to String cannot fail");
}
extensions.insert(AuthenticatedWorker {
identity: OpaqueId::parse(value).expect("certificate digest is a valid identity"),
});
}
impl FromRequest for AuthenticatedWorker {
type Error = WorkerApiError;
type Future = Ready<Result<Self, Self::Error>>;
fn from_request(request: &HttpRequest, _payload: &mut Payload) -> Self::Future {
ready(
request
.conn_data::<Self>()
.cloned()
.ok_or_else(WorkerApiError::unauthenticated),
)
}
}
#[derive(Clone)]
pub struct WorkerApiState {
attempts: AttemptService,
}
impl WorkerApiState {
#[must_use]
pub fn new(attempts: AttemptService) -> Self {
Self { attempts }
}
}
pub fn configure_worker_api(config: &mut web::ServiceConfig) {
config.service(
web::scope("/internal/v1")
.app_data(
web::JsonConfig::default()
.limit(1024 * 1024)
.error_handler(|_, _| WorkerApiError::invalid_request().into()),
)
.route("/jobs/{job_id}/claim", web::post().to(claim))
.route("/attempts/{attempt_id}/renew", web::post().to(renew))
.route("/attempts/{attempt_id}/activate", web::post().to(activate))
.route(
"/attempts/{attempt_id}/resolved-source",
web::post().to(pin_resolved_source),
)
.route(
"/attempts/{attempt_id}/checkpoint",
web::post().to(persist_checkpoint),
)
.route(
"/attempts/{attempt_id}/progress",
web::post().to(report_progress),
)
.route(
"/attempts/{attempt_id}/outcome",
web::post().to(report_outcome),
),
);
}
async fn claim(
_: AuthenticatedWorker,
state: web::Data<WorkerApiState>,
path: web::Path<String>,
request: web::Json<ClaimRequestDto>,
) -> Result<HttpResponse, WorkerApiError> {
let job_id = parse_id(path.into_inner())?;
let outcome = state
.attempts
.claim(&job_id, &request.worker_incarnation, &request.operation_id)
.await
.map_err(WorkerApiError::from_attempt)?;
let ClaimOutcome::Claimed { authority, .. } = outcome else {
return Err(WorkerApiError::precondition_failed());
};
let execution = load_execution_envelope(&state.attempts, &authority).await?;
Ok(HttpResponse::Ok().json(ClaimResponseDto {
authority: authority_dto(&authority)?,
execution,
}))
}
async fn renew(
_: AuthenticatedWorker,
state: web::Data<WorkerApiState>,
path: web::Path<String>,
request: web::Json<AttemptRenewRequestDto>,
) -> Result<HttpResponse, WorkerApiError> {
let authority = request_authority(
&state,
path.into_inner(),
request.fence.get(),
&request.worker_incarnation,
)
.await?;
match state
.attempts
.renew_operation(&authority, &request.operation_id)
.await
.map_err(WorkerApiError::from_attempt)?
{
RenewalMutationOutcome::Renewed { lease, .. } => {
Ok(HttpResponse::Ok().json(lease_dto(lease)?))
}
RenewalMutationOutcome::AuthorityLost { .. } => Err(WorkerApiError::precondition_failed()),
}
}
async fn activate(
_: AuthenticatedWorker,
state: web::Data<WorkerApiState>,
path: web::Path<String>,
request: web::Json<AttemptActivateRequestDto>,
) -> Result<HttpResponse, WorkerApiError> {
let authority = request_authority(
&state,
path.into_inner(),
request.fence.get(),
&request.worker_incarnation,
)
.await?;
let lease = match state
.attempts
.activate_operation(&authority, &request.operation_id)
.await
.map_err(WorkerApiError::from_attempt)?
{
ActivationMutationOutcome::Activated { lease, .. } => lease,
ActivationMutationOutcome::PreconditionFailed => {
return Err(WorkerApiError::precondition_failed());
}
};
let execution = load_execution_envelope(&state.attempts, &authority).await?;
Ok(HttpResponse::Ok().json(ActivationResponseDto {
authority: authority_dto(&authority)?,
execution,
lease: lease_dto(lease)?,
}))
}
async fn pin_resolved_source(
_: AuthenticatedWorker,
state: web::Data<WorkerApiState>,
path: web::Path<String>,
request: web::Json<AttemptResolvedSourceRequestDto>,
) -> Result<HttpResponse, WorkerApiError> {
let authority = request_authority(
&state,
path.into_inner(),
request.fence.get(),
&request.worker_incarnation,
)
.await?;
accepted(
state
.attempts
.pin_resolved_source(
&authority,
&request.operation_id,
descriptor_from_resolved(&request.resolved_source_ref)?,
)
.await
.map_err(WorkerApiError::from_attempt)?,
)
}
async fn persist_checkpoint(
_: AuthenticatedWorker,
state: web::Data<WorkerApiState>,
path: web::Path<String>,
request: web::Json<AttemptCheckpointRequestDto>,
) -> Result<HttpResponse, WorkerApiError> {
let authority = request_authority(
&state,
path.into_inner(),
request.fence.get(),
&request.worker_incarnation,
)
.await?;
accepted(
state
.attempts
.persist_checkpoint(
&authority,
&request.operation_id,
CheckpointRecord {
descriptor: descriptor_from_checkpoint(&request.checkpoint_ref)?,
resolved_source_id: request.resolved_source_id.clone(),
validated_bytes: request.validated_bytes.get(),
validated_members: request.validated_members.get(),
},
)
.await
.map_err(WorkerApiError::from_attempt)?,
)
}
async fn report_progress(
_: AuthenticatedWorker,
state: web::Data<WorkerApiState>,
path: web::Path<String>,
request: web::Json<AttemptProgressRequestDto>,
) -> Result<HttpResponse, WorkerApiError> {
let authority = request_authority(
&state,
path.into_inner(),
request.fence.get(),
&request.worker_incarnation,
)
.await?;
accepted(
state
.attempts
.report_progress(
&authority,
&request.operation_id,
ProgressRecord {
completed_bytes: request.progress.completed_bytes.get(),
total_bytes: request.progress.total_bytes.map(SafeIntegerDto::get),
completed_members: request.progress.completed_members.get(),
total_members: request.progress.total_members.map(SafeIntegerDto::get),
},
)
.await
.map_err(WorkerApiError::from_attempt)?,
)
}
async fn report_outcome(
_: AuthenticatedWorker,
state: web::Data<WorkerApiState>,
path: web::Path<String>,
request: web::Json<AttemptOutcomeRequestDto>,
) -> Result<HttpResponse, WorkerApiError> {
let authority = request_authority(
&state,
path.into_inner(),
request.fence.get(),
&request.worker_incarnation,
)
.await?;
let outcome = match &request.outcome {
AttemptOutcomeDto::Success(success) => {
let result = &success.result;
state
.attempts
.publish_success_operation(
&authority,
&request.operation_id,
PublicationInput {
attempt_id: authority.attempt_id.clone(),
manifest_id: result.manifest_id.clone(),
schema_version: u32::try_from(result.schema_version.get())
.map_err(|_| WorkerApiError::invalid_request())?,
manifest: StoredObject {
identity: ObjectIdentity {
target_store_id: result.manifest.store.clone(),
bucket: result.manifest.bucket.clone(),
key: result.manifest.key.clone(),
version_id: result.manifest.version_id.clone(),
},
content_sha256: ContentSha256::parse(&result.manifest.checksum)
.map_err(|_| WorkerApiError::invalid_request())?,
size: result.manifest.size.get(),
},
member_count: result.member_count.get(),
total_bytes: result.total_bytes.get(),
},
)
.await
}
AttemptOutcomeDto::Failure(failure) => {
state
.attempts
.publish_failure_operation(&authority, &request.operation_id, failure.failure.class)
.await
}
}
.map_err(WorkerApiError::from_attempt)?;
let phase = match outcome {
OutcomePublication::Accepted { phase, .. } => phase,
OutcomePublication::Rejected { reason, .. } => {
return Err(WorkerApiError::from_rejection(reason));
}
};
Ok(HttpResponse::Ok().json(JobPhaseProjectionDto { phase }))
}
fn accepted(outcome: AttemptMutationOutcome) -> Result<HttpResponse, WorkerApiError> {
match outcome {
AttemptMutationOutcome::Accepted(_) => Ok(HttpResponse::NoContent().finish()),
AttemptMutationOutcome::Rejected { reason, .. } => {
Err(WorkerApiError::from_rejection(reason))
}
}
}
async fn request_authority(
state: &WorkerApiState,
attempt_id: String,
fence: u64,
worker: &OpaqueId,
) -> Result<AttemptAuthority, WorkerApiError> {
let attempt_id = parse_id(attempt_id)?;
state
.attempts
.historical_authority(&attempt_id)
.await
.map_err(WorkerApiError::from_attempt)?
.filter(|authority| authority.fence == fence && authority.worker_incarnation == *worker)
.ok_or_else(WorkerApiError::precondition_failed)
}
async fn load_execution_envelope(
attempts: &AttemptService,
authority: &AttemptAuthority,
) -> Result<ExecutionEnvelopeDto, WorkerApiError> {
let row = attempts
.execution_envelope(&authority.attempt_id)
.await
.map_err(WorkerApiError::from_attempt)?
.ok_or_else(WorkerApiError::precondition_failed)?;
if row.job_id != authority.job_id
|| row.attempt_id != authority.attempt_id
|| row.fence != authority.fence
|| row.worker_incarnation != authority.worker_incarnation
{
return Err(WorkerApiError::precondition_failed());
}
let source: SourceProjectionDto =
serde_json::from_value(row.source).map_err(|_| WorkerApiError::unavailable())?;
let target_prefix = row.target_prefix.join("/");
let resolved_source_ref = row.resolved_source.map(|descriptor| {
NullableDto(Some(descriptor_ref(
descriptor,
DurableDescriptorKindDto::ResolvedSource,
)))
});
let checkpoint_ref = row.checkpoint.and_then(|checkpoint| match checkpoint {
CheckpointProjection::Reusable { checkpoint } => Some(NullableDto(Some(descriptor_ref(
checkpoint.descriptor,
DurableDescriptorKindDto::Checkpoint,
)))),
CheckpointProjection::Complete { .. } => None,
});
Ok(ExecutionEnvelopeDto {
id: row.id,
job_id: authority.job_id.clone(),
attempt_id: authority.attempt_id.clone(),
fence: positive(authority.fence)?,
worker_incarnation: authority.worker_incarnation.clone(),
source,
sensitive_source_ref: row.sensitive_source_ref,
credential_version: NullableDto(match (row.credential_id, row.credential_version_id) {
(Some(credential_id), Some(version_id)) => Some(CredentialGrantVersionDto {
credential_id,
version_id,
}),
(None, None) => None,
_ => return Err(WorkerApiError::unavailable()),
}),
resolved_source_ref: resolved_source_ref.unwrap_or(NullableDto(None)),
target: TargetDto {
store: row.target_store.clone(),
bucket: row.target_bucket.clone(),
prefix: target_prefix,
},
staging_target: StagingTargetDto {
store: row.target_store,
bucket: row.target_bucket,
prefix: format!(
"{RESERVED_NAMESPACE_SEGMENT}/a/{}/",
authority.attempt_id.as_str()
),
},
execution_grant_ref: row.execution_grant_id,
checkpoint_ref: checkpoint_ref.unwrap_or(NullableDto(None)),
})
}
fn descriptor_ref(
descriptor: DurableDescriptor,
kind: DurableDescriptorKindDto,
) -> DurableDescriptorRefDto {
DurableDescriptorRefDto {
id: descriptor.id,
kind,
schema_version: positive(descriptor.schema_version.into()).expect("validated descriptor"),
byte_length: safe(descriptor.byte_length).expect("validated descriptor"),
item_count: descriptor
.item_count
.map(|value| safe(value).expect("validated descriptor")),
checksum: descriptor.checksum,
}
}
fn descriptor_from_resolved(
value: &ResolvedSourceDescriptorRefDto,
) -> Result<DurableDescriptor, WorkerApiError> {
let ResolvedSourceDescriptorKindDto::ResolvedSource = value.kind;
Ok(DurableDescriptor {
id: value.id.clone(),
schema_version: u32::try_from(value.schema_version.get())
.map_err(|_| WorkerApiError::invalid_request())?,
byte_length: value.byte_length.get(),
item_count: value.item_count.map(SafeIntegerDto::get),
checksum: value.checksum.clone(),
})
}
fn descriptor_from_checkpoint(
value: &CheckpointDescriptorRefDto,
) -> Result<DurableDescriptor, WorkerApiError> {
let CheckpointDescriptorKindDto::Checkpoint = value.kind;
Ok(DurableDescriptor {
id: value.id.clone(),
schema_version: u32::try_from(value.schema_version.get())
.map_err(|_| WorkerApiError::invalid_request())?,
byte_length: value.byte_length.get(),
item_count: value.item_count.map(SafeIntegerDto::get),
checksum: value.checksum.clone(),
})
}
fn authority_dto(authority: &AttemptAuthority) -> Result<AttemptAuthorityDto, WorkerApiError> {
Ok(AttemptAuthorityDto {
attempt_id: authority.attempt_id.clone(),
fence: positive(authority.fence)?,
worker_incarnation: authority.worker_incarnation.clone(),
})
}
fn lease_dto(lease: LeaseProjection) -> Result<LeaseProjectionDto, WorkerApiError> {
let observed = timestamp(lease.observed_at_ms)?;
let expires = timestamp(lease.expires_at_ms)?;
let renew_at = observed + (expires - observed) / 2;
Ok(LeaseProjectionDto {
expires_at: datetime(expires)?,
renew_after: datetime(renew_at)?,
})
}
fn timestamp(milliseconds: u64) -> Result<OffsetDateTime, WorkerApiError> {
let nanoseconds = i128::from(milliseconds) * 1_000_000;
OffsetDateTime::from_unix_timestamp_nanos(nanoseconds)
.map_err(|_| WorkerApiError::unavailable())
}
fn datetime(value: OffsetDateTime) -> Result<DateTimeDto, WorkerApiError> {
let value = value
.format(&Rfc3339)
.map_err(|_| WorkerApiError::unavailable())?;
DateTimeDto::parse(value).map_err(|_| WorkerApiError::unavailable())
}
fn positive(value: u64) -> Result<PositiveIntegerDto, WorkerApiError> {
PositiveIntegerDto::new(value).map_err(|_| WorkerApiError::unavailable())
}
fn safe(value: u64) -> Result<SafeIntegerDto, WorkerApiError> {
SafeIntegerDto::new(value).map_err(|_| WorkerApiError::unavailable())
}
fn parse_id(value: impl Into<String>) -> Result<OpaqueId, WorkerApiError> {
OpaqueId::parse(value).map_err(|_| WorkerApiError::invalid_request())
}
#[derive(Debug, thiserror::Error)]
#[error("worker API request failed")]
pub struct WorkerApiError {
status: StatusCode,
code: ApiErrorCodeDto,
message: &'static str,
}
impl WorkerApiError {
fn invalid_request() -> Self {
Self::new(
StatusCode::BAD_REQUEST,
ApiErrorCodeDto::InvalidRequest,
"invalid request",
)
}
fn unauthenticated() -> Self {
Self::new(
StatusCode::UNAUTHORIZED,
ApiErrorCodeDto::Unauthenticated,
"authentication required",
)
}
fn precondition_failed() -> Self {
Self::new(
StatusCode::CONFLICT,
ApiErrorCodeDto::PreconditionFailed,
"precondition failed",
)
}
fn unavailable() -> Self {
Self::new(
StatusCode::SERVICE_UNAVAILABLE,
ApiErrorCodeDto::Unavailable,
"service unavailable",
)
}
fn new(status: StatusCode, code: ApiErrorCodeDto, message: &'static str) -> Self {
Self {
status,
code,
message,
}
}
fn from_attempt(error: AttemptError) -> Self {
match error {
AttemptError::Invalid(_) => Self::invalid_request(),
AttemptError::OperationConflict => Self::precondition_failed(),
AttemptError::Coordination(_) | AttemptError::Durable(_) => Self::unavailable(),
}
}
fn from_rejection(_reason: MutationRejection) -> Self {
Self::precondition_failed()
}
}
impl ResponseError for WorkerApiError {
fn status_code(&self) -> StatusCode {
self.status
}
fn error_response(&self) -> HttpResponse {
HttpResponse::build(self.status)
.content_type("application/problem+json")
.json(ApiErrorDto {
code: self.code,
message: self.message.into(),
request_id: request_id(),
fields: None,
})
}
}
fn request_id() -> OpaqueId {
let mut bytes = [0_u8; 16];
getrandom::fill(&mut bytes).expect("operating system randomness is required");
let mut value = String::from("r");
for byte in bytes {
use std::fmt::Write as _;
write!(value, "{byte:02x}").expect("writing to String cannot fail");
}
OpaqueId::parse(value).expect("generated request identifier is valid")
}

735
crates/remora-api/tests/worker_api.rs Normal file
View file

@ -0,0 +1,735 @@
use std::{net::TcpListener, sync::Arc, time::Duration};
use actix_web::{App, HttpResponse, HttpServer, web};
use rcgen::{
BasicConstraints, Certificate, CertificateParams, ExtendedKeyUsagePurpose, IsCa, Issuer,
KeyPair, KeyUsagePurpose,
};
use remora_admission::{
AdmissionConfig, AdmissionDecision, AdmissionService, migrate as admission_migrate,
};
use remora_api::{
AuthenticatedWorker, WorkerApiState, authenticate_rustls_worker, configure_worker_api,
};
use remora_attempt::{AttemptConfig, AttemptService, RecoveryConfig, migrate as attempt_migrate};
use remora_contracts::{
ApiErrorDto, AttemptActivateRequestDto, AttemptCheckpointRequestDto, AttemptFailureOutcomeDto,
AttemptOutcomeDto, AttemptOutcomeRequestDto, AttemptProgressRequestDto, AttemptRenewRequestDto,
AttemptResolvedSourceRequestDto, AttemptRuntimeFailureDto, CheckpointDescriptorKindDto,
CheckpointDescriptorRefDto, ClaimRequestDto, ClaimResponseDto, HttpSourceSubmissionDto,
JobPhaseProjectionDto, JobSubmissionDto, PositiveIntegerDto, ProgressValueDto,
ResolvedSourceDescriptorKindDto, ResolvedSourceDescriptorRefDto, ResultDescriptorDto,
ResultKindDto, SafeIntegerDto, SourceSubmissionDto,
};
use remora_coordination::{CoordinationStore, StoreConfig};
use remora_credentials::migrate as credentials_migrate;
use remora_domain::{
BucketName, JobPhase, OpaqueId, RuntimeFailureClass, StorageScope, TargetPrefix, TargetStoreId,
};
use remora_durable::{PoolConfig, connect, migrate as durable_migrate};
use remora_identity::{DefaultTarget, Principal};
use remora_result::migrate as result_migrate;
use remora_target::attempt_manifest_key;
use reqwest::{Certificate as ClientCertificate, Client, Identity};
use rustls::{
RootCertStore, ServerConfig,
pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer},
server::WebPkiClientVerifier,
};
use sqlx::PgPool;
struct Harness {
pool: PgPool,
coordination: CoordinationStore,
}
impl Harness {
async fn new(name: &str) -> Option<Self> {
let database_url = required_env("DATABASE_URL")?;
let coordination_url = required_env("REMORA_COORDINATION_URL")?;
let pool = connect(
&database_url,
PoolConfig {
max_connections: 16,
acquire_timeout: Duration::from_secs(10),
idle_timeout: Some(Duration::from_secs(30)),
},
)
.await
.unwrap();
durable_migrate(&pool).await.unwrap();
credentials_migrate(&pool).await.unwrap();
admission_migrate(&pool).await.unwrap();
attempt_migrate(&pool).await.unwrap();
result_migrate(&pool).await.unwrap();
sqlx::query("TRUNCATE TABLE jobs CASCADE")
.execute(&pool)
.await
.unwrap();
let coordination = CoordinationStore::connect(
&coordination_url,
format!("{}wp18:{name}:{}:", coordination_namespace(), nonce()),
StoreConfig {
global_limit: 16,
lease_ttl: Duration::from_secs(10),
reserve_ttl: Duration::from_secs(10),
progress_ttl: Duration::from_secs(10),
cancel_ttl: None,
},
)
.await
.unwrap();
Some(Self { pool, coordination })
}
fn service(&self) -> AttemptService {
AttemptService::new(
self.pool.clone(),
self.coordination.clone(),
AttemptConfig {
max_attempts: 3,
local_limit: 8,
},
)
.unwrap()
}
async fn job(&self) -> OpaqueId {
let store = TargetStoreId::parse("primary").unwrap();
let bucket = BucketName::parse("remora-results").unwrap();
let prefix = TargetPrefix::parse("users/alice", ["_remora"]).unwrap();
let principal = Principal {
id: unique_id("owner"),
issuer: "https://issuer.example".into(),
subject: "subject".into(),
administrator: false,
scopes: [StorageScope {
store: store.clone(),
bucket: bucket.clone(),
prefix: prefix.clone(),
}]
.into_iter()
.collect(),
};
let service = AdmissionService::new(
self.pool.clone(),
AdmissionConfig {
default_target: DefaultTarget {
store,
bucket,
prefix,
},
reserved_target_segments: vec!["_remora".into()],
policy_version: "worker-api-v1".into(),
},
)
.unwrap();
let outcome = service
.submit(
&principal,
unique_id("submit").as_str(),
JobSubmissionDto {
source: SourceSubmissionDto::Http(HttpSourceSubmissionDto {
url: "https://example.invalid/artifact".into(),
result_path: "artifact.bin".into(),
credential_id: None,
}),
target: None,
label: Some("worker-api-test".into()),
},
)
.await
.unwrap();
let AdmissionDecision::Accepted(job) = outcome.decision else {
panic!("job rejected")
};
self.coordination
.enqueue(job.job_id.as_str())
.await
.unwrap();
job.job_id
}
async fn seal_result(&self, attempt_id: &OpaqueId) -> ResultDescriptorDto {
let manifest_id = unique_id("manifest");
let checksum = format!("sha256:{}", "11".repeat(32));
let key = attempt_manifest_key(attempt_id, &manifest_id);
sqlx::query(
"INSERT INTO result_inventories (
attempt_id, state, manifest_schema_version, manifest_id, manifest_store,
manifest_bucket, manifest_key, manifest_version_id, manifest_sha256,
manifest_size, member_count, total_bytes
) VALUES ($1, 'sealed', 1, $2, 'primary', 'remora-results', $3,
'version-1', $4, 128, 1, 4096)",
)
.bind(attempt_id.as_str())
.bind(manifest_id.as_str())
.bind(&key)
.bind(&checksum)
.execute(&self.pool)
.await
.unwrap();
ResultDescriptorDto {
kind: ResultKindDto::Manifest,
manifest_id,
schema_version: positive(1),
manifest: remora_contracts::ObjectIdentityDto {
store: "primary".into(),
bucket: "remora-results".into(),
key,
version_id: Some("version-1".into()),
checksum,
size: safe(128),
},
member_count: positive(1),
total_bytes: safe(4096),
}
}
}
#[tokio::test]
async fn worker_operations_are_authenticated_fenced_replayable_and_redacted() {
let Some(harness) = Harness::new("flow").await else {
return;
};
let state = WorkerApiState::new(harness.service());
let (base, server, client) = start_authenticated_server(state).await;
let job_id = harness.job().await;
let worker = unique_id("worker");
let claim_operation = unique_id("operation");
let claim = ClaimRequestDto {
operation_id: claim_operation.clone(),
worker_incarnation: worker.clone(),
};
let unauthenticated = actix_web::test::init_service(App::new().route(
"/",
web::post().to(|_: AuthenticatedWorker| async { HttpResponse::NoContent().finish() }),
))
.await;
let response = actix_web::test::call_service(
&unauthenticated,
actix_web::test::TestRequest::post().uri("/").to_request(),
)
.await;
assert_eq!(response.status(), 401);
assert_eq!(
response.headers().get("content-type").unwrap(),
"application/problem+json"
);
let claim_url = format!("{base}/internal/v1/jobs/{job_id}/claim");
let response = client.post(&claim_url).json(&claim).send().await.unwrap();
assert_eq!(response.status(), 200);
let claimed: ClaimResponseDto = response.json().await.unwrap();
assert_eq!(claimed.authority.worker_incarnation, worker);
assert_eq!(claimed.execution.attempt_id, claimed.authority.attempt_id);
assert_eq!(claimed.execution.job_id, job_id);
assert!(
claimed
.execution
.staging_target
.prefix
.contains(claimed.authority.attempt_id.as_str())
);
let changed_claim = client
.post(&claim_url)
.json(&ClaimRequestDto {
operation_id: claim_operation,
worker_incarnation: unique_id("worker"),
})
.send()
.await
.unwrap();
assert_eq!(changed_claim.status(), 409);
assert_eq!(
changed_claim.headers().get("content-type").unwrap(),
"application/problem+json"
);
let attempt_url = format!(
"{base}/internal/v1/attempts/{}",
claimed.authority.attempt_id
);
let activation_operation = unique_id("operation");
let activation = AttemptActivateRequestDto {
operation_id: activation_operation,
fence: claimed.authority.fence,
worker_incarnation: worker.clone(),
};
let response = client
.post(format!("{attempt_url}/activate"))
.json(&activation)
.send()
.await
.unwrap();
assert_eq!(response.status(), 200);
let activation_body: serde_json::Value = response.json().await.unwrap();
let activated: remora_contracts::ActivationResponseDto =
serde_json::from_value(activation_body.clone()).unwrap();
assert!(activated.lease.renew_after.as_str() < activated.lease.expires_at.as_str());
let renewal_operation = unique_id("operation");
let renewal = AttemptRenewRequestDto {
operation_id: renewal_operation.clone(),
fence: claimed.authority.fence,
worker_incarnation: worker.clone(),
};
let first_renewal = client
.post(format!("{attempt_url}/renew"))
.json(&renewal)
.send()
.await
.unwrap();
assert_eq!(first_renewal.status(), 200);
let first_lease: serde_json::Value = first_renewal.json().await.unwrap();
let replayed_lease: serde_json::Value = client
.post(format!("{attempt_url}/renew"))
.json(&renewal)
.send()
.await
.unwrap()
.json()
.await
.unwrap();
assert_eq!(replayed_lease, first_lease);
let replayed_activation: serde_json::Value = client
.post(format!("{attempt_url}/activate"))
.json(&activation)
.send()
.await
.unwrap()
.json()
.await
.unwrap();
assert_eq!(replayed_activation, activation_body);
let authority_mismatches = [
(
format!("{base}/internal/v1/attempts/{}/renew", unique_id("attempt")),
claimed.authority.fence,
worker.clone(),
),
(
format!("{attempt_url}/renew"),
positive(claimed.authority.fence.get() + 1),
worker.clone(),
),
(
format!("{attempt_url}/renew"),
claimed.authority.fence,
unique_id("worker"),
),
];
for (url, fence, worker_incarnation) in authority_mismatches {
let response = client
.post(url)
.json(&AttemptRenewRequestDto {
operation_id: unique_id("operation"),
fence,
worker_incarnation,
})
.send()
.await
.unwrap();
assert_eq!(response.status(), 409);
}
let resolved_id = unique_id("resolved");
let resolved = AttemptResolvedSourceRequestDto {
operation_id: unique_id("operation"),
fence: claimed.authority.fence,
worker_incarnation: worker.clone(),
resolved_source_ref: ResolvedSourceDescriptorRefDto {
id: resolved_id.clone(),
kind: ResolvedSourceDescriptorKindDto::ResolvedSource,
schema_version: positive(1),
byte_length: safe(64),
item_count: Some(safe(1)),
checksum: format!("sha256:{}", "22".repeat(32)),
},
};
assert_eq!(
client
.post(format!("{attempt_url}/resolved-source"))
.json(&resolved)
.send()
.await
.unwrap()
.status(),
204
);
let checkpoint = AttemptCheckpointRequestDto {
operation_id: unique_id("operation"),
fence: claimed.authority.fence,
worker_incarnation: worker.clone(),
checkpoint_ref: CheckpointDescriptorRefDto {
id: unique_id("checkpoint"),
kind: CheckpointDescriptorKindDto::Checkpoint,
schema_version: positive(1),
byte_length: safe(32),
item_count: Some(safe(1)),
checksum: format!("sha256:{}", "33".repeat(32)),
},
resolved_source_id: resolved_id,
validated_bytes: safe(1024),
validated_members: safe(1),
};
assert_eq!(
client
.post(format!("{attempt_url}/checkpoint"))
.json(&checkpoint)
.send()
.await
.unwrap()
.status(),
204
);
let progress = AttemptProgressRequestDto {
operation_id: unique_id("operation"),
fence: claimed.authority.fence,
worker_incarnation: worker.clone(),
progress: ProgressValueDto {
completed_bytes: safe(2048),
total_bytes: Some(safe(4096)),
completed_members: safe(1),
total_members: Some(safe(1)),
},
};
assert_eq!(
client
.post(format!("{attempt_url}/progress"))
.json(&progress)
.send()
.await
.unwrap()
.status(),
204
);
let replayed_claim: ClaimResponseDto = client
.post(&claim_url)
.json(&claim)
.send()
.await
.unwrap()
.json()
.await
.unwrap();
assert_eq!(
serde_json::to_value(replayed_claim).unwrap(),
serde_json::to_value(&claimed).unwrap()
);
let failure = AttemptOutcomeRequestDto {
operation_id: unique_id("operation"),
fence: claimed.authority.fence,
worker_incarnation: worker.clone(),
outcome: AttemptOutcomeDto::Failure(AttemptFailureOutcomeDto {
failure: AttemptRuntimeFailureDto {
class: RuntimeFailureClass::TransientSource,
message: "secret-value-must-not-escape".into(),
safe_details: None,
},
}),
};
let outcome_url = format!("{attempt_url}/outcome");
let response = client
.post(&outcome_url)
.json(&failure)
.send()
.await
.unwrap();
assert_eq!(response.status(), 200);
let bytes = response.bytes().await.unwrap();
assert!(!String::from_utf8_lossy(&bytes).contains("secret-value-must-not-escape"));
let projection: JobPhaseProjectionDto = serde_json::from_slice(&bytes).unwrap();
assert_eq!(projection.phase, JobPhase::RetryWait);
let report = harness
.service()
.reconcile_once(&RecoveryConfig {
retry_delay: Duration::ZERO,
cleanup_claim_ttl: Duration::from_secs(1),
batch_size: 128,
})
.await
.unwrap();
assert_eq!(report.retries_scheduled, 1);
let replay = client
.post(&outcome_url)
.json(&failure)
.send()
.await
.unwrap();
assert_eq!(replay.status(), 200);
assert_eq!(replay.bytes().await.unwrap(), bytes);
let cross_kind = client
.post(format!("{attempt_url}/progress"))
.json(&AttemptProgressRequestDto {
operation_id: renewal_operation,
fence: claimed.authority.fence,
worker_incarnation: worker,
progress: ProgressValueDto {
completed_bytes: safe(2048),
total_bytes: Some(safe(4096)),
completed_members: safe(1),
total_members: Some(safe(1)),
},
})
.send()
.await
.unwrap();
assert_eq!(cross_kind.status(), 409);
assert_eq!(
cross_kind.headers().get("content-type").unwrap(),
"application/problem+json"
);
let error: ApiErrorDto = cross_kind.json().await.unwrap();
assert_eq!(error.message, "precondition failed");
server.stop(true).await;
}
#[tokio::test]
async fn success_outcome_publishes_the_exact_manifest_and_replays() {
let Some(harness) = Harness::new("success").await else {
return;
};
let state = WorkerApiState::new(harness.service());
let (base, server, client) = start_authenticated_server(state).await;
let job_id = harness.job().await;
let worker = unique_id("worker");
let claim: ClaimResponseDto = client
.post(format!("{base}/internal/v1/jobs/{job_id}/claim"))
.json(&ClaimRequestDto {
operation_id: unique_id("operation"),
worker_incarnation: worker.clone(),
})
.send()
.await
.unwrap()
.json()
.await
.unwrap();
let attempt_url = format!("{base}/internal/v1/attempts/{}", claim.authority.attempt_id);
assert_eq!(
client
.post(format!("{attempt_url}/activate"))
.json(&AttemptActivateRequestDto {
operation_id: unique_id("operation"),
fence: claim.authority.fence,
worker_incarnation: worker.clone(),
})
.send()
.await
.unwrap()
.status(),
200
);
let result = harness.seal_result(&claim.authority.attempt_id).await;
let operation = unique_id("operation");
let outcome = AttemptOutcomeRequestDto {
operation_id: operation,
fence: claim.authority.fence,
worker_incarnation: worker,
outcome: AttemptOutcomeDto::Success(remora_contracts::AttemptSuccessOutcomeDto { result }),
};
let outcome_url = format!("{attempt_url}/outcome");
for _ in 0..2 {
let response = client
.post(&outcome_url)
.json(&outcome)
.send()
.await
.unwrap();
assert_eq!(response.status(), 200);
assert_eq!(
response
.json::<JobPhaseProjectionDto>()
.await
.unwrap()
.phase,
JobPhase::Succeeded
);
}
server.stop(true).await;
}
#[tokio::test]
async fn mutual_tls_requires_a_trusted_client_certificate() {
let certificates = certificates();
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap();
let server = HttpServer::new(|| {
App::new().route(
"/identity",
web::get().to(|identity: AuthenticatedWorker| async move {
HttpResponse::Ok().body(identity.identity.to_string())
}),
)
})
.on_connect(authenticate_rustls_worker)
.listen_rustls_0_23(listener, certificates.server_config)
.unwrap()
.run();
let handle = server.handle();
tokio::spawn(server);
let root = ClientCertificate::from_der(certificates.ca.der()).unwrap();
let untrusted = Client::builder()
.add_root_certificate(root.clone())
.build()
.unwrap();
assert!(
untrusted
.get(format!("https://localhost:{}/identity", address.port()))
.send()
.await
.is_err()
);
let mut client_pem = certificates.client.pem();
client_pem.push_str(&certificates.client_key.serialize_pem());
let trusted = Client::builder()
.add_root_certificate(root)
.identity(Identity::from_pem(client_pem.as_bytes()).unwrap())
.build()
.unwrap();
let response = trusted
.get(format!("https://localhost:{}/identity", address.port()))
.send()
.await
.unwrap();
assert_eq!(response.status(), 200);
let expected = certificate_identity(certificates.client.der().as_ref());
assert_eq!(response.text().await.unwrap(), expected);
handle.stop(true).await;
}
async fn start_authenticated_server(
state: WorkerApiState,
) -> (String, actix_web::dev::ServerHandle, Client) {
let certificates = certificates();
let client = trusted_client(&certificates);
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap();
let server = HttpServer::new(move || {
App::new()
.app_data(web::Data::new(state.clone()))
.configure(configure_worker_api)
})
.on_connect(authenticate_rustls_worker)
.listen_rustls_0_23(listener, certificates.server_config)
.unwrap()
.run();
let handle = server.handle();
tokio::spawn(server);
(
format!("https://localhost:{}", address.port()),
handle,
client,
)
}
fn trusted_client(certificates: &Certificates) -> Client {
let root = ClientCertificate::from_der(certificates.ca.der()).unwrap();
let mut client_pem = certificates.client.pem();
client_pem.push_str(&certificates.client_key.serialize_pem());
Client::builder()
.add_root_certificate(root)
.identity(Identity::from_pem(client_pem.as_bytes()).unwrap())
.build()
.unwrap()
}
fn certificate_identity(certificate: &[u8]) -> String {
use sha2::{Digest, Sha256};
let mut value = String::from("w");
for byte in Sha256::digest(certificate) {
use std::fmt::Write as _;
write!(value, "{byte:02x}").unwrap();
}
value
}
struct Certificates {
ca: Certificate,
client: Certificate,
client_key: KeyPair,
server_config: ServerConfig,
}
fn certificates() -> Certificates {
let _ = rustls::crypto::ring::default_provider().install_default();
let mut ca_params = CertificateParams::new(Vec::new()).unwrap();
ca_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained);
ca_params.key_usages = vec![
KeyUsagePurpose::DigitalSignature,
KeyUsagePurpose::KeyCertSign,
KeyUsagePurpose::CrlSign,
];
let ca_key = KeyPair::generate().unwrap();
let ca = ca_params.self_signed(&ca_key).unwrap();
let issuer = Issuer::new(ca_params, ca_key);
let server_key = KeyPair::generate().unwrap();
let mut server_params = CertificateParams::new(vec!["localhost".into()]).unwrap();
server_params.key_usages = vec![KeyUsagePurpose::DigitalSignature];
server_params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ServerAuth];
let server = server_params.signed_by(&server_key, &issuer).unwrap();
let client_key = KeyPair::generate().unwrap();
let mut client_params = CertificateParams::new(Vec::new()).unwrap();
client_params.key_usages = vec![KeyUsagePurpose::DigitalSignature];
client_params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ClientAuth];
let client = client_params.signed_by(&client_key, &issuer).unwrap();
let mut roots = RootCertStore::empty();
roots.add(ca.der().clone()).unwrap();
let verifier = WebPkiClientVerifier::builder(Arc::new(roots))
.build()
.unwrap();
let server_key_der = PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(server_key.serialize_der()));
let server_config = ServerConfig::builder()
.with_client_cert_verifier(verifier)
.with_single_cert(vec![server.der().clone()], server_key_der)
.unwrap();
Certificates {
ca,
client,
client_key,
server_config,
}
}
fn required_env(name: &str) -> Option<String> {
match std::env::var(name) {
Ok(value) if !value.is_empty() => Some(value),
_ if std::env::var_os("REMORA_WORKER_API_REQUIRE").is_some() => {
panic!("{name} is required")
}
_ => None,
}
}
fn coordination_namespace() -> String {
std::env::var("REMORA_COORDINATION_NAMESPACE").unwrap_or_else(|_| "remora:test:".into())
}
fn nonce() -> u128 {
use std::sync::atomic::{AtomicU64, Ordering};
static COUNTER: AtomicU64 = AtomicU64::new(0);
(u128::from(std::process::id()) << 64) | u128::from(COUNTER.fetch_add(1, Ordering::Relaxed))
}
fn unique_id(prefix: &str) -> OpaqueId {
OpaqueId::parse(format!("{prefix}{:032x}", nonce())).unwrap()
}
fn positive(value: u64) -> PositiveIntegerDto {
PositiveIntegerDto::new(value).unwrap()
}
fn safe(value: u64) -> SafeIntegerDto {
SafeIntegerDto::new(value).unwrap()
}

2
crates/remora-attempt/migrations/0004_worker_api.sql Normal file
View file

@ -0,0 +1,2 @@
ALTER TABLE attempts
ADD COLUMN execution_envelope JSONB;

5
crates/remora-attempt/src/lib.rs
View file

@ -20,6 +20,7 @@ pub use recovery_types::{
};
pub use service::{AttemptConfig, AttemptService};
pub use types::{
ActivationOutcome, AttemptAuthority, AttemptFaultInjector, AttemptFaultPoint, AttemptState,
ClaimOutcome, ClaimRejection, RecoveryOutcome, RenewalOutcome,
ActivationMutationOutcome, ActivationOutcome, AttemptAuthority, AttemptFaultInjector,
AttemptFaultPoint, AttemptState, ClaimOutcome, ClaimRejection, ExecutionEnvelopeRecord,
LeaseProjection, OutcomePublication, RecoveryOutcome, RenewalMutationOutcome, RenewalOutcome,
};

76
crates/remora-attempt/src/mutations.rs
View file

@ -14,7 +14,7 @@ use time::OffsetDateTime;
use crate::{
AttemptAuthority, AttemptError, AttemptMutationOutcome, AttemptService, CheckpointProjection,
CheckpointRecord, CommittedResultRecord, DurableDescriptor, FailureRecord,
JobMutationProjection, MutationRejection, ProgressRecord,
JobMutationProjection, MutationRejection, OutcomePublication, ProgressRecord,
};
impl AttemptService {
@ -64,6 +64,18 @@ impl AttemptService {
.await
}
pub async fn publish_success_operation(
&self,
authority: &AttemptAuthority,
operation_id: &OpaqueId,
result: PublicationInput,
) -> Result<OutcomePublication, AttemptError> {
let outcome = self
.publish_success(authority, operation_id, result)
.await?;
self.publication_outcome(operation_id, outcome).await
}
pub async fn persist_checkpoint(
&self,
authority: &AttemptAuthority,
@ -124,6 +136,16 @@ impl AttemptService {
.await
}
pub async fn publish_failure_operation(
&self,
authority: &AttemptAuthority,
operation_id: &OpaqueId,
class: RuntimeFailureClass,
) -> Result<OutcomePublication, AttemptError> {
let outcome = self.publish_failure(authority, operation_id, class).await?;
self.publication_outcome(operation_id, outcome).await
}
pub async fn report_progress(
&self,
authority: &AttemptAuthority,
@ -227,7 +249,7 @@ impl AttemptService {
.execute(tx.as_mut())
.await?;
enqueue(tx, "job.updated", json!({ "job_id": authority.job_id })).await?;
accepted()
accepted_phase(JobPhase::Succeeded)
})
},
)
@ -307,7 +329,11 @@ impl AttemptService {
json!({ "job_id": authority.job_id }),
)
.await?;
accepted()
accepted_phase(if retry {
JobPhase::RetryWait
} else {
JobPhase::Failed
})
})
},
)
@ -329,6 +355,43 @@ impl AttemptService {
row.map(ProjectionRow::try_into_projection).transpose()
}
async fn publication_outcome(
&self,
operation_id: &OpaqueId,
outcome: AttemptMutationOutcome,
) -> Result<OutcomePublication, AttemptError> {
match outcome {
AttemptMutationOutcome::Rejected {
reason,
disposition,
} => Ok(OutcomePublication::Rejected {
reason,
disposition,
}),
AttemptMutationOutcome::Accepted(disposition) => {
let response = sqlx::query_scalar::<_, Json<Value>>(
"SELECT response_body FROM mutation_records
WHERE scope_subject = $1 AND scope_operation = 'attempt.mutation'
AND operation_key = $2",
)
.bind(self.operation_subject.as_str())
.bind(operation_id.as_str())
.fetch_one(&self.pool)
.await
.map_err(DurableError::from)?;
let phase = serde_json::from_value(
response
.0
.get("phase")
.cloned()
.ok_or_else(|| AttemptError::Invalid("missing outcome phase".into()))?,
)
.map_err(|error| AttemptError::Invalid(error.to_string()))?;
Ok(OutcomePublication::Accepted { disposition, phase })
}
}
}
async fn execute_durable_mutation<T, D>(
&self,
authority: &AttemptAuthority,
@ -423,6 +486,13 @@ fn accepted() -> Result<(MutationStatus, Value), DurableError> {
Ok((MutationStatus::Accepted, json!({ "accepted": true })))
}
fn accepted_phase(phase: JobPhase) -> Result<(MutationStatus, Value), DurableError> {
Ok((
MutationStatus::Accepted,
json!({ "accepted": true, "phase": phase }),
))
}
fn rejected(reason: MutationRejection) -> Result<(MutationStatus, Value), DurableError> {
Ok((MutationStatus::Rejected, json!({ "reason": reason })))
}

324
crates/remora-attempt/src/service.rs
View file

@ -14,9 +14,10 @@ use sqlx::{PgPool, types::Json};
use time::OffsetDateTime;
use crate::{
ActivationOutcome, AttemptAuthority, AttemptError, AttemptFaultInjector, AttemptFaultPoint,
AttemptState, CheckpointProjection, ClaimOutcome, ClaimRejection, RecoveryOutcome,
RenewalOutcome,
ActivationMutationOutcome, ActivationOutcome, AttemptAuthority, AttemptError,
AttemptFaultInjector, AttemptFaultPoint, AttemptState, CheckpointProjection, ClaimOutcome,
ClaimRejection, ExecutionEnvelopeRecord, LeaseProjection, RecoveryOutcome,
RenewalMutationOutcome, RenewalOutcome,
};
struct NoFaults;
@ -179,6 +180,23 @@ impl AttemptService {
authority: &AttemptAuthority,
operation_id: &OpaqueId,
) -> Result<ActivationOutcome, AttemptError> {
Ok(
match self.activate_operation(authority, operation_id).await? {
ActivationMutationOutcome::Activated { disposition, .. } => {
ActivationOutcome::Activated(disposition)
}
ActivationMutationOutcome::PreconditionFailed => {
ActivationOutcome::PreconditionFailed
}
},
)
}
pub async fn activate_operation(
&self,
authority: &AttemptAuthority,
operation_id: &OpaqueId,
) -> Result<ActivationMutationOutcome, AttemptError> {
validate_authority(authority)?;
let request_fingerprint = fingerprint(&ActivationRequest {
job_id: &authority.job_id,
@ -203,16 +221,17 @@ impl AttemptService {
request_fingerprint,
move |tx| {
Box::pin(async move {
let accepted =
let lease =
activate_attempt(tx, &coordination, &authority_for_effect, faults.as_ref())
.await?;
Ok((
if accepted {
if lease.is_some() {
MutationStatus::Accepted
} else {
MutationStatus::Rejected
},
json!({ "activated": accepted }),
serde_json::to_value(ActivationStored { lease })
.map_err(DurableError::effect)?,
))
})
},
@ -225,7 +244,14 @@ impl AttemptService {
.exact_authority_is_executing_and_live(authority)
.await?
{
return Ok(ActivationOutcome::Activated(MutationDisposition::First));
let lease = self
.live_lease(authority)
.await?
.ok_or_else(|| AttemptError::Invalid("live lease disappeared".into()))?;
return Ok(ActivationMutationOutcome::Activated {
disposition: MutationDisposition::First,
lease,
});
}
self.release_exact(authority).await;
return Err(error.into());
@ -237,16 +263,24 @@ impl AttemptService {
}
if outcome.status == MutationStatus::Rejected {
self.release_exact(authority).await;
return Ok(ActivationOutcome::PreconditionFailed);
return Ok(ActivationMutationOutcome::PreconditionFailed);
}
if !self
.exact_authority_is_executing_and_live(authority)
.await?
{
self.release_exact(authority).await;
return Ok(ActivationOutcome::PreconditionFailed);
return Ok(ActivationMutationOutcome::PreconditionFailed);
}
Ok(ActivationOutcome::Activated(outcome.disposition))
let stored: ActivationStored = serde_json::from_value(outcome.response)
.map_err(|error| AttemptError::Invalid(error.to_string()))?;
let lease = stored
.lease
.ok_or_else(|| AttemptError::Invalid("accepted activation omitted its lease".into()))?;
Ok(ActivationMutationOutcome::Activated {
disposition: outcome.disposition,
lease,
})
}
pub async fn renew(
@ -269,6 +303,103 @@ impl AttemptService {
Ok(RenewalOutcome::Renewed)
}
pub async fn renew_operation(
&self,
authority: &AttemptAuthority,
operation_id: &OpaqueId,
) -> Result<RenewalMutationOutcome, AttemptError> {
validate_authority(authority)?;
let request_fingerprint = fingerprint(&ActivationRequest {
job_id: &authority.job_id,
attempt_id: &authority.attempt_id,
fence: authority.fence,
worker_incarnation: &authority.worker_incarnation,
})
.map_err(|error| AttemptError::Invalid(error.to_string()))?;
let eligibility = authority_binding(authority);
let authority = authority.clone();
let coordination = self.coordination.clone();
let faults = self.faults.clone();
let outcome = execute_mutation(
&self.pool,
MutationScope {
subject: &self.operation_subject,
operation: "attempt.mutation",
key: operation_id.as_str(),
},
"renew_attempt",
&eligibility,
request_fingerprint,
move |tx| {
Box::pin(async move {
let Some(row) = lock_authority(tx, &authority.attempt_id).await? else {
return Ok((
MutationStatus::Rejected,
serde_json::to_value(RenewalMutationOutcome::AuthorityLost {
disposition: None,
})
.map_err(DurableError::effect)?,
));
};
if row.state != AttemptState::Executing
|| !row.matches(
&authority.job_id,
authority.fence,
&authority.worker_incarnation,
)
{
return Ok((
MutationStatus::Rejected,
serde_json::to_value(RenewalMutationOutcome::AuthorityLost {
disposition: None,
})
.map_err(DurableError::effect)?,
));
}
faults.hit(AttemptFaultPoint::RenewalBeforeLease);
let expected = lease_owner(&authority).map_err(DurableError::effect)?;
let renewed = coordination
.renew(authority.job_id.as_str(), &expected)
.await
.map_err(DurableError::effect)?;
faults.hit(AttemptFaultPoint::RenewalAfterLease);
let live = coordination
.get_live_lease(authority.job_id.as_str())
.await
.map_err(DurableError::effect)?;
let response = match live
.filter(|lease| renewed.ok() && lease.record.owner() == expected)
{
Some(lease) => RenewalMutationOutcome::Renewed {
lease: LeaseProjection {
expires_at_ms: lease.record.expires_at,
observed_at_ms: lease.now_ms,
},
disposition: None,
},
None => RenewalMutationOutcome::AuthorityLost { disposition: None },
};
let status = if matches!(response, RenewalMutationOutcome::Renewed { .. }) {
MutationStatus::Accepted
} else {
MutationStatus::Rejected
};
Ok((
status,
serde_json::to_value(response).map_err(DurableError::effect)?,
))
})
},
)
.await?;
if outcome.disposition == MutationDisposition::Conflict {
return Err(AttemptError::OperationConflict);
}
let response: RenewalMutationOutcome = serde_json::from_value(outcome.response)
.map_err(|error| AttemptError::Invalid(error.to_string()))?;
Ok(response.with_disposition(outcome.disposition))
}
pub async fn recover_inactive(
&self,
authority: &AttemptAuthority,
@ -372,6 +503,45 @@ impl AttemptService {
.map_err(Into::into)
}
pub async fn historical_authority(
&self,
attempt_id: &OpaqueId,
) -> Result<Option<AttemptAuthority>, AttemptError> {
load_historical_authority(&self.pool, attempt_id)
.await
.map_err(Into::into)
}
pub async fn execution_envelope(
&self,
attempt_id: &OpaqueId,
) -> Result<Option<ExecutionEnvelopeRecord>, AttemptError> {
let value = sqlx::query_scalar::<_, Option<Json<ExecutionEnvelopeRecord>>>(
"SELECT execution_envelope FROM attempts WHERE attempt_id = $1",
)
.bind(attempt_id.as_str())
.fetch_optional(&self.pool)
.await?
.flatten();
Ok(value.map(|value| value.0))
}
pub async fn live_lease(
&self,
authority: &AttemptAuthority,
) -> Result<Option<LeaseProjection>, AttemptError> {
let expected = lease_owner(authority)?;
Ok(self
.coordination
.get_live_lease(authority.job_id.as_str())
.await?
.filter(|lease| lease.record.owner() == expected)
.map(|lease| LeaseProjection {
expires_at_ms: lease.record.expires_at,
observed_at_ms: lease.now_ms,
}))
}
pub fn release_local(&self, job_id: &OpaqueId) -> bool {
self.local_permits.release_all(job_id.as_str())
}
@ -451,6 +621,11 @@ struct ActivationRequest<'a> {
worker_incarnation: &'a OpaqueId,
}
#[derive(serde::Deserialize, Serialize)]
struct ActivationStored {
lease: Option<LeaseProjection>,
}
struct AuthorityRow {
job_id: OpaqueId,
fence: u64,
@ -534,11 +709,14 @@ async fn authorize_attempt(
_ => 0,
};
let now = OffsetDateTime::now_utc();
let envelope =
load_execution_envelope_record(tx, input.job_id, input.attempt_id, fence, input.worker)
.await?;
sqlx::query(
"INSERT INTO attempts (
attempt_id, job_id, fence, worker_incarnation, state,
initial_progress, authorized_at, executing_at, reaped_at
) VALUES ($1, $2, $3, $4, 'authorized', $5, $6, NULL, NULL)",
initial_progress, authorized_at, executing_at, reaped_at, execution_envelope
) VALUES ($1, $2, $3, $4, 'authorized', $5, $6, NULL, NULL, $7)",
)
.bind(input.attempt_id.as_str())
.bind(input.job_id.as_str())
@ -546,6 +724,7 @@ async fn authorize_attempt(
.bind(input.worker.as_str())
.bind(i64::try_from(initial_progress).map_err(DurableError::effect)?)
.bind(now)
.bind(Json(&envelope))
.execute(tx.as_mut())
.await?;
sqlx::query(
@ -572,21 +751,79 @@ async fn authorize_attempt(
})
}
async fn load_execution_envelope_record(
tx: &mut remora_durable::PgTx<'_>,
job_id: &OpaqueId,
attempt_id: &OpaqueId,
fence: u64,
worker: &OpaqueId,
) -> Result<ExecutionEnvelopeRecord, DurableError> {
type Row = (
Json<serde_json::Value>,
String,
String,
String,
Json<Vec<String>>,
String,
Option<String>,
Option<String>,
Option<Json<crate::DurableDescriptor>>,
Option<Json<CheckpointProjection>>,
);
let row = sqlx::query_as::<_, Row>(
"SELECT j.source_projection, j.sensitive_source_ref, j.target_store,
j.target_bucket, j.target_prefix, j.execution_grant_id,
g.credential_id, g.credential_version_id, j.resolved_source, j.checkpoint
FROM jobs j
JOIN execution_grants g ON g.grant_id = j.execution_grant_id
WHERE j.job_id = $1",
)
.bind(job_id.as_str())
.fetch_one(tx.as_mut())
.await?;
let grant_id = OpaqueId::parse(row.5).map_err(DurableError::effect)?;
Ok(ExecutionEnvelopeRecord {
id: grant_id.clone(),
job_id: job_id.clone(),
attempt_id: attempt_id.clone(),
fence,
worker_incarnation: worker.clone(),
source: row.0.0,
sensitive_source_ref: OpaqueId::parse(row.1).map_err(DurableError::effect)?,
credential_id: row
.6
.map(OpaqueId::parse)
.transpose()
.map_err(DurableError::effect)?,
credential_version_id: row
.7
.map(OpaqueId::parse)
.transpose()
.map_err(DurableError::effect)?,
resolved_source: row.8.map(|value| value.0),
target_store: row.2,
target_bucket: row.3,
target_prefix: row.4.0,
execution_grant_id: grant_id,
checkpoint: row.9.map(|value| value.0),
})
}
async fn activate_attempt(
tx: &mut remora_durable::PgTx<'_>,
coordination: &CoordinationStore,
authority: &AttemptAuthority,
faults: &dyn AttemptFaultInjector,
) -> Result<bool, DurableError> {
) -> Result<Option<LeaseProjection>, DurableError> {
let Some(row) = lock_authority(tx, &authority.attempt_id).await? else {
return Ok(false);
return Ok(None);
};
if !row.matches(
&authority.job_id,
authority.fence,
&authority.worker_incarnation,
) {
return Ok(false);
return Ok(None);
}
if row.state == AttemptState::Executing {
let live = coordination
@ -595,16 +832,28 @@ async fn activate_attempt(
.map_err(DurableError::effect)?;
let expected = lease_owner(authority).map_err(DurableError::effect)?;
if !live.is_some_and(|lease| lease.record.owner() == expected) {
return Ok(false);
return Ok(None);
}
return Ok(coordination
if !coordination
.renew(authority.job_id.as_str(), &expected)
.await
.map_err(DurableError::effect)?
.ok());
.ok()
{
return Ok(None);
}
return Ok(coordination
.get_live_lease(authority.job_id.as_str())
.await
.map_err(DurableError::effect)?
.filter(|lease| lease.record.owner() == expected)
.map(|lease| LeaseProjection {
expires_at_ms: lease.record.expires_at,
observed_at_ms: lease.now_ms,
}));
}
if row.state != AttemptState::Authorized {
return Ok(false);
return Ok(None);
}
faults.hit(AttemptFaultPoint::ActivationBeforeLease);
let owner = lease_owner(authority).map_err(DurableError::effect)?;
@ -620,7 +869,7 @@ async fn activate_attempt(
.await
.map_err(DurableError::effect)?;
if !coordinated.ok() {
return Ok(false);
return Ok(None);
}
faults.hit(AttemptFaultPoint::ActivationAfterLease);
let live = coordination
@ -628,9 +877,9 @@ async fn activate_attempt(
.await
.map_err(DurableError::effect)?;
let expected = lease_owner(authority).map_err(DurableError::effect)?;
if !live.is_some_and(|lease| lease.record.owner() == expected) {
return Ok(false);
}
let Some(live) = live.filter(|lease| lease.record.owner() == expected) else {
return Ok(None);
};
let now = OffsetDateTime::now_utc();
sqlx::query(
"UPDATE attempts SET state = 'executing', executing_at = $2
@ -640,7 +889,10 @@ async fn activate_attempt(
.bind(now)
.execute(tx.as_mut())
.await?;
Ok(true)
Ok(Some(LeaseProjection {
expires_at_ms: live.record.expires_at,
observed_at_ms: live.now_ms,
}))
}
async fn lock_authority(
@ -692,6 +944,30 @@ async fn load_authority(
.transpose()
}
async fn load_historical_authority(
pool: &PgPool,
attempt_id: &OpaqueId,
) -> Result<Option<AttemptAuthority>, DurableError> {
let row = sqlx::query_as::<_, (String, String, i64, String, String)>(
"SELECT job_id, attempt_id, fence, worker_incarnation, state
FROM attempts WHERE attempt_id = $1",
)
.bind(attempt_id.as_str())
.fetch_optional(pool)
.await?;
row.map(|(job, attempt, fence, worker, state)| {
Ok(AttemptAuthority {
job_id: OpaqueId::parse(job).map_err(DurableError::effect)?,
attempt_id: OpaqueId::parse(attempt).map_err(DurableError::effect)?,
fence: u64::try_from(fence).map_err(DurableError::effect)?,
worker_incarnation: OpaqueId::parse(worker).map_err(DurableError::effect)?,
state: AttemptState::parse(&state)
.ok_or_else(|| DurableError::Config(format!("unknown attempt state '{state}'")))?,
})
})
.transpose()
}
fn authority_row(row: (String, i64, String, String)) -> Result<AuthorityRow, DurableError> {
Ok(AuthorityRow {
job_id: OpaqueId::parse(row.0).map_err(DurableError::effect)?,

73
crates/remora-attempt/src/types.rs
View file

@ -1,4 +1,4 @@
use remora_domain::OpaqueId;
use remora_domain::{JobPhase, OpaqueId};
use remora_durable::MutationDisposition;
use serde::{Deserialize, Serialize};
@ -95,12 +95,83 @@ pub enum ActivationOutcome {
PreconditionFailed,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum ActivationMutationOutcome {
Activated {
disposition: MutationDisposition,
lease: LeaseProjection,
},
PreconditionFailed,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum RenewalOutcome {
Renewed,
AuthorityLost,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
pub struct LeaseProjection {
pub expires_at_ms: u64,
pub observed_at_ms: u64,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(tag = "status", rename_all = "snake_case")]
pub enum RenewalMutationOutcome {
Renewed {
lease: LeaseProjection,
#[serde(skip)]
disposition: Option<MutationDisposition>,
},
AuthorityLost {
#[serde(skip)]
disposition: Option<MutationDisposition>,
},
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum OutcomePublication {
Accepted {
disposition: MutationDisposition,
phase: JobPhase,
},
Rejected {
reason: crate::MutationRejection,
disposition: MutationDisposition,
},
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
pub struct ExecutionEnvelopeRecord {
pub id: OpaqueId,
pub job_id: OpaqueId,
pub attempt_id: OpaqueId,
pub fence: u64,
pub worker_incarnation: OpaqueId,
pub source: serde_json::Value,
pub sensitive_source_ref: OpaqueId,
pub credential_id: Option<OpaqueId>,
pub credential_version_id: Option<OpaqueId>,
pub resolved_source: Option<crate::DurableDescriptor>,
pub target_store: String,
pub target_bucket: String,
pub target_prefix: Vec<String>,
pub execution_grant_id: OpaqueId,
pub checkpoint: Option<crate::CheckpointProjection>,
}
impl RenewalMutationOutcome {
pub(crate) fn with_disposition(mut self, value: MutationDisposition) -> Self {
match &mut self {
Self::Renewed { disposition, .. } | Self::AuthorityLost { disposition } => {
*disposition = Some(value);
}
}
self
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum RecoveryOutcome {
StillOwned,

3
crates/remora-contracts/src/dto.rs
View file

@ -315,6 +315,8 @@ pub enum SafeDetailValueDto {
#[serde(deny_unknown_fields)]
pub struct ResultDescriptorDto {
pub kind: ResultKindDto,
pub manifest_id: OpaqueId,
pub schema_version: PositiveIntegerDto,
pub manifest: ObjectIdentityDto,
pub member_count: PositiveIntegerDto,
pub total_bytes: SafeIntegerDto,
@ -339,6 +341,7 @@ pub struct ObjectIdentityDto {
)]
pub version_id: Option<String>,
pub checksum: String,
pub size: SafeIntegerDto,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]

3
crates/remora-contracts/src/internal.rs
View file

@ -157,6 +157,9 @@ pub struct AttemptCheckpointRequestDto {
pub fence: PositiveIntegerDto,
pub worker_incarnation: OpaqueId,
pub checkpoint_ref: CheckpointDescriptorRefDto,
pub resolved_source_id: OpaqueId,
pub validated_bytes: SafeIntegerDto,
pub validated_members: SafeIntegerDto,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]

274
crates/remora-contracts/tests/schema_contract.rs
View file

@ -1,5 +1,12 @@
use std::{collections::BTreeSet, fs};
use remora_contracts::{
ActivationResponseDto, AttemptActivateRequestDto, AttemptCheckpointRequestDto,
AttemptOutcomeRequestDto, AttemptProgressRequestDto, AttemptRenewRequestDto,
AttemptResolvedSourceRequestDto, ClaimRequestDto, ClaimResponseDto, JobPhaseProjectionDto,
LeaseProjectionDto,
};
use serde::{Serialize, de::DeserializeOwned};
use serde_json::Value;
#[test]
@ -48,6 +55,273 @@ fn api_contract_documents_parse_and_local_references_resolve() {
validate_openapi_corpus(&component_schema, &corpus, "JobFailure", "job_failures");
}
#[test]
fn worker_routes_and_dtos_conform_to_the_mutable_openapi_contract() {
let openapi: Value =
serde_yaml_ng::from_str(&fs::read_to_string("../../docs/api/openapi.yaml").unwrap())
.unwrap();
let component_schema = openapi_component_schema(&openapi);
let mutation = serde_json::json!({
"operation_id": "operation_0001",
"fence": 1,
"worker_incarnation": "worker_0001"
});
let descriptor = serde_json::json!({
"id": "descriptor_0001",
"kind": "resolved_source",
"schema_version": 1,
"byte_length": 64,
"item_count": 1,
"checksum": format!("sha256:{}", "22".repeat(32))
});
let execution = serde_json::json!({
"id": "grant_0001",
"job_id": "job_00000001",
"attempt_id": "attempt_0001",
"fence": 1,
"worker_incarnation": "worker_0001",
"source": {
"kind": "http",
"display_url": "https://example.invalid/artifact",
"result_path": "artifact.bin"
},
"sensitive_source_ref": "source_00001",
"credential_version": null,
"resolved_source_ref": null,
"target": {
"store": "primary",
"bucket": "remora-results",
"prefix": "users/alice"
},
"staging_target": {
"store": "primary",
"bucket": "remora-results",
"prefix": "_remora/a/attempt_0001/"
},
"execution_grant_ref": "grant_0001",
"checkpoint_ref": null
});
let authority = serde_json::json!({
"attempt_id": "attempt_0001",
"fence": 1,
"worker_incarnation": "worker_0001"
});
let lease = serde_json::json!({
"expires_at": "2026-07-17T00:01:00Z",
"renew_after": "2026-07-17T00:00:30Z"
});
let claim_request = serde_json::json!({
"operation_id": "operation_0001",
"worker_incarnation": "worker_0001"
});
assert_dto_and_schema::<ClaimRequestDto>(&component_schema, "ClaimRequest", claim_request);
let claim_response = serde_json::json!({
"authority": authority,
"execution": execution
});
assert_dto_and_schema::<ClaimResponseDto>(
&component_schema,
"ClaimResponse",
claim_response.clone(),
);
assert_dto_and_schema::<AttemptRenewRequestDto>(
&component_schema,
"AttemptRenewRequest",
mutation.clone(),
);
assert_dto_and_schema::<AttemptActivateRequestDto>(
&component_schema,
"AttemptActivateRequest",
mutation.clone(),
);
assert_dto_and_schema::<ActivationResponseDto>(
&component_schema,
"ActivationResponse",
serde_json::json!({
"authority": claim_response["authority"],
"execution": claim_response["execution"],
"lease": lease.clone()
}),
);
assert_dto_and_schema::<LeaseProjectionDto>(
&component_schema,
"LeaseProjection",
lease.clone(),
);
assert_dto_and_schema::<AttemptResolvedSourceRequestDto>(
&component_schema,
"AttemptResolvedSourceRequest",
merged(
&mutation,
serde_json::json!({ "resolved_source_ref": descriptor.clone() }),
),
);
let mut checkpoint_descriptor = descriptor.clone();
checkpoint_descriptor["kind"] = Value::String("checkpoint".into());
assert_dto_and_schema::<AttemptCheckpointRequestDto>(
&component_schema,
"AttemptCheckpointRequest",
merged(
&mutation,
serde_json::json!({
"checkpoint_ref": checkpoint_descriptor,
"resolved_source_id": "descriptor_0001",
"validated_bytes": 1024,
"validated_members": 1
}),
),
);
assert_dto_and_schema::<AttemptProgressRequestDto>(
&component_schema,
"AttemptProgressRequest",
merged(
&mutation,
serde_json::json!({
"progress": {
"completed_bytes": 1024,
"total_bytes": 4096,
"completed_members": 1,
"total_members": 1
}
}),
),
);
assert_dto_and_schema::<AttemptOutcomeRequestDto>(
&component_schema,
"AttemptOutcomeRequest",
merged(
&mutation,
serde_json::json!({
"outcome": {
"kind": "failure",
"failure": {
"class": "transient_source",
"message": "temporarily unavailable"
}
}
}),
),
);
assert_dto_and_schema::<JobPhaseProjectionDto>(
&component_schema,
"JobPhaseProjection",
serde_json::json!({ "phase": "RetryWait" }),
);
let routes = [
(
"/internal/v1/jobs/{job_id}/claim",
"ClaimRequest",
"200",
Some("ClaimResponse"),
),
(
"/internal/v1/attempts/{attempt_id}/renew",
"AttemptRenewRequest",
"200",
Some("LeaseProjection"),
),
(
"/internal/v1/attempts/{attempt_id}/activate",
"AttemptActivateRequest",
"200",
Some("ActivationResponse"),
),
(
"/internal/v1/attempts/{attempt_id}/resolved-source",
"AttemptResolvedSourceRequest",
"204",
None,
),
(
"/internal/v1/attempts/{attempt_id}/checkpoint",
"AttemptCheckpointRequest",
"204",
None,
),
(
"/internal/v1/attempts/{attempt_id}/progress",
"AttemptProgressRequest",
"204",
None,
),
(
"/internal/v1/attempts/{attempt_id}/outcome",
"AttemptOutcomeRequest",
"200",
Some("JobPhaseProjection"),
),
];
for (path, request, status, response) in routes {
assert_worker_route(&openapi, path, request, status, response);
}
}
fn assert_dto_and_schema<T>(component_schema: &Value, definition: &str, value: Value)
where
T: DeserializeOwned + Serialize,
{
let dto: T = serde_json::from_value(value.clone()).unwrap();
assert_eq!(serde_json::to_value(dto).unwrap(), value);
validate_openapi_value(component_schema, definition, &value);
}
fn validate_openapi_value(component_schema: &Value, definition: &str, value: &Value) {
let mut schema = component_schema.clone();
schema.as_object_mut().unwrap().insert(
"$ref".to_owned(),
Value::String(format!("#/$defs/{definition}")),
);
jsonschema::draft202012::options()
.should_validate_formats(true)
.build(&schema)
.unwrap()
.validate(value)
.unwrap();
}
fn merged(base: &Value, extension: Value) -> Value {
let mut value = base.clone();
value
.as_object_mut()
.unwrap()
.extend(extension.as_object().unwrap().clone());
value
}
fn assert_worker_route(
openapi: &Value,
path: &str,
request_schema: &str,
success_status: &str,
response_schema: Option<&str>,
) {
let operation = &openapi["paths"][path]["post"];
assert_eq!(
operation.pointer("/security/0/serviceAuth"),
Some(&serde_json::json!([]))
);
assert_eq!(
operation.pointer("/requestBody/content/application~1json/schema/$ref"),
Some(&Value::String(format!(
"#/components/schemas/{request_schema}"
)))
);
assert!(operation["responses"].get("409").is_some());
assert!(operation["responses"].get("default").is_some());
let success = &operation["responses"][success_status];
match response_schema {
Some(response_schema) => assert_eq!(
success.pointer("/content/application~1json/schema/$ref"),
Some(&Value::String(format!(
"#/components/schemas/{response_schema}"
)))
),
None => assert!(success.get("content").is_none()),
}
}
fn openapi_component_schema(openapi: &Value) -> Value {
let mut definitions = openapi
.pointer("/components/schemas")

6
crates/remora-target/src/multipart.rs
View file

@ -42,6 +42,7 @@ const PATH_SET: &AsciiSet = &CONTROLS
.add(b'}');
/// One version or delete marker listed under a key/prefix.
#[cfg(feature = "test-utils")]
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ObjectVersion {
pub key: String,
@ -81,6 +82,7 @@ impl MultipartClient {
/// Enable bucket versioning when the peer supports it.
///
/// Peers that return NotImplemented are reported as `false` without error.
#[cfg(feature = "test-utils")]
pub async fn enable_bucket_versioning(&self) -> Result<bool, TargetError> {
let body = r#"<?xml version="1.0" encoding="UTF-8"?>
<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
@ -341,6 +343,7 @@ impl MultipartClient {
}
/// List object versions and delete markers under an optional key prefix.
#[cfg(feature = "test-utils")]
pub async fn list_object_versions(
&self,
prefix: &str,
@ -410,6 +413,7 @@ impl MultipartClient {
}
/// Delete one exact object version or delete marker.
#[cfg(feature = "test-utils")]
pub async fn delete_object_version(
&self,
key: &str,
@ -556,6 +560,7 @@ struct CompletePart {
etag: String,
}
#[cfg(feature = "test-utils")]
#[derive(Debug, Deserialize)]
struct ListVersionsResult {
#[serde(rename = "Version", default)]
@ -570,6 +575,7 @@ struct ListVersionsResult {
next_version_id_marker: Option<String>,
}
#[cfg(feature = "test-utils")]
#[derive(Debug, Deserialize)]
struct ListedVersion {
#[serde(rename = "Key", default)]