Implement crash-safe streaming worker runtime

crates/remora-worker/Cargo.toml

@@ -4,3 +4,41 @@ edition.workspace = true
 publish = false
 rust-version.workspace = true
 version.workspace = true
+
+[dependencies]
+base64.workspace = true
+bytes = "1.12.1"
+getrandom = "0.3.4"
+http = "1.4.0"
+remora-contracts = { path = "../remora-contracts" }
+remora-coordination = { path = "../remora-coordination" }
+remora-credentials = { path = "../remora-credentials" }
+remora-domain.workspace = true
+remora-hf-source = { path = "../remora-hf-source" }
+remora-http-source = { path = "../remora-http-source" }
+remora-object-source = { path = "../remora-object-source" }
+remora-outbound = { path = "../remora-outbound" }
+remora-result = { path = "../remora-result" }
+remora-stream = { path = "../remora-stream" }
+remora-target = { path = "../remora-target" }
+reqwest = { version = "=0.13.4", default-features = false, features = ["json", "rustls"] }
+serde.workspace = true
+serde_json.workspace = true
+serde_json_canonicalizer.workspace = true
+sha2.workspace = true
+sqlx.workspace = true
+thiserror.workspace = true
+time.workspace = true
+tokio.workspace = true
+
+[dev-dependencies]
+actix-tls = "=3.5.0"
+actix-web = { version = "=4.14.0", features = ["rustls-0_23"] }
+futures-util = "0.3.32"
+rcgen = "=0.14.8"
+remora-admission = { path = "../remora-admission" }
+remora-api = { path = "../remora-api" }
+remora-attempt = { path = "../remora-attempt" }
+remora-durable.workspace = true
+remora-identity = { path = "../remora-identity" }
+rustls = "=0.23.42"

crates/remora-worker/migrations/0001_descriptors.sql

@@ -0,0 +1,14 @@
+CREATE TABLE worker_descriptors (
+    descriptor_id TEXT PRIMARY KEY,
+    attempt_id TEXT NOT NULL REFERENCES attempts (attempt_id) ON DELETE CASCADE,
+    kind TEXT NOT NULL CHECK (kind IN ('resolved_source', 'checkpoint')),
+    schema_version INTEGER NOT NULL CHECK (schema_version >= 1),
+    payload JSONB NOT NULL,
+    byte_length BIGINT NOT NULL CHECK (byte_length >= 0),
+    item_count BIGINT CHECK (item_count >= 0),
+    checksum TEXT NOT NULL,
+    created_at TIMESTAMPTZ NOT NULL
+);
+
+CREATE INDEX worker_descriptors_attempt_idx
+    ON worker_descriptors (attempt_id, kind, created_at, descriptor_id);

crates/remora-worker/src/client.rs

@@ -0,0 +1,241 @@
+use remora_contracts::{
+    ActivationResponseDto, AttemptActivateRequestDto, AttemptCheckpointRequestDto,
+    AttemptOutcomeRequestDto, AttemptProgressRequestDto, AttemptRenewRequestDto,
+    AttemptResolvedSourceRequestDto, AttemptStagingMultipartRequestDto,
+    AttemptStagingObjectRequestDto, AttemptStagingWriteIntentRequestDto, AttemptStopRequestDto,
+    ClaimRequestDto, ClaimResponseDto, JobPhaseProjectionDto, LeaseProjectionDto, ProgressValueDto,
+    ResolvedSourceDescriptorRefDto,
+};
+use remora_domain::OpaqueId;
+use reqwest::{Client, Response, StatusCode};
+use serde::{Serialize, de::DeserializeOwned};
+
+use crate::WorkerError;
+
+#[derive(Clone)]
+pub struct WorkerApiClient {
+    base_url: String,
+    client: Client,
+}
+
+impl WorkerApiClient {
+    pub fn new(base_url: impl Into<String>, client: Client) -> Result<Self, WorkerError> {
+        let base_url = base_url.into().trim_end_matches('/').to_owned();
+        let parsed =
+            reqwest::Url::parse(&base_url).map_err(|_| WorkerError::InvalidConfiguration)?;
+        if parsed.scheme() != "https" || parsed.host_str().is_none() {
+            return Err(WorkerError::InvalidConfiguration);
+        }
+        Ok(Self { base_url, client })
+    }
+
+    pub async fn claim(
+        &self,
+        job_id: &OpaqueId,
+        operation_id: OpaqueId,
+        worker_incarnation: OpaqueId,
+    ) -> Result<ClaimResponseDto, WorkerError> {
+        self.post_json(
+            format!("/internal/v1/jobs/{job_id}/claim"),
+            ClaimRequestDto {
+                operation_id,
+                worker_incarnation,
+            },
+        )
+        .await
+    }
+
+    pub async fn activate(
+        &self,
+        attempt_id: &OpaqueId,
+        request: AttemptActivateRequestDto,
+    ) -> Result<ActivationResponseDto, WorkerError> {
+        self.post_json(
+            format!("/internal/v1/attempts/{attempt_id}/activate"),
+            request,
+        )
+        .await
+    }
+
+    pub async fn renew(
+        &self,
+        attempt_id: &OpaqueId,
+        request: AttemptRenewRequestDto,
+    ) -> Result<LeaseProjectionDto, WorkerError> {
+        self.post_json(format!("/internal/v1/attempts/{attempt_id}/renew"), request)
+            .await
+    }
+
+    pub async fn pin_resolved_source(
+        &self,
+        attempt_id: &OpaqueId,
+        operation_id: OpaqueId,
+        fence: remora_contracts::PositiveIntegerDto,
+        worker_incarnation: OpaqueId,
+        resolved_source_ref: ResolvedSourceDescriptorRefDto,
+    ) -> Result<(), WorkerError> {
+        self.post_empty(
+            format!("/internal/v1/attempts/{attempt_id}/resolved-source"),
+            AttemptResolvedSourceRequestDto {
+                operation_id,
+                fence,
+                worker_incarnation,
+                resolved_source_ref,
+            },
+        )
+        .await
+    }
+
+    pub async fn persist_checkpoint(
+        &self,
+        attempt_id: &OpaqueId,
+        request: AttemptCheckpointRequestDto,
+    ) -> Result<(), WorkerError> {
+        self.post_empty(
+            format!("/internal/v1/attempts/{attempt_id}/checkpoint"),
+            request,
+        )
+        .await
+    }
+
+    pub async fn report_progress(
+        &self,
+        attempt_id: &OpaqueId,
+        operation_id: OpaqueId,
+        fence: remora_contracts::PositiveIntegerDto,
+        worker_incarnation: OpaqueId,
+        progress: ProgressValueDto,
+    ) -> Result<(), WorkerError> {
+        self.post_empty(
+            format!("/internal/v1/attempts/{attempt_id}/progress"),
+            AttemptProgressRequestDto {
+                operation_id,
+                fence,
+                worker_incarnation,
+                progress,
+            },
+        )
+        .await
+    }
+
+    pub async fn report_outcome(
+        &self,
+        attempt_id: &OpaqueId,
+        request: AttemptOutcomeRequestDto,
+    ) -> Result<JobPhaseProjectionDto, WorkerError> {
+        self.post_json(
+            format!("/internal/v1/attempts/{attempt_id}/outcome"),
+            request,
+        )
+        .await
+    }
+
+    pub async fn stop(
+        &self,
+        attempt_id: &OpaqueId,
+        request: AttemptStopRequestDto,
+    ) -> Result<(), WorkerError> {
+        self.post_empty(format!("/internal/v1/attempts/{attempt_id}/stop"), request)
+            .await
+    }
+
+    pub async fn register_staging_object(
+        &self,
+        attempt_id: &OpaqueId,
+        request: AttemptStagingObjectRequestDto,
+    ) -> Result<(), WorkerError> {
+        self.post_empty(
+            format!("/internal/v1/attempts/{attempt_id}/staging/object"),
+            request,
+        )
+        .await
+    }
+
+    pub async fn register_staging_write_intent(
+        &self,
+        attempt_id: &OpaqueId,
+        request: AttemptStagingWriteIntentRequestDto,
+    ) -> Result<(), WorkerError> {
+        self.post_empty(
+            format!("/internal/v1/attempts/{attempt_id}/staging/write-intent"),
+            request,
+        )
+        .await
+    }
+
+    pub async fn register_staging_multipart(
+        &self,
+        attempt_id: &OpaqueId,
+        request: AttemptStagingMultipartRequestDto,
+    ) -> Result<(), WorkerError> {
+        self.post_empty(
+            format!("/internal/v1/attempts/{attempt_id}/staging/multipart"),
+            request,
+        )
+        .await
+    }
+
+    async fn post_json<T, R>(&self, path: String, body: T) -> Result<R, WorkerError>
+    where
+        T: Clone + Serialize,
+        R: DeserializeOwned,
+    {
+        let response = self.send_with_retry(path, body).await?;
+        response
+            .json()
+            .await
+            .map_err(|_| WorkerError::InvalidResponse)
+    }
+
+    async fn post_empty<T>(&self, path: String, body: T) -> Result<(), WorkerError>
+    where
+        T: Clone + Serialize,
+    {
+        self.send_with_retry(path, body).await.map(|_| ())
+    }
+
+    async fn send_with_retry<T>(&self, path: String, body: T) -> Result<Response, WorkerError>
+    where
+        T: Clone + Serialize,
+    {
+        let url = format!("{}{path}", self.base_url);
+        let send = || self.client.post(&url).json(&body).send();
+        match send().await {
+            Ok(response) => classify(response),
+            Err(_) => send()
+                .await
+                .map_err(|_| WorkerError::ApiUnavailable)
+                .and_then(classify),
+        }
+    }
+}
+
+fn classify(response: Response) -> Result<Response, WorkerError> {
+    match response.status() {
+        status if status.is_success() => Ok(response),
+        StatusCode::BAD_REQUEST => Err(WorkerError::InvalidRequest),
+        StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN => Err(WorkerError::Unauthenticated),
+        StatusCode::CONFLICT => Err(WorkerError::PreconditionFailed),
+        _ => Err(WorkerError::ApiUnavailable),
+    }
+}
+
+pub(crate) fn operation_id() -> OpaqueId {
+    generated_id('o')
+}
+
+pub(crate) fn incarnation_id() -> OpaqueId {
+    generated_id('w')
+}
+
+fn generated_id(prefix: char) -> OpaqueId {
+    let mut bytes = [0_u8; 16];
+    getrandom::fill(&mut bytes).expect("operating system randomness is required");
+    let mut value = String::with_capacity(33);
+    value.push(prefix);
+    for byte in bytes {
+        use std::fmt::Write as _;
+        write!(value, "{byte:02x}").expect("writing to String cannot fail");
+    }
+    OpaqueId::parse(value).expect("generated identifier is valid")
+}

crates/remora-worker/src/descriptor.rs

@@ -0,0 +1,198 @@
+use remora_contracts::{
+    CheckpointDescriptorKindDto, CheckpointDescriptorRefDto, DurableDescriptorKindDto,
+    DurableDescriptorRefDto, PositiveIntegerDto, ResolvedSourceDescriptorKindDto,
+    ResolvedSourceDescriptorRefDto, SafeIntegerDto,
+};
+use remora_domain::OpaqueId;
+use serde::Serialize;
+use serde_json::Value;
+use sha2::{Digest, Sha256};
+use sqlx::PgPool;
+use time::OffsetDateTime;
+
+use crate::WorkerError;
+
+const MIGRATION_TABLE: &str = "_remora_worker_migrations";
+
+#[derive(Clone)]
+pub struct DescriptorStore {
+    pool: PgPool,
+}
+
+impl DescriptorStore {
+    #[must_use]
+    pub fn new(pool: PgPool) -> Self {
+        Self { pool }
+    }
+
+    pub async fn store_resolved<T: Serialize>(
+        &self,
+        attempt_id: &OpaqueId,
+        payload: &T,
+        item_count: Option<u64>,
+    ) -> Result<ResolvedSourceDescriptorRefDto, WorkerError> {
+        let stored = self
+            .store(attempt_id, "resolved_source", payload, item_count)
+            .await?;
+        Ok(ResolvedSourceDescriptorRefDto {
+            id: stored.id,
+            kind: ResolvedSourceDescriptorKindDto::ResolvedSource,
+            schema_version: stored.schema_version,
+            byte_length: stored.byte_length,
+            item_count: stored.item_count,
+            checksum: stored.checksum,
+        })
+    }
+
+    pub async fn store_checkpoint<T: Serialize>(
+        &self,
+        attempt_id: &OpaqueId,
+        payload: &T,
+        item_count: Option<u64>,
+    ) -> Result<CheckpointDescriptorRefDto, WorkerError> {
+        let stored = self
+            .store(attempt_id, "checkpoint", payload, item_count)
+            .await?;
+        Ok(CheckpointDescriptorRefDto {
+            id: stored.id,
+            kind: CheckpointDescriptorKindDto::Checkpoint,
+            schema_version: stored.schema_version,
+            byte_length: stored.byte_length,
+            item_count: stored.item_count,
+            checksum: stored.checksum,
+        })
+    }
+
+    pub async fn load_ref(
+        &self,
+        job_id: &OpaqueId,
+        descriptor: &DurableDescriptorRefDto,
+    ) -> Result<Option<Value>, WorkerError> {
+        type Row = (String, i32, Value, i64, Option<i64>, String);
+        let row = sqlx::query_as::<_, Row>(
+            "SELECT d.kind, d.schema_version, d.payload, d.byte_length, d.item_count, d.checksum
+             FROM worker_descriptors d
+             JOIN attempts a ON a.attempt_id = d.attempt_id
+             WHERE d.descriptor_id = $1 AND a.job_id = $2",
+        )
+        .bind(descriptor.id.as_str())
+        .bind(job_id.as_str())
+        .fetch_optional(&self.pool)
+        .await
+        .map_err(|_| WorkerError::ApiUnavailable)?;
+        let Some((kind, schema_version, payload, byte_length, item_count, checksum)) = row else {
+            return Ok(None);
+        };
+        let encoded = canonical_bytes(&payload)?;
+        let expected_kind = match descriptor.kind {
+            DurableDescriptorKindDto::ResolvedSource => "resolved_source",
+            DurableDescriptorKindDto::Checkpoint => "checkpoint",
+        };
+        if kind != expected_kind
+            || u64::try_from(schema_version).ok() != Some(descriptor.schema_version.get())
+            || u64::try_from(byte_length).ok() != Some(descriptor.byte_length.get())
+            || item_count.map(u64::try_from).transpose().ok().flatten()
+                != descriptor.item_count.map(SafeIntegerDto::get)
+            || usize::try_from(byte_length).ok() != Some(encoded.len())
+            || checksum != descriptor.checksum
+            || digest(&encoded) != checksum
+        {
+            return Err(WorkerError::InvalidResponse);
+        }
+        Ok(Some(payload))
+    }
+
+    async fn store<T: Serialize>(
+        &self,
+        attempt_id: &OpaqueId,
+        kind: &str,
+        payload: &T,
+        item_count: Option<u64>,
+    ) -> Result<StoredDescriptor, WorkerError> {
+        let payload = serde_json::to_value(payload).map_err(|_| WorkerError::InvalidRequest)?;
+        let encoded = canonical_bytes(&payload)?;
+        let id = descriptor_id();
+        let checksum = digest(&encoded);
+        let byte_length = SafeIntegerDto::new(
+            u64::try_from(encoded.len()).map_err(|_| WorkerError::InvalidRequest)?,
+        )
+        .map_err(|_| WorkerError::InvalidRequest)?;
+        let item_count = item_count
+            .map(SafeIntegerDto::new)
+            .transpose()
+            .map_err(|_| WorkerError::InvalidRequest)?;
+        sqlx::query(
+            "INSERT INTO worker_descriptors (
+                descriptor_id, attempt_id, kind, schema_version, payload,
+                byte_length, item_count, checksum, created_at
+             ) VALUES ($1, $2, $3, 1, $4, $5, $6, $7, $8)",
+        )
+        .bind(id.as_str())
+        .bind(attempt_id.as_str())
+        .bind(kind)
+        .bind(&payload)
+        .bind(i64::try_from(byte_length.get()).map_err(|_| WorkerError::InvalidRequest)?)
+        .bind(
+            item_count
+                .map(SafeIntegerDto::get)
+                .map(i64::try_from)
+                .transpose()
+                .map_err(|_| WorkerError::InvalidRequest)?,
+        )
+        .bind(&checksum)
+        .bind(OffsetDateTime::now_utc())
+        .execute(&self.pool)
+        .await
+        .map_err(|_| WorkerError::ApiUnavailable)?;
+        Ok(StoredDescriptor {
+            id,
+            schema_version: PositiveIntegerDto::new(1).expect("one is positive"),
+            byte_length,
+            item_count,
+            checksum,
+        })
+    }
+}
+
+pub async fn migrate(pool: &PgPool) -> Result<(), sqlx::migrate::MigrateError> {
+    migrator().run(pool).await
+}
+
+#[must_use]
+pub fn migrator() -> sqlx::migrate::Migrator {
+    let mut migrator = sqlx::migrate!("./migrations");
+    migrator.dangerous_set_table_name(MIGRATION_TABLE);
+    migrator
+}
+
+struct StoredDescriptor {
+    id: OpaqueId,
+    schema_version: PositiveIntegerDto,
+    byte_length: SafeIntegerDto,
+    item_count: Option<SafeIntegerDto>,
+    checksum: String,
+}
+
+fn canonical_bytes(value: &Value) -> Result<Vec<u8>, WorkerError> {
+    serde_json_canonicalizer::to_vec(value).map_err(|_| WorkerError::InvalidRequest)
+}
+
+fn digest(bytes: &[u8]) -> String {
+    let mut value = String::from("sha256:");
+    for byte in Sha256::digest(bytes) {
+        use std::fmt::Write as _;
+        write!(value, "{byte:02x}").expect("writing to String cannot fail");
+    }
+    value
+}
+
+fn descriptor_id() -> OpaqueId {
+    let mut bytes = [0_u8; 16];
+    getrandom::fill(&mut bytes).expect("operating system randomness is required");
+    let mut value = String::from("d");
+    for byte in bytes {
+        use std::fmt::Write as _;
+        write!(value, "{byte:02x}").expect("writing to String cannot fail");
+    }
+    OpaqueId::parse(value).expect("generated descriptor identifier is valid")
+}

crates/remora-worker/src/error.rs

@@ -0,0 +1,17 @@
+#[derive(Debug, thiserror::Error)]
+pub enum WorkerError {
+    #[error("worker configuration is invalid")]
+    InvalidConfiguration,
+    #[error("worker API request is invalid")]
+    InvalidRequest,
+    #[error("worker API authentication failed")]
+    Unauthenticated,
+    #[error("attempt authority is unavailable")]
+    PreconditionFailed,
+    #[error("worker API is unavailable")]
+    ApiUnavailable,
+    #[error("coordination store is unavailable")]
+    CoordinationUnavailable,
+    #[error("worker API returned an invalid response")]
+    InvalidResponse,
+}

crates/remora-worker/src/lib.rs

@@ -1,5 +1,22 @@
+mod client;
+mod descriptor;
+mod error;
+mod executor;
+mod runtime;
+
+pub use client::WorkerApiClient;
+pub use descriptor::{DescriptorStore, migrate, migrator};
+pub use error::WorkerError;
+pub use executor::{AdapterExecutor, ObjectSourceEndpoint};
+pub use runtime::{
+    AttemptContext, AttemptExecution, AttemptExecutor, ExecutionFailure, WorkerConfig,
+    WorkerRunOutcome, WorkerRuntime,
+};
+
 pub const COMPONENT_NAME: &str = "remora-worker";
 
+use client::{incarnation_id, operation_id};
+
 #[cfg(test)]
 mod tests {
     use super::COMPONENT_NAME;

crates/remora-worker/src/runtime.rs

@@ -0,0 +1,488 @@
+use std::{future::Future, pin::Pin, sync::Arc, time::Duration};
+
+use remora_contracts::{
+    AttemptActivateRequestDto, AttemptAuthorityDto, AttemptFailureOutcomeDto, AttemptOutcomeDto,
+    AttemptOutcomeRequestDto, AttemptRenewRequestDto, AttemptRuntimeFailureDto,
+    AttemptStagingMultipartRequestDto, AttemptStagingObjectRequestDto,
+    AttemptStagingWriteIntentRequestDto, AttemptStopRequestDto, AttemptSuccessOutcomeDto,
+    ExecutionEnvelopeDto, ResultDescriptorDto, StagingMultipartIdentityDto,
+    StagingObjectIdentityDto, StagingWriteIntentDto,
+};
+use remora_coordination::CoordinationStore;
+use remora_domain::{JobPhase, OpaqueId, RuntimeFailureClass};
+use remora_stream::CancelToken;
+use tokio::{sync::Semaphore, task::JoinSet, time};
+
+use crate::{WorkerApiClient, WorkerError, incarnation_id, operation_id};
+
+#[derive(Clone, Debug)]
+pub struct WorkerConfig {
+    pub local_limit: usize,
+    pub idle_delay: Duration,
+    pub renew_interval: Duration,
+    pub cancel_poll_interval: Duration,
+}
+
+impl Default for WorkerConfig {
+    fn default() -> Self {
+        Self {
+            local_limit: 4,
+            idle_delay: Duration::from_millis(250),
+            renew_interval: Duration::from_secs(10),
+            cancel_poll_interval: Duration::from_millis(500),
+        }
+    }
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub enum WorkerRunOutcome {
+    Idle,
+    Succeeded,
+    Failed(JobPhase),
+    Cancelled,
+    AuthorityLost,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct ExecutionFailure {
+    pub class: RuntimeFailureClass,
+    pub message: &'static str,
+}
+
+impl ExecutionFailure {
+    #[must_use]
+    pub const fn new(class: RuntimeFailureClass, message: &'static str) -> Self {
+        Self { class, message }
+    }
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub enum AttemptExecution {
+    Succeeded(ResultDescriptorDto),
+    Failed(ExecutionFailure),
+    Cancelled,
+}
+
+pub trait AttemptExecutor: Send + Sync + 'static {
+    fn execute(
+        &self,
+        context: AttemptContext,
+        envelope: ExecutionEnvelopeDto,
+    ) -> Pin<Box<dyn Future<Output = AttemptExecution> + Send>>;
+}
+
+#[derive(Clone)]
+pub struct AttemptContext {
+    api: WorkerApiClient,
+    coordination: CoordinationStore,
+    authority: AttemptAuthorityDto,
+    job_id: OpaqueId,
+    cancel: CancelToken,
+}
+
+impl AttemptContext {
+    #[must_use]
+    pub fn authority(&self) -> &AttemptAuthorityDto {
+        &self.authority
+    }
+
+    #[must_use]
+    pub fn job_id(&self) -> &OpaqueId {
+        &self.job_id
+    }
+
+    #[must_use]
+    pub fn cancellation(&self) -> &CancelToken {
+        &self.cancel
+    }
+
+    pub async fn require_live_authority(&self) -> Result<(), WorkerError> {
+        let lease = self
+            .coordination
+            .get_live_lease(self.job_id.as_str())
+            .await
+            .map_err(|_| WorkerError::CoordinationUnavailable)?;
+        let live = lease.is_some_and(|lease| {
+            lease.record.attempt_id == self.authority.attempt_id.as_str()
+                && lease.record.fence == self.authority.fence.get()
+                && lease.record.worker_id == self.authority.worker_incarnation.as_str()
+        });
+        if !live || self.cancel.is_cancelled() {
+            return Err(WorkerError::PreconditionFailed);
+        }
+        Ok(())
+    }
+
+    pub async fn report_progress(
+        &self,
+        progress: remora_contracts::ProgressValueDto,
+    ) -> Result<(), WorkerError> {
+        self.require_live_authority().await?;
+        self.api
+            .report_progress(
+                &self.authority.attempt_id,
+                operation_id(),
+                self.authority.fence,
+                self.authority.worker_incarnation.clone(),
+                progress,
+            )
+            .await
+    }
+
+    pub async fn pin_resolved_source(
+        &self,
+        descriptor: remora_contracts::ResolvedSourceDescriptorRefDto,
+    ) -> Result<(), WorkerError> {
+        self.require_live_authority().await?;
+        self.api
+            .pin_resolved_source(
+                &self.authority.attempt_id,
+                operation_id(),
+                self.authority.fence,
+                self.authority.worker_incarnation.clone(),
+                descriptor,
+            )
+            .await
+    }
+
+    pub async fn persist_checkpoint(
+        &self,
+        checkpoint_ref: remora_contracts::CheckpointDescriptorRefDto,
+        resolved_source_id: OpaqueId,
+        validated_bytes: remora_contracts::SafeIntegerDto,
+        validated_members: remora_contracts::SafeIntegerDto,
+        staging_multipart: Option<StagingMultipartIdentityDto>,
+    ) -> Result<(), WorkerError> {
+        self.require_live_authority().await?;
+        self.api
+            .persist_checkpoint(
+                &self.authority.attempt_id,
+                remora_contracts::AttemptCheckpointRequestDto {
+                    operation_id: operation_id(),
+                    fence: self.authority.fence,
+                    worker_incarnation: self.authority.worker_incarnation.clone(),
+                    checkpoint_ref,
+                    resolved_source_id,
+                    validated_bytes,
+                    validated_members,
+                    staging_multipart,
+                },
+            )
+            .await
+    }
+
+    pub async fn register_staging_object(
+        &self,
+        object: StagingObjectIdentityDto,
+        completed_upload_id: Option<String>,
+    ) -> Result<(), WorkerError> {
+        self.require_live_authority().await?;
+        self.api
+            .register_staging_object(
+                &self.authority.attempt_id,
+                AttemptStagingObjectRequestDto {
+                    operation_id: operation_id(),
+                    fence: self.authority.fence,
+                    worker_incarnation: self.authority.worker_incarnation.clone(),
+                    object,
+                    completed_upload_id,
+                },
+            )
+            .await
+    }
+
+    pub async fn register_staging_write_intent(
+        &self,
+        intent: StagingWriteIntentDto,
+    ) -> Result<(), WorkerError> {
+        self.require_live_authority().await?;
+        self.api
+            .register_staging_write_intent(
+                &self.authority.attempt_id,
+                AttemptStagingWriteIntentRequestDto {
+                    operation_id: operation_id(),
+                    fence: self.authority.fence,
+                    worker_incarnation: self.authority.worker_incarnation.clone(),
+                    intent,
+                },
+            )
+            .await
+    }
+
+    pub async fn register_staging_multipart(
+        &self,
+        multipart: StagingMultipartIdentityDto,
+    ) -> Result<(), WorkerError> {
+        self.require_live_authority().await?;
+        self.api
+            .register_staging_multipart(
+                &self.authority.attempt_id,
+                AttemptStagingMultipartRequestDto {
+                    operation_id: operation_id(),
+                    fence: self.authority.fence,
+                    worker_incarnation: self.authority.worker_incarnation.clone(),
+                    multipart,
+                },
+            )
+            .await
+    }
+}
+
+#[derive(Clone)]
+pub struct WorkerRuntime {
+    incarnation: OpaqueId,
+    coordination: CoordinationStore,
+    api: WorkerApiClient,
+    executor: Arc<dyn AttemptExecutor>,
+    permits: Arc<Semaphore>,
+    config: WorkerConfig,
+}
+
+impl WorkerRuntime {
+    pub fn new(
+        coordination: CoordinationStore,
+        api: WorkerApiClient,
+        executor: Arc<dyn AttemptExecutor>,
+        config: WorkerConfig,
+    ) -> Result<Self, WorkerError> {
+        if config.local_limit == 0
+            || config.idle_delay.is_zero()
+            || config.renew_interval.is_zero()
+            || config.renew_interval >= coordination.config().lease_ttl
+            || config.cancel_poll_interval.is_zero()
+        {
+            return Err(WorkerError::InvalidConfiguration);
+        }
+        Ok(Self {
+            incarnation: incarnation_id(),
+            coordination,
+            api,
+            executor,
+            permits: Arc::new(Semaphore::new(config.local_limit)),
+            config,
+        })
+    }
+
+    #[must_use]
+    pub fn incarnation(&self) -> &OpaqueId {
+        &self.incarnation
+    }
+
+    pub async fn run_once(&self) -> Result<WorkerRunOutcome, WorkerError> {
+        let permit = self
+            .permits
+            .clone()
+            .try_acquire_owned()
+            .map_err(|_| WorkerError::PreconditionFailed)?;
+        let Some(job) = self
+            .coordination
+            .dequeue()
+            .await
+            .map_err(|_| WorkerError::CoordinationUnavailable)?
+        else {
+            return Ok(WorkerRunOutcome::Idle);
+        };
+        let job_id = OpaqueId::parse(job).map_err(|_| WorkerError::InvalidResponse)?;
+        let outcome = self.process(job_id).await;
+        drop(permit);
+        outcome
+    }
+
+    pub async fn run(&self, shutdown: CancelToken) -> Result<(), WorkerError> {
+        let mut tasks = JoinSet::new();
+        while !shutdown.is_cancelled() {
+            while tasks.len() < self.config.local_limit {
+                let runtime = self.clone();
+                tasks.spawn(async move {
+                    let outcome = runtime.run_once().await;
+                    if matches!(outcome, Ok(WorkerRunOutcome::Idle)) {
+                        time::sleep(runtime.config.idle_delay).await;
+                    }
+                    outcome
+                });
+            }
+            tokio::select! {
+                () = shutdown.cancelled() => break,
+                result = tasks.join_next() => {
+                    if let Some(result) = result {
+                        let _ = result.map_err(|_| WorkerError::ApiUnavailable)?;
+                    }
+                }
+            }
+        }
+        shutdown.cancel();
+        tasks.abort_all();
+        while let Some(result) = tasks.join_next().await {
+            match result {
+                Err(error) if error.is_cancelled() => {}
+                Err(_) => return Err(WorkerError::ApiUnavailable),
+                Ok(outcome) => {
+                    outcome?;
+                }
+            }
+        }
+        Ok(())
+    }
+
+    async fn process(&self, job_id: OpaqueId) -> Result<WorkerRunOutcome, WorkerError> {
+        let claim = self
+            .api
+            .claim(&job_id, operation_id(), self.incarnation.clone())
+            .await?;
+        validate_claim(&job_id, &self.incarnation, &claim)?;
+        let activation = self
+            .api
+            .activate(
+                &claim.authority.attempt_id,
+                AttemptActivateRequestDto {
+                    operation_id: operation_id(),
+                    fence: claim.authority.fence,
+                    worker_incarnation: self.incarnation.clone(),
+                },
+            )
+            .await?;
+        if activation.authority != claim.authority || activation.execution != claim.execution {
+            return Err(WorkerError::InvalidResponse);
+        }
+        let cancel = CancelToken::new();
+        let context = AttemptContext {
+            api: self.api.clone(),
+            coordination: self.coordination.clone(),
+            authority: activation.authority.clone(),
+            job_id: job_id.clone(),
+            cancel: cancel.clone(),
+        };
+        context.require_live_authority().await?;
+        let mut execution = self.executor.execute(context.clone(), activation.execution);
+        let mut renewal = time::interval(self.config.renew_interval);
+        renewal.reset();
+        let mut cancellation = time::interval(self.config.cancel_poll_interval);
+        cancellation.reset();
+        let mut authority_lost = false;
+        let completed = loop {
+            tokio::select! {
+                result = &mut execution => break result,
+                _ = renewal.tick() => {
+                    if self.api.renew(
+                        &claim.authority.attempt_id,
+                        AttemptRenewRequestDto {
+                            operation_id: operation_id(),
+                            fence: claim.authority.fence,
+                            worker_incarnation: self.incarnation.clone(),
+                        },
+                    ).await.is_err() {
+                        authority_lost = true;
+                        cancel.cancel();
+                    }
+                }
+                _ = cancellation.tick() => {
+                    let requested = self.coordination.cancel_notice(job_id.as_str())
+                        .await
+                        .map_err(|_| WorkerError::CoordinationUnavailable)?;
+                    if requested {
+                        cancel.cancel();
+                    }
+                }
+            }
+        };
+        if authority_lost {
+            let _ = self.stop_attempt(&claim.authority).await;
+            return Ok(WorkerRunOutcome::AuthorityLost);
+        }
+        if cancel.is_cancelled() || matches!(completed, AttemptExecution::Cancelled) {
+            self.stop_attempt(&claim.authority).await?;
+            return Ok(WorkerRunOutcome::Cancelled);
+        }
+        if context.require_live_authority().await.is_err() {
+            let _ = self.stop_attempt(&claim.authority).await;
+            return Ok(WorkerRunOutcome::AuthorityLost);
+        }
+        let outcome = match completed {
+            AttemptExecution::Succeeded(result) => {
+                AttemptOutcomeDto::Success(AttemptSuccessOutcomeDto { result })
+            }
+            AttemptExecution::Failed(failure) => {
+                AttemptOutcomeDto::Failure(AttemptFailureOutcomeDto {
+                    failure: AttemptRuntimeFailureDto {
+                        class: failure.class,
+                        message: failure.message.into(),
+                        safe_details: None,
+                    },
+                })
+            }
+            AttemptExecution::Cancelled => unreachable!(),
+        };
+        let phase = self
+            .api
+            .report_outcome(
+                &claim.authority.attempt_id,
+                AttemptOutcomeRequestDto {
+                    operation_id: operation_id(),
+                    fence: claim.authority.fence,
+                    worker_incarnation: self.incarnation.clone(),
+                    outcome,
+                },
+            )
+            .await?
+            .phase;
+        Ok(if phase == JobPhase::Succeeded {
+            WorkerRunOutcome::Succeeded
+        } else {
+            WorkerRunOutcome::Failed(phase)
+        })
+    }
+
+    async fn stop_attempt(&self, authority: &AttemptAuthorityDto) -> Result<(), WorkerError> {
+        self.api
+            .stop(
+                &authority.attempt_id,
+                AttemptStopRequestDto {
+                    operation_id: operation_id(),
+                    fence: authority.fence,
+                    worker_incarnation: authority.worker_incarnation.clone(),
+                },
+            )
+            .await
+    }
+}
+
+fn validate_claim(
+    job_id: &OpaqueId,
+    incarnation: &OpaqueId,
+    claim: &remora_contracts::ClaimResponseDto,
+) -> Result<(), WorkerError> {
+    if claim.authority.worker_incarnation != *incarnation
+        || claim.execution.job_id != *job_id
+        || claim.execution.attempt_id != claim.authority.attempt_id
+        || claim.execution.fence != claim.authority.fence
+        || claim.execution.worker_incarnation != claim.authority.worker_incarnation
+        || claim.execution.id != claim.execution.execution_grant_ref
+        || claim.execution.staging_target.store != claim.execution.target.store
+        || claim.execution.staging_target.bucket != claim.execution.target.bucket
+        || !valid_staging_target(&claim.execution)
+    {
+        return Err(WorkerError::InvalidResponse);
+    }
+    Ok(())
+}
+
+fn valid_staging_target(execution: &remora_contracts::ExecutionEnvelopeDto) -> bool {
+    let prefix = &execution.staging_target.prefix;
+    let root = if execution.target.prefix.is_empty() {
+        format!("{}/a/", remora_target::RESERVED_NAMESPACE_SEGMENT)
+    } else {
+        format!(
+            "{}/{}/a/",
+            execution.target.prefix,
+            remora_target::RESERVED_NAMESPACE_SEGMENT
+        )
+    };
+    let Some(origin) = prefix
+        .strip_prefix(&root)
+        .and_then(|value| value.strip_suffix('/'))
+        .and_then(|value| OpaqueId::parse(value).ok())
+    else {
+        return false;
+    };
+    origin == execution.attempt_id || execution.checkpoint_ref.0.is_some()
+}