Ship production deployment and runtime observability

Cargo.lock

@@ -3295,9 +3295,11 @@ dependencies = [
  "remora-durable",
  "remora-identity",
  "remora-result",
+ "remora-runtime",
  "remora-target",
  "reqwest 0.13.4",
  "rustls",
+ "rustls-pemfile",
  "serde",
  "serde_json",
  "sha2 0.11.0",
@@ -3306,6 +3308,7 @@ dependencies = [
  "time",
  "tokio",
  "tokio-tungstenite",
+ "tracing",
 ]
 
 [[package]]
@@ -3530,6 +3533,20 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "remora-runtime"
+version = "0.0.0"
+dependencies = [
+ "actix-web",
+ "getrandom 0.3.4",
+ "remora-coordination",
+ "serde",
+ "sqlx",
+ "tokio",
+ "tracing",
+ "tracing-subscriber",
+]
+
 [[package]]
 name = "remora-stream"
 version = "0.0.0"
@@ -3575,6 +3592,7 @@ dependencies = [
  "futures-util",
  "getrandom 0.3.4",
  "http 1.4.2",
+ "ipnet",
  "rcgen",
  "remora-admission",
  "remora-api",
@@ -3590,6 +3608,7 @@ dependencies = [
  "remora-object-source",
  "remora-outbound",
  "remora-result",
+ "remora-runtime",
  "remora-stream",
  "remora-target",
  "reqwest 0.13.4",
@@ -3602,6 +3621,7 @@ dependencies = [
  "thiserror",
  "time",
  "tokio",
+ "tracing",
 ]
 
 [[package]]
@@ -3865,6 +3885,15 @@ dependencies = [
  "security-framework",
 ]
 
+[[package]]
+name = "rustls-pemfile"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50"
+dependencies = [
+ "rustls-pki-types",
+]
+
 [[package]]
 name = "rustls-pki-types"
 version = "1.15.0"

Cargo.toml

@@ -15,6 +15,7 @@ members = [
     "crates/remora-object-source",
     "crates/remora-outbound",
     "crates/remora-result",
+    "crates/remora-runtime",
     "crates/remora-stream",
     "crates/remora-target",
     "crates/remora-worker",
@@ -48,6 +49,8 @@ sqlx = { version = "0.9.0", default-features = false, features = [
 ] }
 thiserror = "2.0.17"
 time = { version = "0.3.47", features = ["formatting", "macros", "parsing", "serde"] }
-tokio = { version = "1.52.3", features = ["macros", "rt-multi-thread", "sync", "time"] }
+tokio = { version = "1.52.3", features = ["macros", "rt-multi-thread", "signal", "sync", "time"] }
+tracing = "0.1.44"
+tracing-subscriber = { version = "0.3.22", features = ["env-filter", "json"] }
 unicode-general-category = "1.1.0"
 unicode-normalization = "0.1.25"

Dockerfile

@@ -0,0 +1,44 @@
+# syntax=docker/dockerfile:1.7@sha256:a57df69d0ea827fb7266491f2813635de6f17269be881f696fbfdf2d83dda33e
+
+ARG RUST_IMAGE=docker.io/library/rust:1.97.0-bookworm@sha256:8fa55b2f3ddf97471ab6a767bfa3f37e6bad0986ba823e75fea57e2a2a5c3073
+ARG NODE_IMAGE=docker.io/library/node:24.18.0-bookworm@sha256:5711a0d445a1af54af9589066c646df387d1831a608226f4cd694fc59e745059
+ARG CARGO_BUILD_JOBS=4
+
+FROM ${RUST_IMAGE} AS rust-build
+ARG CARGO_BUILD_JOBS
+WORKDIR /workspace
+COPY Cargo.toml Cargo.lock rust-toolchain.toml ./
+COPY .config .config
+COPY contracts contracts
+COPY crates crates
+COPY docs/api docs/api
+RUN cargo build --release --locked --jobs ${CARGO_BUILD_JOBS} --package remora-api --package remora-worker --bins
+
+FROM ${RUST_IMAGE} AS api
+COPY --from=rust-build /workspace/target/release/remora-api /usr/local/bin/remora-api
+USER 65532:65532
+ENTRYPOINT ["/usr/local/bin/remora-api"]
+
+FROM ${RUST_IMAGE} AS worker
+COPY --from=rust-build /workspace/target/release/remora-worker /usr/local/bin/remora-worker
+USER 65532:65532
+ENTRYPOINT ["/usr/local/bin/remora-worker"]
+
+FROM ${NODE_IMAGE} AS web-build
+ENV NODE_OPTIONS=--max-old-space-size=4096
+WORKDIR /workspace
+RUN corepack enable
+COPY web/package.json web/pnpm-lock.yaml web/pnpm-workspace.yaml ./
+COPY web/tools tools
+RUN pnpm install --frozen-lockfile
+COPY web/index.html web/tsconfig*.json web/vite.config.ts ./
+COPY web/public public
+COPY web/src src
+RUN pnpm build
+
+FROM ${NODE_IMAGE} AS web
+WORKDIR /app
+COPY --from=web-build /workspace/dist dist
+COPY web/server.mjs server.mjs
+USER 65532:65532
+ENTRYPOINT ["node", "/app/server.mjs"]

Dockerfile.test

@@ -328,6 +328,13 @@ COPY crates crates
 COPY --from=kind-worker-api-verified /tmp/remora-api-tests.tar.zst /tests/remora-api-tests.tar.zst
 CMD ["bash", "-ec", "test -s /run/remora-fixture/profile.env && test -n \"$DATABASE_URL\" && test -n \"$REMORA_COORDINATION_URL\" && test -n \"$REMORA_OIDC_ISSUER\" && case $REMORA_COORDINATION_PROVIDER in redis|dragonfly) ;; *) exit 1 ;; esac && case $REMORA_OIDC_PROVIDER in keycloak|zitadel) ;; *) exit 1 ;; esac && export REMORA_LIVE_EVENTS_REQUIRE=1 && cargo-nextest nextest run --archive-file /tests/remora-api-tests.tar.zst --workspace-remap /workspace --test-threads=1 --no-fail-fast -E 'binary(live_events)'"]
 
+FROM rust-tools AS kind-deployment-runner
+WORKDIR /workspace
+COPY Cargo.toml ./
+COPY crates crates
+COPY --from=kind-worker-api-verified /tmp/remora-api-tests.tar.zst /tests/remora-api-tests.tar.zst
+CMD ["bash", "-ec", "test -s /run/remora-fixture/profile.env && test -n \"$REMORA_DEPLOYMENT_API_URL\" && export REMORA_DEPLOYMENT_REQUIRE=1 && cargo-nextest nextest run --archive-file /tests/remora-api-tests.tar.zst --workspace-remap /workspace --test-threads=1 -E 'binary(deployment)'"]
+
 FROM rust-tools AS kind-worker-build
 WORKDIR /workspace
 COPY Cargo.toml Cargo.lock rust-toolchain.toml ./

Makefile

@@ -1,4 +1,4 @@
-.PHONY: check-dependencies check-fixture-lib check-kind-sample check-durable check-coordination check-identity check-credentials check-admission check-attempt check-attempt-mutations check-recovery check-worker-api check-public-api check-live-events check-worker check-stream check-target check-result check-deletion check-outbound check-http-source check-object-source check-hf-source check-fast check-full check-fast-host check-full-host rust-check web-install web-check web-test-fast web-test-full
+.PHONY: check-dependencies check-fixture-lib check-kind-sample check-durable check-coordination check-identity check-credentials check-admission check-attempt check-attempt-mutations check-recovery check-worker-api check-public-api check-live-events check-worker check-stream check-target check-result check-deletion check-outbound check-http-source check-object-source check-hf-source check-deployment check-fast check-full check-fast-host check-full-host rust-check web-install web-check web-test-fast web-test-full
 
 CONTAINER_ENGINE ?= nerdctl
 CONTAINER_RUN_FLAGS ?= --memory=16g --memory-swap=24g --cpus=4
@@ -118,3 +118,6 @@ check-object-source:
 
 check-hf-source:
 	./tests/kind/run-hf-source-job.sh
+
+check-deployment:
+	./tests/kind/run-deployment.sh

crates/remora-api/Cargo.toml

@@ -14,13 +14,17 @@ getrandom = "0.3.4"
 remora-admission = { path = "../remora-admission" }
 remora-attempt = { path = "../remora-attempt" }
 remora-contracts = { path = "../remora-contracts" }
+remora-coordination = { path = "../remora-coordination" }
 remora-credentials = { path = "../remora-credentials" }
 remora-deletion = { path = "../remora-deletion" }
 remora-domain.workspace = true
 remora-durable.workspace = true
 remora-identity = { path = "../remora-identity" }
 remora-result = { path = "../remora-result" }
+remora-runtime = { path = "../remora-runtime" }
 remora-target = { path = "../remora-target" }
+rustls = "=0.23.42"
+rustls-pemfile = "=2.2.0"
 serde.workspace = true
 serde_json.workspace = true
 sha2.workspace = true
@@ -28,6 +32,7 @@ sqlx.workspace = true
 thiserror.workspace = true
 time.workspace = true
 tokio.workspace = true
+tracing.workspace = true
 
 [dev-dependencies]
 base64 = "=0.22.1"
@@ -35,5 +40,4 @@ jsonwebtoken = "=9.3.1"
 rcgen = "=0.14.8"
 remora-coordination = { path = "../remora-coordination" }
 reqwest = { version = "=0.13.4", default-features = false, features = ["form", "json", "rustls"] }
-rustls = "=0.23.42"
 tokio-tungstenite = "=0.30.0"

crates/remora-api/src/application.rs

@@ -0,0 +1,675 @@
+use std::{
+    fs,
+    io::{self, BufReader},
+    net::{SocketAddr, TcpListener},
+    path::{Path, PathBuf},
+    sync::Arc,
+    time::Duration,
+};
+
+use actix_web::{App, HttpServer, middleware, web};
+use remora_admission::{AdmissionConfig, AdmissionService};
+use remora_attempt::{
+    AttemptConfig, AttemptService, CleanupIdentity, RecoveryConfig, StagingMultipart, StagingObject,
+};
+use remora_coordination::{CoordinationStore, StoreConfig};
+use remora_credentials::{CredentialRegistry, VaultClient, VaultConfig};
+use remora_deletion::{DeletionConfig, DeletionService, DrainOutcome, TargetRegistry};
+use remora_domain::{BucketName, OpaqueId, TargetPrefix, TargetStoreId};
+use remora_durable::{PoolConfig, connect};
+use remora_identity::{DefaultTarget, OidcConfig, OidcIdentityProvider};
+use remora_runtime::{
+    ManagementConfig, RuntimeError, RuntimeStatus, env_duration, env_parse, env_string,
+    required_env, spawn_periodic, start_management_server, trace_request,
+};
+use remora_target::{MultipartIdentity, ObjectIdentity, TargetStore, TargetStoreConfig};
+use rustls::{
+    RootCertStore, ServerConfig,
+    pki_types::{CertificateDer, PrivateKeyDer},
+    server::WebPkiClientVerifier,
+};
+use sqlx::PgPool;
+use tokio::task::JoinHandle;
+
+use crate::{
+    PublicApiConfig, PublicApiServices, PublicApiState, WorkerApiState, authenticate_rustls_worker,
+    configure_public_api, configure_worker_api,
+};
+
+#[derive(Clone)]
+pub struct ApiApplicationConfig {
+    pub public_bind: SocketAddr,
+    pub worker_bind: SocketAddr,
+    pub management: ManagementConfig,
+    pub database_url: String,
+    pub database_connections: u32,
+    pub coordination_url: String,
+    pub coordination_namespace: String,
+    pub coordination: StoreConfig,
+    pub attempt: AttemptConfig,
+    pub recovery: RecoveryConfig,
+    pub recovery_interval: Duration,
+    pub credential_reconcile_interval: Duration,
+    pub deletion_interval: Duration,
+    pub default_target_prefix: String,
+    pub reserved_target_segments: Vec<String>,
+    pub admission_policy_version: String,
+    pub public_api: PublicApiConfig,
+    pub oidc: OidcConfig,
+    pub target: TargetStoreConfig,
+    pub vault: VaultConfig,
+    pub worker_tls: WorkerTlsConfig,
+}
+
+#[derive(Clone, Debug)]
+pub struct WorkerTlsConfig {
+    pub certificate: PathBuf,
+    pub private_key: PathBuf,
+    pub client_ca: PathBuf,
+}
+
+pub struct ApiApplication {
+    config: ApiApplicationConfig,
+}
+
+impl ApiApplicationConfig {
+    pub fn from_env() -> Result<Self, RuntimeError> {
+        let reserved_target_segments = comma_values(
+            "REMORA_RESERVED_TARGET_SEGMENTS",
+            &env_string("REMORA_RESERVED_TARGET_SEGMENTS", "_remora"),
+        )?;
+        let oidc_audience = required_env("REMORA_OIDC_AUDIENCE")?;
+        let oidc_resource_client = env_string("REMORA_OIDC_RESOURCE_CLIENT", &oidc_audience);
+        let mut oidc = OidcConfig::new(required_env("REMORA_OIDC_ISSUER")?, oidc_audience)
+            .map_err(runtime_error)?
+            .with_resource_access_client(oidc_resource_client)
+            .with_reserved_segments(reserved_target_segments.clone());
+        for claim in comma_values(
+            "REMORA_OIDC_PROJECT_ROLE_CLAIMS",
+            &env_string(
+                "REMORA_OIDC_PROJECT_ROLE_CLAIMS",
+                remora_identity::ZITADEL_PROJECT_ROLES_CLAIM,
+            ),
+        )? {
+            oidc = oidc.with_project_role_claim(claim);
+        }
+        let event_origins = comma_values(
+            "REMORA_EVENT_ALLOWED_ORIGINS",
+            &required_env("REMORA_EVENT_ALLOWED_ORIGINS")?,
+        )?;
+        let cursor_secret_path = PathBuf::from(required_env("REMORA_CURSOR_SECRET_FILE")?);
+        let cursor_secret = fs::read(&cursor_secret_path)
+            .map_err(|error| file_error("REMORA_CURSOR_SECRET_FILE", error))?;
+        let global_limit = env_parse("REMORA_GLOBAL_ATTEMPT_LIMIT", "64")?;
+        let lease_ttl = env_duration("REMORA_LEASE_TTL_SECONDS", 30)?;
+        let target = TargetStoreConfig::from_env().map_err(runtime_error)?;
+        let vault = VaultConfig::from_env().map_err(runtime_error)?;
+        Ok(Self {
+            public_bind: env_parse("REMORA_PUBLIC_BIND", "0.0.0.0:8080")?,
+            worker_bind: env_parse("REMORA_WORKER_BIND", "0.0.0.0:8443")?,
+            management: ManagementConfig::from_env(crate::COMPONENT_NAME)?,
+            database_url: required_env("DATABASE_URL")?,
+            database_connections: env_parse("REMORA_DATABASE_MAX_CONNECTIONS", "32")?,
+            coordination_url: required_env("REMORA_COORDINATION_URL")?,
+            coordination_namespace: required_env("REMORA_COORDINATION_NAMESPACE")?,
+            coordination: StoreConfig {
+                global_limit,
+                lease_ttl,
+                reserve_ttl: env_duration("REMORA_RESERVE_TTL_SECONDS", 30)?,
+                progress_ttl: env_duration("REMORA_PROGRESS_TTL_SECONDS", 60)?,
+                cancel_ttl: None,
+            },
+            attempt: AttemptConfig {
+                max_attempts: env_parse("REMORA_MAX_ATTEMPTS", "3")?,
+            },
+            recovery: RecoveryConfig {
+                retry_delay: env_duration("REMORA_RETRY_DELAY_SECONDS", 1)?,
+                cleanup_claim_ttl: env_duration("REMORA_CLEANUP_CLAIM_TTL_SECONDS", 30)?,
+                batch_size: env_parse("REMORA_RECOVERY_BATCH_SIZE", "128")?,
+            },
+            recovery_interval: env_duration("REMORA_RECOVERY_INTERVAL_SECONDS", 2)?,
+            credential_reconcile_interval: env_duration(
+                "REMORA_CREDENTIAL_RECONCILE_INTERVAL_SECONDS",
+                10,
+            )?,
+            deletion_interval: env_duration("REMORA_DELETION_INTERVAL_SECONDS", 2)?,
+            default_target_prefix: env_string("REMORA_DEFAULT_TARGET_PREFIX", ""),
+            reserved_target_segments,
+            admission_policy_version: env_string(
+                "REMORA_ADMISSION_POLICY_VERSION",
+                "deployment-v1",
+            ),
+            public_api: PublicApiConfig {
+                cursor_secret,
+                member_access_ttl: env_duration("REMORA_MEMBER_ACCESS_TTL_SECONDS", 60)?,
+                event_retention: env_duration("REMORA_EVENT_RETENTION_SECONDS", 3600)?,
+                event_poll_interval: duration_millis("REMORA_EVENT_POLL_MILLIS", 250)?,
+                event_heartbeat_interval: env_duration("REMORA_EVENT_HEARTBEAT_SECONDS", 15)?,
+                event_ticket_ttl: env_duration("REMORA_EVENT_TICKET_TTL_SECONDS", 30)?,
+                event_allowed_origins: event_origins,
+            },
+            oidc,
+            target,
+            vault,
+            worker_tls: WorkerTlsConfig {
+                certificate: PathBuf::from(required_env("REMORA_WORKER_TLS_CERT_FILE")?),
+                private_key: PathBuf::from(required_env("REMORA_WORKER_TLS_KEY_FILE")?),
+                client_ca: PathBuf::from(required_env("REMORA_WORKER_TLS_CLIENT_CA_FILE")?),
+            },
+        })
+    }
+}
+
+impl ApiApplication {
+    #[must_use]
+    pub fn new(config: ApiApplicationConfig) -> Self {
+        Self { config }
+    }
+
+    pub async fn run(self) -> Result<(), RuntimeError> {
+        let _ = rustls::crypto::ring::default_provider().install_default();
+        let pool = connect(
+            &self.config.database_url,
+            PoolConfig {
+                max_connections: self.config.database_connections,
+                acquire_timeout: Duration::from_secs(10),
+                idle_timeout: Some(Duration::from_secs(60)),
+            },
+        )
+        .await
+        .map_err(runtime_error)?;
+        migrate_all(&pool).await?;
+        let coordination = CoordinationStore::connect(
+            &self.config.coordination_url,
+            &self.config.coordination_namespace,
+            self.config.coordination.clone(),
+        )
+        .await
+        .map_err(runtime_error)?;
+        let target = TargetStore::connect(self.config.target.clone()).map_err(runtime_error)?;
+        let readiness_target = target.clone();
+        let target_registry = TargetRegistry::new([target.clone()]);
+        let vault = VaultClient::new(self.config.vault.clone()).map_err(runtime_error)?;
+        let readiness_vault = vault.clone();
+        let credentials = CredentialRegistry::new(pool.clone(), vault);
+        let attempts = AttemptService::new(
+            pool.clone(),
+            coordination.clone(),
+            self.config.attempt.clone(),
+        )
+        .map_err(runtime_error)?;
+        let deletion = DeletionService::new(
+            pool.clone(),
+            target_registry.clone(),
+            DeletionConfig::default(),
+        )
+        .map_err(runtime_error)?;
+        let state = PublicApiState::new(
+            pool.clone(),
+            PublicApiServices {
+                identity: OidcIdentityProvider::new(self.config.oidc.clone())
+                    .map_err(runtime_error)?,
+                admission: admission_service(&self.config, &pool)?,
+                attempts: attempts.clone(),
+                credentials: credentials.clone(),
+                deletion: deletion.clone(),
+                targets: target_registry.clone(),
+            },
+            self.config.public_api.clone(),
+        )
+        .map_err(runtime_error)?;
+        let status = RuntimeStatus::new(crate::COMPONENT_NAME, pool.clone(), coordination.clone())
+            .with_readiness_check(move || {
+                let target = readiness_target.clone();
+                async move { target.check().await.is_ok() }
+            })
+            .with_readiness_check(move || {
+                let vault = readiness_vault.clone();
+                async move { vault.check().await.is_ok() }
+            });
+        let maintenance = start_maintenance(
+            &self.config,
+            &pool,
+            &status,
+            attempts.clone(),
+            credentials,
+            deletion,
+            target_registry,
+        );
+        let public = public_server(self.config.public_bind, state)?;
+        let worker = worker_server(
+            self.config.worker_bind,
+            WorkerApiState::new(attempts),
+            &self.config.worker_tls,
+        )?;
+        let management = start_management_server(self.config.management, status.clone())
+            .map_err(runtime_error)?;
+        status.set_accepting_traffic(true);
+        tracing::info!(
+            public_bind = %self.config.public_bind,
+            worker_bind = %self.config.worker_bind,
+            "api application ready"
+        );
+        wait_for_shutdown().await?;
+        status.set_accepting_traffic(false);
+        public.stop(true).await;
+        worker.stop(true).await;
+        management.stop(true).await;
+        for task in maintenance {
+            task.abort();
+        }
+        pool.close().await;
+        Ok(())
+    }
+}
+
+fn admission_service(
+    config: &ApiApplicationConfig,
+    pool: &PgPool,
+) -> Result<AdmissionService, RuntimeError> {
+    let store =
+        TargetStoreId::parse(config.target.target_store_id.clone()).map_err(runtime_error)?;
+    let bucket = BucketName::parse(config.target.bucket.clone()).map_err(runtime_error)?;
+    let prefix = TargetPrefix::parse(
+        &config.default_target_prefix,
+        config.reserved_target_segments.iter().map(String::as_str),
+    )
+    .map_err(runtime_error)?;
+    AdmissionService::new(
+        pool.clone(),
+        AdmissionConfig {
+            default_target: DefaultTarget {
+                store,
+                bucket,
+                prefix,
+            },
+            reserved_target_segments: config.reserved_target_segments.clone(),
+            policy_version: config.admission_policy_version.clone(),
+        },
+    )
+    .map_err(runtime_error)
+}
+
+async fn migrate_all(pool: &PgPool) -> Result<(), RuntimeError> {
+    remora_durable::migrate(pool).await.map_err(runtime_error)?;
+    remora_identity::migrate(pool)
+        .await
+        .map_err(runtime_error)?;
+    remora_credentials::migrate(pool)
+        .await
+        .map_err(runtime_error)?;
+    remora_admission::migrate(pool)
+        .await
+        .map_err(runtime_error)?;
+    remora_attempt::migrate(pool).await.map_err(runtime_error)?;
+    remora_result::migrate(pool).await.map_err(runtime_error)?;
+    remora_deletion::migrate(pool)
+        .await
+        .map_err(runtime_error)?;
+    crate::migrate(pool).await.map_err(runtime_error)
+}
+
+fn start_maintenance(
+    config: &ApiApplicationConfig,
+    pool: &PgPool,
+    status: &RuntimeStatus,
+    attempts: AttemptService,
+    credentials: CredentialRegistry,
+    deletion: DeletionService,
+    targets: TargetRegistry,
+) -> Vec<JoinHandle<()>> {
+    let recovery = config.recovery.clone();
+    let cleanup_attempts = attempts.clone();
+    let cleanup_recovery = recovery.clone();
+    let cleanup_targets = targets.clone();
+    let attempt_task = spawn_periodic(
+        "attempt_recovery",
+        config.recovery_interval,
+        status.clone(),
+        move || {
+            let attempts = attempts.clone();
+            let recovery = recovery.clone();
+            async move {
+                attempts
+                    .reconcile_once(&recovery)
+                    .await
+                    .map(|_| ())
+                    .map_err(|error| error.to_string())
+            }
+        },
+    );
+    let credential_task = spawn_periodic(
+        "credential_recovery",
+        config.credential_reconcile_interval,
+        status.clone(),
+        move || {
+            let credentials = credentials.clone();
+            async move {
+                credentials
+                    .reconcile_secret_write_intents()
+                    .await
+                    .map(|_| ())
+                    .map_err(|error| error.to_string())
+            }
+        },
+    );
+    let deletion_pool = pool.clone();
+    let deletion_task = spawn_periodic(
+        "result_deletion",
+        config.deletion_interval,
+        status.clone(),
+        move || {
+            let deletion = deletion.clone();
+            let pool = deletion_pool.clone();
+            async move { process_deletions(&pool, &deletion).await }
+        },
+    );
+    let cleanup_task = spawn_periodic(
+        "staging_cleanup",
+        config.recovery_interval,
+        status.clone(),
+        move || {
+            let attempts = cleanup_attempts.clone();
+            let recovery = cleanup_recovery.clone();
+            let targets = cleanup_targets.clone();
+            async move { process_staging_cleanup(&attempts, &recovery, &targets).await }
+        },
+    );
+    let event_pool = pool.clone();
+    let retention = config.public_api.event_retention;
+    let event_task = spawn_periodic(
+        "event_history_prune",
+        Duration::from_secs(60),
+        status.clone(),
+        move || {
+            let pool = event_pool.clone();
+            async move {
+                let retention =
+                    time::Duration::try_from(retention).map_err(|error| error.to_string())?;
+                crate::prune_live_event_history(&pool, time::OffsetDateTime::now_utc() - retention)
+                    .await
+                    .map(|_| ())
+                    .map_err(|error| error.to_string())
+            }
+        },
+    );
+    vec![
+        attempt_task,
+        credential_task,
+        deletion_task,
+        cleanup_task,
+        event_task,
+    ]
+}
+
+async fn process_deletions(pool: &PgPool, deletion: &DeletionService) -> Result<(), String> {
+    let operation_ids = sqlx::query_scalar::<_, String>(
+        "SELECT operation_id FROM deletion_operations
+         WHERE state IN ('queued', 'running', 'retry_wait', 'blocked')
+         ORDER BY updated_at, operation_id LIMIT 32",
+    )
+    .fetch_all(pool)
+    .await
+    .map_err(|error| error.to_string())?;
+    for value in operation_ids {
+        let operation_id = OpaqueId::parse(value).map_err(|error| error.to_string())?;
+        match deletion.process_next(&operation_id).await {
+            Ok(
+                DrainOutcome::Idle
+                | DrainOutcome::ItemCompleted
+                | DrainOutcome::Blocked
+                | DrainOutcome::Completed,
+            ) => {}
+            Err(error) => return Err(error.to_string()),
+        }
+    }
+    Ok(())
+}
+
+async fn process_staging_cleanup(
+    attempts: &AttemptService,
+    recovery: &RecoveryConfig,
+    targets: &TargetRegistry,
+) -> Result<(), String> {
+    let Some(claim) = attempts
+        .claim_cleanup(recovery)
+        .await
+        .map_err(|error| error.to_string())?
+    else {
+        return Ok(());
+    };
+    let (target_store, bucket) = cleanup_target(&claim.identity);
+    let Some(target) = targets.resolve(target_store, bucket) else {
+        attempts
+            .retry_cleanup(&claim, "target_configuration")
+            .await
+            .map_err(|error| error.to_string())?;
+        return Err("cleanup target configuration is unavailable".into());
+    };
+    let effect = match &claim.identity {
+        CleanupIdentity::Object(object) => target
+            .delete_exact(&ObjectIdentity {
+                target_store_id: object.target_store.clone(),
+                bucket: object.bucket.clone(),
+                key: object.key.clone(),
+                version_id: object.version_id.clone(),
+            })
+            .await
+            .map(|_| None),
+        CleanupIdentity::Multipart(multipart) => target
+            .abort_multipart_exact(&MultipartIdentity {
+                target_store_id: multipart.target_store.clone(),
+                bucket: multipart.bucket.clone(),
+                key: multipart.key.clone(),
+                upload_id: multipart.upload_id.clone(),
+            })
+            .await
+            .map(|_| None),
+        CleanupIdentity::WriteIntent(intent) => {
+            target.discover_staging_write(&intent.key).await.map(Some)
+        }
+    };
+    match effect {
+        Ok(Some(discovered)) => {
+            let objects = discovered
+                .objects
+                .into_iter()
+                .map(|object| StagingObject {
+                    target_store: object.target_store_id,
+                    bucket: object.bucket,
+                    key: object.key,
+                    version_id: object.version_id,
+                })
+                .collect();
+            let multiparts = discovered
+                .multiparts
+                .into_iter()
+                .map(|multipart| StagingMultipart {
+                    target_store: multipart.target_store_id,
+                    bucket: multipart.bucket,
+                    key: multipart.key,
+                    upload_id: multipart.upload_id,
+                })
+                .collect();
+            attempts
+                .reconcile_write_intent(&claim, objects, multiparts)
+                .await
+                .map(|_| ())
+                .map_err(|error| error.to_string())
+        }
+        Ok(None) => attempts
+            .complete_cleanup(&claim)
+            .await
+            .map(|_| ())
+            .map_err(|error| error.to_string()),
+        Err(_) => {
+            attempts
+                .retry_cleanup(&claim, "target_unavailable")
+                .await
+                .map_err(|error| error.to_string())?;
+            Err("cleanup target operation failed".into())
+        }
+    }
+}
+
+fn cleanup_target(identity: &CleanupIdentity) -> (&str, &str) {
+    match identity {
+        CleanupIdentity::WriteIntent(intent) => (&intent.target_store, &intent.bucket),
+        CleanupIdentity::Object(object) => (&object.target_store, &object.bucket),
+        CleanupIdentity::Multipart(multipart) => (&multipart.target_store, &multipart.bucket),
+    }
+}
+
+fn public_server(
+    bind: SocketAddr,
+    state: PublicApiState,
+) -> Result<actix_web::dev::ServerHandle, RuntimeError> {
+    let listener = TcpListener::bind(bind).map_err(runtime_error)?;
+    let server = HttpServer::new(move || {
+        App::new()
+            .wrap(middleware::Logger::new("%a \"%m %U\" %s %b %D"))
+            .wrap(middleware::from_fn(trace_request))
+            .app_data(web::Data::new(state.clone()))
+            .configure(configure_public_api)
+    })
+    .disable_signals()
+    .listen(listener)
+    .map_err(runtime_error)?
+    .run();
+    let handle = server.handle();
+    tokio::spawn(server);
+    Ok(handle)
+}
+
+fn worker_server(
+    bind: SocketAddr,
+    state: WorkerApiState,
+    tls: &WorkerTlsConfig,
+) -> Result<actix_web::dev::ServerHandle, RuntimeError> {
+    let listener = TcpListener::bind(bind).map_err(runtime_error)?;
+    let server = HttpServer::new(move || {
+        App::new()
+            .wrap(middleware::Logger::new("%a \"%m %U\" %s %b %D"))
+            .wrap(middleware::from_fn(trace_request))
+            .app_data(web::Data::new(state.clone()))
+            .configure(configure_worker_api)
+    })
+    .disable_signals()
+    .on_connect(authenticate_rustls_worker)
+    .listen_rustls_0_23(listener, server_tls(tls)?)
+    .map_err(runtime_error)?
+    .run();
+    let handle = server.handle();
+    tokio::spawn(server);
+    Ok(handle)
+}
+
+fn server_tls(config: &WorkerTlsConfig) -> Result<ServerConfig, RuntimeError> {
+    let server_certificates = certificates(&config.certificate)?;
+    let key = private_key(&config.private_key)?;
+    let mut roots = RootCertStore::empty();
+    for certificate in certificates(&config.client_ca)? {
+        roots.add(certificate).map_err(runtime_error)?;
+    }
+    let verifier = WebPkiClientVerifier::builder(Arc::new(roots))
+        .build()
+        .map_err(runtime_error)?;
+    ServerConfig::builder()
+        .with_client_cert_verifier(verifier)
+        .with_single_cert(server_certificates, key)
+        .map_err(runtime_error)
+}
+
+fn certificates(path: &Path) -> Result<Vec<CertificateDer<'static>>, RuntimeError> {
+    let file = fs::File::open(path).map_err(|error| path_error(path, error))?;
+    let certificates = rustls_pemfile::certs(&mut BufReader::new(file))
+        .collect::<Result<Vec<_>, _>>()
+        .map_err(runtime_error)?;
+    if certificates.is_empty() {
+        return Err(RuntimeError::message(format!(
+            "{} contains no certificates",
+            path.display()
+        )));
+    }
+    Ok(certificates)
+}
+
+fn private_key(path: &Path) -> Result<PrivateKeyDer<'static>, RuntimeError> {
+    let file = fs::File::open(path).map_err(|error| path_error(path, error))?;
+    rustls_pemfile::private_key(&mut BufReader::new(file))
+        .map_err(runtime_error)?
+        .ok_or_else(|| RuntimeError::message(format!("{} contains no private key", path.display())))
+}
+
+async fn wait_for_shutdown() -> Result<(), RuntimeError> {
+    #[cfg(unix)]
+    {
+        let mut terminate =
+            tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
+                .map_err(runtime_error)?;
+        tokio::select! {
+            result = tokio::signal::ctrl_c() => result.map_err(runtime_error),
+            _ = terminate.recv() => Ok(()),
+        }
+    }
+    #[cfg(not(unix))]
+    tokio::signal::ctrl_c().await.map_err(runtime_error)
+}
+
+fn comma_values(name: &str, input: &str) -> Result<Vec<String>, RuntimeError> {
+    let values = input
+        .split(',')
+        .map(str::trim)
+        .map(str::to_owned)
+        .collect::<Vec<_>>();
+    if values.is_empty() || values.iter().any(String::is_empty) {
+        return Err(RuntimeError::message(format!(
+            "{name} must contain non-empty comma-separated values"
+        )));
+    }
+    Ok(values)
+}
+
+fn duration_millis(name: &str, default: u64) -> Result<Duration, RuntimeError> {
+    let milliseconds = env_parse(name, &default.to_string())?;
+    if milliseconds == 0 {
+        return Err(RuntimeError::message(format!(
+            "{name} must be greater than zero"
+        )));
+    }
+    Ok(Duration::from_millis(milliseconds))
+}
+
+fn runtime_error(error: impl std::fmt::Display) -> RuntimeError {
+    RuntimeError::message(error.to_string())
+}
+
+fn file_error(name: &str, error: io::Error) -> RuntimeError {
+    RuntimeError::message(format!("cannot read {name}: {}", error.kind()))
+}
+
+fn path_error(path: &Path, error: io::Error) -> RuntimeError {
+    RuntimeError::message(format!("cannot read {}: {}", path.display(), error.kind()))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn rejects_empty_comma_values() {
+        assert!(comma_values("TEST", "one,,two").is_err());
+    }
+
+    #[test]
+    fn file_errors_do_not_include_file_contents() {
+        let error = file_error(
+            "REMORA_CURSOR_SECRET_FILE",
+            io::Error::from(io::ErrorKind::NotFound),
+        );
+        assert_eq!(
+            error.to_string(),
+            "cannot read REMORA_CURSOR_SECRET_FILE: entity not found"
+        );
+    }
+}

crates/remora-api/src/lib.rs

@@ -1,7 +1,9 @@
+mod application;
 mod migrate;
 mod public;
 mod worker;
 
+pub use application::{ApiApplication, ApiApplicationConfig};
 pub use migrate::{migrate, migrator};
 pub use public::{
     PublicApiConfig, PublicApiServices, PublicApiState, configure_public_api,

crates/remora-api/src/main.rs

@@ -1,3 +1,9 @@
-fn main() {
+use remora_api::{ApiApplication, ApiApplicationConfig};
-    println!("{}", remora_api::COMPONENT_NAME);
+
+#[actix_web::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    remora_runtime::init_logging()?;
+    let config = ApiApplicationConfig::from_env()?;
+    ApiApplication::new(config).run().await?;
+    Ok(())
 }

crates/remora-api/src/worker.rs

@@ -96,6 +96,7 @@ pub fn configure_worker_api(config: &mut web::ServiceConfig) {
                     .limit(1024 * 1024)
                     .error_handler(|_, _| WorkerApiError::invalid_request().into()),
             )
+            .route("/health/ready", web::get().to(worker_ready))
             .route("/jobs/{job_id}/claim", web::post().to(claim))
             .route("/attempts/{attempt_id}/renew", web::post().to(renew))
             .route("/attempts/{attempt_id}/activate", web::post().to(activate))
@@ -131,6 +132,10 @@ pub fn configure_worker_api(config: &mut web::ServiceConfig) {
     );
 }
 
+async fn worker_ready(_: AuthenticatedWorker) -> HttpResponse {
+    HttpResponse::Ok().finish()
+}
+
 async fn register_staging_object(
     _: AuthenticatedWorker,
     state: web::Data<WorkerApiState>,

crates/remora-api/tests/deployment.rs

@@ -0,0 +1,45 @@
+#[allow(dead_code)]
+mod support;
+
+use remora_contracts::JobResourceDto;
+use reqwest::StatusCode;
+use serde_json::{Value, json};
+
+use support::{mint_provider_token, required_env};
+
+#[tokio::test]
+async fn deployed_api_accepts_an_authorized_job_for_coordination_rebuild() {
+    if std::env::var("REMORA_DEPLOYMENT_REQUIRE").is_err() {
+        return;
+    }
+    let base = required_env("REMORA_DEPLOYMENT_API_URL")
+        .trim_end_matches('/')
+        .to_owned();
+    let client = reqwest::Client::builder()
+        .redirect(reqwest::redirect::Policy::none())
+        .build()
+        .unwrap();
+    let unauthenticated = client.get(format!("{base}/jobs")).send().await.unwrap();
+    assert_eq!(unauthenticated.status(), StatusCode::UNAUTHORIZED);
+
+    let token = mint_provider_token().await.unwrap();
+    let submitted = client
+        .post(format!("{base}/jobs"))
+        .bearer_auth(token)
+        .header("Idempotency-Key", "deployment-coordination-rebuild")
+        .json(&json!({
+            "source": {
+                "kind": "http",
+                "url": "https://example.com/deployment-smoke",
+                "result_path": "artifact.bin"
+            },
+            "label": "deployment-coordination-rebuild"
+        }))
+        .send()
+        .await
+        .unwrap();
+    assert_eq!(submitted.status(), StatusCode::ACCEPTED);
+    let body: Value = submitted.json().await.unwrap();
+    serde_json::from_value::<JobResourceDto>(body.clone()).unwrap();
+    assert_eq!(body["phase"], "Queued");
+}

crates/remora-api/tests/live_events.rs

@@ -115,10 +115,7 @@ impl Harness {
         let attempts = AttemptService::new(
             pool.clone(),
             coordination.clone(),
-            AttemptConfig {
+            AttemptConfig { max_attempts: 3 },
-                max_attempts: 3,
-                local_limit: 8,
-            },
         )
         .unwrap();
         let admission = AdmissionService::new(

crates/remora-api/tests/public_api.rs

@@ -117,10 +117,7 @@ impl Harness {
         let attempts = AttemptService::new(
             pool.clone(),
             coordination.clone(),
-            AttemptConfig {
+            AttemptConfig { max_attempts: 3 },
-                max_attempts: 3,
-                local_limit: 8,
-            },
         )
         .unwrap();
         let vault = VaultClient::new(VaultConfig::from_env().unwrap()).unwrap();

crates/remora-api/tests/worker_api.rs

@@ -88,10 +88,7 @@ impl Harness {
         AttemptService::new(
             self.pool.clone(),
             self.coordination.clone(),
-            AttemptConfig {
+            AttemptConfig { max_attempts: 3 },
-                max_attempts: 3,
-                local_limit: 8,
-            },
         )
         .unwrap()
     }

crates/remora-attempt/src/recovery.rs

@@ -97,7 +97,6 @@ impl AttemptService {
                 .coordination
                 .release(authority.job_id.as_str(), &lease_owner(authority))
                 .await;
-            self.release_local(&authority.job_id);
         }
         Ok(outcome)
     }
@@ -154,7 +153,6 @@ impl AttemptService {
                 .coordination
                 .release(authority.job_id.as_str(), &lease_owner(authority))
                 .await;
-            self.release_local(&authority.job_id);
         }
         Ok(decision)
     }
@@ -649,7 +647,6 @@ impl AttemptService {
         };
         let mut report = ReconcileReport::default();
         if row.phase == "retry_wait" {
-            self.release_local_if_stopped(job_id).await?;
             if row.retry_not_before.is_none() {
                 let due =
                     OffsetDateTime::now_utc() + time_duration(config.retry_delay, "retry_delay")?;
@@ -731,7 +728,6 @@ impl AttemptService {
             }
         }
         if parse_phase(&row.phase)?.is_terminal() {
-            self.release_local_if_stopped(job_id).await?;
             let expired = self.coordination.expire_lease(job_id.as_str()).await?;
             self.coordination.clean_terminal(job_id.as_str()).await?;
             if expired.code.is_success() {
@@ -774,9 +770,6 @@ impl AttemptService {
         })
         .await?;
         let _ = self.coordination.expire_lease(job_id.as_str()).await?;
-        if reaped.release_local {
-            self.release_local(job_id);
-        }
         Ok(reaped.changed)
     }
 
@@ -861,22 +854,6 @@ impl AttemptService {
         .await?;
         Ok(members.rows_affected() + manifests.rows_affected())
     }
-
-    async fn release_local_if_stopped(&self, job_id: &OpaqueId) -> Result<(), AttemptError> {
-        let stopped = sqlx::query_scalar::<_, bool>(
-            "SELECT NOT EXISTS (
-                SELECT 1 FROM attempts
-                WHERE job_id = $1 AND state = 'executing' AND worker_down_at IS NULL
-             )",
-        )
-        .bind(job_id.as_str())
-        .fetch_one(&self.pool)
-        .await?;
-        if stopped {
-            self.release_local(job_id);
-        }
-        Ok(())
-    }
 }
 
 async fn cooperative_stop_transaction(
@@ -1242,10 +1219,7 @@ async fn reap_lost_attempt(
         .await?;
         enqueue(tx, "job.updated", json!({ "job_id": job_id })).await?;
     }
-    Ok(ReapEffect {
+    Ok(ReapEffect { changed: true })
-        changed: true,
-        release_local: row.state == "authorized" || row.worker_down_at.is_some(),
-    })
 }
 
 async fn validate_staging_key(
@@ -1393,7 +1367,6 @@ struct LockedReapRow {
 #[derive(Default)]
 struct ReapEffect {
     changed: bool,
-    release_local: bool,
 }
 
 #[derive(FromRow)]

crates/remora-attempt/src/service.rs

@@ -2,7 +2,7 @@ use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, Ordering};
 
 use remora_coordination::{
-    ActivateOptions, CoordinationStore, LeaseOwner, LocalPermitGate, MAX_SAFE_INTEGER, OpCode,
+    ActivateOptions, CoordinationStore, LeaseOwner, MAX_SAFE_INTEGER, OpCode,
 };
 use remora_domain::{OpaqueId, fingerprint, ports::MutationScope};
 use remora_durable::{
@@ -29,15 +29,11 @@ impl AttemptFaultInjector for NoFaults {
 #[derive(Clone, Debug)]
 pub struct AttemptConfig {
     pub max_attempts: u64,
-    pub local_limit: u64,
 }
 
 impl Default for AttemptConfig {
     fn default() -> Self {
-        Self {
+        Self { max_attempts: 3 }
-            max_attempts: 3,
-            local_limit: 4,
-        }
     }
 }
 
@@ -45,7 +41,6 @@ impl Default for AttemptConfig {
 pub struct AttemptService {
     pub(crate) pool: PgPool,
     pub(crate) coordination: CoordinationStore,
-    local_permits: LocalPermitGate,
     pub(crate) operation_subject: OpaqueId,
     pub(crate) faults: Arc<dyn AttemptFaultInjector>,
     pub(crate) config: AttemptConfig,
@@ -71,15 +66,9 @@ impl AttemptService {
                 "max_attempts must fit the coordination safe-integer range".into(),
             ));
         }
-        if config.local_limit == 0 {
-            return Err(AttemptError::Invalid(
-                "local_limit must be greater than zero".into(),
-            ));
-        }
         Ok(Self {
             pool,
             coordination,
-            local_permits: LocalPermitGate::new(config.local_limit),
             operation_subject: OpaqueId::parse("remora_attempt_service")
                 .map_err(|error| AttemptError::Invalid(error.to_string()))?,
             faults,
@@ -114,12 +103,6 @@ impl AttemptService {
         worker_incarnation: &OpaqueId,
         operation_id: &OpaqueId,
     ) -> Result<ClaimOutcome, AttemptError> {
-        if self.local_permits.acquire(job_id.as_str()).is_err() {
-            return Ok(ClaimOutcome::Rejected {
-                reason: ClaimRejection::LocalCapacity,
-                disposition: None,
-            });
-        }
         let request = ClaimRequest {
             job_id,
             worker_incarnation,
@@ -177,22 +160,16 @@ impl AttemptService {
                 if reserved.load(Ordering::Acquire) {
                     self.compensate_reservation(job_id, worker_incarnation)
                         .await;
-                } else {
-                    self.local_permits.release(job_id.as_str());
                 }
                 return Err(error.into());
             }
         };
         self.faults.hit(AttemptFaultPoint::ClaimAfterDurable);
         if outcome.disposition == MutationDisposition::Conflict {
-            self.local_permits.release(job_id.as_str());
             return Err(AttemptError::OperationConflict);
         }
         let decision: ClaimOutcome = serde_json::from_value(outcome.response)
             .map_err(|error| AttemptError::Invalid(error.to_string()))?;
-        if matches!(decision, ClaimOutcome::Rejected { .. }) {
-            self.local_permits.release(job_id.as_str());
-        }
         Ok(decision.with_disposition(outcome.disposition))
     }
 
@@ -564,21 +541,11 @@ impl AttemptService {
             }))
     }
 
-    pub fn release_local(&self, job_id: &OpaqueId) -> bool {
-        self.local_permits.release_all(job_id.as_str())
-    }
-
-    #[must_use]
-    pub fn local_used(&self) -> u64 {
-        self.local_permits.used()
-    }
-
     async fn compensate_reservation(&self, job_id: &OpaqueId, worker: &OpaqueId) {
         let _ = self
             .coordination
             .free_reserve(job_id.as_str(), Some(worker.as_str()))
             .await;
-        self.local_permits.release(job_id.as_str());
     }
 
     pub(crate) async fn release_exact(&self, authority: &AttemptAuthority) {

crates/remora-attempt/src/types.rs

@@ -58,7 +58,6 @@ pub struct AttemptAuthority {
 pub enum ClaimRejection {
     JobUnavailable,
     AttemptsExhausted,
-    LocalCapacity,
     GlobalCapacity,
     ReservationHeld,
 }

crates/remora-attempt/tests/authority.rs

@@ -61,31 +61,20 @@ impl Harness {
         Some(Self { pool, coordination })
     }
 
-    fn service(&self, max_attempts: u64, local_limit: u64) -> AttemptService {
+    fn service(&self, max_attempts: u64) -> AttemptService {
         AttemptService::new(
             self.pool.clone(),
             self.coordination.clone(),
-            AttemptConfig {
+            AttemptConfig { max_attempts },
-                max_attempts,
-                local_limit,
-            },
         )
         .expect("attempt service")
     }
 
-    fn service_with_fault(
+    fn service_with_fault(&self, point: AttemptFaultPoint, max_attempts: u64) -> AttemptService {
-        &self,
-        point: AttemptFaultPoint,
-        max_attempts: u64,
-        local_limit: u64,
-    ) -> AttemptService {
         AttemptService::new_with_fault_injector(
             self.pool.clone(),
             self.coordination.clone(),
-            AttemptConfig {
+            AttemptConfig { max_attempts },
-                max_attempts,
-                local_limit,
-            },
             Arc::new(PanicOnce::new(point)),
         )
         .expect("fault-injected attempt service")
@@ -154,7 +143,7 @@ async fn claim_replay_conflict_and_fence_allocation_are_durable() {
     let job = harness.job().await;
     let worker = unique_id("worker");
     let operation = unique_id("claimop");
-    let service = harness.service(2, 2);
+    let service = harness.service(2);
 
     let first = claimed(
         service
@@ -189,7 +178,6 @@ async fn claim_replay_conflict_and_fence_allocation_are_durable() {
         service.recover_inactive(&authority).await.unwrap(),
         RecoveryOutcome::Reaped
     );
-    service.release_local(&job);
     harness.coordination.enqueue(job.as_str()).await.unwrap();
     let second = claimed(
         service
@@ -201,7 +189,6 @@ async fn claim_replay_conflict_and_fence_allocation_are_durable() {
     sleep(Duration::from_millis(2200)).await;
     let second_authority = service.authority(&second.2).await.unwrap().unwrap();
     service.recover_inactive(&second_authority).await.unwrap();
-    service.release_local(&job);
     harness.coordination.enqueue(job.as_str()).await.unwrap();
     let exhausted = service
         .claim(&job, &worker, &unique_id("claimop"))
@@ -217,7 +204,7 @@ async fn activation_replay_requires_the_complete_live_authority_tuple() {
     };
     let job = harness.job().await;
     let worker = unique_id("worker");
-    let service = harness.service(3, 2);
+    let service = harness.service(3);
     let claim_operation = unique_id("claimop");
     let (_, _, attempt) = claimed(
         service
@@ -240,10 +227,7 @@ async fn activation_replay_requires_the_complete_live_authority_tuple() {
     let unavailable = AttemptService::new(
         unavailable_pool,
         harness.coordination.clone(),
-        AttemptConfig {
+        AttemptConfig { max_attempts: 3 },
-            max_attempts: 3,
-            local_limit: 2,
-        },
     )
     .unwrap();
     assert!(
@@ -338,7 +322,7 @@ async fn activation_and_inactive_recovery_serialize_on_durable_authority() {
     };
     let job = harness.job().await;
     let worker = unique_id("worker");
-    let service = harness.service(3, 2);
+    let service = harness.service(3);
     let (_, _, attempt) = claimed(
         service
             .claim(&job, &worker, &unique_id("claimop"))
@@ -380,11 +364,11 @@ async fn activation_and_inactive_recovery_serialize_on_durable_authority() {
 }
 
 #[tokio::test]
-async fn local_and_global_permits_bound_claims_and_survive_lease_expiry() {
+async fn global_permits_bound_claims_and_survive_lease_expiry() {
     let Some(harness) = Harness::new("permits", Duration::from_millis(500), 1).await else {
         return;
     };
-    let service = harness.service(3, 1);
+    let service = harness.service(3);
     let worker = unique_id("worker");
     let first_job = harness.job().await;
     let (_, _, attempt) = claimed(
@@ -399,8 +383,6 @@ async fn local_and_global_permits_bound_claims_and_survive_lease_expiry() {
         .await
         .unwrap();
     sleep(Duration::from_millis(600)).await;
-    assert_eq!(service.local_used(), 1);
-
     let second_job = harness.job().await;
     assert!(matches!(
         service
@@ -409,13 +391,11 @@ async fn local_and_global_permits_bound_claims_and_survive_lease_expiry() {
             .unwrap(),
         ClaimOutcome::Rejected { .. }
     ));
-    service.release_local(&first_job);
     harness
         .coordination
         .expire_lease(first_job.as_str())
         .await
         .unwrap();
-    assert_eq!(service.local_used(), 0);
 }
 
 #[tokio::test]
@@ -425,7 +405,7 @@ async fn precommit_lease_is_compensated_when_durable_authority_was_reaped() {
     };
     let job = harness.job().await;
     let worker = unique_id("worker");
-    let service = harness.service(3, 2);
+    let service = harness.service(3);
     let (_, _, attempt) = claimed(
         service
             .claim(&job, &worker, &unique_id("claimop"))
@@ -479,11 +459,11 @@ async fn crash_injection_covers_every_cross_store_and_lost_response_boundary() {
         return;
     };
     let worker = unique_id("worker");
-    let normal = harness.service(32, 32);
+    let normal = harness.service(32);
 
     let before_reserve_job = harness.job().await;
     crash_claim(
-        harness.service_with_fault(AttemptFaultPoint::ClaimBeforeReserve, 32, 32),
+        harness.service_with_fault(AttemptFaultPoint::ClaimBeforeReserve, 32),
         before_reserve_job.clone(),
         worker.clone(),
         unique_id("claimop"),
@@ -500,7 +480,7 @@ async fn crash_injection_covers_every_cross_store_and_lost_response_boundary() {
 
     let after_reserve_job = harness.job().await;
     crash_claim(
-        harness.service_with_fault(AttemptFaultPoint::ClaimAfterReserve, 32, 32),
+        harness.service_with_fault(AttemptFaultPoint::ClaimAfterReserve, 32),
         after_reserve_job.clone(),
         worker.clone(),
         unique_id("claimop"),
@@ -523,7 +503,7 @@ async fn crash_injection_covers_every_cross_store_and_lost_response_boundary() {
     let lost_claim_job = harness.job().await;
     let lost_claim_operation = unique_id("claimop");
     crash_claim(
-        harness.service_with_fault(AttemptFaultPoint::ClaimAfterDurable, 32, 32),
+        harness.service_with_fault(AttemptFaultPoint::ClaimAfterDurable, 32),
         lost_claim_job.clone(),
         worker.clone(),
         lost_claim_operation.clone(),
@@ -544,7 +524,7 @@ async fn crash_injection_covers_every_cross_store_and_lost_response_boundary() {
 
     let before_lease_operation = unique_id("activateop");
     crash_activate(
-        harness.service_with_fault(AttemptFaultPoint::ActivationBeforeLease, 32, 32),
+        harness.service_with_fault(AttemptFaultPoint::ActivationBeforeLease, 32),
         lost_claim_authority.clone(),
         before_lease_operation.clone(),
     )
@@ -579,7 +559,7 @@ async fn crash_injection_covers_every_cross_store_and_lost_response_boundary() {
         .unwrap();
     let after_lease_operation = unique_id("activateop");
     crash_activate(
-        harness.service_with_fault(AttemptFaultPoint::ActivationAfterLease, 32, 32),
+        harness.service_with_fault(AttemptFaultPoint::ActivationAfterLease, 32),
         after_lease_authority.clone(),
         after_lease_operation.clone(),
     )
@@ -614,7 +594,7 @@ async fn crash_injection_covers_every_cross_store_and_lost_response_boundary() {
         .unwrap();
     let lost_activation_operation = unique_id("activateop");
     crash_activate(
-        harness.service_with_fault(AttemptFaultPoint::ActivationAfterDurable, 32, 32),
+        harness.service_with_fault(AttemptFaultPoint::ActivationAfterDurable, 32),
         lost_activation_authority.clone(),
         lost_activation_operation.clone(),
     )
@@ -633,12 +613,12 @@ async fn crash_injection_covers_every_cross_store_and_lost_response_boundary() {
     );
 
     crash_renew(
-        harness.service_with_fault(AttemptFaultPoint::RenewalBeforeLease, 32, 32),
+        harness.service_with_fault(AttemptFaultPoint::RenewalBeforeLease, 32),
         executing.clone(),
     )
     .await;
     crash_renew(
-        harness.service_with_fault(AttemptFaultPoint::RenewalAfterLease, 32, 32),
+        harness.service_with_fault(AttemptFaultPoint::RenewalAfterLease, 32),
         executing.clone(),
     )
     .await;
@@ -666,7 +646,7 @@ async fn crash_injection_covers_every_cross_store_and_lost_response_boundary() {
     sleep(Duration::from_millis(600)).await;
     for (point, authority) in recovery_cases {
         let used_before = harness.coordination.global_used().await.unwrap();
-        crash_recovery(harness.service_with_fault(point, 32, 32), authority.clone()).await;
+        crash_recovery(harness.service_with_fault(point, 32), authority.clone()).await;
         let expected = if point == AttemptFaultPoint::RecoveryAfterDurable {
             RecoveryOutcome::AlreadyInactive
         } else {

crates/remora-attempt/tests/mutations.rs

@@ -67,10 +67,7 @@ impl Harness {
         AttemptService::new(
             self.pool.clone(),
             self.coordination.clone(),
-            AttemptConfig {
+            AttemptConfig { max_attempts },
-                max_attempts,
-                local_limit: 8,
-            },
         )
         .expect("attempt service")
     }
@@ -637,7 +634,6 @@ async fn failure_disposition_retries_with_budget_and_becomes_terminal_when_exhau
         )
         .await
         .unwrap();
-    service.release_local(&job);
     harness.coordination.enqueue(job.as_str()).await.unwrap();
     let second = harness.executing(&service, &job).await;
     assert_eq!(second.fence, 2);

crates/remora-attempt/tests/recovery.rs

@@ -163,10 +163,7 @@ impl Harness {
         AttemptService::new(
             self.pool.clone(),
             self.coordination.clone(),
-            AttemptConfig {
+            AttemptConfig { max_attempts },
-                max_attempts,
-                local_limit: 16,
-            },
         )
         .expect("attempt service")
     }
@@ -304,10 +301,7 @@ async fn activation_and_recovery_share_authority_serialization() {
     let service = AttemptService::new_with_fault_injector(
         harness.pool.clone(),
         harness.coordination.clone(),
-        AttemptConfig {
+        AttemptConfig { max_attempts: 3 },
-            max_attempts: 3,
-            local_limit: 16,
-        },
         activation_gate.clone(),
     )
     .unwrap();
@@ -371,10 +365,7 @@ async fn activation_and_recovery_share_authority_serialization() {
     let service = AttemptService::new_with_fault_injector(
         harness.pool.clone(),
         harness.coordination.clone(),
-        AttemptConfig {
+        AttemptConfig { max_attempts: 3 },
-            max_attempts: 3,
-            local_limit: 16,
-        },
         recovery_gate.clone(),
     )
     .unwrap();
@@ -485,10 +476,7 @@ async fn coordination_partition_rolls_back_and_restart_converges() {
     let service = AttemptService::new(
         harness.pool.clone(),
         store.clone(),
-        AttemptConfig {
+        AttemptConfig { max_attempts: 3 },
-            max_attempts: 3,
-            local_limit: 16,
-        },
     )
     .unwrap();
     let job = harness.job().await;
@@ -529,10 +517,7 @@ async fn coordination_partition_rolls_back_and_restart_converges() {
     let restarted = AttemptService::new(
         harness.pool.clone(),
         resumed_store.clone(),
-        AttemptConfig {
+        AttemptConfig { max_attempts: 3 },
-            max_attempts: 3,
-            local_limit: 16,
-        },
     )
     .unwrap();
     assert_eq!(