Add exact resumable object source streaming

Cargo.lock

@@ -2346,6 +2346,21 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "remora-object-source"
+version = "0.0.0"
+dependencies = [
+ "bytes",
+ "futures-util",
+ "opendal",
+ "remora-domain",
+ "remora-outbound",
+ "remora-stream",
+ "remora-target",
+ "thiserror",
+ "tokio",
+]
+
 [[package]]
 name = "remora-outbound"
 version = "0.0.0"

Cargo.toml

@@ -9,6 +9,7 @@ members = [
     "crates/remora-durable",
     "crates/remora-identity",
     "crates/remora-http-source",
+    "crates/remora-object-source",
     "crates/remora-outbound",
     "crates/remora-result",
     "crates/remora-stream",

Dockerfile.test

@@ -243,6 +243,35 @@ COPY crates crates
 COPY --from=kind-http-source-verified /tmp/remora-http-source-tests.tar.zst /tests/remora-http-source-tests.tar.zst
 CMD ["cargo-nextest", "nextest", "run", "--archive-file", "/tests/remora-http-source-tests.tar.zst", "--workspace-remap", "/workspace"]
 
+FROM rust-tools AS kind-object-source-build
+WORKDIR /workspace
+COPY Cargo.toml Cargo.lock rust-toolchain.toml ./
+COPY .config .config
+COPY contracts contracts
+COPY crates crates
+COPY docs/api docs/api
+RUN cargo nextest archive \
+    --package remora-object-source \
+    --build-jobs 4 \
+    --locked \
+    --archive-file /tmp/remora-object-source-tests.tar.zst
+
+FROM kind-object-source-build AS kind-object-source-verified
+RUN cargo clippy \
+    --package remora-object-source \
+    --all-targets \
+    --all-features \
+    --locked \
+    -- -D warnings \
+    && cargo test --package remora-object-source --doc --locked
+
+FROM rust-tools AS kind-object-source-runner
+WORKDIR /workspace
+COPY Cargo.toml ./
+COPY crates crates
+COPY --from=kind-object-source-verified /tmp/remora-object-source-tests.tar.zst /tests/remora-object-source-tests.tar.zst
+CMD ["cargo-nextest", "nextest", "run", "--archive-file", "/tests/remora-object-source-tests.tar.zst", "--workspace-remap", "/workspace", "--test-threads=1"]
+
 FROM ${NODE_IMAGE} AS node-runtime
 
 FROM ${PLAYWRIGHT_IMAGE} AS web-dependencies

Makefile

@@ -1,4 +1,4 @@
-.PHONY: check-dependencies check-fixture-lib check-kind-sample check-durable check-coordination check-identity check-credentials check-admission check-stream check-target check-result check-outbound check-http-source check-fast check-full check-fast-host check-full-host rust-check web-install web-check web-test-fast web-test-full
+.PHONY: check-dependencies check-fixture-lib check-kind-sample check-durable check-coordination check-identity check-credentials check-admission check-stream check-target check-result check-outbound check-http-source check-object-source check-fast check-full check-fast-host check-full-host rust-check web-install web-check web-test-fast web-test-full
 
 CONTAINER_ENGINE ?= nerdctl
 CONTAINER_RUN_FLAGS ?= --memory=16g --memory-swap=24g --cpus=4
@@ -88,3 +88,6 @@ check-outbound:
 
 check-http-source:
 	./tests/kind/run-http-source-job.sh
+
+check-object-source:
+	./tests/kind/run-object-source-job.sh

crates/remora-object-source/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "remora-object-source"
+edition.workspace = true
+publish = false
+rust-version.workspace = true
+version.workspace = true
+
+[dependencies]
+bytes = "1.12.1"
+futures-util = "0.3.32"
+opendal = { version = "0.58.0", default-features = false, features = [
+    "executors-tokio",
+    "http-transport-reqwest",
+    "http-transport-reqwest-rustls",
+    "services-s3",
+] }
+remora-domain.workspace = true
+remora-outbound = { path = "../remora-outbound" }
+remora-stream = { path = "../remora-stream" }
+remora-target = { path = "../remora-target" }
+thiserror.workspace = true
+
+[dev-dependencies]
+remora-target = { path = "../remora-target", features = ["test-utils"] }
+tokio = { workspace = true, features = ["macros", "rt-multi-thread", "time"] }

crates/remora-object-source/src/body.rs

@@ -0,0 +1,84 @@
+use bytes::Bytes;
+use futures_util::StreamExt;
+use opendal::FuturesBytesStream;
+use remora_stream::{SourceBody, StreamError, StreamSide};
+
+pub struct ObjectSourceBody {
+    stream: FuturesBytesStream,
+    pending: Bytes,
+    pending_offset: usize,
+    remaining: u64,
+    finished: bool,
+}
+
+impl ObjectSourceBody {
+    pub(crate) fn new(stream: FuturesBytesStream, expected_length: u64) -> Self {
+        Self {
+            stream,
+            pending: Bytes::new(),
+            pending_offset: 0,
+            remaining: expected_length,
+            finished: false,
+        }
+    }
+
+    fn copy_pending(&mut self, output: &mut [u8]) -> usize {
+        let available = &self.pending[self.pending_offset..];
+        let count = available.len().min(output.len());
+        output[..count].copy_from_slice(&available[..count]);
+        self.pending_offset += count;
+        self.remaining -= count as u64;
+        if self.pending_offset == self.pending.len() {
+            self.pending = Bytes::new();
+            self.pending_offset = 0;
+        }
+        count
+    }
+}
+
+impl SourceBody for ObjectSourceBody {
+    async fn read(&mut self, output: &mut [u8]) -> Result<usize, StreamError> {
+        if output.is_empty() {
+            return Ok(0);
+        }
+        if self.pending_offset < self.pending.len() {
+            return Ok(self.copy_pending(output));
+        }
+        if self.finished {
+            return Ok(0);
+        }
+        match self.stream.next().await {
+            Some(Ok(chunk)) if chunk.len() as u64 <= self.remaining => {
+                self.pending = chunk;
+                Ok(self.copy_pending(output))
+            }
+            Some(Ok(_)) => Err(StreamError::SourceChanged),
+            Some(Err(error)) => Err(map_stream_error(&error)),
+            None if self.remaining != 0 => Err(StreamError::transient_source(
+                "object source ended before its declared length",
+            )),
+            None => {
+                self.finished = true;
+                Ok(0)
+            }
+        }
+    }
+}
+
+fn map_stream_error(error: &std::io::Error) -> StreamError {
+    let kind = error
+        .get_ref()
+        .and_then(|source| source.downcast_ref::<opendal::Error>())
+        .map(opendal::Error::kind);
+    match kind {
+        Some(opendal::ErrorKind::PermissionDenied) => StreamError::Authorization {
+            side: StreamSide::Source,
+        },
+        Some(
+            opendal::ErrorKind::ConditionNotMatch
+            | opendal::ErrorKind::NotFound
+            | opendal::ErrorKind::RangeNotSatisfied,
+        ) => StreamError::SourceChanged,
+        _ => StreamError::transient_source("object source read failed"),
+    }
+}

crates/remora-object-source/src/config.rs

@@ -0,0 +1,84 @@
+use std::fmt;
+
+use remora_outbound::CanonicalUrl;
+use remora_target::TargetStoreConfig;
+
+use crate::ObjectSourceError;
+
+#[derive(Clone, Eq, PartialEq)]
+pub struct ObjectSourceConfig {
+    endpoint: CanonicalUrl,
+    bucket: String,
+    region: String,
+    access_key_id: String,
+    secret_access_key: String,
+}
+
+impl ObjectSourceConfig {
+    pub fn new(
+        endpoint: &str,
+        bucket: impl Into<String>,
+        region: impl Into<String>,
+        access_key_id: impl Into<String>,
+        secret_access_key: impl Into<String>,
+    ) -> Result<Self, ObjectSourceError> {
+        let endpoint =
+            CanonicalUrl::parse(endpoint).map_err(|_| ObjectSourceError::InvalidRequest)?;
+        let bucket = bucket.into();
+        let region = region.into();
+        let access_key_id = access_key_id.into();
+        let secret_access_key = secret_access_key.into();
+        if bucket.is_empty() || access_key_id.is_empty() || secret_access_key.is_empty() {
+            return Err(ObjectSourceError::InvalidRequest);
+        }
+        Ok(Self {
+            endpoint,
+            bucket,
+            region,
+            access_key_id,
+            secret_access_key,
+        })
+    }
+
+    pub fn from_target(config: &TargetStoreConfig) -> Result<Self, ObjectSourceError> {
+        Self::new(
+            &config.endpoint,
+            config.bucket.clone(),
+            config.region.clone(),
+            config.access_key_id.clone(),
+            config.secret_access_key.clone(),
+        )
+    }
+
+    #[must_use]
+    pub fn endpoint(&self) -> &CanonicalUrl {
+        &self.endpoint
+    }
+
+    #[must_use]
+    pub fn bucket(&self) -> &str {
+        &self.bucket
+    }
+
+    pub(crate) fn region(&self) -> &str {
+        &self.region
+    }
+
+    pub(crate) fn access_key_id(&self) -> &str {
+        &self.access_key_id
+    }
+
+    pub(crate) fn secret_access_key(&self) -> &str {
+        &self.secret_access_key
+    }
+}
+
+impl fmt::Debug for ObjectSourceConfig {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        formatter
+            .debug_struct("ObjectSourceConfig")
+            .field("bucket", &self.bucket)
+            .field("region", &self.region)
+            .finish_non_exhaustive()
+    }
+}

crates/remora-object-source/src/error.rs

@@ -0,0 +1,83 @@
+use remora_stream::{StreamError, StreamSide};
+use remora_target::TargetError;
+
+#[derive(Clone, Copy, Debug, Eq, thiserror::Error, PartialEq)]
+pub enum ObjectSourceError {
+    #[error("object source request is invalid")]
+    InvalidRequest,
+    #[error("object source access is denied")]
+    SourceAuthorization,
+    #[error("destination access is denied")]
+    DestinationAuthorization,
+    #[error("object source was not found")]
+    SourceNotFound,
+    #[error("object source changed")]
+    SourceChanged,
+    #[error("destination resume state changed")]
+    DestinationChanged,
+    #[error("object source is temporarily unavailable")]
+    TransientSource,
+    #[error("destination is temporarily unavailable")]
+    TransientDestination,
+    #[error("object source adapter failed")]
+    Internal,
+}
+
+impl ObjectSourceError {
+    #[must_use]
+    pub fn into_stream_error(self) -> StreamError {
+        match self {
+            Self::SourceAuthorization => StreamError::Authorization {
+                side: StreamSide::Source,
+            },
+            Self::DestinationAuthorization => StreamError::Authorization {
+                side: StreamSide::Target,
+            },
+            Self::SourceNotFound => StreamError::SourceNotFound,
+            Self::SourceChanged => StreamError::SourceChanged,
+            Self::TransientSource => StreamError::transient_source("object source unavailable"),
+            Self::TransientDestination => StreamError::transient_target("destination unavailable"),
+            Self::DestinationChanged => {
+                StreamError::internal("destination resume proof is no longer reusable")
+            }
+            Self::InvalidRequest | Self::Internal => {
+                StreamError::internal("object source contract violation")
+            }
+        }
+    }
+}
+
+pub(crate) fn map_open_error(error: opendal::Error) -> ObjectSourceError {
+    use opendal::ErrorKind;
+
+    match error.kind() {
+        ErrorKind::NotFound => ObjectSourceError::SourceNotFound,
+        ErrorKind::PermissionDenied => ObjectSourceError::SourceAuthorization,
+        ErrorKind::ConditionNotMatch | ErrorKind::RangeNotSatisfied => {
+            ObjectSourceError::SourceChanged
+        }
+        ErrorKind::RateLimited => ObjectSourceError::TransientSource,
+        ErrorKind::Unsupported | ErrorKind::ConfigInvalid => ObjectSourceError::InvalidRequest,
+        _ => ObjectSourceError::TransientSource,
+    }
+}
+
+pub(crate) fn map_resume_error(error: opendal::Error) -> ObjectSourceError {
+    match map_open_error(error) {
+        ObjectSourceError::SourceNotFound => ObjectSourceError::SourceChanged,
+        other => other,
+    }
+}
+
+pub(crate) fn map_destination_error(error: TargetError) -> ObjectSourceError {
+    match error {
+        TargetError::Authorization => ObjectSourceError::DestinationAuthorization,
+        TargetError::NotFound | TargetError::AlreadyExists | TargetError::Integrity(_) => {
+            ObjectSourceError::DestinationChanged
+        }
+        TargetError::Capacity | TargetError::Transient(_) => {
+            ObjectSourceError::TransientDestination
+        }
+        TargetError::Validation(_) | TargetError::Internal(_) => ObjectSourceError::Internal,
+    }
+}

crates/remora-object-source/src/identity.rs

@@ -0,0 +1,312 @@
+use std::fmt;
+
+use remora_domain::MemberPath;
+use remora_outbound::CanonicalUrl;
+use remora_stream::{IntegrityDigest, Sha256Accountant};
+use remora_target::{CompletedPart, ContentSha256, MultipartIdentity};
+
+use crate::ObjectSourceError;
+
+#[derive(Clone, Eq, Hash, PartialEq)]
+pub struct ObjectValidator(String);
+
+impl ObjectValidator {
+    pub fn parse(value: &str) -> Result<Self, ObjectSourceError> {
+        if value.is_empty() || value.len() > 1024 || value.chars().any(char::is_control) {
+            return Err(ObjectSourceError::InvalidRequest);
+        }
+        Ok(Self(value.to_owned()))
+    }
+
+    #[must_use]
+    pub fn as_str(&self) -> &str {
+        &self.0
+    }
+}
+
+impl fmt::Debug for ObjectValidator {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        formatter.write_str("ObjectValidator([redacted])")
+    }
+}
+
+#[derive(Clone, Eq, PartialEq)]
+pub struct ObjectSourceIdentity {
+    endpoint: CanonicalUrl,
+    bucket: String,
+    key: String,
+    size: u64,
+    version_id: Option<String>,
+    validator: Option<ObjectValidator>,
+}
+
+impl ObjectSourceIdentity {
+    pub fn from_parts(
+        endpoint: CanonicalUrl,
+        bucket: impl Into<String>,
+        key: impl Into<String>,
+        size: u64,
+        version_id: Option<String>,
+        validator: Option<ObjectValidator>,
+    ) -> Result<Self, ObjectSourceError> {
+        let bucket = bucket.into();
+        let key = key.into();
+        let version_id = version_id.filter(|value| !value.is_empty() && value != "null");
+        if bucket.is_empty()
+            || bucket.chars().any(char::is_control)
+            || key.is_empty()
+            || key.starts_with('/')
+            || key.chars().any(char::is_control)
+            || version_id
+                .as_deref()
+                .is_some_and(|value| value.chars().any(char::is_control))
+        {
+            return Err(ObjectSourceError::InvalidRequest);
+        }
+        Ok(Self {
+            endpoint,
+            bucket,
+            key,
+            size,
+            version_id,
+            validator,
+        })
+    }
+
+    #[must_use]
+    pub fn endpoint(&self) -> &CanonicalUrl {
+        &self.endpoint
+    }
+
+    #[must_use]
+    pub fn bucket(&self) -> &str {
+        &self.bucket
+    }
+
+    #[must_use]
+    pub fn key(&self) -> &str {
+        &self.key
+    }
+
+    #[must_use]
+    pub fn size(&self) -> u64 {
+        self.size
+    }
+
+    #[must_use]
+    pub fn version_id(&self) -> Option<&str> {
+        self.version_id.as_deref()
+    }
+
+    #[must_use]
+    pub fn validator(&self) -> Option<&ObjectValidator> {
+        self.validator.as_ref()
+    }
+
+    #[must_use]
+    pub fn is_resumable(&self) -> bool {
+        self.version_id.is_some() || self.validator.is_some()
+    }
+}
+
+impl fmt::Debug for ObjectSourceIdentity {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        formatter
+            .debug_struct("ObjectSourceIdentity")
+            .field("size", &self.size)
+            .field("versioned", &self.version_id.is_some())
+            .field("has_validator", &self.validator.is_some())
+            .finish_non_exhaustive()
+    }
+}
+
+#[derive(Clone, Eq, PartialEq)]
+pub struct DestinationResumeProof {
+    endpoint: CanonicalUrl,
+    identity: MultipartIdentity,
+    completed_part_count: u32,
+    expected_content_sha256: Option<ContentSha256>,
+    completed_bytes: u64,
+    completed_parts_integrity: IntegrityDigest,
+}
+
+impl DestinationResumeProof {
+    pub fn new(
+        endpoint: CanonicalUrl,
+        identity: MultipartIdentity,
+        completed_parts: Vec<CompletedPart>,
+        expected_content_sha256: Option<ContentSha256>,
+    ) -> Result<Self, ObjectSourceError> {
+        if identity.target_store_id.is_empty()
+            || identity.bucket.is_empty()
+            || identity.key.is_empty()
+            || identity.upload_id.is_empty()
+        {
+            return Err(ObjectSourceError::InvalidRequest);
+        }
+        let (completed_part_count, completed_bytes, completed_parts_integrity) =
+            summarize_parts(&completed_parts)?;
+        Ok(Self {
+            endpoint,
+            identity,
+            completed_part_count,
+            expected_content_sha256,
+            completed_bytes,
+            completed_parts_integrity,
+        })
+    }
+
+    #[must_use]
+    pub fn identity(&self) -> &MultipartIdentity {
+        &self.identity
+    }
+
+    #[must_use]
+    pub fn endpoint(&self) -> &CanonicalUrl {
+        &self.endpoint
+    }
+
+    #[must_use]
+    pub fn completed_part_count(&self) -> u32 {
+        self.completed_part_count
+    }
+
+    #[must_use]
+    pub fn expected_content_sha256(&self) -> Option<&ContentSha256> {
+        self.expected_content_sha256.as_ref()
+    }
+
+    #[must_use]
+    pub fn completed_bytes(&self) -> u64 {
+        self.completed_bytes
+    }
+
+    pub(crate) fn matches_parts(&self, parts: &[CompletedPart]) -> bool {
+        summarize_parts(parts).is_ok_and(|(count, bytes, integrity)| {
+            count == self.completed_part_count
+                && bytes == self.completed_bytes
+                && integrity == self.completed_parts_integrity
+        })
+    }
+}
+
+impl fmt::Debug for DestinationResumeProof {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        formatter
+            .debug_struct("DestinationResumeProof")
+            .field("completed_part_count", &self.completed_part_count)
+            .field("completed_bytes", &self.completed_bytes)
+            .finish_non_exhaustive()
+    }
+}
+
+fn summarize_parts(
+    parts: &[CompletedPart],
+) -> Result<(u32, u64, IntegrityDigest), ObjectSourceError> {
+    if parts.is_empty() || parts.len() > u32::MAX as usize {
+        return Err(ObjectSourceError::InvalidRequest);
+    }
+    let mut previous = None;
+    let mut completed_bytes = 0_u64;
+    let mut accountant = Sha256Accountant::new();
+    for part in parts {
+        let size = part
+            .size
+            .filter(|size| *size > 0)
+            .ok_or(ObjectSourceError::InvalidRequest)?;
+        if part.part_number == 0
+            || previous.is_some_and(|number| part.part_number <= number)
+            || part.etag.is_empty()
+        {
+            return Err(ObjectSourceError::InvalidRequest);
+        }
+        previous = Some(part.part_number);
+        completed_bytes = completed_bytes
+            .checked_add(size)
+            .ok_or(ObjectSourceError::InvalidRequest)?;
+        let etag_length =
+            u64::try_from(part.etag.len()).map_err(|_| ObjectSourceError::InvalidRequest)?;
+        accountant.update(&part.part_number.to_be_bytes());
+        accountant.update(&size.to_be_bytes());
+        accountant.update(&etag_length.to_be_bytes());
+        accountant.update(part.etag.as_bytes());
+    }
+    let count = u32::try_from(parts.len()).map_err(|_| ObjectSourceError::InvalidRequest)?;
+    let integrity = accountant
+        .finalize()
+        .digest()
+        .ok_or(ObjectSourceError::Internal)?
+        .clone();
+    Ok((count, completed_bytes, integrity))
+}
+
+#[derive(Clone, Eq, PartialEq)]
+pub struct ObjectResumeCheckpoint {
+    source: ObjectSourceIdentity,
+    result_path: MemberPath,
+    offset: u64,
+    prefix_integrity: IntegrityDigest,
+    destination: DestinationResumeProof,
+}
+
+impl ObjectResumeCheckpoint {
+    pub fn new(
+        source: ObjectSourceIdentity,
+        result_path: MemberPath,
+        offset: u64,
+        prefix_integrity: IntegrityDigest,
+        destination: DestinationResumeProof,
+    ) -> Result<Self, ObjectSourceError> {
+        if !source.is_resumable()
+            || offset == 0
+            || offset >= source.size
+            || destination.completed_bytes != offset
+        {
+            return Err(ObjectSourceError::InvalidRequest);
+        }
+        Ok(Self {
+            source,
+            result_path,
+            offset,
+            prefix_integrity,
+            destination,
+        })
+    }
+
+    #[must_use]
+    pub fn source(&self) -> &ObjectSourceIdentity {
+        &self.source
+    }
+
+    #[must_use]
+    pub fn result_path(&self) -> &MemberPath {
+        &self.result_path
+    }
+
+    #[must_use]
+    pub fn offset(&self) -> u64 {
+        self.offset
+    }
+
+    #[must_use]
+    pub fn prefix_integrity(&self) -> &IntegrityDigest {
+        &self.prefix_integrity
+    }
+
+    #[must_use]
+    pub fn destination(&self) -> &DestinationResumeProof {
+        &self.destination
+    }
+}
+
+impl fmt::Debug for ObjectResumeCheckpoint {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        formatter
+            .debug_struct("ObjectResumeCheckpoint")
+            .field("source", &self.source)
+            .field("result_path", &self.result_path.canonical())
+            .field("offset", &self.offset)
+            .field("destination", &self.destination)
+            .finish_non_exhaustive()
+    }
+}

crates/remora-object-source/src/lib.rs

@@ -0,0 +1,15 @@
+//! S3-compatible object source adapter with exact source and destination resume proof.
+
+mod body;
+mod config;
+mod error;
+mod identity;
+mod source;
+
+pub use body::ObjectSourceBody;
+pub use config::ObjectSourceConfig;
+pub use error::ObjectSourceError;
+pub use identity::{
+    DestinationResumeProof, ObjectResumeCheckpoint, ObjectSourceIdentity, ObjectValidator,
+};
+pub use source::{ObjectSource, ObjectSourceRequest, OpenedObjectSource, ValidatedObjectResume};

crates/remora-object-source/src/source.rs

@@ -0,0 +1,261 @@
+use std::fmt;
+
+use opendal::{Operator, services::S3};
+use remora_domain::MemberPath;
+use remora_stream::IntegrityDigest;
+use remora_target::{MultipartSession, TargetStore};
+
+use crate::{
+    ObjectResumeCheckpoint, ObjectSourceBody, ObjectSourceConfig, ObjectSourceError,
+    ObjectSourceIdentity, ObjectValidator,
+    error::{map_destination_error, map_open_error, map_resume_error},
+};
+
+pub struct ObjectSourceRequest {
+    key: String,
+    result_path: MemberPath,
+}
+
+impl ObjectSourceRequest {
+    pub fn new(key: impl Into<String>, result_path: &str) -> Result<Self, ObjectSourceError> {
+        let key = key.into();
+        if key.is_empty() || key.starts_with('/') || key.chars().any(char::is_control) {
+            return Err(ObjectSourceError::InvalidRequest);
+        }
+        Ok(Self {
+            key,
+            result_path: MemberPath::parse(result_path)
+                .map_err(|_| ObjectSourceError::InvalidRequest)?,
+        })
+    }
+}
+
+impl fmt::Debug for ObjectSourceRequest {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        formatter
+            .debug_struct("ObjectSourceRequest")
+            .field("result_path", &self.result_path.canonical())
+            .finish_non_exhaustive()
+    }
+}
+
+pub struct OpenedObjectSource {
+    result_path: MemberPath,
+    identity: ObjectSourceIdentity,
+    body: ObjectSourceBody,
+}
+
+impl OpenedObjectSource {
+    #[must_use]
+    pub fn result_path(&self) -> &MemberPath {
+        &self.result_path
+    }
+
+    #[must_use]
+    pub fn identity(&self) -> &ObjectSourceIdentity {
+        &self.identity
+    }
+
+    #[must_use]
+    pub fn into_body(self) -> ObjectSourceBody {
+        self.body
+    }
+}
+
+impl fmt::Debug for OpenedObjectSource {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        formatter
+            .debug_struct("OpenedObjectSource")
+            .field("result_path", &self.result_path.canonical())
+            .field("identity", &self.identity)
+            .finish_non_exhaustive()
+    }
+}
+
+pub struct ValidatedObjectResume {
+    source: OpenedObjectSource,
+    destination: MultipartSession,
+}
+
+impl fmt::Debug for ValidatedObjectResume {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        formatter
+            .debug_struct("ValidatedObjectResume")
+            .field("source", &self.source)
+            .field("destination_validated", &true)
+            .finish_non_exhaustive()
+    }
+}
+
+impl ValidatedObjectResume {
+    #[must_use]
+    pub fn source(&self) -> &OpenedObjectSource {
+        &self.source
+    }
+
+    #[must_use]
+    pub fn destination(&self) -> &MultipartSession {
+        &self.destination
+    }
+
+    #[must_use]
+    pub fn into_parts(self) -> (OpenedObjectSource, MultipartSession) {
+        (self.source, self.destination)
+    }
+}
+
+pub struct ObjectSource {
+    config: ObjectSourceConfig,
+    operator: Operator,
+}
+
+impl ObjectSource {
+    pub fn connect(config: ObjectSourceConfig) -> Result<Self, ObjectSourceError> {
+        opendal::install_default();
+        let builder = S3::default()
+            .bucket(config.bucket())
+            .endpoint(config.endpoint().as_url().as_str())
+            .region(config.region())
+            .access_key_id(config.access_key_id())
+            .secret_access_key(config.secret_access_key())
+            .disable_config_load()
+            .disable_ec2_metadata();
+        let operator = Operator::new(builder).map_err(map_open_error)?;
+        Ok(Self { config, operator })
+    }
+
+    pub async fn open(
+        &self,
+        request: ObjectSourceRequest,
+    ) -> Result<OpenedObjectSource, ObjectSourceError> {
+        let metadata = self
+            .operator
+            .stat(&request.key)
+            .await
+            .map_err(map_open_error)?;
+        let identity = ObjectSourceIdentity::from_parts(
+            self.config.endpoint().clone(),
+            self.config.bucket().to_owned(),
+            request.key,
+            metadata.content_length(),
+            normalized_version(metadata.version()),
+            metadata.etag().map(ObjectValidator::parse).transpose()?,
+        )?;
+        let body = self.open_body(&identity, 0, false).await?;
+        Ok(OpenedObjectSource {
+            result_path: request.result_path,
+            identity,
+            body,
+        })
+    }
+
+    pub async fn resume(
+        &self,
+        checkpoint: &ObjectResumeCheckpoint,
+        observed_prefix: &IntegrityDigest,
+        target: &TargetStore,
+    ) -> Result<ValidatedObjectResume, ObjectSourceError> {
+        if checkpoint.prefix_integrity() != observed_prefix {
+            return Err(ObjectSourceError::SourceChanged);
+        }
+        self.ensure_identity(checkpoint.source())?;
+        let proof = checkpoint.destination();
+        let target_endpoint = remora_outbound::CanonicalUrl::parse(&target.config().endpoint)
+            .map_err(|_| ObjectSourceError::DestinationChanged)?;
+        if proof.endpoint() != &target_endpoint {
+            return Err(ObjectSourceError::DestinationChanged);
+        }
+        let (destination, live_parts) = target
+            .resume_multipart(
+                proof.identity().clone(),
+                proof.expected_content_sha256().cloned(),
+            )
+            .await
+            .map_err(map_destination_error)?;
+        if !proof.matches_parts(&live_parts) {
+            return Err(ObjectSourceError::DestinationChanged);
+        }
+        self.validate_current(checkpoint.source()).await?;
+        let body = self
+            .open_body(checkpoint.source(), checkpoint.offset(), true)
+            .await?;
+        Ok(ValidatedObjectResume {
+            source: OpenedObjectSource {
+                result_path: checkpoint.result_path().clone(),
+                identity: checkpoint.source().clone(),
+                body,
+            },
+            destination,
+        })
+    }
+
+    async fn validate_current(
+        &self,
+        identity: &ObjectSourceIdentity,
+    ) -> Result<(), ObjectSourceError> {
+        let mut stat = self.operator.stat_with(identity.key());
+        if let Some(version) = identity.version_id() {
+            stat = stat.version(version);
+        } else if let Some(validator) = identity.validator() {
+            stat = stat.if_match(validator.as_str());
+        } else {
+            return Err(ObjectSourceError::SourceChanged);
+        }
+        let metadata = stat.await.map_err(map_resume_error)?;
+        if metadata.content_length() != identity.size() {
+            return Err(ObjectSourceError::SourceChanged);
+        }
+        if identity.version_id().is_none()
+            && metadata.etag() != identity.validator().map(ObjectValidator::as_str)
+        {
+            return Err(ObjectSourceError::SourceChanged);
+        }
+        Ok(())
+    }
+
+    async fn open_body(
+        &self,
+        identity: &ObjectSourceIdentity,
+        offset: u64,
+        resume: bool,
+    ) -> Result<ObjectSourceBody, ObjectSourceError> {
+        let mut reader = self.operator.reader_with(identity.key()).chunk(256 * 1024);
+        if let Some(version) = identity.version_id() {
+            reader = reader.version(version);
+        } else if let Some(validator) = identity.validator() {
+            reader = reader.if_match(validator.as_str());
+        }
+        let reader = if resume {
+            reader.await.map_err(map_resume_error)?
+        } else {
+            reader.await.map_err(map_open_error)?
+        };
+        let stream = if resume {
+            reader
+                .into_bytes_stream(offset..identity.size())
+                .await
+                .map_err(map_resume_error)?
+        } else {
+            reader
+                .into_bytes_stream(offset..identity.size())
+                .await
+                .map_err(map_open_error)?
+        };
+        Ok(ObjectSourceBody::new(stream, identity.size() - offset))
+    }
+
+    fn ensure_identity(&self, identity: &ObjectSourceIdentity) -> Result<(), ObjectSourceError> {
+        if identity.endpoint() != self.config.endpoint()
+            || identity.bucket() != self.config.bucket()
+        {
+            return Err(ObjectSourceError::SourceChanged);
+        }
+        Ok(())
+    }
+}
+
+fn normalized_version(version: Option<&str>) -> Option<String> {
+    version
+        .filter(|value| !value.is_empty() && *value != "null")
+        .map(str::to_owned)
+}

crates/remora-object-source/tests/compatibility.rs

@@ -0,0 +1,363 @@
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use remora_domain::OpaqueId;
+use remora_object_source::{
+    DestinationResumeProof, ObjectResumeCheckpoint, ObjectSource, ObjectSourceConfig,
+    ObjectSourceError, ObjectSourceRequest,
+};
+use remora_outbound::CanonicalUrl;
+use remora_stream::{Sha256Accountant, SourceBody, StreamError};
+use remora_target::{
+    CreateObjectRequest, ObjectIdentity, TargetStore, TargetStoreConfig, attempt_object_key,
+};
+
+const PART_SIZE: usize = 5 * 1024 * 1024;
+
+struct Prepared {
+    checkpoint: ObjectResumeCheckpoint,
+    source_object: ObjectIdentity,
+}
+
+fn require_env() -> bool {
+    std::env::var("REMORA_OBJECT_SOURCE_REQUIRE").is_ok()
+}
+
+fn stores_from_env() -> Option<(TargetStoreConfig, TargetStore, ObjectSource)> {
+    let config = match TargetStoreConfig::from_env() {
+        Ok(config) => config,
+        Err(error) => {
+            if require_env() {
+                panic!("object source env is required: {error}");
+            }
+            eprintln!("skipping object source compatibility test: {error}");
+            return None;
+        }
+    };
+    let target = TargetStore::connect(config.clone()).expect("target");
+    let source = ObjectSource::connect(ObjectSourceConfig::from_target(&config).expect("config"))
+        .expect("source");
+    Some((config, target, source))
+}
+
+fn unique_id(prefix: &str) -> OpaqueId {
+    let nanos = SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .expect("time")
+        .as_nanos();
+    OpaqueId::parse(format!("{prefix}{nanos:x}")).expect("opaque id")
+}
+
+fn key(label: &str) -> String {
+    attempt_object_key(
+        &unique_id(&format!("att{label}")),
+        &unique_id(&format!("obj{label}")),
+        None,
+    )
+}
+
+fn request(key: &str) -> CreateObjectRequest {
+    CreateObjectRequest::parse(key).expect("request")
+}
+
+fn digest(bytes: &[u8]) -> remora_stream::IntegrityDigest {
+    let mut accountant = Sha256Accountant::new();
+    accountant.update(bytes);
+    accountant.finalize().digest().expect("digest").clone()
+}
+
+async fn read_all(
+    mut body: remora_object_source::ObjectSourceBody,
+) -> Result<Vec<u8>, StreamError> {
+    let mut result = Vec::new();
+    let mut buffer = vec![0_u8; 64 * 1024];
+    loop {
+        let count = body.read(&mut buffer).await?;
+        if count == 0 {
+            return Ok(result);
+        }
+        result.extend_from_slice(&buffer[..count]);
+    }
+}
+
+async fn prepare(
+    target: &TargetStore,
+    source: &ObjectSource,
+    label: &str,
+    payload: &[u8],
+) -> Prepared {
+    let source_key = key(&format!("src{label}"));
+    let source_object = target
+        .put_create_only(&request(&source_key), payload.to_vec())
+        .await
+        .expect("source put")
+        .identity;
+    let opened = source
+        .open(ObjectSourceRequest::new(&source_key, "dir/cafe\u{301}.bin").expect("source request"))
+        .await
+        .expect("source open");
+    let destination_key = key(&format!("dst{label}"));
+    let session = target
+        .begin_multipart(&request(&destination_key))
+        .await
+        .expect("multipart begin");
+    target
+        .upload_part(&session, 1, payload[..PART_SIZE].to_vec().into())
+        .await
+        .expect("part upload");
+    let parts = target.list_parts(&session).await.expect("part list");
+    let proof = DestinationResumeProof::new(
+        CanonicalUrl::parse(&target.config().endpoint).expect("target endpoint"),
+        session.identity().clone(),
+        parts,
+        None,
+    )
+    .expect("destination proof");
+    drop(session);
+    let checkpoint = ObjectResumeCheckpoint::new(
+        opened.identity().clone(),
+        opened.result_path().clone(),
+        PART_SIZE as u64,
+        digest(&payload[..PART_SIZE]),
+        proof,
+    )
+    .expect("checkpoint");
+    Prepared {
+        checkpoint,
+        source_object,
+    }
+}
+
+async fn abort_checkpoint(target: &TargetStore, checkpoint: &ObjectResumeCheckpoint) {
+    if let Ok((session, _)) = target
+        .resume_multipart(checkpoint.destination().identity().clone(), None)
+        .await
+    {
+        target.abort_multipart(session).await.expect("abort");
+    }
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn object_source_and_destination_resume_matrix() {
+    let Some((config, target, source)) = stores_from_env() else {
+        return;
+    };
+    let payload = [vec![b'a'; PART_SIZE], vec![b'b'; 8192]].concat();
+    let replacement = [vec![b'x'; PART_SIZE], vec![b'y'; 8192]].concat();
+
+    let full_key = key("full");
+    let full = target
+        .put_create_only(&request(&full_key), payload.clone())
+        .await
+        .expect("full put");
+    let opened = source
+        .open(ObjectSourceRequest::new(&full_key, "member.bin").expect("request"))
+        .await
+        .expect("open");
+    assert_eq!(
+        opened.identity().endpoint(),
+        &CanonicalUrl::parse(&config.endpoint).expect("canonical fixture endpoint")
+    );
+    assert_eq!(opened.identity().bucket(), config.bucket);
+    assert_eq!(opened.identity().key(), full_key);
+    assert_eq!(opened.identity().size(), payload.len() as u64);
+    assert!(opened.identity().is_resumable());
+    assert_eq!(read_all(opened.into_body()).await.expect("body"), payload);
+    target
+        .delete_exact(&full.identity)
+        .await
+        .expect("full delete");
+
+    let success = prepare(&target, &source, "success", &payload).await;
+    drop(source);
+    let restarted_source =
+        ObjectSource::connect(ObjectSourceConfig::from_target(&config).expect("restart config"))
+            .expect("restart source");
+    let restarted_target = TargetStore::connect(config.clone()).expect("restart target");
+    let mut substituted_config = config.clone();
+    substituted_config.endpoint = "http://substituted.invalid:3900".to_owned();
+    let substituted_target = TargetStore::connect(substituted_config).expect("substituted target");
+    assert_eq!(
+        restarted_source
+            .resume(
+                &success.checkpoint,
+                &digest(&payload[..PART_SIZE]),
+                &substituted_target,
+            )
+            .await
+            .expect_err("destination endpoint changed"),
+        ObjectSourceError::DestinationChanged
+    );
+    let resumed = restarted_source
+        .resume(
+            &success.checkpoint,
+            &digest(&payload[..PART_SIZE]),
+            &restarted_target,
+        )
+        .await
+        .expect("resume after restart");
+    assert_eq!(resumed.source().result_path().canonical(), "dir/café.bin");
+    let (resumed_source, destination) = resumed.into_parts();
+    assert_eq!(
+        read_all(resumed_source.into_body()).await.expect("suffix"),
+        payload[PART_SIZE..]
+    );
+    restarted_target
+        .abort_multipart(destination)
+        .await
+        .expect("success abort");
+    restarted_target
+        .delete_exact(&success.source_object)
+        .await
+        .expect("success source delete");
+
+    let changed_source = prepare(
+        &restarted_target,
+        &restarted_source,
+        "sourcechange",
+        &payload,
+    )
+    .await;
+    restarted_target
+        .put_overwrite(
+            &request(&changed_source.source_object.key),
+            replacement.clone(),
+        )
+        .await
+        .expect("source overwrite");
+    assert_eq!(
+        restarted_source
+            .resume(
+                &changed_source.checkpoint,
+                &digest(&payload[..PART_SIZE]),
+                &restarted_target,
+            )
+            .await
+            .expect_err("source changed"),
+        ObjectSourceError::SourceChanged
+    );
+    abort_checkpoint(&restarted_target, &changed_source.checkpoint).await;
+    restarted_target
+        .delete_exact(&changed_source.source_object)
+        .await
+        .expect("changed source delete");
+
+    let changed_part = prepare(&restarted_target, &restarted_source, "partchange", &payload).await;
+    let (session, _) = restarted_target
+        .resume_multipart(
+            changed_part.checkpoint.destination().identity().clone(),
+            None,
+        )
+        .await
+        .expect("part session");
+    restarted_target
+        .upload_part(&session, 1, vec![b'z'; PART_SIZE].into())
+        .await
+        .expect("part overwrite");
+    drop(session);
+    assert_eq!(
+        restarted_source
+            .resume(
+                &changed_part.checkpoint,
+                &digest(&payload[..PART_SIZE]),
+                &restarted_target,
+            )
+            .await
+            .expect_err("destination changed"),
+        ObjectSourceError::DestinationChanged
+    );
+    abort_checkpoint(&restarted_target, &changed_part.checkpoint).await;
+    restarted_target
+        .delete_exact(&changed_part.source_object)
+        .await
+        .expect("part source delete");
+
+    let aborted = prepare(&restarted_target, &restarted_source, "aborted", &payload).await;
+    let (session, _) = restarted_target
+        .resume_multipart(aborted.checkpoint.destination().identity().clone(), None)
+        .await
+        .expect("aborted session");
+    restarted_target
+        .abort_multipart(session)
+        .await
+        .expect("abort");
+    assert_eq!(
+        restarted_source
+            .resume(
+                &aborted.checkpoint,
+                &digest(&payload[..PART_SIZE]),
+                &restarted_target,
+            )
+            .await
+            .expect_err("aborted upload"),
+        ObjectSourceError::DestinationChanged
+    );
+    restarted_target
+        .delete_exact(&aborted.source_object)
+        .await
+        .expect("aborted source delete");
+
+    let completed = prepare(&restarted_target, &restarted_source, "completed", &payload).await;
+    let (session, parts) = restarted_target
+        .resume_multipart(completed.checkpoint.destination().identity().clone(), None)
+        .await
+        .expect("completion session");
+    let completed_destination = restarted_target
+        .complete_multipart(session, &parts)
+        .await
+        .expect("ambiguous completion");
+    assert_eq!(
+        restarted_source
+            .resume(
+                &completed.checkpoint,
+                &digest(&payload[..PART_SIZE]),
+                &restarted_target,
+            )
+            .await
+            .expect_err("completed upload"),
+        ObjectSourceError::DestinationChanged
+    );
+    restarted_target
+        .delete_exact(&completed_destination.identity)
+        .await
+        .expect("completed destination delete");
+    restarted_target
+        .delete_exact(&completed.source_object)
+        .await
+        .expect("completed source delete");
+
+    if restarted_target
+        .enable_bucket_versioning()
+        .await
+        .expect("enable versioning")
+    {
+        let versioned = prepare(&restarted_target, &restarted_source, "versioned", &payload).await;
+        assert!(versioned.checkpoint.source().version_id().is_some());
+        restarted_target
+            .put_overwrite(&request(&versioned.source_object.key), replacement)
+            .await
+            .expect("versioned overwrite");
+        let resumed = restarted_source
+            .resume(
+                &versioned.checkpoint,
+                &digest(&payload[..PART_SIZE]),
+                &restarted_target,
+            )
+            .await
+            .expect("version-pinned resume");
+        let (source, destination) = resumed.into_parts();
+        assert_eq!(
+            read_all(source.into_body()).await.expect("old version"),
+            payload[PART_SIZE..]
+        );
+        restarted_target
+            .abort_multipart(destination)
+            .await
+            .expect("versioned abort");
+        restarted_target
+            .purge_versions_with_prefix("_remora")
+            .await
+            .expect("version purge");
+    } else if std::env::var("REMORA_TARGET_PROVIDER").as_deref() == Ok("versitygw") {
+        panic!("VersityGW must support version-pinned WP-12 qualification");
+    }
+}

crates/remora-object-source/tests/validation.rs

@@ -0,0 +1,122 @@
+use remora_domain::MemberPath;
+use remora_object_source::{
+    DestinationResumeProof, ObjectResumeCheckpoint, ObjectSourceConfig, ObjectSourceError,
+    ObjectSourceIdentity, ObjectSourceRequest, ObjectValidator,
+};
+use remora_outbound::CanonicalUrl;
+use remora_stream::{Sha256Accountant, SourceBody};
+use remora_target::{CompletedPart, MultipartIdentity};
+
+fn digest(bytes: &[u8]) -> remora_stream::IntegrityDigest {
+    let mut accountant = Sha256Accountant::new();
+    accountant.update(bytes);
+    accountant.finalize().digest().expect("digest").clone()
+}
+
+fn source_identity(size: u64) -> ObjectSourceIdentity {
+    ObjectSourceIdentity::from_parts(
+        CanonicalUrl::parse("https://objects.example.test").expect("endpoint"),
+        "source-bucket",
+        "private/key.bin",
+        size,
+        None,
+        Some(ObjectValidator::parse("\"validator\"").expect("validator")),
+    )
+    .expect("identity")
+}
+
+fn destination(size: u64) -> DestinationResumeProof {
+    DestinationResumeProof::new(
+        CanonicalUrl::parse("https://target.example.test").expect("target endpoint"),
+        MultipartIdentity {
+            target_store_id: "target-a".to_owned(),
+            bucket: "target-bucket".to_owned(),
+            key: "_remora/a/attempt/o/object".to_owned(),
+            upload_id: "upload-1".to_owned(),
+        },
+        vec![CompletedPart {
+            part_number: 1,
+            etag: "part-etag".to_owned(),
+            size: Some(size),
+        }],
+        None,
+    )
+    .expect("destination")
+}
+
+#[test]
+fn checkpoint_requires_exact_source_and_destination_boundary() {
+    let path = MemberPath::parse("dir/cafe\u{301}.bin").expect("path");
+    let checkpoint = ObjectResumeCheckpoint::new(
+        source_identity(10),
+        path,
+        4,
+        digest(b"abcd"),
+        destination(4),
+    )
+    .expect("checkpoint");
+    assert_eq!(checkpoint.result_path().canonical(), "dir/café.bin");
+    assert_eq!(checkpoint.destination().completed_bytes(), 4);
+    assert_eq!(checkpoint.destination().completed_part_count(), 1);
+    assert_eq!(
+        ObjectResumeCheckpoint::new(
+            source_identity(10),
+            MemberPath::parse("member.bin").expect("path"),
+            5,
+            digest(b"abcde"),
+            destination(4),
+        ),
+        Err(ObjectSourceError::InvalidRequest)
+    );
+}
+
+#[test]
+fn restart_only_identity_cannot_form_checkpoint() {
+    let identity = ObjectSourceIdentity::from_parts(
+        CanonicalUrl::parse("https://objects.example.test").expect("endpoint"),
+        "source-bucket",
+        "key.bin",
+        10,
+        None,
+        None,
+    )
+    .expect("identity");
+    assert!(!identity.is_resumable());
+    assert_eq!(
+        ObjectResumeCheckpoint::new(
+            identity,
+            MemberPath::parse("member.bin").expect("path"),
+            4,
+            digest(b"abcd"),
+            destination(4),
+        ),
+        Err(ObjectSourceError::InvalidRequest)
+    );
+}
+
+#[test]
+fn validation_and_debug_surfaces_hide_keys_and_credentials() {
+    let config = ObjectSourceConfig::new(
+        "https://objects.example.test",
+        "bucket",
+        "region",
+        "access-secret",
+        "credential-secret",
+    )
+    .expect("config");
+    let request = ObjectSourceRequest::new("private/token/object", "member.bin").expect("request");
+    let combined = format!("{config:?} {request:?} {:?}", source_identity(10));
+    assert!(!combined.contains("private"));
+    assert!(!combined.contains("token"));
+    assert!(!combined.contains("access-secret"));
+    assert!(!combined.contains("credential-secret"));
+    assert!(ObjectSourceRequest::new("key", "../bad").is_err());
+    assert!(ObjectValidator::parse("bad\r\nvalidator").is_err());
+}
+
+fn assert_source_body<T: SourceBody>() {}
+
+#[test]
+fn source_body_implements_streaming_port() {
+    assert_source_body::<remora_object_source::ObjectSourceBody>();
+}

tests/kind/object-source-job.yaml.in

@@ -0,0 +1,34 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: @@JOB_NAME@@
+  namespace: remora-testbed
+spec:
+  activeDeadlineSeconds: 900
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 300
+  template:
+    spec:
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      restartPolicy: Never
+      containers:
+        - name: tests
+          image: @@TEST_IMAGE@@
+          imagePullPolicy: Never
+          envFrom:
+            - configMapRef: {name: @@PROFILE_NAME@@}
+          env:
+            - name: REMORA_OBJECT_SOURCE_REQUIRE
+              value: "1"
+          volumeMounts:
+            - {name: fixture-profile, mountPath: /run/remora-fixture, readOnly: true}
+          resources:
+            requests: {cpu: 250m, memory: 512Mi}
+            limits: {cpu: "2", memory: 2Gi}
+      volumes:
+        - name: fixture-profile
+          configMap:
+            name: @@PROFILE_NAME@@-file
+            items:
+              - {key: profile.env, path: profile.env}

tests/kind/run-object-source-job.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
+source "$ROOT/tests/kind/lib.sh"
+
+USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
+SUCCEEDED=false
+OWNS_FIXTURE=false
+
+cleanup() {
+  local status=$?
+  trap - EXIT INT TERM
+  if [[ "${REMORA_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
+    printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
+    printf 'retained context: %s\n' "$CONTEXT"
+    exit "$status"
+  fi
+  if [[ "$OWNS_FIXTURE" = true ]]; then
+    destroy_fixture || status=1
+  fi
+  if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
+    printf 'user kubeconfig changed unexpectedly\n' >&2
+    status=1
+  fi
+  exit "$status"
+}
+
+trap cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+acquire_fixture_lock
+REMORA_REAL_NERDCTL="$(command -v nerdctl)"
+export REMORA_REAL_NERDCTL
+"$REMORA_REAL_NERDCTL" --namespace default build \
+  --file "$ROOT/Dockerfile.test" \
+  --target kind-object-source-runner \
+  --tag remora-kind-object-source:local \
+  "$ROOT"
+OWNS_FIXTURE=true
+setup_fixture target
+load_local_image remora-kind-object-source:local
+run_job_matrix \
+  object-source \
+  "$TESTBED/object-source-job.yaml.in" \
+  remora-kind-object-source:local \
+  target
+SUCCEEDED=true
+printf 'object source adapter package test Jobs passed\n'