ms2k/remora
SHA256
Watch
0
Code Releases Activity

Add exact-identity target storage adapter

Browse source
This commit is contained in:
ms2k 2026-07-17 02:54:18 +09:00
commit 854e6e1c17
11 changed files with 3263 additions and 41 deletions
Download patch file Download diff file Expand all files Collapse all files

40
crates/remora-target/Cargo.toml Normal file
View file

@ -0,0 +1,40 @@
[package]
name = "remora-target"
edition.workspace = true
publish = false
rust-version.workspace = true
version.workspace = true
[features]
default = []
# Fixture-only helpers (overwrite, version purge). Not for production callers.
test-utils = []
[dependencies]
aws-credential-types = "1.2.11"
aws-sigv4 = "1.3.7"
bytes = "1.11.1"
futures-util = "0.3.32"
http = "1.4.0"
opendal = { version = "0.58.0", default-features = false, features = [
"executors-tokio",
"http-transport-reqwest",
"http-transport-reqwest-rustls",
"services-s3",
] }
percent-encoding = "2.3.2"
quick-xml = { version = "0.38.4", features = ["serialize"] }
remora-domain.workspace = true
remora-stream = { path = "../remora-stream" }
reqwest = { version = "0.12.28", default-features = false, features = [
"rustls-tls-webpki-roots",
] }
serde = { workspace = true, features = ["derive"] }
sha2.workspace = true
thiserror.workspace = true
tokio = { workspace = true, features = ["macros", "sync", "time"] }
[dev-dependencies]
remora-target = { path = ".", features = ["test-utils"] }
sha2.workspace = true
tokio = { workspace = true, features = ["macros", "rt-multi-thread", "sync", "time"] }

92
crates/remora-target/src/body.rs Normal file
View file

@ -0,0 +1,92 @@
use opendal::Writer;
use remora_stream::{StreamError, TargetBody, TargetFinish};
use crate::digest::{ContentHasher, ContentSha256, check_expected};
use crate::error::{TargetError, map_opendal};
use crate::identity::ObjectIdentity;
/// Streaming create-only target body backed by an OpenDAL writer.
///
/// SHA-256 is accumulated over every written byte. The open writer cannot carry
/// final digest metadata (unknown until finish); the independently computed
/// digest is returned on finish for durable persistence.
pub struct OpenDalTargetBody {
writer: Writer,
hasher: ContentHasher,
bytes_written: u64,
identity: ObjectIdentity,
expected_sha256: Option<ContentSha256>,
}
impl OpenDalTargetBody {
pub(crate) fn new(
writer: Writer,
identity: ObjectIdentity,
expected_sha256: Option<ContentSha256>,
) -> Self {
Self {
writer,
hasher: ContentHasher::new(),
bytes_written: 0,
identity,
expected_sha256,
}
}
#[must_use]
pub fn identity(&self) -> &ObjectIdentity {
&self.identity
}
/// Finish the write and return identity plus independently computed digest.
pub async fn finish_object(mut self) -> Result<StoredWrite, TargetError> {
let metadata = self.writer.close().await.map_err(map_opendal)?;
let version_id = metadata.version().map(str::to_owned);
let content_sha256 = self.hasher.finalize();
check_expected(self.expected_sha256.as_ref(), &content_sha256)?;
let content_length = metadata.content_length();
let bytes_written = if content_length > 0 {
content_length
} else {
self.bytes_written
};
Ok(StoredWrite {
identity: self.identity.with_version(version_id),
bytes_written,
content_sha256,
})
}
}
impl TargetBody for OpenDalTargetBody {
async fn write_all(&mut self, data: &[u8]) -> Result<(), StreamError> {
if data.is_empty() {
return Ok(());
}
self.writer
.write(data.to_vec())
.await
.map_err(|error| map_opendal(error).into_stream_error())?;
self.hasher.update(data);
self.bytes_written = self
.bytes_written
.saturating_add(u64::try_from(data.len()).unwrap_or(u64::MAX));
Ok(())
}
async fn finish(self) -> Result<TargetFinish, StreamError> {
let stored = self.finish_object().await?;
Ok(TargetFinish {
bytes_written: stored.bytes_written,
})
}
}
/// Result of completing a create-only stream write.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct StoredWrite {
pub identity: ObjectIdentity,
pub bytes_written: u64,
/// Independently computed SHA-256 of written bytes.
pub content_sha256: ContentSha256,
}

142
crates/remora-target/src/config.rs Normal file
View file

@ -0,0 +1,142 @@
use std::fmt;
use crate::error::TargetError;
/// Connection settings for one S3-compatible target store.
#[derive(Clone, Eq, PartialEq)]
pub struct TargetStoreConfig {
/// Stable target-store identity used in authorization and object identities.
pub target_store_id: String,
pub endpoint: String,
pub bucket: String,
pub region: String,
pub access_key_id: String,
pub secret_access_key: String,
/// Optional operator root under the bucket. Empty means bucket root.
pub root: String,
}
impl fmt::Debug for TargetStoreConfig {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("TargetStoreConfig")
.field("target_store_id", &self.target_store_id)
.field("endpoint", &self.endpoint)
.field("bucket", &self.bucket)
.field("region", &self.region)
.field("access_key_id", &self.access_key_id)
.field("secret_access_key", &"REDACTED")
.field("root", &self.root)
.finish()
}
}
impl TargetStoreConfig {
/// Load fixture or deployment settings from environment variables.
///
/// Recognized variables: `REMORA_TARGET_STORE_ID` (or `REMORA_TARGET_PROVIDER`),
/// `REMORA_TARGET_ENDPOINT`, `REMORA_TARGET_BUCKET`, `REMORA_TARGET_REGION`,
/// `REMORA_TARGET_ACCESS_KEY`, `REMORA_TARGET_SECRET_KEY`,
/// `REMORA_TARGET_EXCLUSIVE_WRITER`, and optional `REMORA_TARGET_ROOT`.
pub fn from_env() -> Result<Self, TargetError> {
let target_store_id = std::env::var("REMORA_TARGET_STORE_ID")
.ok()
.filter(|value| !value.is_empty())
.or_else(|| {
std::env::var("REMORA_TARGET_PROVIDER")
.ok()
.filter(|value| !value.is_empty())
})
.ok_or_else(|| {
TargetError::validation(
"REMORA_TARGET_STORE_ID or REMORA_TARGET_PROVIDER is required",
)
})?;
let endpoint = required_env("REMORA_TARGET_ENDPOINT")?;
let bucket = required_env("REMORA_TARGET_BUCKET")?;
let region = std::env::var("REMORA_TARGET_REGION")
.ok()
.filter(|value| !value.is_empty())
.unwrap_or_else(|| "us-east-1".to_owned());
let access_key_id = required_env("REMORA_TARGET_ACCESS_KEY")?;
let secret_access_key = required_env("REMORA_TARGET_SECRET_KEY")?;
let root = std::env::var("REMORA_TARGET_ROOT")
.ok()
.filter(|value| !value.is_empty())
.unwrap_or_default();
let exclusive_writer = required_env("REMORA_TARGET_EXCLUSIVE_WRITER")?
.parse::<bool>()
.map_err(|_| {
TargetError::validation("REMORA_TARGET_EXCLUSIVE_WRITER must be true or false")
})?;
if !exclusive_writer {
return Err(TargetError::validation(
"target reserved namespace must have Remora as its exclusive writer",
));
}
Self::new_exclusive(
target_store_id,
endpoint,
bucket,
region,
access_key_id,
secret_access_key,
root,
)
}
pub fn new_exclusive(
target_store_id: impl Into<String>,
endpoint: impl Into<String>,
bucket: impl Into<String>,
region: impl Into<String>,
access_key_id: impl Into<String>,
secret_access_key: impl Into<String>,
root: impl Into<String>,
) -> Result<Self, TargetError> {
let target_store_id = target_store_id.into();
let endpoint = endpoint.into().trim_end_matches('/').to_owned();
let bucket = bucket.into();
let region = region.into();
let access_key_id = access_key_id.into();
let secret_access_key = secret_access_key.into();
let root = normalize_root(root.into());
if target_store_id.is_empty() {
return Err(TargetError::validation("target store id is required"));
}
if endpoint.is_empty() {
return Err(TargetError::validation("target endpoint is required"));
}
if bucket.is_empty() {
return Err(TargetError::validation("target bucket is required"));
}
if access_key_id.is_empty() || secret_access_key.is_empty() {
return Err(TargetError::validation("target credentials are required"));
}
Ok(Self {
target_store_id,
endpoint,
bucket,
region,
access_key_id,
secret_access_key,
root,
})
}
}
fn required_env(name: &str) -> Result<String, TargetError> {
std::env::var(name)
.ok()
.filter(|value| !value.is_empty())
.ok_or_else(|| TargetError::validation(format!("{name} is required")))
}
fn normalize_root(root: String) -> String {
let trimmed = root.trim_matches('/');
if trimmed.is_empty() {
String::new()
} else {
format!("/{trimmed}/")
}
}

117
crates/remora-target/src/digest.rs Normal file
View file

@ -0,0 +1,117 @@
use sha2::{Digest, Sha256};
use crate::error::TargetError;
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
pub struct ContentSha256(String);
impl ContentSha256 {
pub fn parse(value: impl AsRef<str>) -> Result<Self, TargetError> {
let value = value.as_ref().trim();
let hex = value
.strip_prefix("sha256:")
.or_else(|| value.strip_prefix("SHA256:"))
.unwrap_or(value);
if hex.len() != 64 || !hex.bytes().all(|byte| byte.is_ascii_hexdigit()) {
return Err(TargetError::validation("invalid sha256 digest"));
}
Ok(Self(format!("sha256:{}", hex.to_ascii_lowercase())))
}
#[must_use]
pub fn as_str(&self) -> &str {
&self.0
}
}
impl std::fmt::Display for ContentSha256 {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
self.0.fmt(formatter)
}
}
/// Format a raw SHA-256 digest as `sha256:` + lowercase hex.
#[must_use]
fn format_sha256(bytes: &[u8]) -> ContentSha256 {
let mut out = String::with_capacity(7 + bytes.len() * 2);
out.push_str("sha256:");
for byte in bytes {
use std::fmt::Write as _;
let _ = write!(out, "{byte:02x}");
}
ContentSha256(out)
}
/// Independently hash `data` and return the Remora digest string.
#[must_use]
pub fn hash_bytes(data: &[u8]) -> ContentSha256 {
format_sha256(Sha256::digest(data).as_ref())
}
/// Running SHA-256 over written bytes (never trusts a caller claim).
#[derive(Clone, Default)]
pub struct ContentHasher {
hasher: Sha256,
}
impl ContentHasher {
#[must_use]
pub fn new() -> Self {
Self::default()
}
pub fn update(&mut self, data: &[u8]) {
if data.is_empty() {
return;
}
self.hasher.update(data);
}
#[must_use]
pub fn finalize(self) -> ContentSha256 {
format_sha256(self.hasher.finalize().as_ref())
}
}
/// If the caller supplied an expected digest, require an exact match to the
/// independently computed value. Omitted expectation is allowed.
pub fn check_expected(
expected: Option<&ContentSha256>,
computed: &ContentSha256,
) -> Result<(), TargetError> {
let Some(expected) = expected else {
return Ok(());
};
if expected != computed {
return Err(TargetError::integrity(
"independently computed sha256 does not match caller expectation",
));
}
Ok(())
}
#[must_use]
pub fn sha256_equal(left: &str, right: &str) -> bool {
match (ContentSha256::parse(left), ContentSha256::parse(right)) {
(Ok(left), Ok(right)) => left == right,
_ => false,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn digest_parsing_canonicalizes_and_rejects_invalid_values() {
let upper = ContentSha256::parse(
"SHA256:BA7816BF8F01CFEA414140DE5DAE2223B00361A396177A9CB410FF61F20015AD",
)
.expect("valid digest");
assert_eq!(
upper.as_str(),
"sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"
);
assert!(ContentSha256::parse("sha256:not-a-digest").is_err());
}
}

186
crates/remora-target/src/error.rs Normal file
View file

@ -0,0 +1,186 @@
use remora_stream::{StreamError, StreamSide};
/// Target storage adapter errors.
///
/// Diagnostic text is for operator logs only after secret scrubbing. Public
/// projections go through [`remora_stream::classify_stream_error`].
#[derive(Debug, thiserror::Error)]
pub enum TargetError {
#[error("target authorization failure")]
Authorization,
#[error("object already exists under create-only identity")]
AlreadyExists,
#[error("object not found")]
NotFound,
#[error("integrity failure: {0}")]
Integrity(String),
#[error("capacity exhausted on target")]
Capacity,
#[error("transient target failure: {0}")]
Transient(String),
#[error("validation failure: {0}")]
Validation(String),
#[error("internal target error: {0}")]
Internal(String),
}
impl TargetError {
pub fn validation(detail: impl Into<String>) -> Self {
Self::Validation(detail.into())
}
pub fn integrity(detail: impl Into<String>) -> Self {
Self::Integrity(detail.into())
}
pub fn transient(detail: impl Into<String>) -> Self {
Self::Transient(detail.into())
}
pub fn internal(detail: impl Into<String>) -> Self {
Self::Internal(detail.into())
}
/// Map to a stream error for copy-loop and adapter I/O boundaries.
#[must_use]
pub fn into_stream_error(self) -> StreamError {
match self {
Self::Authorization => StreamError::Authorization {
side: StreamSide::Target,
},
Self::AlreadyExists => StreamError::integrity("create-only identity already exists"),
Self::NotFound => StreamError::internal("target object not found"),
Self::Integrity(detail) => StreamError::Integrity { detail },
Self::Capacity => StreamError::Capacity {
side: StreamSide::Target,
},
Self::Transient(detail) => StreamError::Transient {
side: StreamSide::Target,
detail,
},
Self::Validation(detail) => StreamError::internal(detail),
Self::Internal(detail) => StreamError::Internal { detail },
}
}
}
impl From<TargetError> for StreamError {
fn from(value: TargetError) -> Self {
value.into_stream_error()
}
}
/// Map OpenDAL errors without embedding raw exception bodies in public paths.
pub(crate) fn map_opendal(error: opendal::Error) -> TargetError {
use opendal::ErrorKind;
let detail = redact_sensitive(error.to_string());
match error.kind() {
ErrorKind::NotFound => TargetError::NotFound,
ErrorKind::PermissionDenied => TargetError::Authorization,
ErrorKind::AlreadyExists | ErrorKind::ConditionNotMatch => TargetError::AlreadyExists,
ErrorKind::RateLimited => TargetError::Capacity,
ErrorKind::Unsupported | ErrorKind::ConfigInvalid => TargetError::Validation(detail),
ErrorKind::Unexpected
| ErrorKind::IsADirectory
| ErrorKind::NotADirectory
| ErrorKind::IsSameFile
| ErrorKind::RangeNotSatisfied => {
if is_likely_transient(&detail) {
TargetError::Transient(detail)
} else {
TargetError::Internal(detail)
}
}
_ => TargetError::Internal(detail),
}
}
pub(crate) fn map_http_status(status: u16, body: &str) -> TargetError {
let detail = redact_sensitive(body);
match status {
401 | 403 => TargetError::Authorization,
404 => TargetError::NotFound,
409 | 412 => TargetError::AlreadyExists,
429 | 503 => TargetError::Capacity,
500 | 502 | 504 => TargetError::Transient(format!("http {status}: {detail}")),
_ if status >= 500 => TargetError::Transient(format!("http {status}: {detail}")),
_ => TargetError::Internal(format!("http {status}: {detail}")),
}
}
/// Scrub credential-bearing fragments from operator-facing diagnostic text.
pub(crate) fn redact_sensitive(input: impl AsRef<str>) -> String {
let mut out = input.as_ref().to_owned();
const NEEDLES: &[&str] = &[
"Authorization:",
"authorization:",
"AWSAccessKeyId=",
"X-Amz-Credential=",
"X-Amz-Signature=",
"X-Amz-Security-Token=",
"x-amz-security-token=",
"Credential=",
"Signature=",
"SecretAccessKey",
"secret_access_key",
"secret-access-key",
"Bearer ",
];
while let Some((idx, needle)) = NEEDLES
.iter()
.filter_map(|needle| out.find(needle).map(|idx| (idx, *needle)))
.min_by_key(|(idx, _)| *idx)
{
let value_start = idx + needle.len();
// Authorization headers are multi-token; scrub through end of line.
let value_end = if needle.eq_ignore_ascii_case("Authorization:") {
out[value_start..]
.find(['\r', '\n'])
.map(|offset| value_start + offset)
.unwrap_or(out.len())
} else {
out[value_start..]
.find(|character: char| {
character.is_whitespace()
|| matches!(character, '"' | '\'' | ',' | ';' | '&' | '<' | '>')
})
.map(|offset| value_start + offset)
.unwrap_or(out.len())
};
out.replace_range(idx..value_end, "[redacted]");
}
if out.len() > 512 {
out.truncate(512);
out.push('…');
}
out
}
fn is_likely_transient(detail: &str) -> bool {
let lower = detail.to_ascii_lowercase();
lower.contains("timeout")
|| lower.contains("temporar")
|| lower.contains("connection")
|| lower.contains("reset")
|| lower.contains("unavailable")
|| lower.contains("slow down")
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn redact_sensitive_scrubs_long_and_repeated_secrets() {
let long_sig = "a".repeat(80);
let input = format!(
"X-Amz-Signature={long_sig}&X-Amz-Credential=AKIAEXAMPLE/path Authorization: AWS4-HMAC-SHA256 Credential=AKIAEXAMPLE, SignedHeaders=host, Signature={long_sig} Bearer super-secret-token-value-here"
);
let redacted = redact_sensitive(input);
assert!(!redacted.contains("AKIAEXAMPLE"));
assert!(!redacted.contains(&long_sig));
assert!(!redacted.contains("super-secret-token-value-here"));
assert!(redacted.contains("[redacted]"));
}
}

137
crates/remora-target/src/identity.rs Normal file
View file

@ -0,0 +1,137 @@
use remora_domain::OpaqueId;
use crate::error::TargetError;
/// Deployment-reserved target namespace segment.
///
/// User target prefixes must not contain this segment. Attempt objects and
/// manifests are written only under this service namespace.
pub const RESERVED_NAMESPACE_SEGMENT: &str = "_remora";
/// User-metadata key for Remora-computed SHA-256 digests.
pub const REMORA_SHA256_METADATA_KEY: &str = "remora-sha256";
/// Exact object identity retained for reads, deletes, and time-limited access.
///
/// Bound to a target-store identity so identities cannot be applied across stores.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ObjectIdentity {
pub target_store_id: String,
pub bucket: String,
pub key: String,
pub version_id: Option<String>,
}
impl ObjectIdentity {
#[must_use]
pub fn with_version(mut self, version_id: Option<String>) -> Self {
self.version_id = version_id.filter(|value| !value.is_empty());
self
}
}
/// Exact multipart identity retained for destination resume and cleanup.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct MultipartIdentity {
pub target_store_id: String,
pub bucket: String,
pub key: String,
pub upload_id: String,
}
/// One completed multipart part as re-listed from the target store.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct CompletedPart {
pub part_number: u32,
pub etag: String,
pub size: Option<u64>,
}
/// Construct a create-only attempt object key under the reserved namespace.
///
/// Layout: `_remora/a/{attempt_id}/o/{object_id}[/{relative_path}]`
#[must_use]
pub fn attempt_object_key(
attempt_id: &OpaqueId,
object_id: &OpaqueId,
relative_path: Option<&str>,
) -> String {
let mut key = format!(
"{RESERVED_NAMESPACE_SEGMENT}/a/{}/o/{}",
attempt_id.as_str(),
object_id.as_str()
);
if let Some(path) = relative_path.filter(|value| !value.is_empty()) {
let trimmed = path.trim_matches('/');
if !trimmed.is_empty() {
key.push('/');
key.push_str(trimmed);
}
}
key
}
/// Construct a reserved-namespace key for a manifest descriptor object.
#[must_use]
pub fn attempt_manifest_key(attempt_id: &OpaqueId, manifest_id: &OpaqueId) -> String {
format!(
"{RESERVED_NAMESPACE_SEGMENT}/a/{}/m/{}",
attempt_id.as_str(),
manifest_id.as_str()
)
}
/// Return true when `key` is under the reserved service namespace.
#[must_use]
pub fn is_reserved_key(key: &str) -> bool {
let trimmed = key.trim_start_matches('/');
trimmed == RESERVED_NAMESPACE_SEGMENT
|| trimmed.starts_with(&format!("{RESERVED_NAMESPACE_SEGMENT}/"))
}
/// Validate a reserved attempt object or manifest identity layout.
pub fn validate_attempt_key(key: &str) -> Result<(), TargetError> {
if key.is_empty() || key.starts_with('/') || key.contains("//") {
return Err(TargetError::validation("invalid object key"));
}
if !is_reserved_key(key) {
return Err(TargetError::validation(
"object key must be under the reserved service namespace",
));
}
let segments = key.split('/').collect::<Vec<_>>();
if segments.len() < 5
|| segments[0] != RESERVED_NAMESPACE_SEGMENT
|| segments[1] != "a"
|| OpaqueId::parse(segments[2]).is_err()
|| OpaqueId::parse(segments[4]).is_err()
|| !matches!(segments[3], "o" | "m")
|| (segments[3] == "m" && segments.len() != 5)
{
return Err(TargetError::validation(
"object key must contain attempt and object identities",
));
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn attempt_keys_are_reserved_and_unique() {
let attempt = OpaqueId::parse("attempt01aaaaaaaa").unwrap();
let object = OpaqueId::parse("object01aaaaaaaaa").unwrap();
let key = attempt_object_key(&attempt, &object, Some("한글/é/member"));
assert!(is_reserved_key(&key));
assert!(validate_attempt_key(&key).is_ok());
assert!(key.contains("한글"));
assert_eq!(
attempt_manifest_key(&attempt, &object),
"_remora/a/attempt01aaaaaaaa/m/object01aaaaaaaaa"
);
assert!(validate_attempt_key("public/data").is_err());
assert!(validate_attempt_key("_remora/arbitrary").is_err());
}
}

27
crates/remora-target/src/lib.rs Normal file
View file

@ -0,0 +1,27 @@
//! Target storage adapter (WP-08).
//!
//! OpenDAL-backed create-only attempt objects, multipart recovery, exact
//! identity reads and deletes, independent SHA-256, version retention, and
//! exact time-limited access against S3-compatible target stores.
mod body;
mod config;
mod digest;
mod error;
mod identity;
mod multipart;
mod store;
pub use body::{OpenDalTargetBody, StoredWrite};
pub use config::TargetStoreConfig;
pub use digest::{ContentSha256, hash_bytes, sha256_equal};
pub use error::TargetError;
pub use identity::{
CompletedPart, MultipartIdentity, ObjectIdentity, REMORA_SHA256_METADATA_KEY,
RESERVED_NAMESPACE_SEGMENT, attempt_manifest_key, attempt_object_key, is_reserved_key,
validate_attempt_key,
};
pub use store::{
CreateObjectRequest, MultipartSession, ObjectMetadata, StoredObject, TargetStore,
TimeLimitedAccess, finish_create_only,
};

579
crates/remora-target/src/multipart.rs Normal file
View file

@ -0,0 +1,579 @@
use std::fmt;
use std::time::{Duration, SystemTime};
use aws_credential_types::Credentials;
use aws_sigv4::http_request::{
PayloadChecksumKind, SignableBody, SignableRequest, SigningSettings, UriPathNormalizationMode,
sign,
};
use aws_sigv4::sign::v4;
use bytes::Bytes;
use http::header::{CONTENT_LENGTH, CONTENT_TYPE, HeaderName, HeaderValue};
use http::{Method, Request};
use percent_encoding::{AsciiSet, CONTROLS, utf8_percent_encode};
use quick_xml::de::from_str;
use quick_xml::se::to_string;
use serde::{Deserialize, Serialize};
use crate::config::TargetStoreConfig;
use crate::error::{TargetError, map_http_status, redact_sensitive};
use crate::identity::{CompletedPart, MultipartIdentity, REMORA_SHA256_METADATA_KEY};
/// Encode path segments for path-style S3 URLs.
const PATH_SET: &AsciiSet = &CONTROLS
.add(b' ')
.add(b'"')
.add(b'#')
.add(b'%')
.add(b'&')
.add(b'+')
.add(b',')
.add(b'<')
.add(b'>')
.add(b'?')
.add(b'@')
.add(b'[')
.add(b'\\')
.add(b']')
.add(b'^')
.add(b'`')
.add(b'{')
.add(b'|')
.add(b'}');
/// One version or delete marker listed under a key/prefix.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ObjectVersion {
pub key: String,
pub version_id: String,
pub is_delete_marker: bool,
}
/// Explicit multipart control plane for destination recovery.
///
/// OpenDAL streams multipart writes but does not expose list-parts recovery.
/// This client issues path-style S3 multipart APIs with the same credentials
/// as the OpenDAL operator so exact upload IDs and re-listed parts are durable.
#[derive(Clone)]
pub struct MultipartClient {
config: TargetStoreConfig,
http: reqwest::Client,
}
impl fmt::Debug for MultipartClient {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("MultipartClient")
.field("config", &self.config)
.finish_non_exhaustive()
}
}
impl MultipartClient {
pub fn new(config: TargetStoreConfig) -> Result<Self, TargetError> {
let http = reqwest::Client::builder()
.timeout(Duration::from_secs(120))
.build()
.map_err(|error| TargetError::internal(redact_sensitive(error.to_string())))?;
Ok(Self { config, http })
}
/// Enable bucket versioning when the peer supports it.
///
/// Peers that return NotImplemented are reported as `false` without error.
pub async fn enable_bucket_versioning(&self) -> Result<bool, TargetError> {
let body = r#"<?xml version="1.0" encoding="UTF-8"?>
<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Status>Enabled</Status>
</VersioningConfiguration>"#;
let uri = format!(
"{}/{}/?versioning",
self.config.endpoint, self.config.bucket
);
let request = Request::builder()
.method(Method::PUT)
.uri(uri)
.header(CONTENT_TYPE, "application/xml")
.header(CONTENT_LENGTH, body.len().to_string())
.body(Bytes::from_static(body.as_bytes()))
.map_err(|error| TargetError::internal(error.to_string()))?;
let response = self.send(request).await?;
let status = response.status().as_u16();
if status / 100 == 2 {
return Ok(true);
}
let text = response.text().await.unwrap_or_default();
if status == 501
|| text.contains("NotImplemented")
|| text.contains("XNotImplemented")
|| text.contains("NotSupported")
{
return Ok(false);
}
Err(map_http_status(status, &text))
}
/// Start a multipart upload. `content_sha256` is optional S3 user metadata
/// for an independently computed digest only.
pub async fn initiate(
&self,
key: &str,
content_sha256: Option<&str>,
) -> Result<MultipartIdentity, TargetError> {
let path = object_path(&self.config, key);
let mut builder = Request::builder()
.method(Method::POST)
.uri(format!("{}{}?uploads", self.config.endpoint, path));
if let Some(digest) = content_sha256 {
let header = format!("x-amz-meta-{REMORA_SHA256_METADATA_KEY}");
builder = builder.header(
HeaderName::from_bytes(header.as_bytes())
.map_err(|_| TargetError::internal("invalid metadata header name"))?,
HeaderValue::from_str(digest)
.map_err(|_| TargetError::validation("invalid sha256 metadata"))?,
);
}
let request = builder
.header(CONTENT_LENGTH, "0")
.body(Bytes::new())
.map_err(|error| TargetError::internal(error.to_string()))?;
let response = self.send(request).await?;
let status = response.status().as_u16();
let body = response
.text()
.await
.map_err(|error| TargetError::transient(redact_sensitive(error.to_string())))?;
if status / 100 != 2 {
return Err(map_http_status(status, &body));
}
let parsed: InitiateMultipartUploadResult = from_str(&body)
.map_err(|error| TargetError::internal(redact_sensitive(error.to_string())))?;
if parsed.upload_id.is_empty() {
return Err(TargetError::internal(
"multipart initiate missing upload id",
));
}
Ok(MultipartIdentity {
target_store_id: self.config.target_store_id.clone(),
bucket: self.config.bucket.clone(),
key: key.to_owned(),
upload_id: parsed.upload_id,
})
}
pub async fn upload_part(
&self,
identity: &MultipartIdentity,
part_number: u32,
body: bytes::Bytes,
) -> Result<CompletedPart, TargetError> {
if part_number == 0 {
return Err(TargetError::validation("part number must be >= 1"));
}
let path = object_path(&self.config, &identity.key);
let uri = format!(
"{}{}?partNumber={part_number}&uploadId={}",
self.config.endpoint,
path,
encode_query(&identity.upload_id)
);
let len = body.len();
let request = Request::builder()
.method(Method::PUT)
.uri(uri)
.header(CONTENT_LENGTH, len.to_string())
.body(body)
.map_err(|error| TargetError::internal(error.to_string()))?;
let response = self.send(request).await?;
let status = response.status().as_u16();
if status / 100 != 2 {
let body = response.text().await.unwrap_or_default();
return Err(map_http_status(status, &body));
}
let etag = response
.headers()
.get("etag")
.and_then(|value| value.to_str().ok())
.map(|value| value.trim_matches('"').to_owned())
.ok_or_else(|| TargetError::internal("multipart part missing etag"))?;
Ok(CompletedPart {
part_number,
etag,
size: Some(len as u64),
})
}
pub async fn list_parts(
&self,
identity: &MultipartIdentity,
) -> Result<Vec<CompletedPart>, TargetError> {
let mut parts = Vec::new();
let mut part_number_marker: Option<u32> = None;
loop {
let path = object_path(&self.config, &identity.key);
let mut uri = format!(
"{}{}?uploadId={}",
self.config.endpoint,
path,
encode_query(&identity.upload_id)
);
if let Some(marker) = part_number_marker {
uri.push_str(&format!("&part-number-marker={marker}"));
}
let request = Request::builder()
.method(Method::GET)
.uri(uri)
.body(Bytes::new())
.map_err(|error| TargetError::internal(error.to_string()))?;
let response = self.send(request).await?;
let status = response.status().as_u16();
let body = response
.text()
.await
.map_err(|error| TargetError::transient(redact_sensitive(error.to_string())))?;
if status / 100 != 2 {
return Err(map_http_status(status, &body));
}
let parsed: ListPartsResult = from_str(&body)
.map_err(|error| TargetError::internal(redact_sensitive(error.to_string())))?;
for part in parsed.parts {
parts.push(CompletedPart {
part_number: part.part_number,
etag: part.etag.trim_matches('"').to_owned(),
size: Some(part.size),
});
}
if parsed.is_truncated {
part_number_marker = parsed
.next_part_number_marker
.or_else(|| parts.last().map(|part| part.part_number));
if part_number_marker.is_none() {
break;
}
} else {
break;
}
}
parts.sort_by_key(|part| part.part_number);
Ok(parts)
}
pub async fn complete(
&self,
identity: &MultipartIdentity,
parts: &[CompletedPart],
if_not_exists: bool,
) -> Result<Option<String>, TargetError> {
if parts.is_empty() {
return Err(TargetError::validation(
"multipart complete requires at least one part",
));
}
let payload = CompleteMultipartUpload {
parts: parts
.iter()
.map(|part| CompletePart {
part_number: part.part_number,
etag: format!("\"{}\"", part.etag.trim_matches('"')),
})
.collect(),
};
let xml = to_string(&payload)
.map_err(|error| TargetError::internal(redact_sensitive(error.to_string())))?;
let path = object_path(&self.config, &identity.key);
let uri = format!(
"{}{}?uploadId={}",
self.config.endpoint,
path,
encode_query(&identity.upload_id)
);
let mut builder = Request::builder()
.method(Method::POST)
.uri(uri)
.header(CONTENT_TYPE, "application/xml")
.header(CONTENT_LENGTH, xml.len().to_string());
if if_not_exists {
builder = builder.header("If-None-Match", "*");
}
let request = builder
.body(Bytes::from(xml))
.map_err(|error| TargetError::internal(error.to_string()))?;
let response = self.send(request).await?;
let status = response.status().as_u16();
let version_id = response
.headers()
.get("x-amz-version-id")
.and_then(|value| value.to_str().ok())
.map(str::to_owned);
let body = response
.text()
.await
.map_err(|error| TargetError::transient(redact_sensitive(error.to_string())))?;
if status / 100 != 2 {
return Err(map_http_status(status, &body));
}
Ok(version_id.filter(|value| !value.is_empty()))
}
pub async fn abort(&self, identity: &MultipartIdentity) -> Result<(), TargetError> {
let path = object_path(&self.config, &identity.key);
let uri = format!(
"{}{}?uploadId={}",
self.config.endpoint,
path,
encode_query(&identity.upload_id)
);
let request = Request::builder()
.method(Method::DELETE)
.uri(uri)
.body(Bytes::new())
.map_err(|error| TargetError::internal(error.to_string()))?;
let response = self.send(request).await?;
let status = response.status().as_u16();
if status == 404 {
return Ok(());
}
if status / 100 != 2 {
let body = response.text().await.unwrap_or_default();
return Err(map_http_status(status, &body));
}
Ok(())
}
/// List object versions and delete markers under an optional key prefix.
pub async fn list_object_versions(
&self,
prefix: &str,
) -> Result<Vec<ObjectVersion>, TargetError> {
let mut versions = Vec::new();
let mut key_marker = String::new();
let mut version_id_marker = String::new();
loop {
let mut uri = format!("{}/{}/?versions", self.config.endpoint, self.config.bucket);
if !prefix.is_empty() {
uri.push_str(&format!("&prefix={}", encode_query(prefix)));
}
if !key_marker.is_empty() {
uri.push_str(&format!("&key-marker={}", encode_query(&key_marker)));
}
if !version_id_marker.is_empty() {
uri.push_str(&format!(
"&version-id-marker={}",
encode_query(&version_id_marker)
));
}
let request = Request::builder()
.method(Method::GET)
.uri(uri)
.body(Bytes::new())
.map_err(|error| TargetError::internal(error.to_string()))?;
let response = self.send(request).await?;
let status = response.status().as_u16();
let body = response
.text()
.await
.map_err(|error| TargetError::transient(redact_sensitive(error.to_string())))?;
if status / 100 != 2 {
return Err(map_http_status(status, &body));
}
let parsed: ListVersionsResult = from_str(&body)
.map_err(|error| TargetError::internal(redact_sensitive(error.to_string())))?;
for version in parsed.versions {
if !version.version_id.is_empty() {
versions.push(ObjectVersion {
key: version.key,
version_id: version.version_id,
is_delete_marker: false,
});
}
}
for marker in parsed.delete_markers {
if !marker.version_id.is_empty() {
versions.push(ObjectVersion {
key: marker.key,
version_id: marker.version_id,
is_delete_marker: true,
});
}
}
if parsed.is_truncated {
key_marker = parsed.next_key_marker.unwrap_or_default();
version_id_marker = parsed.next_version_id_marker.unwrap_or_default();
if key_marker.is_empty() && version_id_marker.is_empty() {
break;
}
} else {
break;
}
}
Ok(versions)
}
/// Delete one exact object version or delete marker.
pub async fn delete_object_version(
&self,
key: &str,
version_id: &str,
) -> Result<(), TargetError> {
let path = object_path(&self.config, key);
let uri = format!(
"{}{}?versionId={}",
self.config.endpoint,
path,
encode_query(version_id)
);
let request = Request::builder()
.method(Method::DELETE)
.uri(uri)
.body(Bytes::new())
.map_err(|error| TargetError::internal(error.to_string()))?;
let response = self.send(request).await?;
let status = response.status().as_u16();
if status == 404 || status / 100 == 2 {
return Ok(());
}
let body = response.text().await.unwrap_or_default();
Err(map_http_status(status, &body))
}
async fn send(&self, request: Request<Bytes>) -> Result<reqwest::Response, TargetError> {
let (parts, body) = sign_request(&self.config, request)?.into_parts();
self.http
.request(parts.method, parts.uri.to_string())
.headers(parts.headers)
.body(body)
.send()
.await
.map_err(|error| TargetError::transient(redact_sensitive(error.to_string())))
}
}
fn object_path(config: &TargetStoreConfig, key: &str) -> String {
let key = key.trim_start_matches('/');
let rooted = if config.root.is_empty() {
key.to_owned()
} else {
let root = config.root.trim_matches('/');
if root.is_empty() {
key.to_owned()
} else {
format!("{root}/{key}")
}
};
let encoded = rooted
.split('/')
.map(|segment| utf8_percent_encode(segment, PATH_SET).to_string())
.collect::<Vec<_>>()
.join("/");
format!("/{}/{}", config.bucket, encoded)
}
fn encode_query(value: &str) -> String {
utf8_percent_encode(value, PATH_SET).to_string()
}
fn sign_request(
config: &TargetStoreConfig,
mut request: Request<Bytes>,
) -> Result<Request<Bytes>, TargetError> {
let identity = Credentials::new(
&config.access_key_id,
&config.secret_access_key,
None,
None,
"remora-target",
)
.into();
let mut settings = SigningSettings::default();
// S3-compatible stores (including Garage) require x-amz-content-sha256 and
// reject URI path normalization in the canonical request.
settings.payload_checksum_kind = PayloadChecksumKind::XAmzSha256;
settings.uri_path_normalization_mode = UriPathNormalizationMode::Disabled;
let signing_params = v4::SigningParams::builder()
.identity(&identity)
.region(&config.region)
.name("s3")
.time(SystemTime::now())
.settings(settings)
.build()
.map_err(|error| TargetError::internal(redact_sensitive(error.to_string())))?
.into();
let signable = SignableRequest::new(
request.method().as_str(),
request.uri().to_string(),
request
.headers()
.iter()
.filter_map(|(name, value)| value.to_str().ok().map(|value| (name.as_str(), value))),
SignableBody::Bytes(request.body().as_ref()),
)
.map_err(|error| TargetError::internal(redact_sensitive(error.to_string())))?;
let (instructions, _signature) = sign(signable, &signing_params)
.map_err(|error| TargetError::internal(redact_sensitive(error.to_string())))?
.into_parts();
instructions.apply_to_request_http1x(&mut request);
Ok(request)
}
#[derive(Debug, Deserialize)]
struct InitiateMultipartUploadResult {
#[serde(rename = "UploadId", default)]
upload_id: String,
}
#[derive(Debug, Deserialize)]
struct ListPartsResult {
#[serde(rename = "Part", default)]
parts: Vec<ListPart>,
#[serde(rename = "IsTruncated", default)]
is_truncated: bool,
#[serde(rename = "NextPartNumberMarker", default)]
next_part_number_marker: Option<u32>,
}
#[derive(Debug, Deserialize)]
struct ListPart {
#[serde(rename = "PartNumber")]
part_number: u32,
#[serde(rename = "ETag")]
etag: String,
#[serde(rename = "Size", default)]
size: u64,
}
#[derive(Debug, Serialize)]
#[serde(rename = "CompleteMultipartUpload")]
struct CompleteMultipartUpload {
#[serde(rename = "Part")]
parts: Vec<CompletePart>,
}
#[derive(Debug, Serialize)]
struct CompletePart {
#[serde(rename = "PartNumber")]
part_number: u32,
#[serde(rename = "ETag")]
etag: String,
}
#[derive(Debug, Deserialize)]
struct ListVersionsResult {
#[serde(rename = "Version", default)]
versions: Vec<ListedVersion>,
#[serde(rename = "DeleteMarker", default)]
delete_markers: Vec<ListedVersion>,
#[serde(rename = "IsTruncated", default)]
is_truncated: bool,
#[serde(rename = "NextKeyMarker", default)]
next_key_marker: Option<String>,
#[serde(rename = "NextVersionIdMarker", default)]
next_version_id_marker: Option<String>,
}
#[derive(Debug, Deserialize)]
struct ListedVersion {
#[serde(rename = "Key", default)]
key: String,
#[serde(rename = "VersionId", default)]
version_id: String,
}

548
crates/remora-target/src/store.rs Normal file
View file

@ -0,0 +1,548 @@
use std::collections::HashMap;
use std::num::NonZeroUsize;
use std::time::Duration;
use futures_util::TryStreamExt;
use opendal::services::S3;
use opendal::{Metadata, Operator, Writer};
use remora_domain::OpaqueId;
use remora_domain::ports::StorageAdapter;
use crate::body::{OpenDalTargetBody, StoredWrite};
use crate::config::TargetStoreConfig;
use crate::digest::{ContentHasher, ContentSha256, check_expected, hash_bytes};
use crate::error::{TargetError, map_opendal, redact_sensitive};
use crate::identity::{
CompletedPart, MultipartIdentity, ObjectIdentity, REMORA_SHA256_METADATA_KEY,
attempt_manifest_key, attempt_object_key, validate_attempt_key,
};
use crate::multipart::MultipartClient;
/// Request for a create-only object write under a unique reserved key.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct CreateObjectRequest {
key: String,
expected_content_sha256: Option<ContentSha256>,
chunk_size: Option<NonZeroUsize>,
}
impl CreateObjectRequest {
pub fn parse(key: impl Into<String>) -> Result<Self, TargetError> {
let key = key.into();
validate_attempt_key(&key)?;
Ok(Self {
key,
expected_content_sha256: None,
chunk_size: None,
})
}
#[must_use]
pub fn for_attempt_object(
attempt_id: &OpaqueId,
object_id: &OpaqueId,
relative_path: Option<&str>,
) -> Self {
Self {
key: attempt_object_key(attempt_id, object_id, relative_path),
expected_content_sha256: None,
chunk_size: None,
}
}
#[must_use]
pub fn for_attempt_manifest(attempt_id: &OpaqueId, manifest_id: &OpaqueId) -> Self {
Self {
key: attempt_manifest_key(attempt_id, manifest_id),
expected_content_sha256: None,
chunk_size: None,
}
}
#[must_use]
pub fn with_expected_content_sha256(mut self, expected: ContentSha256) -> Self {
self.expected_content_sha256 = Some(expected);
self
}
#[must_use]
pub fn with_chunk_size(mut self, chunk_size: NonZeroUsize) -> Self {
self.chunk_size = Some(chunk_size);
self
}
#[must_use]
pub fn key(&self) -> &str {
&self.key
}
}
/// Stored object returned after a create-only write completes.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct StoredObject {
pub identity: ObjectIdentity,
/// Independently computed SHA-256 of the written bytes.
pub content_sha256: ContentSha256,
pub size: u64,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ObjectMetadata {
pub identity: ObjectIdentity,
pub content_sha256: Option<ContentSha256>,
pub size: u64,
}
/// Time-limited exact-object access URL.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct TimeLimitedAccess {
pub method: String,
pub url: String,
pub expires_in: Duration,
}
/// Recoverable multipart upload bound to one exact target identity.
pub struct MultipartSession {
identity: MultipartIdentity,
expected_sha256: Option<ContentSha256>,
}
impl std::fmt::Debug for MultipartSession {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter
.debug_struct("MultipartSession")
.field("identity", &self.identity)
.field("expected_sha256", &self.expected_sha256)
.finish_non_exhaustive()
}
}
impl MultipartSession {
#[must_use]
pub fn identity(&self) -> &MultipartIdentity {
&self.identity
}
}
/// OpenDAL-backed target storage adapter for attempt-scoped object writes.
#[derive(Clone)]
pub struct TargetStore {
config: TargetStoreConfig,
operator: Operator,
multipart: MultipartClient,
}
impl TargetStore {
pub fn connect(config: TargetStoreConfig) -> Result<Self, TargetError> {
opendal::install_default();
let mut builder = S3::default()
.bucket(&config.bucket)
.endpoint(&config.endpoint)
.region(&config.region)
.access_key_id(&config.access_key_id)
.secret_access_key(&config.secret_access_key)
.disable_config_load()
.disable_ec2_metadata();
if !config.root.is_empty() {
builder = builder.root(&config.root);
}
let operator = Operator::new(builder)
.map_err(|error| TargetError::validation(redact_sensitive(error.to_string())))?;
let multipart = MultipartClient::new(config.clone())?;
Ok(Self {
config,
operator,
multipart,
})
}
pub fn connect_from_env() -> Result<Self, TargetError> {
Self::connect(TargetStoreConfig::from_env()?)
}
#[must_use]
pub fn config(&self) -> &TargetStoreConfig {
&self.config
}
/// Open a create-only streaming writer for one attempt object.
pub async fn open_create_only(
&self,
request: &CreateObjectRequest,
) -> Result<OpenDalTargetBody, TargetError> {
validate_attempt_key(&request.key)?;
self.require_absent(&request.key).await?;
let writer = self.open_writer(request).await?;
Ok(OpenDalTargetBody::new(
writer,
self.new_object_identity(&request.key, None),
request.expected_content_sha256.clone(),
))
}
/// Create-only put of a complete buffer.
///
/// Independently hashes `body`, persists that digest as user metadata, and
/// retains the version ID from the write response when present.
pub async fn put_create_only(
&self,
request: &CreateObjectRequest,
body: impl Into<bytes::Bytes>,
) -> Result<StoredObject, TargetError> {
validate_attempt_key(&request.key)?;
self.require_absent(&request.key).await?;
let body = body.into();
let computed = hash_bytes(&body);
check_expected(request.expected_content_sha256.as_ref(), &computed)?;
let size = body.len() as u64;
let write = self
.operator
.write_with(&request.key, body)
.if_not_exists(true)
.user_metadata(sha256_metadata(&computed));
let metadata = write.await.map_err(map_opendal)?;
Ok(self.stored_from_write(&request.key, &computed, &metadata, size))
}
/// Exact identity read. Version is required when present on the identity.
pub async fn read_exact(&self, identity: &ObjectIdentity) -> Result<bytes::Bytes, TargetError> {
self.ensure_object_identity(identity)?;
let mut read = self.operator.read_with(&identity.key);
if let Some(version) = identity.version_id.as_deref() {
read = read.version(version);
}
let buffer = read.await.map_err(map_opendal)?;
Ok(buffer.to_bytes())
}
/// Exact identity delete. Missing objects are success for cleanup.
pub async fn delete_exact(&self, identity: &ObjectIdentity) -> Result<(), TargetError> {
self.ensure_object_identity(identity)?;
let mut delete = self.operator.delete_with(&identity.key);
if let Some(version) = identity.version_id.as_deref() {
delete = delete.version(version);
}
match delete.await {
Ok(()) => Ok(()),
Err(error) if matches!(error.kind(), opendal::ErrorKind::NotFound) => Ok(()),
Err(error) => Err(map_opendal(error)),
}
}
/// Head an exact identity, returning size and retained SHA-256 metadata.
pub async fn head_exact(
&self,
identity: &ObjectIdentity,
) -> Result<ObjectMetadata, TargetError> {
self.ensure_object_identity(identity)?;
let mut stat = self.operator.stat_with(&identity.key);
if let Some(version) = identity.version_id.as_deref() {
stat = stat.version(version);
}
let metadata = stat.await.map_err(map_opendal)?;
// Prefer exact version retained on the identity when the head omits it.
let version_id = metadata
.version()
.map(str::to_owned)
.or_else(|| identity.version_id.clone());
let content_sha256 = observed_sha256(&metadata)?;
Ok(ObjectMetadata {
identity: ObjectIdentity {
target_store_id: self.config.target_store_id.clone(),
bucket: self.config.bucket.clone(),
key: identity.key.clone(),
version_id,
},
content_sha256,
size: metadata.content_length(),
})
}
/// Issue a time-limited URL bound to one exact object identity.
pub async fn presign_read(
&self,
identity: &ObjectIdentity,
expires_in: Duration,
) -> Result<TimeLimitedAccess, TargetError> {
self.ensure_object_identity(identity)?;
if expires_in.is_zero() {
return Err(TargetError::validation(
"time-limited access expiry must be positive",
));
}
let mut read_options = opendal::options::ReadOptions::default();
if let Some(version) = identity.version_id.as_deref() {
read_options.version = Some(version.to_owned());
}
let signed = self
.operator
.presign_read_options(&identity.key, expires_in, read_options)
.await
.map_err(map_opendal)?;
Ok(TimeLimitedAccess {
method: signed.method().as_str().to_owned(),
url: signed.uri().to_string(),
expires_in,
})
}
/// Begin a create-only multipart upload under a reserved key.
///
/// The completed exact object is independently stream-hashed after S3
/// accepts the freshly re-listed part set.
pub async fn begin_multipart(
&self,
request: &CreateObjectRequest,
) -> Result<MultipartSession, TargetError> {
self.require_absent(&request.key).await?;
let identity = self.multipart.initiate(&request.key, None).await?;
Ok(MultipartSession {
identity,
expected_sha256: request.expected_content_sha256.clone(),
})
}
/// Reconstruct a multipart session from its durable identity after loss.
pub async fn resume_multipart(
&self,
identity: MultipartIdentity,
expected_content_sha256: Option<ContentSha256>,
) -> Result<(MultipartSession, Vec<CompletedPart>), TargetError> {
self.ensure_multipart_identity(&identity)?;
validate_attempt_key(&identity.key)?;
let parts = self.multipart.list_parts(&identity).await?;
Ok((
MultipartSession {
identity,
expected_sha256: expected_content_sha256,
},
parts,
))
}
pub async fn upload_part(
&self,
session: &MultipartSession,
part_number: u32,
body: bytes::Bytes,
) -> Result<CompletedPart, TargetError> {
self.ensure_multipart_identity(&session.identity)?;
self.multipart
.upload_part(&session.identity, part_number, body)
.await
}
pub async fn list_parts(
&self,
session: &MultipartSession,
) -> Result<Vec<CompletedPart>, TargetError> {
self.ensure_multipart_identity(&session.identity)?;
self.multipart.list_parts(&session.identity).await
}
pub async fn complete_multipart(
&self,
session: MultipartSession,
parts: &[CompletedPart],
) -> Result<StoredObject, TargetError> {
self.ensure_multipart_identity(&session.identity)?;
self.require_absent(&session.identity.key).await?;
let relisted = self.multipart.list_parts(&session.identity).await?;
if relisted != parts {
return Err(TargetError::integrity(
"multipart completion parts do not match exact target re-list",
));
}
let version_id = self
.multipart
.complete(&session.identity, parts, true)
.await?;
let object = self.new_object_identity(&session.identity.key, version_id);
let (computed, size) = self.hash_exact(&object).await?;
if let Err(error) = check_expected(session.expected_sha256.as_ref(), &computed) {
let _ = self.delete_exact(&object).await;
return Err(error);
}
Ok(StoredObject {
identity: object,
content_sha256: computed,
size,
})
}
pub async fn abort_multipart(&self, session: MultipartSession) -> Result<(), TargetError> {
self.ensure_multipart_identity(&session.identity)?;
self.multipart.abort(&session.identity).await
}
async fn open_writer(&self, request: &CreateObjectRequest) -> Result<Writer, TargetError> {
let chunk = request
.chunk_size
.map(NonZeroUsize::get)
.unwrap_or(5 * 1024 * 1024);
self.operator
.writer_with(&request.key)
.if_not_exists(true)
.chunk(chunk)
.await
.map_err(map_opendal)
}
async fn require_absent(&self, key: &str) -> Result<(), TargetError> {
match self.operator.stat(key).await {
Ok(_) => Err(TargetError::AlreadyExists),
Err(error) if matches!(error.kind(), opendal::ErrorKind::NotFound) => Ok(()),
Err(error) => Err(map_opendal(error)),
}
}
async fn hash_exact(
&self,
identity: &ObjectIdentity,
) -> Result<(ContentSha256, u64), TargetError> {
self.ensure_object_identity(identity)?;
let mut reader = self.operator.reader_with(&identity.key).chunk(1024 * 1024);
if let Some(version) = identity.version_id.as_deref() {
reader = reader.version(version);
}
let reader = reader.await.map_err(map_opendal)?;
let mut stream = reader.into_bytes_stream(..).await.map_err(map_opendal)?;
let mut hasher = ContentHasher::new();
let mut size = 0u64;
while let Some(chunk) = stream
.try_next()
.await
.map_err(|error| TargetError::transient(redact_sensitive(error.to_string())))?
{
hasher.update(&chunk);
size = size.saturating_add(u64::try_from(chunk.len()).unwrap_or(u64::MAX));
}
Ok((hasher.finalize(), size))
}
fn stored_from_write(
&self,
key: &str,
content_sha256: &ContentSha256,
metadata: &Metadata,
written_size: u64,
) -> StoredObject {
StoredObject {
identity: self.new_object_identity(key, metadata.version().map(str::to_owned)),
content_sha256: content_sha256.clone(),
size: metadata.content_length().max(written_size),
}
}
fn new_object_identity(&self, key: &str, version_id: Option<String>) -> ObjectIdentity {
ObjectIdentity {
target_store_id: self.config.target_store_id.clone(),
bucket: self.config.bucket.clone(),
key: key.to_owned(),
version_id: version_id.filter(|value| !value.is_empty()),
}
}
fn ensure_object_identity(&self, identity: &ObjectIdentity) -> Result<(), TargetError> {
if identity.target_store_id != self.config.target_store_id {
return Err(TargetError::validation(
"object identity target store does not match configured store",
));
}
if identity.bucket != self.config.bucket {
return Err(TargetError::validation(
"object identity bucket does not match configured target",
));
}
Ok(())
}
fn ensure_multipart_identity(&self, identity: &MultipartIdentity) -> Result<(), TargetError> {
if identity.target_store_id != self.config.target_store_id {
return Err(TargetError::validation(
"multipart identity target store does not match configured store",
));
}
if identity.bucket != self.config.bucket {
return Err(TargetError::validation(
"multipart identity bucket does not match configured target",
));
}
Ok(())
}
}
#[cfg(feature = "test-utils")]
impl TargetStore {
/// Overwrite put without create-only guards (fixture version matrix only).
pub async fn put_overwrite(
&self,
request: &CreateObjectRequest,
body: impl Into<bytes::Bytes>,
) -> Result<StoredObject, TargetError> {
validate_attempt_key(&request.key)?;
let body = body.into();
let computed = hash_bytes(&body);
check_expected(request.expected_content_sha256.as_ref(), &computed)?;
let size = body.len() as u64;
let write = self
.operator
.write_with(&request.key, body)
.user_metadata(sha256_metadata(&computed));
let metadata = write.await.map_err(map_opendal)?;
Ok(self.stored_from_write(&request.key, &computed, &metadata, size))
}
/// Enable bucket versioning when supported (fixture only).
pub async fn enable_bucket_versioning(&self) -> Result<bool, TargetError> {
self.multipart.enable_bucket_versioning().await
}
/// Delete every version under a prefix (fixture teardown only).
pub async fn purge_versions_with_prefix(&self, prefix: &str) -> Result<usize, TargetError> {
let versions = self.multipart.list_object_versions(prefix).await?;
let count = versions.len();
for version in versions {
self.multipart
.delete_object_version(&version.key, &version.version_id)
.await?;
}
Ok(count)
}
}
impl StorageAdapter for TargetStore {
type Error = TargetError;
type Request = CreateObjectRequest;
type StoredObject = StoredObject;
async fn create(&self, request: &Self::Request) -> Result<Self::StoredObject, Self::Error> {
self.put_create_only(request, bytes::Bytes::new()).await
}
}
fn sha256_metadata(digest: &ContentSha256) -> HashMap<String, String> {
HashMap::from([(
REMORA_SHA256_METADATA_KEY.to_owned(),
digest.as_str().to_owned(),
)])
}
fn observed_sha256(metadata: &Metadata) -> Result<Option<ContentSha256>, TargetError> {
let Some(map) = metadata.user_metadata() else {
return Ok(None);
};
map.get(REMORA_SHA256_METADATA_KEY)
.or_else(|| {
map.iter()
.find(|(key, _)| key.eq_ignore_ascii_case(REMORA_SHA256_METADATA_KEY))
.map(|(_, value)| value)
})
.map(ContentSha256::parse)
.transpose()
}
/// Finish helper re-exported for callers that hold a streaming body.
pub async fn finish_create_only(body: OpenDalTargetBody) -> Result<StoredWrite, TargetError> {
body.finish_object().await
}

19
crates/remora-target/tests/redaction.rs Normal file
View file

@ -0,0 +1,19 @@
use remora_target::TargetStoreConfig;
#[test]
fn target_store_config_debug_redacts_secret() {
let config = TargetStoreConfig::new_exclusive(
"garage",
"http://garage:3900",
"remora-lane-001",
"garage",
"synthetic-access-key",
"synthetic-secret-key-must-not-leak",
"",
)
.expect("config");
let debug = format!("{config:?}");
assert!(!debug.contains("synthetic-secret-key-must-not-leak"));
assert!(debug.contains("REDACTED"));
assert!(debug.contains("synthetic-access-key"));
}

436
crates/remora-target/tests/substrate.rs Normal file
View file

@ -0,0 +1,436 @@
use std::num::NonZeroUsize;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use remora_domain::OpaqueId;
use remora_stream::TargetBody;
use remora_target::{
CreateObjectRequest, ObjectIdentity, RESERVED_NAMESPACE_SEGMENT, TargetError, TargetStore,
TargetStoreConfig, attempt_object_key, hash_bytes, is_reserved_key, validate_attempt_key,
};
fn require_env() -> bool {
std::env::var("REMORA_TARGET_REQUIRE").is_ok()
}
fn store_from_env() -> Option<TargetStore> {
match TargetStoreConfig::from_env() {
Ok(config) => Some(TargetStore::connect(config).expect("connect target")),
Err(error) => {
if require_env() {
panic!("target env required when REMORA_TARGET_REQUIRE is set: {error}");
}
eprintln!("skipping target test: {error}");
None
}
}
}
fn unique_id(prefix: &str) -> OpaqueId {
let nanos = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("time")
.as_nanos();
let value = format!("{prefix}{nanos:x}");
OpaqueId::parse(&value).unwrap_or_else(|_| {
OpaqueId::parse(format!("{prefix}aaaaaaaa")).expect("fallback opaque id")
})
}
fn attempt_key(label: &str) -> String {
attempt_object_key(
&unique_id(&format!("att{label}")),
&unique_id(&format!("obj{label}")),
None,
)
}
fn create_request(key: impl Into<String>) -> CreateObjectRequest {
CreateObjectRequest::parse(key).expect("valid create-only request")
}
#[tokio::test]
async fn create_only_independently_hashes_and_rejects_wrong_expectation() {
let Some(store) = store_from_env() else {
return;
};
let body = b"immutable-member-v1".as_slice();
let digest = hash_bytes(body);
let key = attempt_key("csha");
assert!(is_reserved_key(&key));
assert!(validate_attempt_key(&key).is_ok());
assert!(validate_attempt_key("not-reserved/key").is_err());
let wrong = store
.put_create_only(
&create_request(key.clone()).with_expected_content_sha256(hash_bytes(b"other-payload")),
body,
)
.await;
assert!(
matches!(wrong, Err(TargetError::Integrity(_))),
"wrong expectation must fail: {wrong:?}"
);
let stored = store
.put_create_only(
&create_request(key.clone()).with_expected_content_sha256(digest.clone()),
body,
)
.await
.expect("create-only put");
assert_eq!(stored.identity.key, key);
assert_eq!(
stored.identity.target_store_id,
store.config().target_store_id
);
assert_eq!(stored.content_sha256, digest);
assert_eq!(
store.read_exact(&stored.identity).await.expect("read"),
body
);
let head = store.head_exact(&stored.identity).await.expect("head");
assert_eq!(head.content_sha256, Some(digest));
let conflict = store
.put_create_only(&create_request(key.clone()), b"replacement".as_slice())
.await;
assert!(
matches!(conflict, Err(TargetError::AlreadyExists)),
"expected create-only conflict, got {conflict:?}"
);
// Cross-store identity must fail closed.
let mut foreign = stored.identity.clone();
foreign.target_store_id = "other-store".into();
let cross = store.read_exact(&foreign).await;
assert!(
matches!(cross, Err(TargetError::Validation(_))),
"cross-store identity: {cross:?}"
);
}
#[tokio::test]
async fn streaming_create_only_computes_digest() {
let Some(store) = store_from_env() else {
return;
};
let payload = vec![0x5Au8; 256 * 1024];
let expected = hash_bytes(&payload);
let key = attempt_key("strm");
let mut body = store
.open_create_only(
&create_request(key.clone())
.with_expected_content_sha256(expected.clone())
.with_chunk_size(NonZeroUsize::new(64 * 1024).expect("non-zero chunk size")),
)
.await
.expect("open writer");
body.write_all(&payload).await.expect("write");
let stored = body.finish_object().await.expect("finish");
assert_eq!(stored.bytes_written, payload.len() as u64);
assert_eq!(stored.content_sha256, expected);
assert_eq!(stored.identity.key, key);
assert_eq!(
store.read_exact(&stored.identity).await.expect("read"),
payload
);
}
#[tokio::test]
async fn multipart_relist_complete_abort_and_create_only() {
let Some(store) = store_from_env() else {
return;
};
let replaced_part_one = vec![b'x'; 5 * 1024 * 1024];
let part_one = vec![b'a'; 5 * 1024 * 1024];
let part_two = vec![b'b'; 1024];
let combined = [part_one.as_slice(), part_two.as_slice()].concat();
let digest = hash_bytes(&combined);
let key = attempt_key("mpu");
let multipart_request =
create_request(key.clone()).with_expected_content_sha256(digest.clone());
let session = store
.begin_multipart(&multipart_request)
.await
.expect("begin");
let p2 = store
.upload_part(&session, 2, part_two.clone().into())
.await
.expect("part 2");
let replaced_p1 = store
.upload_part(&session, 1, replaced_part_one.into())
.await
.expect("original part 1");
let stale = store.list_parts(&session).await.expect("stale part list");
let p1 = store
.upload_part(&session, 1, part_one.clone().into())
.await
.expect("replacement part 1");
let durable_identity = session.identity().clone();
drop(session);
let (session, listed) = store
.resume_multipart(durable_identity.clone(), Some(digest.clone()))
.await
.expect("resume and re-list");
assert_eq!(listed.len(), 2);
assert_eq!(listed[0].etag, p1.etag);
assert_eq!(listed[1].etag, p2.etag);
assert_ne!(listed[0].etag, replaced_p1.etag);
let stale_completion = store.complete_multipart(session, &stale).await;
assert!(
matches!(stale_completion, Err(TargetError::Integrity(_))),
"completion must reject a replaced part: {stale_completion:?}"
);
let (session, listed) = store
.resume_multipart(durable_identity, Some(digest.clone()))
.await
.expect("resume after stale completion");
let completed = store
.complete_multipart(session, &listed)
.await
.expect("complete");
assert_eq!(completed.identity.key, key);
assert_eq!(completed.content_sha256, digest);
assert_eq!(
store
.head_exact(&completed.identity)
.await
.expect("head multipart")
.content_sha256,
None
);
let read = store
.read_exact(&completed.identity)
.await
.expect("read completed");
assert_eq!(read.as_ref(), combined.as_slice());
let occupied = store.begin_multipart(&multipart_request).await;
assert!(
matches!(occupied, Err(TargetError::AlreadyExists)),
"multipart begin on occupied key: {occupied:?}"
);
let abort_key = attempt_key("mpa");
let abort_request = create_request(abort_key);
let abort_session = store
.begin_multipart(&abort_request)
.await
.expect("begin abort");
store
.upload_part(&abort_session, 1, part_one.into())
.await
.expect("abort part");
store.abort_multipart(abort_session).await.expect("abort");
}
#[tokio::test]
async fn unicode_key_exact_delete() {
let Some(store) = store_from_env() else {
return;
};
let key = attempt_object_key(
&unique_id("attuni"),
&unique_id("objuni"),
Some("한글/é/member"),
);
let body = "정규화".as_bytes();
let stored = store
.put_create_only(&create_request(key.clone()), body)
.await
.expect("unicode put");
assert_eq!(stored.content_sha256, hash_bytes(body));
store
.delete_exact(&stored.identity)
.await
.expect("exact delete");
let missing = store.read_exact(&stored.identity).await;
assert!(
matches!(missing, Err(TargetError::NotFound)),
"expected not found after delete, got {missing:?}"
);
}
#[tokio::test]
async fn time_limited_exact_access_rejects_tamper_and_expiry() {
let Some(store) = store_from_env() else {
return;
};
let body = b"presign-body";
let key = attempt_key("prs");
let sibling_key = attempt_key("sib");
let stored = store
.put_create_only(&create_request(key.clone()), body.as_slice())
.await
.expect("put");
store
.put_create_only(&create_request(sibling_key.clone()), b"sibling".as_slice())
.await
.expect("sibling put");
let access = store
.presign_read(&stored.identity, Duration::from_secs(2))
.await
.expect("presign");
let client = reqwest::Client::new();
let ok = client
.get(&access.url)
.send()
.await
.expect("presign get")
.error_for_status()
.expect("presign status")
.bytes()
.await
.expect("presign body");
assert_eq!(ok.as_ref(), body);
let tampered = access.url.replacen(&key, &sibling_key, 1);
assert_ne!(tampered, access.url);
let tamper_status = client
.get(&tampered)
.send()
.await
.expect("tampered get")
.status()
.as_u16();
assert!(
matches!(tamper_status, 400 | 403),
"tampered URL must not authorize sibling, got {tamper_status}"
);
tokio::time::sleep(Duration::from_secs(3)).await;
let expired_status = client
.get(&access.url)
.send()
.await
.expect("expired get")
.status()
.as_u16();
assert!(
matches!(expired_status, 400 | 403),
"expired URL must fail, got {expired_status}"
);
}
#[tokio::test]
async fn version_addressed_and_stale_version_matrix() {
let Some(store) = store_from_env() else {
return;
};
if !store
.enable_bucket_versioning()
.await
.expect("enable versioning")
{
if require_env() {
let provider =
std::env::var("REMORA_TARGET_PROVIDER").unwrap_or_else(|_| "unknown".into());
if provider == "versitygw" {
panic!("versitygw must support bucket versioning for WP-08 matrix");
}
}
eprintln!(
"target provider {} cannot enable versioning; skipping version matrix",
std::env::var("REMORA_TARGET_PROVIDER").unwrap_or_else(|_| "unknown".into())
);
return;
}
let key = attempt_key("ver");
let first = store
.put_create_only(&create_request(key.clone()), b"one".as_slice())
.await
.expect("first version");
let first_version = first
.identity
.version_id
.clone()
.expect("version id after enable");
let second = store
.put_overwrite(&create_request(key.clone()), b"two".as_slice())
.await
.expect("second version");
let second_version = second
.identity
.version_id
.clone()
.expect("second version id");
assert_ne!(first_version, second_version);
let first_identity = ObjectIdentity {
target_store_id: store.config().target_store_id.clone(),
bucket: store.config().bucket.clone(),
key: key.clone(),
version_id: Some(first_version),
};
let second_identity = ObjectIdentity {
target_store_id: store.config().target_store_id.clone(),
bucket: store.config().bucket.clone(),
key: key.clone(),
version_id: Some(second_version),
};
assert_eq!(
store.read_exact(&first_identity).await.expect("v1"),
b"one".as_slice()
);
assert_eq!(
store.read_exact(&second_identity).await.expect("v2"),
b"two".as_slice()
);
let versioned_access = store
.presign_read(&first_identity, Duration::from_secs(60))
.await
.expect("versioned presign");
let client = reqwest::Client::new();
let body = client
.get(&versioned_access.url)
.send()
.await
.expect("versioned get")
.bytes()
.await
.expect("versioned body");
assert_eq!(body.as_ref(), b"one");
store
.delete_exact(&first_identity)
.await
.expect("delete v1");
assert!(matches!(
store.read_exact(&first_identity).await,
Err(TargetError::NotFound)
));
let stale_key = attempt_key("stv");
let keep = store
.put_create_only(&create_request(stale_key.clone()), b"keep".as_slice())
.await
.expect("stale put");
let version = keep.identity.version_id.clone().expect("stale version id");
let bogus = ObjectIdentity {
target_store_id: store.config().target_store_id.clone(),
bucket: store.config().bucket.clone(),
key: stale_key,
version_id: Some(format!("{version}-stale")),
};
let result = store.read_exact(&bogus).await;
assert!(
matches!(
result,
Err(TargetError::NotFound | TargetError::Internal(_))
),
"stale version must fail closed: {result:?}"
);
store
.purge_versions_with_prefix(RESERVED_NAMESPACE_SEGMENT)
.await
.expect("purge versions");
}