ms2k/remora
SHA256
Watch
0
Code Releases Activity

Add exact-identity target storage adapter

Browse source
This commit is contained in:
ms2k 2026-07-17 02:54:18 +09:00
commit 854e6e1c17
13 changed files with 3263 additions and 41 deletions
Download patch file Download diff file Expand all files Collapse all files

938
Cargo.lock generated
View file

File diff suppressed because it is too large Load diff

1
Cargo.toml
View file

@ -8,6 +8,7 @@ members = [
"crates/remora-durable", "crates/remora-durable",
"crates/remora-identity", "crates/remora-identity",
"crates/remora-stream", "crates/remora-stream",
"crates/remora-target",
"crates/remora-worker", "crates/remora-worker",
] ]
resolver = "3" resolver = "3"

40
crates/remora-target/Cargo.toml Normal file
View file

@ -0,0 +1,40 @@
[package]
name = "remora-target"
edition.workspace = true
publish = false
rust-version.workspace = true
version.workspace = true
[features]
default = []
# Fixture-only helpers (overwrite, version purge). Not for production callers.
test-utils = []
[dependencies]
aws-credential-types = "1.2.11"
aws-sigv4 = "1.3.7"
bytes = "1.11.1"
futures-util = "0.3.32"
http = "1.4.0"
opendal = { version = "0.58.0", default-features = false, features = [
"executors-tokio",
"http-transport-reqwest",
"http-transport-reqwest-rustls",
"services-s3",
] }
percent-encoding = "2.3.2"
quick-xml = { version = "0.38.4", features = ["serialize"] }
remora-domain.workspace = true
remora-stream = { path = "../remora-stream" }
reqwest = { version = "0.12.28", default-features = false, features = [
"rustls-tls-webpki-roots",
] }
serde = { workspace = true, features = ["derive"] }
sha2.workspace = true
thiserror.workspace = true
tokio = { workspace = true, features = ["macros", "sync", "time"] }
[dev-dependencies]
remora-target = { path = ".", features = ["test-utils"] }
sha2.workspace = true
tokio = { workspace = true, features = ["macros", "rt-multi-thread", "sync", "time"] }

92
crates/remora-target/src/body.rs Normal file
View file

@ -0,0 +1,92 @@
use opendal::Writer;
use remora_stream::{StreamError, TargetBody, TargetFinish};
use crate::digest::{ContentHasher, ContentSha256, check_expected};
use crate::error::{TargetError, map_opendal};
use crate::identity::ObjectIdentity;
/// Streaming create-only target body backed by an OpenDAL writer.
///
/// SHA-256 is accumulated over every written byte. The open writer cannot carry
/// final digest metadata (unknown until finish); the independently computed
/// digest is returned on finish for durable persistence.
pub struct OpenDalTargetBody {
writer: Writer,
hasher: ContentHasher,
bytes_written: u64,
identity: ObjectIdentity,
expected_sha256: Option<ContentSha256>,
}
impl OpenDalTargetBody {
pub(crate) fn new(
writer: Writer,
identity: ObjectIdentity,
expected_sha256: Option<ContentSha256>,
) -> Self {
Self {
writer,
hasher: ContentHasher::new(),
bytes_written: 0,
identity,
expected_sha256,
}
}
#[must_use]
pub fn identity(&self) -> &ObjectIdentity {
&self.identity
}
/// Finish the write and return identity plus independently computed digest.
pub async fn finish_object(mut self) -> Result<StoredWrite, TargetError> {
let metadata = self.writer.close().await.map_err(map_opendal)?;
let version_id = metadata.version().map(str::to_owned);
let content_sha256 = self.hasher.finalize();
check_expected(self.expected_sha256.as_ref(), &content_sha256)?;
let content_length = metadata.content_length();
let bytes_written = if content_length > 0 {
content_length
} else {
self.bytes_written
};
Ok(StoredWrite {
identity: self.identity.with_version(version_id),
bytes_written,
content_sha256,
})
}
}
impl TargetBody for OpenDalTargetBody {
async fn write_all(&mut self, data: &[u8]) -> Result<(), StreamError> {
if data.is_empty() {
return Ok(());
}
self.writer
.write(data.to_vec())
.await
.map_err(|error| map_opendal(error).into_stream_error())?;
self.hasher.update(data);
self.bytes_written = self
.bytes_written
.saturating_add(u64::try_from(data.len()).unwrap_or(u64::MAX));
Ok(())
}
async fn finish(self) -> Result<TargetFinish, StreamError> {
let stored = self.finish_object().await?;
Ok(TargetFinish {
bytes_written: stored.bytes_written,
})
}
}
/// Result of completing a create-only stream write.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct StoredWrite {
pub identity: ObjectIdentity,
pub bytes_written: u64,
/// Independently computed SHA-256 of written bytes.
pub content_sha256: ContentSha256,
}

142
crates/remora-target/src/config.rs Normal file
View file

@ -0,0 +1,142 @@
use std::fmt;
use crate::error::TargetError;
/// Connection settings for one S3-compatible target store.
#[derive(Clone, Eq, PartialEq)]
pub struct TargetStoreConfig {
/// Stable target-store identity used in authorization and object identities.
pub target_store_id: String,
pub endpoint: String,
pub bucket: String,
pub region: String,
pub access_key_id: String,
pub secret_access_key: String,
/// Optional operator root under the bucket. Empty means bucket root.
pub root: String,
}
impl fmt::Debug for TargetStoreConfig {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("TargetStoreConfig")
.field("target_store_id", &self.target_store_id)
.field("endpoint", &self.endpoint)
.field("bucket", &self.bucket)
.field("region", &self.region)
.field("access_key_id", &self.access_key_id)
.field("secret_access_key", &"REDACTED")
.field("root", &self.root)
.finish()
}
}
impl TargetStoreConfig {
/// Load fixture or deployment settings from environment variables.
///
/// Recognized variables: `REMORA_TARGET_STORE_ID` (or `REMORA_TARGET_PROVIDER`),
/// `REMORA_TARGET_ENDPOINT`, `REMORA_TARGET_BUCKET`, `REMORA_TARGET_REGION`,
/// `REMORA_TARGET_ACCESS_KEY`, `REMORA_TARGET_SECRET_KEY`,
/// `REMORA_TARGET_EXCLUSIVE_WRITER`, and optional `REMORA_TARGET_ROOT`.
pub fn from_env() -> Result<Self, TargetError> {
let target_store_id = std::env::var("REMORA_TARGET_STORE_ID")
.ok()
.filter(|value| !value.is_empty())
.or_else(|| {
std::env::var("REMORA_TARGET_PROVIDER")
.ok()
.filter(|value| !value.is_empty())
})
.ok_or_else(|| {
TargetError::validation(
"REMORA_TARGET_STORE_ID or REMORA_TARGET_PROVIDER is required",
)
})?;
let endpoint = required_env("REMORA_TARGET_ENDPOINT")?;
let bucket = required_env("REMORA_TARGET_BUCKET")?;
let region = std::env::var("REMORA_TARGET_REGION")
.ok()
.filter(|value| !value.is_empty())
.unwrap_or_else(|| "us-east-1".to_owned());
let access_key_id = required_env("REMORA_TARGET_ACCESS_KEY")?;
let secret_access_key = required_env("REMORA_TARGET_SECRET_KEY")?;
let root = std::env::var("REMORA_TARGET_ROOT")
.ok()
.filter(|value| !value.is_empty())
.unwrap_or_default();
let exclusive_writer = required_env("REMORA_TARGET_EXCLUSIVE_WRITER")?
.parse::<bool>()
.map_err(|_| {
TargetError::validation("REMORA_TARGET_EXCLUSIVE_WRITER must be true or false")
})?;
if !exclusive_writer {
return Err(TargetError::validation(
"target reserved namespace must have Remora as its exclusive writer",
));
}
Self::new_exclusive(
target_store_id,
endpoint,
bucket,
region,
access_key_id,
secret_access_key,
root,
)
}
pub fn new_exclusive(
target_store_id: impl Into<String>,
endpoint: impl Into<String>,
bucket: impl Into<String>,
region: impl Into<String>,
access_key_id: impl Into<String>,
secret_access_key: impl Into<String>,
root: impl Into<String>,
) -> Result<Self, TargetError> {
let target_store_id = target_store_id.into();
let endpoint = endpoint.into().trim_end_matches('/').to_owned();
let bucket = bucket.into();
let region = region.into();
let access_key_id = access_key_id.into();
let secret_access_key = secret_access_key.into();
let root = normalize_root(root.into());
if target_store_id.is_empty() {
return Err(TargetError::validation("target store id is required"));
}
if endpoint.is_empty() {
return Err(TargetError::validation("target endpoint is required"));
}
if bucket.is_empty() {
return Err(TargetError::validation("target bucket is required"));
}
if access_key_id.is_empty() || secret_access_key.is_empty() {
return Err(TargetError::validation("target credentials are required"));
}
Ok(Self {
target_store_id,
endpoint,
bucket,
region,
access_key_id,
secret_access_key,
root,
})
}
}
fn required_env(name: &str) -> Result<String, TargetError> {
std::env::var(name)
.ok()
.filter(|value| !value.is_empty())
.ok_or_else(|| TargetError::validation(format!("{name} is required")))
}
fn normalize_root(root: String) -> String {
let trimmed = root.trim_matches('/');
if trimmed.is_empty() {
String::new()
} else {
format!("/{trimmed}/")
}
}

117
crates/remora-target/src/digest.rs Normal file
View file

@ -0,0 +1,117 @@
use sha2::{Digest, Sha256};
use crate::error::TargetError;
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
pub struct ContentSha256(String);
impl ContentSha256 {
pub fn parse(value: impl AsRef<str>) -> Result<Self, TargetError> {
let value = value.as_ref().trim();
let hex = value
.strip_prefix("sha256:")
.or_else(|| value.strip_prefix("SHA256:"))
.unwrap_or(value);
if hex.len() != 64 || !hex.bytes().all(|byte| byte.is_ascii_hexdigit()) {
return Err(TargetError::validation("invalid sha256 digest"));
}
Ok(Self(format!("sha256:{}", hex.to_ascii_lowercase())))
}
#[must_use]
pub fn as_str(&self) -> &str {
&self.0
}
}
impl std::fmt::Display for ContentSha256 {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
self.0.fmt(formatter)
}
}
/// Format a raw SHA-256 digest as `sha256:` + lowercase hex.
#[must_use]
fn format_sha256(bytes: &[u8]) -> ContentSha256 {
let mut out = String::with_capacity(7 + bytes.len() * 2);
out.push_str("sha256:");
for byte in bytes {
use std::fmt::Write as _;
let _ = write!(out, "{byte:02x}");
}
ContentSha256(out)
}
/// Independently hash `data` and return the Remora digest string.
#[must_use]
pub fn hash_bytes(data: &[u8]) -> ContentSha256 {
format_sha256(Sha256::digest(data).as_ref())
}
/// Running SHA-256 over written bytes (never trusts a caller claim).
#[derive(Clone, Default)]
pub struct ContentHasher {
hasher: Sha256,
}
impl ContentHasher {
#[must_use]
pub fn new() -> Self {
Self::default()
}
pub fn update(&mut self, data: &[u8]) {
if data.is_empty() {
return;
}
self.hasher.update(data);
}
#[must_use]
pub fn finalize(self) -> ContentSha256 {
format_sha256(self.hasher.finalize().as_ref())
}
}
/// If the caller supplied an expected digest, require an exact match to the
/// independently computed value. Omitted expectation is allowed.
pub fn check_expected(
expected: Option<&ContentSha256>,
computed: &ContentSha256,
) -> Result<(), TargetError> {
let Some(expected) = expected else {
return Ok(());
};
if expected != computed {
return Err(TargetError::integrity(
"independently computed sha256 does not match caller expectation",
));
}
Ok(())
}
#[must_use]
pub fn sha256_equal(left: &str, right: &str) -> bool {
match (ContentSha256::parse(left), ContentSha256::parse(right)) {
(Ok(left), Ok(right)) => left == right,
_ => false,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn digest_parsing_canonicalizes_and_rejects_invalid_values() {
let upper = ContentSha256::parse(
"SHA256:BA7816BF8F01CFEA414140DE5DAE2223B00361A396177A9CB410FF61F20015AD",
)
.expect("valid digest");
assert_eq!(
upper.as_str(),
"sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"
);
assert!(ContentSha256::parse("sha256:not-a-digest").is_err());
}
}

186
crates/remora-target/src/error.rs Normal file
View file

@ -0,0 +1,186 @@
use remora_stream::{StreamError, StreamSide};
/// Target storage adapter errors.
///
/// Diagnostic text is for operator logs only after secret scrubbing. Public
/// projections go through [`remora_stream::classify_stream_error`].
#[derive(Debug, thiserror::Error)]
pub enum TargetError {
#[error("target authorization failure")]
Authorization,
#[error("object already exists under create-only identity")]
AlreadyExists,
#[error("object not found")]
NotFound,
#[error("integrity failure: {0}")]
Integrity(String),
#[error("capacity exhausted on target")]
Capacity,
#[error("transient target failure: {0}")]
Transient(String),
#[error("validation failure: {0}")]
Validation(String),
#[error("internal target error: {0}")]
Internal(String),
}
impl TargetError {
pub fn validation(detail: impl Into<String>) -> Self {
Self::Validation(detail.into())
}
pub fn integrity(detail: impl Into<String>) -> Self {
Self::Integrity(detail.into())
}
pub fn transient(detail: impl Into<String>) -> Self {
Self::Transient(detail.into())
}
pub fn internal(detail: impl Into<String>) -> Self {
Self::Internal(detail.into())
}
/// Map to a stream error for copy-loop and adapter I/O boundaries.
#[must_use]
pub fn into_stream_error(self) -> StreamError {
match self {
Self::Authorization => StreamError::Authorization {
side: StreamSide::Target,
},
Self::AlreadyExists => StreamError::integrity("create-only identity already exists"),
Self::NotFound => StreamError::internal("target object not found"),
Self::Integrity(detail) => StreamError::Integrity { detail },
Self::Capacity => StreamError::Capacity {
side: StreamSide::Target,
},
Self::Transient(detail) => StreamError::Transient {
side: StreamSide::Target,
detail,
},
Self::Validation(detail) => StreamError::internal(detail),
Self::Internal(detail) => StreamError::Internal { detail },
}
}
}
impl From<TargetError> for StreamError {
fn from(value: TargetError) -> Self {
value.into_stream_error()
}
}
/// Map OpenDAL errors without embedding raw exception bodies in public paths.
pub(crate) fn map_opendal(error: opendal::Error) -> TargetError {
use opendal::ErrorKind;
let detail = redact_sensitive(error.to_string());
match error.kind() {
ErrorKind::NotFound => TargetError::NotFound,
ErrorKind::PermissionDenied => TargetError::Authorization,
ErrorKind::AlreadyExists | ErrorKind::ConditionNotMatch => TargetError::AlreadyExists,
ErrorKind::RateLimited => TargetError::Capacity,
ErrorKind::Unsupported | ErrorKind::ConfigInvalid => TargetError::Validation(detail),
ErrorKind::Unexpected
| ErrorKind::IsADirectory
| ErrorKind::NotADirectory
| ErrorKind::IsSameFile
| ErrorKind::RangeNotSatisfied => {
if is_likely_transient(&detail) {
TargetError::Transient(detail)
} else {
TargetError::Internal(detail)
}
}
_ => TargetError::Internal(detail),
}
}
pub(crate) fn map_http_status(status: u16, body: &str) -> TargetError {
let detail = redact_sensitive(body);
match status {
401 | 403 => TargetError::Authorization,
404 => TargetError::NotFound,
409 | 412 => TargetError::AlreadyExists,
429 | 503 => TargetError::Capacity,
500 | 502 | 504 => TargetError::Transient(format!("http {status}: {detail}")),
_ if status >= 500 => TargetError::Transient(format!("http {status}: {detail}")),
_ => TargetError::Internal(format!("http {status}: {detail}")),
}
}
/// Scrub credential-bearing fragments from operator-facing diagnostic text.
pub(crate) fn redact_sensitive(input: impl AsRef<str>) -> String {
let mut out = input.as_ref().to_owned();
const NEEDLES: &[&str] = &[
"Authorization:",
"authorization:",
"AWSAccessKeyId=",
"X-Amz-Credential=",
"X-Amz-Signature=",
"X-Amz-Security-Token=",
"x-amz-security-token=",
"Credential=",
"Signature=",
"SecretAccessKey",
"secret_access_key",
"secret-access-key",
"Bearer ",
];
while let Some((idx, needle)) = NEEDLES
.iter()
.filter_map(|needle| out.find(needle).map(|idx| (idx, *needle)))
.min_by_key(|(idx, _)| *idx)
{
let value_start = idx + needle.len();
// Authorization headers are multi-token; scrub through end of line.
let value_end = if needle.eq_ignore_ascii_case("Authorization:") {
out[value_start..]
.find(['\r', '\n'])
.map(|offset| value_start + offset)
.unwrap_or(out.len())
} else {
out[value_start..]
.find(|character: char| {
character.is_whitespace()
|| matches!(character, '"' | '\'' | ',' | ';' | '&' | '<' | '>')
})
.map(|offset| value_start + offset)
.unwrap_or(out.len())
};
out.replace_range(idx..value_end, "[redacted]");
}
if out.len() > 512 {
out.truncate(512);
out.push('…');
}
out
}
fn is_likely_transient(detail: &str) -> bool {
let lower = detail.to_ascii_lowercase();
lower.contains("timeout")
|| lower.contains("temporar")
|| lower.contains("connection")
|| lower.contains("reset")
|| lower.contains("unavailable")
|| lower.contains("slow down")
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn redact_sensitive_scrubs_long_and_repeated_secrets() {
let long_sig = "a".repeat(80);
let input = format!(
"X-Amz-Signature={long_sig}&X-Amz-Credential=AKIAEXAMPLE/path Authorization: AWS4-HMAC-SHA256 Credential=AKIAEXAMPLE, SignedHeaders=host, Signature={long_sig} Bearer super-secret-token-value-here"
);
let redacted = redact_sensitive(input);
assert!(!redacted.contains("AKIAEXAMPLE"));
assert!(!redacted.contains(&long_sig));
assert!(!redacted.contains("super-secret-token-value-here"));
assert!(redacted.contains("[redacted]"));
}
}

137
crates/remora-target/src/identity.rs Normal file
View file

@ -0,0 +1,137 @@
use remora_domain::OpaqueId;
use crate::error::TargetError;
/// Deployment-reserved target namespace segment.
///
/// User target prefixes must not contain this segment. Attempt objects and
/// manifests are written only under this service namespace.
pub const RESERVED_NAMESPACE_SEGMENT: &str = "_remora";
/// User-metadata key for Remora-computed SHA-256 digests.
pub const REMORA_SHA256_METADATA_KEY: &str = "remora-sha256";
/// Exact object identity retained for reads, deletes, and time-limited access.
///
/// Bound to a target-store identity so identities cannot be applied across stores.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ObjectIdentity {
pub target_store_id: String,
pub bucket: String,
pub key: String,
pub version_id: Option<String>,
}
impl ObjectIdentity {
#[must_use]
pub fn with_version(mut self, version_id: Option<String>) -> Self {
self.version_id = version_id.filter(|value| !value.is_empty());
self
}
}
/// Exact multipart identity retained for destination resume and cleanup.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct MultipartIdentity {
pub target_store_id: String,
pub bucket: String,
pub key: String,
pub upload_id: String,
}
/// One completed multipart part as re-listed from the target store.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct CompletedPart {
pub part_number: u32,
pub etag: String,
pub size: Option<u64>,
}
/// Construct a create-only attempt object key under the reserved namespace.
///
/// Layout: `_remora/a/{attempt_id}/o/{object_id}[/{relative_path}]`
#[must_use]
pub fn attempt_object_key(
attempt_id: &OpaqueId,
object_id: &OpaqueId,
relative_path: Option<&str>,
) -> String {
let mut key = format!(
"{RESERVED_NAMESPACE_SEGMENT}/a/{}/o/{}",
attempt_id.as_str(),
object_id.as_str()
);
if let Some(path) = relative_path.filter(|value| !value.is_empty()) {
let trimmed = path.trim_matches('/');
if !trimmed.is_empty() {
key.push('/');
key.push_str(trimmed);
}
}
key
}
/// Construct a reserved-namespace key for a manifest descriptor object.
#[must_use]
pub fn attempt_manifest_key(attempt_id: &OpaqueId, manifest_id: &OpaqueId) -> String {
format!(
"{RESERVED_NAMESPACE_SEGMENT}/a/{}/m/{}",
attempt_id.as_str(),
manifest_id.as_str()
)
}
/// Return true when `key` is under the reserved service namespace.
#[must_use]
pub fn is_reserved_key(key: &str) -> bool {
let trimmed = key.trim_start_matches('/');
trimmed == RESERVED_NAMESPACE_SEGMENT
|| trimmed.starts_with(&format!("{RESERVED_NAMESPACE_SEGMENT}/"))
}
/// Validate a reserved attempt object or manifest identity layout.
pub fn validate_attempt_key(key: &str) -> Result<(), TargetError> {
if key.is_empty() || key.starts_with('/') || key.contains("//") {
return Err(TargetError::validation("invalid object key"));
}
if !is_reserved_key(key) {
return Err(TargetError::validation(
"object key must be under the reserved service namespace",
));
}
let segments = key.split('/').collect::<Vec<_>>();
if segments.len() < 5
|| segments[0] != RESERVED_NAMESPACE_SEGMENT
|| segments[1] != "a"
|| OpaqueId::parse(segments[2]).is_err()
|| OpaqueId::parse(segments[4]).is_err()
|| !matches!(segments[3], "o" | "m")
|| (segments[3] == "m" && segments.len() != 5)
{
return Err(TargetError::validation(
"object key must contain attempt and object identities",
));
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn attempt_keys_are_reserved_and_unique() {
let attempt = OpaqueId::parse("attempt01aaaaaaaa").unwrap();
let object = OpaqueId::parse("object01aaaaaaaaa").unwrap();
let key = attempt_object_key(&attempt, &object, Some("한글/é/member"));
assert!(is_reserved_key(&key));
assert!(validate_attempt_key(&key).is_ok());
assert!(key.contains("한글"));
assert_eq!(
attempt_manifest_key(&attempt, &object),
"_remora/a/attempt01aaaaaaaa/m/object01aaaaaaaaa"
);
assert!(validate_attempt_key("public/data").is_err());
assert!(validate_attempt_key("_remora/arbitrary").is_err());
}
}

27
crates/remora-target/src/lib.rs Normal file
View file

@ -0,0 +1,27 @@
//! Target storage adapter (WP-08).
//!
//! OpenDAL-backed create-only attempt objects, multipart recovery, exact
//! identity reads and deletes, independent SHA-256, version retention, and
//! exact time-limited access against S3-compatible target stores.
mod body;
mod config;
mod digest;
mod error;
mod identity;
mod multipart;
mod store;
pub use body::{OpenDalTargetBody, StoredWrite};
pub use config::TargetStoreConfig;
pub use digest::{ContentSha256, hash_bytes, sha256_equal};
pub use error::TargetError;
pub use identity::{
CompletedPart, MultipartIdentity, ObjectIdentity, REMORA_SHA256_METADATA_KEY,
RESERVED_NAMESPACE_SEGMENT, attempt_manifest_key, attempt_object_key, is_reserved_key,
validate_attempt_key,
};
pub use store::{
CreateObjectRequest, MultipartSession, ObjectMetadata, StoredObject, TargetStore,
TimeLimitedAccess, finish_create_only,
};

579
crates/remora-target/src/multipart.rs Normal file
View file

@ -0,0 +1,579 @@
use std::fmt;
use std::time::{Duration, SystemTime};
use aws_credential_types::Credentials;
use aws_sigv4::http_request::{
PayloadChecksumKind, SignableBody, SignableRequest, SigningSettings, UriPathNormalizationMode,
sign,
};
use aws_sigv4::sign::v4;
use bytes::Bytes;
use http::header::{CONTENT_LENGTH, CONTENT_TYPE, HeaderName, HeaderValue};
use http::{Method, Request};
use percent_encoding::{AsciiSet, CONTROLS, utf8_percent_encode};
use quick_xml::de::from_str;
use quick_xml::se::to_string;
use serde::{Deserialize, Serialize};
use crate::config::TargetStoreConfig;
use crate::error::{TargetError, map_http_status, redact_sensitive};
use crate::identity::{CompletedPart, MultipartIdentity, REMORA_SHA256_METADATA_KEY};
/// Encode path segments for path-style S3 URLs.
const PATH_SET: &AsciiSet = &CONTROLS
.add(b' ')
.add(b'"')
.add(b'#')
.add(b'%')
.add(b'&')
.add(b'+')
.add(b',')
.add(b'<')
.add(b'>')
.add(b'?')
.add(b'@')
.add(b'[')
.add(b'\\')
.add(b']')
.add(b'^')
.add(b'`')
.add(b'{')
.add(b'|')
.add(b'}');
/// One version or delete marker listed under a key/prefix.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ObjectVersion {
pub key: String,
pub version_id: String,
pub is_delete_marker: bool,
}
/// Explicit multipart control plane for destination recovery.
///
/// OpenDAL streams multipart writes but does not expose list-parts recovery.
/// This client issues path-style S3 multipart APIs with the same credentials
/// as the OpenDAL operator so exact upload IDs and re-listed parts are durable.
#[derive(Clone)]
pub struct MultipartClient {
config: TargetStoreConfig,
http: reqwest::Client,
}
impl fmt::Debug for MultipartClient {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("MultipartClient")
.field("config", &self.config)
.finish_non_exhaustive()
}
}
impl MultipartClient {
pub fn new(config: TargetStoreConfig) -> Result<Self, TargetError> {
let http = reqwest::Client::builder()
.timeout(Duration::from_secs(120))
.build()
.map_err(|error| TargetError::internal(redact_sensitive(error.to_string())))?;
Ok(Self { config, http })
}
/// Enable bucket versioning when the peer supports it.
///
/// Peers that return NotImplemented are reported as `false` without error.
pub async fn enable_bucket_versioning(&self) -> Result<bool, TargetError> {
let body = r#"<?xml version="1.0" encoding="UTF-8"?>
<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Status>Enabled</Status>
</VersioningConfiguration>"#;
let uri = format!(
"{}/{}/?versioning",
self.config.endpoint, self.config.bucket
);
let request = Request::builder()
.method(Method::PUT)
.uri(uri)
.header(CONTENT_TYPE, "application/xml")
.header(CONTENT_LENGTH, body.len().to_string())
.body(Bytes::from_static(body.as_bytes()))
.map_err(|error| TargetError::internal(error.to_string()))?;
let response = self.send(request).await?;
let status = response.status().as_u16();
if status / 100 == 2 {
return Ok(true);
}
let text = response.text().await.unwrap_or_default();
if status == 501
|| text.contains("NotImplemented")
|| text.contains("XNotImplemented")
|| text.contains("NotSupported")
{
return Ok(false);
}
Err(map_http_status(status, &text))
}
/// Start a multipart upload. `content_sha256` is optional S3 user metadata
/// for an independently computed digest only.
pub async fn initiate(
&self,
key: &str,
content_sha256: Option<&str>,
) -> Result<MultipartIdentity, TargetError> {
let path = object_path(&self.config, key);
let mut builder = Request::builder()
.method(Method::POST)
.uri(format!("{}{}?uploads", self.config.endpoint, path));
if let Some(digest) = content_sha256 {
let header = format!("x-amz-meta-{REMORA_SHA256_METADATA_KEY}");
builder = builder.header(
HeaderName::from_bytes(header.as_bytes())
.map_err(|_| TargetError::internal("invalid metadata header name"))?,
HeaderValue::from_str(digest)
.map_err(|_| TargetError::validation("invalid sha256 metadata"))?,
);
}
let request = builder
.header(CONTENT_LENGTH, "0")
.body(Bytes::new())
.map_err(|error| TargetError::internal(error.to_string()))?;
let response = self.send(request).await?;
let status = response.status().as_u16();
let body = response
.text()
.await
.map_err(|error| TargetError::transient(redact_sensitive(error.to_string())))?;
if status / 100 != 2 {
return Err(map_http_status(status, &body));
}
let parsed: InitiateMultipartUploadResult = from_str(&body)
.map_err(|error| TargetError::internal(redact_sensitive(error.to_string())))?;
if parsed.upload_id.is_empty() {
return Err(TargetError::internal(
"multipart initiate missing upload id",
));
}
Ok(MultipartIdentity {
target_store_id: self.config.target_store_id.clone(),
bucket: self.config.bucket.clone(),
key: key.to_owned(),
upload_id: parsed.upload_id,
})
}
pub async fn upload_part(
&self,
identity: &MultipartIdentity,
part_number: u32,
body: bytes::Bytes,
) -> Result<CompletedPart, TargetError> {
if part_number == 0 {
return Err(TargetError::validation("part number must be >= 1"));
}
let path = object_path(&self.config, &identity.key);
let uri = format!(
"{}{}?partNumber={part_number}&uploadId={}",
self.config.endpoint,
path,
encode_query(&identity.upload_id)
);
let len = body.len();
let request = Request::builder()
.method(Method::PUT)
.uri(uri)
.header(CONTENT_LENGTH, len.to_string())
.body(body)
.map_err(|error| TargetError::internal(error.to_string()))?;
let response = self.send(request).await?;
let status = response.status().as_u16();
if status / 100 != 2 {
let body = response.text().await.unwrap_or_default();
return Err(map_http_status(status, &body));
}
let etag = response
.headers()
.get("etag")
.and_then(|value| value.to_str().ok())
.map(|value| value.trim_matches('"').to_owned())
.ok_or_else(|| TargetError::internal("multipart part missing etag"))?;
Ok(CompletedPart {
part_number,
etag,
size: Some(len as u64),
})
}
pub async fn list_parts(
&self,
identity: &MultipartIdentity,
) -> Result<Vec<CompletedPart>, TargetError> {
let mut parts = Vec::new();
let mut part_number_marker: Option<u32> = None;
loop {
let path = object_path(&self.config, &identity.key);
let mut uri = format!(
"{}{}?uploadId={}",
self.config.endpoint,
path,
encode_query(&identity.upload_id)
);
if let Some(marker) = part_number_marker {
uri.push_str(&format!("&part-number-marker={marker}"));
}
let request = Request::builder()
.method(Method::GET)
.uri(uri)
.body(Bytes::new())
.map_err(|error| TargetError::internal(error.to_string()))?;
let response = self.send(request).await?;
let status = response.status().as_u16();
let body = response
.text()
.await
.map_err(|error| TargetError::transient(redact_sensitive(error.to_string())))?;
if status / 100 != 2 {
return Err(map_http_status(status, &body));
}
let parsed: ListPartsResult = from_str(&body)
.map_err(|error| TargetError::internal(redact_sensitive(error.to_string())))?;
for part in parsed.parts {
parts.push(CompletedPart {
part_number: part.part_number,
etag: part.etag.trim_matches('"').to_owned(),
size: Some(part.size),
});
}
if parsed.is_truncated {
part_number_marker = parsed
.next_part_number_marker
.or_else(|| parts.last().map(|part| part.part_number));
if part_number_marker.is_none() {
break;
}
} else {
break;
}
}
parts.sort_by_key(|part| part.part_number);
Ok(parts)
}
pub async fn complete(
&self,
identity: &MultipartIdentity,
parts: &[CompletedPart],
if_not_exists: bool,
) -> Result<Option<String>, TargetError> {
if parts.is_empty() {
return Err(TargetError::validation(
"multipart complete requires at least one part",
));
}
let payload = CompleteMultipartUpload {
parts: parts
.iter()
.map(|part| CompletePart {
part_number: part.part_number,
etag: format!("\"{}\"", part.etag.trim_matches('"')),
})
.collect(),
};
let xml = to_string(&payload)
.map_err(|error| TargetError::internal(redact_sensitive(error.to_string())))?;
let path = object_path(&self.config, &identity.key);
let uri = format!(
"{}{}?uploadId={}",
self.config.endpoint,
path,
encode_query(&identity.upload_id)
);
let mut builder = Request::builder()
.method(Method::POST)
.uri(uri)
.header(CONTENT_TYPE, "application/xml")
.header(CONTENT_LENGTH, xml.len().to_string());
if if_not_exists {
builder = builder.header("If-None-Match", "*");
}
let request = builder
.body(Bytes::from(xml))
.map_err(|error| TargetError::internal(error.to_string()))?;
let response = self.send(request).await?;
let status = response.status().as_u16();
let version_id = response
.headers()
.get("x-amz-version-id")
.and_then(|value| value.to_str().ok())
.map(str::to_owned);
let body = response
.text()
.await
.map_err(|error| TargetError::transient(redact_sensitive(error.to_string())))?;
if status / 100 != 2 {
return Err(map_http_status(status, &body));
}
Ok(version_id.filter(|value| !value.is_empty()))
}
pub async fn abort(&self, identity: &MultipartIdentity) -> Result<(), TargetError> {
let path = object_path(&self.config, &identity.key);
let uri = format!(
"{}{}?uploadId={}",
self.config.endpoint,
path,
encode_query(&identity.upload_id)
);
let request = Request::builder()
.method(Method::DELETE)
.uri(uri)
.body(Bytes::new())
.map_err(|error| TargetError::internal(error.to_string()))?;
let response = self.send(request).await?;
let status = response.status().as_u16();
if status == 404 {
return Ok(());
}
if status / 100 != 2 {
let body = response.text().await.unwrap_or_default();
return Err(map_http_status(status, &body));
}
Ok(())
}
/// List object versions and delete markers under an optional key prefix.
pub async fn list_object_versions(
&self,
prefix: &str,
) -> Result<Vec<ObjectVersion>, TargetError> {
let mut versions = Vec::new();
let mut key_marker = String::new();
let mut version_id_marker = String::new();
loop {
let mut uri = format!("{}/{}/?versions", self.config.endpoint, self.config.bucket);
if !prefix.is_empty() {
uri.push_str(&format!("&prefix={}", encode_query(prefix)));
}
if !key_marker.is_empty() {
uri.push_str(&format!("&key-marker={}", encode_query(&key_marker)));
}
if !version_id_marker.is_empty() {
uri.push_str(&format!(
"&version-id-marker={}",
encode_query(&version_id_marker)
));
}
let request = Request::builder()
.method(Method::GET)
.uri(uri)
.body(Bytes::new())
.map_err(|error| TargetError::internal(error.to_string()))?;
let response = self.send(request).await?;
let status = response.status().as_u16();
let body = response
.text()
.await
.map_err(|error| TargetError::transient(redact_sensitive(error.to_string())))?;
if status / 100 != 2 {
return Err(map_http_status(status, &body));
}
let parsed: ListVersionsResult = from_str(&body)
.map_err(|error| TargetError::internal(redact_sensitive(error.to_string())))?;
for version in parsed.versions {
if !version.version_id.is_empty() {
versions.push(ObjectVersion {
key: version.key,
version_id: version.version_id,
is_delete_marker: false,
});
}
}
for marker in parsed.delete_markers {
if !marker.version_id.is_empty() {
versions.push(ObjectVersion {
key: marker.key,
version_id: marker.version_id,
is_delete_marker: true,
});
}
}
if parsed.is_truncated {
key_marker = parsed.next_key_marker.unwrap_or_default();
version_id_marker = parsed.next_version_id_marker.unwrap_or_default();
if key_marker.is_empty() && version_id_marker.is_empty() {
break;
}
} else {
break;
}
}
Ok(versions)
}
/// Delete one exact object version or delete marker.
pub async fn delete_object_version(
&self,
key: &str,
version_id: &str,
) -> Result<(), TargetError> {
let path = object_path(&self.config, key);
let uri = format!(
"{}{}?versionId={}",
self.config.endpoint,
path,
encode_query(version_id)
);
let request = Request::builder()
.method(Method::DELETE)
.uri(uri)
.body(Bytes::new())
.map_err(|error| TargetError::internal(error.to_string()))?;
let response = self.send(request).await?;
let status = response.status().as_u16();
if status == 404 || status / 100 == 2 {
return Ok(());
}
let body = response.text().await.unwrap_or_default();
Err(map_http_status(status, &body))
}
async fn send(&self, request: Request<Bytes>) -> Result<reqwest::Response, TargetError> {
let (parts, body) = sign_request(&self.config, request)?.into_parts();
self.http
.request(parts.method, parts.uri.to_string())
.headers(parts.headers)
.body(body)
.send()
.await
.map_err(|error| TargetError::transient(redact_sensitive(error.to_string())))
}
}
fn object_path(config: &TargetStoreConfig, key: &str) -> String {
let key = key.trim_start_matches('/');
let rooted = if config.root.is_empty() {
key.to_owned()
} else {
let root = config.root.trim_matches('/');
if root.is_empty() {
key.to_owned()
} else {
format!("{root}/{key}")
}
};
let encoded = rooted
.split('/')
.map(|segment| utf8_percent_encode(segment, PATH_SET).to_string())
.collect::<Vec<_>>()
.join("/");
format!("/{}/{}", config.bucket, encoded)
}
fn encode_query(value: &str) -> String {
utf8_percent_encode(value, PATH_SET).to_string()
}
fn sign_request(
config: &TargetStoreConfig,
mut request: Request<Bytes>,
) -> Result<Request<Bytes>, TargetError> {
let identity = Credentials::new(
&config.access_key_id,
&config.secret_access_key,
None,
None,
"remora-target",
)
.into();
let mut settings = SigningSettings::default();
// S3-compatible stores (including Garage) require x-amz-content-sha256 and
// reject URI path normalization in the canonical request.
settings.payload_checksum_kind = PayloadChecksumKind::XAmzSha256;
settings.uri_path_normalization_mode = UriPathNormalizationMode::Disabled;
let signing_params = v4::SigningParams::builder()
.identity(&identity)
.region(&config.region)
.name("s3")
.time(SystemTime::now())
.settings(settings)
.build()
.map_err(|error| TargetError::internal(redact_sensitive(error.to_string())))?
.into();
let signable = SignableRequest::new(
request.method().as_str(),
request.uri().to_string(),
request
.headers()
.iter()
.filter_map(|(name, value)| value.to_str().ok().map(|value| (name.as_str(), value))),
SignableBody::Bytes(request.body().as_ref()),
)
.map_err(|error| TargetError::internal(redact_sensitive(error.to_string())))?;
let (instructions, _signature) = sign(signable, &signing_params)
.map_err(|error| TargetError::internal(redact_sensitive(error.to_string())))?
.into_parts();
instructions.apply_to_request_http1x(&mut request);
Ok(request)
}
#[derive(Debug, Deserialize)]
struct InitiateMultipartUploadResult {
#[serde(rename = "UploadId", default)]
upload_id: String,
}
#[derive(Debug, Deserialize)]
struct ListPartsResult {
#[serde(rename = "Part", default)]
parts: Vec<ListPart>,
#[serde(rename = "IsTruncated", default)]
is_truncated: bool,
#[serde(rename = "NextPartNumberMarker", default)]
next_part_number_marker: Option<u32>,
}
#[derive(Debug, Deserialize)]
struct ListPart {
#[serde(rename = "PartNumber")]
part_number: u32,
#[serde(rename = "ETag")]
etag: String,
#[serde(rename = "Size", default)]
size: u64,
}
#[derive(Debug, Serialize)]
#[serde(rename = "CompleteMultipartUpload")]
struct CompleteMultipartUpload {
#[serde(rename = "Part")]
parts: Vec<CompletePart>,
}
#[derive(Debug, Serialize)]
struct CompletePart {
#[serde(rename = "PartNumber")]
part_number: u32,
#[serde(rename = "ETag")]
etag: String,
}
#[derive(Debug, Deserialize)]
struct ListVersionsResult {
#[serde(rename = "Version", default)]
versions: Vec<ListedVersion>,
#[serde(rename = "DeleteMarker", default)]
delete_markers: Vec<ListedVersion>,
#[serde(rename = "IsTruncated", default)]
is_truncated: bool,
#[serde(rename = "NextKeyMarker", default)]
next_key_marker: Option<String>,
#[serde(rename = "NextVersionIdMarker", default)]
next_version_id_marker: Option<String>,
}
#[derive(Debug, Deserialize)]
struct ListedVersion {
#[serde(rename = "Key", default)]
key: String,
#[serde(rename = "VersionId", default)]
version_id: String,
}

548
crates/remora-target/src/store.rs Normal file
View file

@ -0,0 +1,548 @@
use std::collections::HashMap;
use std::num::NonZeroUsize;
use std::time::Duration;
use futures_util::TryStreamExt;
use opendal::services::S3;
use opendal::{Metadata, Operator, Writer};
use remora_domain::OpaqueId;
use remora_domain::ports::StorageAdapter;
use crate::body::{OpenDalTargetBody, StoredWrite};
use crate::config::TargetStoreConfig;
use crate::digest::{ContentHasher, ContentSha256, check_expected, hash_bytes};
use crate::error::{TargetError, map_opendal, redact_sensitive};
use crate::identity::{
CompletedPart, MultipartIdentity, ObjectIdentity, REMORA_SHA256_METADATA_KEY,
attempt_manifest_key, attempt_object_key, validate_attempt_key,
};
use crate::multipart::MultipartClient;
/// Request for a create-only object write under a unique reserved key.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct CreateObjectRequest {
key: String,
expected_content_sha256: Option<ContentSha256>,
chunk_size: Option<NonZeroUsize>,
}
impl CreateObjectRequest {
pub fn parse(key: impl Into<String>) -> Result<Self, TargetError> {
let key = key.into();
validate_attempt_key(&key)?;
Ok(Self {
key,
expected_content_sha256: None,
chunk_size: None,
})
}
#[must_use]
pub fn for_attempt_object(
attempt_id: &OpaqueId,
object_id: &OpaqueId,
relative_path: Option<&str>,
) -> Self {
Self {
key: attempt_object_key(attempt_id, object_id, relative_path),
expected_content_sha256: None,
chunk_size: None,
}
}
#[must_use]
pub fn for_attempt_manifest(attempt_id: &OpaqueId, manifest_id: &OpaqueId) -> Self {
Self {
key: attempt_manifest_key(attempt_id, manifest_id),
expected_content_sha256: None,
chunk_size: None,
}
}
#[must_use]
pub fn with_expected_content_sha256(mut self, expected: ContentSha256) -> Self {
self.expected_content_sha256 = Some(expected);
self
}
#[must_use]
pub fn with_chunk_size(mut self, chunk_size: NonZeroUsize) -> Self {
self.chunk_size = Some(chunk_size);
self
}
#[must_use]
pub fn key(&self) -> &str {
&self.key
}
}
/// Stored object returned after a create-only write completes.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct StoredObject {
pub identity: ObjectIdentity,
/// Independently computed SHA-256 of the written bytes.
pub content_sha256: ContentSha256,
pub size: u64,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ObjectMetadata {
pub identity: ObjectIdentity,
pub content_sha256: Option<ContentSha256>,
pub size: u64,
}
/// Time-limited exact-object access URL.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct TimeLimitedAccess {
pub method: String,
pub url: String,
pub expires_in: Duration,
}
/// Recoverable multipart upload bound to one exact target identity.
pub struct MultipartSession {
identity: MultipartIdentity,
expected_sha256: Option<ContentSha256>,
}
impl std::fmt::Debug for MultipartSession {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter
.debug_struct("MultipartSession")
.field("identity", &self.identity)
.field("expected_sha256", &self.expected_sha256)
.finish_non_exhaustive()
}
}
impl MultipartSession {
#[must_use]
pub fn identity(&self) -> &MultipartIdentity {
&self.identity
}
}
/// OpenDAL-backed target storage adapter for attempt-scoped object writes.
#[derive(Clone)]
pub struct TargetStore {
config: TargetStoreConfig,
operator: Operator,
multipart: MultipartClient,
}
impl TargetStore {
pub fn connect(config: TargetStoreConfig) -> Result<Self, TargetError> {
opendal::install_default();
let mut builder = S3::default()
.bucket(&config.bucket)
.endpoint(&config.endpoint)
.region(&config.region)
.access_key_id(&config.access_key_id)
.secret_access_key(&config.secret_access_key)
.disable_config_load()
.disable_ec2_metadata();
if !config.root.is_empty() {
builder = builder.root(&config.root);
}
let operator = Operator::new(builder)
.map_err(|error| TargetError::validation(redact_sensitive(error.to_string())))?;
let multipart = MultipartClient::new(config.clone())?;
Ok(Self {
config,
operator,
multipart,
})
}
pub fn connect_from_env() -> Result<Self, TargetError> {
Self::connect(TargetStoreConfig::from_env()?)
}
#[must_use]
pub fn config(&self) -> &TargetStoreConfig {
&self.config
}
/// Open a create-only streaming writer for one attempt object.
pub async fn open_create_only(
&self,
request: &CreateObjectRequest,
) -> Result<OpenDalTargetBody, TargetError> {
validate_attempt_key(&request.key)?;
self.require_absent(&request.key).await?;
let writer = self.open_writer(request).await?;
Ok(OpenDalTargetBody::new(
writer,
self.new_object_identity(&request.key, None),
request.expected_content_sha256.clone(),
))
}
/// Create-only put of a complete buffer.
///
/// Independently hashes `body`, persists that digest as user metadata, and
/// retains the version ID from the write response when present.
pub async fn put_create_only(
&self,
request: &CreateObjectRequest,
body: impl Into<bytes::Bytes>,
) -> Result<StoredObject, TargetError> {
validate_attempt_key(&request.key)?;
self.require_absent(&request.key).await?;
let body = body.into();
let computed = hash_bytes(&body);
check_expected(request.expected_content_sha256.as_ref(), &computed)?;
let size = body.len() as u64;
let write = self
.operator
.write_with(&request.key, body)
.if_not_exists(true)
.user_metadata(sha256_metadata(&computed));
let metadata = write.await.map_err(map_opendal)?;
Ok(self.stored_from_write(&request.key, &computed, &metadata, size))
}
/// Exact identity read. Version is required when present on the identity.
pub async fn read_exact(&self, identity: &ObjectIdentity) -> Result<bytes::Bytes, TargetError> {
self.ensure_object_identity(identity)?;
let mut read = self.operator.read_with(&identity.key);
if let Some(version) = identity.version_id.as_deref() {
read = read.version(version);
}
let buffer = read.await.map_err(map_opendal)?;
Ok(buffer.to_bytes())
}
/// Exact identity delete. Missing objects are success for cleanup.
pub async fn delete_exact(&self, identity: &ObjectIdentity) -> Result<(), TargetError> {
self.ensure_object_identity(identity)?;
let mut delete = self.operator.delete_with(&identity.key);
if let Some(version) = identity.version_id.as_deref() {
delete = delete.version(version);
}
match delete.await {
Ok(()) => Ok(()),
Err(error) if matches!(error.kind(), opendal::ErrorKind::NotFound) => Ok(()),
Err(error) => Err(map_opendal(error)),
}
}
/// Head an exact identity, returning size and retained SHA-256 metadata.
pub async fn head_exact(
&self,
identity: &ObjectIdentity,
) -> Result<ObjectMetadata, TargetError> {
self.ensure_object_identity(identity)?;
let mut stat = self.operator.stat_with(&identity.key);
if let Some(version) = identity.version_id.as_deref() {
stat = stat.version(version);
}
let metadata = stat.await.map_err(map_opendal)?;
// Prefer exact version retained on the identity when the head omits it.
let version_id = metadata
.version()
.map(str::to_owned)
.or_else(|| identity.version_id.clone());
let content_sha256 = observed_sha256(&metadata)?;
Ok(ObjectMetadata {
identity: ObjectIdentity {
target_store_id: self.config.target_store_id.clone(),
bucket: self.config.bucket.clone(),
key: identity.key.clone(),
version_id,
},
content_sha256,
size: metadata.content_length(),
})
}
/// Issue a time-limited URL bound to one exact object identity.
pub async fn presign_read(
&self,
identity: &ObjectIdentity,
expires_in: Duration,
) -> Result<TimeLimitedAccess, TargetError> {
self.ensure_object_identity(identity)?;
if expires_in.is_zero() {
return Err(TargetError::validation(
"time-limited access expiry must be positive",
));
}
let mut read_options = opendal::options::ReadOptions::default();
if let Some(version) = identity.version_id.as_deref() {
read_options.version = Some(version.to_owned());
}
let signed = self
.operator
.presign_read_options(&identity.key, expires_in, read_options)
.await
.map_err(map_opendal)?;
Ok(TimeLimitedAccess {
method: signed.method().as_str().to_owned(),
url: signed.uri().to_string(),
expires_in,
})
}
/// Begin a create-only multipart upload under a reserved key.
///
/// The completed exact object is independently stream-hashed after S3
/// accepts the freshly re-listed part set.
pub async fn begin_multipart(
&self,
request: &CreateObjectRequest,
) -> Result<MultipartSession, TargetError> {
self.require_absent(&request.key).await?;
let identity = self.multipart.initiate(&request.key, None).await?;
Ok(MultipartSession {
identity,
expected_sha256: request.expected_content_sha256.clone(),
})
}
/// Reconstruct a multipart session from its durable identity after loss.
pub async fn resume_multipart(
&self,
identity: MultipartIdentity,
expected_content_sha256: Option<ContentSha256>,
) -> Result<(MultipartSession, Vec<CompletedPart>), TargetError> {
self.ensure_multipart_identity(&identity)?;
validate_attempt_key(&identity.key)?;
let parts = self.multipart.list_parts(&identity).await?;
Ok((
MultipartSession {
identity,
expected_sha256: expected_content_sha256,
},
parts,
))
}
pub async fn upload_part(
&self,
session: &MultipartSession,
part_number: u32,
body: bytes::Bytes,
) -> Result<CompletedPart, TargetError> {
self.ensure_multipart_identity(&session.identity)?;
self.multipart
.upload_part(&session.identity, part_number, body)
.await
}
pub async fn list_parts(
&self,
session: &MultipartSession,
) -> Result<Vec<CompletedPart>, TargetError> {
self.ensure_multipart_identity(&session.identity)?;
self.multipart.list_parts(&session.identity).await
}
pub async fn complete_multipart(
&self,
session: MultipartSession,
parts: &[CompletedPart],
) -> Result<StoredObject, TargetError> {
self.ensure_multipart_identity(&session.identity)?;
self.require_absent(&session.identity.key).await?;
let relisted = self.multipart.list_parts(&session.identity).await?;
if relisted != parts {
return Err(TargetError::integrity(
"multipart completion parts do not match exact target re-list",
));
}
let version_id = self
.multipart
.complete(&session.identity, parts, true)
.await?;
let object = self.new_object_identity(&session.identity.key, version_id);
let (computed, size) = self.hash_exact(&object).await?;
if let Err(error) = check_expected(session.expected_sha256.as_ref(), &computed) {
let _ = self.delete_exact(&object).await;
return Err(error);
}
Ok(StoredObject {
identity: object,
content_sha256: computed,
size,
})
}
pub async fn abort_multipart(&self, session: MultipartSession) -> Result<(), TargetError> {
self.ensure_multipart_identity(&session.identity)?;
self.multipart.abort(&session.identity).await
}
async fn open_writer(&self, request: &CreateObjectRequest) -> Result<Writer, TargetError> {
let chunk = request
.chunk_size
.map(NonZeroUsize::get)
.unwrap_or(5 * 1024 * 1024);
self.operator
.writer_with(&request.key)
.if_not_exists(true)
.chunk(chunk)
.await
.map_err(map_opendal)
}
async fn require_absent(&self, key: &str) -> Result<(), TargetError> {
match self.operator.stat(key).await {
Ok(_) => Err(TargetError::AlreadyExists),
Err(error) if matches!(error.kind(), opendal::ErrorKind::NotFound) => Ok(()),
Err(error) => Err(map_opendal(error)),
}
}
async fn hash_exact(
&self,
identity: &ObjectIdentity,
) -> Result<(ContentSha256, u64), TargetError> {
self.ensure_object_identity(identity)?;
let mut reader = self.operator.reader_with(&identity.key).chunk(1024 * 1024);
if let Some(version) = identity.version_id.as_deref() {
reader = reader.version(version);
}
let reader = reader.await.map_err(map_opendal)?;
let mut stream = reader.into_bytes_stream(..).await.map_err(map_opendal)?;
let mut hasher = ContentHasher::new();
let mut size = 0u64;
while let Some(chunk) = stream
.try_next()
.await
.map_err(|error| TargetError::transient(redact_sensitive(error.to_string())))?
{
hasher.update(&chunk);
size = size.saturating_add(u64::try_from(chunk.len()).unwrap_or(u64::MAX));
}
Ok((hasher.finalize(), size))
}
fn stored_from_write(
&self,
key: &str,
content_sha256: &ContentSha256,
metadata: &Metadata,
written_size: u64,
) -> StoredObject {
StoredObject {
identity: self.new_object_identity(key, metadata.version().map(str::to_owned)),
content_sha256: content_sha256.clone(),
size: metadata.content_length().max(written_size),
}
}
fn new_object_identity(&self, key: &str, version_id: Option<String>) -> ObjectIdentity {
ObjectIdentity {
target_store_id: self.config.target_store_id.clone(),
bucket: self.config.bucket.clone(),
key: key.to_owned(),
version_id: version_id.filter(|value| !value.is_empty()),
}
}
fn ensure_object_identity(&self, identity: &ObjectIdentity) -> Result<(), TargetError> {
if identity.target_store_id != self.config.target_store_id {
return Err(TargetError::validation(
"object identity target store does not match configured store",
));
}
if identity.bucket != self.config.bucket {
return Err(TargetError::validation(
"object identity bucket does not match configured target",
));
}
Ok(())
}
fn ensure_multipart_identity(&self, identity: &MultipartIdentity) -> Result<(), TargetError> {
if identity.target_store_id != self.config.target_store_id {
return Err(TargetError::validation(
"multipart identity target store does not match configured store",
));
}
if identity.bucket != self.config.bucket {
return Err(TargetError::validation(
"multipart identity bucket does not match configured target",
));
}
Ok(())
}
}
#[cfg(feature = "test-utils")]
impl TargetStore {
/// Overwrite put without create-only guards (fixture version matrix only).
pub async fn put_overwrite(
&self,
request: &CreateObjectRequest,
body: impl Into<bytes::Bytes>,
) -> Result<StoredObject, TargetError> {
validate_attempt_key(&request.key)?;
let body = body.into();
let computed = hash_bytes(&body);
check_expected(request.expected_content_sha256.as_ref(), &computed)?;
let size = body.len() as u64;
let write = self
.operator
.write_with(&request.key, body)
.user_metadata(sha256_metadata(&computed));
let metadata = write.await.map_err(map_opendal)?;
Ok(self.stored_from_write(&request.key, &computed, &metadata, size))
}
/// Enable bucket versioning when supported (fixture only).
pub async fn enable_bucket_versioning(&self) -> Result<bool, TargetError> {
self.multipart.enable_bucket_versioning().await
}
/// Delete every version under a prefix (fixture teardown only).
pub async fn purge_versions_with_prefix(&self, prefix: &str) -> Result<usize, TargetError> {
let versions = self.multipart.list_object_versions(prefix).await?;
let count = versions.len();
for version in versions {
self.multipart
.delete_object_version(&version.key, &version.version_id)
.await?;
}
Ok(count)
}
}
impl StorageAdapter for TargetStore {
type Error = TargetError;
type Request = CreateObjectRequest;
type StoredObject = StoredObject;
async fn create(&self, request: &Self::Request) -> Result<Self::StoredObject, Self::Error> {
self.put_create_only(request, bytes::Bytes::new()).await
}
}
fn sha256_metadata(digest: &ContentSha256) -> HashMap<String, String> {
HashMap::from([(
REMORA_SHA256_METADATA_KEY.to_owned(),
digest.as_str().to_owned(),
)])
}
fn observed_sha256(metadata: &Metadata) -> Result<Option<ContentSha256>, TargetError> {
let Some(map) = metadata.user_metadata() else {
return Ok(None);
};
map.get(REMORA_SHA256_METADATA_KEY)
.or_else(|| {
map.iter()
.find(|(key, _)| key.eq_ignore_ascii_case(REMORA_SHA256_METADATA_KEY))
.map(|(_, value)| value)
})
.map(ContentSha256::parse)
.transpose()
}
/// Finish helper re-exported for callers that hold a streaming body.
pub async fn finish_create_only(body: OpenDalTargetBody) -> Result<StoredWrite, TargetError> {
body.finish_object().await
}

19
crates/remora-target/tests/redaction.rs Normal file
View file

@ -0,0 +1,19 @@
use remora_target::TargetStoreConfig;
#[test]
fn target_store_config_debug_redacts_secret() {
let config = TargetStoreConfig::new_exclusive(
"garage",
"http://garage:3900",
"remora-lane-001",
"garage",
"synthetic-access-key",
"synthetic-secret-key-must-not-leak",
"",
)
.expect("config");
let debug = format!("{config:?}");
assert!(!debug.contains("synthetic-secret-key-must-not-leak"));
assert!(debug.contains("REDACTED"));
assert!(debug.contains("synthetic-access-key"));
}

436
crates/remora-target/tests/substrate.rs Normal file
View file

@ -0,0 +1,436 @@
use std::num::NonZeroUsize;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use remora_domain::OpaqueId;
use remora_stream::TargetBody;
use remora_target::{
CreateObjectRequest, ObjectIdentity, RESERVED_NAMESPACE_SEGMENT, TargetError, TargetStore,
TargetStoreConfig, attempt_object_key, hash_bytes, is_reserved_key, validate_attempt_key,
};
fn require_env() -> bool {
std::env::var("REMORA_TARGET_REQUIRE").is_ok()
}
fn store_from_env() -> Option<TargetStore> {
match TargetStoreConfig::from_env() {
Ok(config) => Some(TargetStore::connect(config).expect("connect target")),
Err(error) => {
if require_env() {
panic!("target env required when REMORA_TARGET_REQUIRE is set: {error}");
}
eprintln!("skipping target test: {error}");
None
}
}
}
fn unique_id(prefix: &str) -> OpaqueId {
let nanos = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("time")
.as_nanos();
let value = format!("{prefix}{nanos:x}");
OpaqueId::parse(&value).unwrap_or_else(|_| {
OpaqueId::parse(format!("{prefix}aaaaaaaa")).expect("fallback opaque id")
})
}
fn attempt_key(label: &str) -> String {
attempt_object_key(
&unique_id(&format!("att{label}")),
&unique_id(&format!("obj{label}")),
None,
)
}
fn create_request(key: impl Into<String>) -> CreateObjectRequest {
CreateObjectRequest::parse(key).expect("valid create-only request")
}
#[tokio::test]
async fn create_only_independently_hashes_and_rejects_wrong_expectation() {
let Some(store) = store_from_env() else {
return;
};
let body = b"immutable-member-v1".as_slice();
let digest = hash_bytes(body);
let key = attempt_key("csha");
assert!(is_reserved_key(&key));
assert!(validate_attempt_key(&key).is_ok());
assert!(validate_attempt_key("not-reserved/key").is_err());
let wrong = store
.put_create_only(
&create_request(key.clone()).with_expected_content_sha256(hash_bytes(b"other-payload")),
body,
)
.await;
assert!(
matches!(wrong, Err(TargetError::Integrity(_))),
"wrong expectation must fail: {wrong:?}"
);
let stored = store
.put_create_only(
&create_request(key.clone()).with_expected_content_sha256(digest.clone()),
body,
)
.await
.expect("create-only put");
assert_eq!(stored.identity.key, key);
assert_eq!(
stored.identity.target_store_id,
store.config().target_store_id
);
assert_eq!(stored.content_sha256, digest);
assert_eq!(
store.read_exact(&stored.identity).await.expect("read"),
body
);
let head = store.head_exact(&stored.identity).await.expect("head");
assert_eq!(head.content_sha256, Some(digest));
let conflict = store
.put_create_only(&create_request(key.clone()), b"replacement".as_slice())
.await;
assert!(
matches!(conflict, Err(TargetError::AlreadyExists)),
"expected create-only conflict, got {conflict:?}"
);
// Cross-store identity must fail closed.
let mut foreign = stored.identity.clone();
foreign.target_store_id = "other-store".into();
let cross = store.read_exact(&foreign).await;
assert!(
matches!(cross, Err(TargetError::Validation(_))),
"cross-store identity: {cross:?}"
);
}
#[tokio::test]
async fn streaming_create_only_computes_digest() {
let Some(store) = store_from_env() else {
return;
};
let payload = vec![0x5Au8; 256 * 1024];
let expected = hash_bytes(&payload);
let key = attempt_key("strm");
let mut body = store
.open_create_only(
&create_request(key.clone())
.with_expected_content_sha256(expected.clone())
.with_chunk_size(NonZeroUsize::new(64 * 1024).expect("non-zero chunk size")),
)
.await
.expect("open writer");
body.write_all(&payload).await.expect("write");
let stored = body.finish_object().await.expect("finish");
assert_eq!(stored.bytes_written, payload.len() as u64);
assert_eq!(stored.content_sha256, expected);
assert_eq!(stored.identity.key, key);
assert_eq!(
store.read_exact(&stored.identity).await.expect("read"),
payload
);
}
#[tokio::test]
async fn multipart_relist_complete_abort_and_create_only() {
let Some(store) = store_from_env() else {
return;
};
let replaced_part_one = vec![b'x'; 5 * 1024 * 1024];
let part_one = vec![b'a'; 5 * 1024 * 1024];
let part_two = vec![b'b'; 1024];
let combined = [part_one.as_slice(), part_two.as_slice()].concat();
let digest = hash_bytes(&combined);
let key = attempt_key("mpu");
let multipart_request =
create_request(key.clone()).with_expected_content_sha256(digest.clone());
let session = store
.begin_multipart(&multipart_request)
.await
.expect("begin");
let p2 = store
.upload_part(&session, 2, part_two.clone().into())
.await
.expect("part 2");
let replaced_p1 = store
.upload_part(&session, 1, replaced_part_one.into())
.await
.expect("original part 1");
let stale = store.list_parts(&session).await.expect("stale part list");
let p1 = store
.upload_part(&session, 1, part_one.clone().into())
.await
.expect("replacement part 1");
let durable_identity = session.identity().clone();
drop(session);
let (session, listed) = store
.resume_multipart(durable_identity.clone(), Some(digest.clone()))
.await
.expect("resume and re-list");
assert_eq!(listed.len(), 2);
assert_eq!(listed[0].etag, p1.etag);
assert_eq!(listed[1].etag, p2.etag);
assert_ne!(listed[0].etag, replaced_p1.etag);
let stale_completion = store.complete_multipart(session, &stale).await;
assert!(
matches!(stale_completion, Err(TargetError::Integrity(_))),
"completion must reject a replaced part: {stale_completion:?}"
);
let (session, listed) = store
.resume_multipart(durable_identity, Some(digest.clone()))
.await
.expect("resume after stale completion");
let completed = store
.complete_multipart(session, &listed)
.await
.expect("complete");
assert_eq!(completed.identity.key, key);
assert_eq!(completed.content_sha256, digest);
assert_eq!(
store
.head_exact(&completed.identity)
.await
.expect("head multipart")
.content_sha256,
None
);
let read = store
.read_exact(&completed.identity)
.await
.expect("read completed");
assert_eq!(read.as_ref(), combined.as_slice());
let occupied = store.begin_multipart(&multipart_request).await;
assert!(
matches!(occupied, Err(TargetError::AlreadyExists)),
"multipart begin on occupied key: {occupied:?}"
);
let abort_key = attempt_key("mpa");
let abort_request = create_request(abort_key);
let abort_session = store
.begin_multipart(&abort_request)
.await
.expect("begin abort");
store
.upload_part(&abort_session, 1, part_one.into())
.await
.expect("abort part");
store.abort_multipart(abort_session).await.expect("abort");
}
#[tokio::test]
async fn unicode_key_exact_delete() {
let Some(store) = store_from_env() else {
return;
};
let key = attempt_object_key(
&unique_id("attuni"),
&unique_id("objuni"),
Some("한글/é/member"),
);
let body = "정규화".as_bytes();
let stored = store
.put_create_only(&create_request(key.clone()), body)
.await
.expect("unicode put");
assert_eq!(stored.content_sha256, hash_bytes(body));
store
.delete_exact(&stored.identity)
.await
.expect("exact delete");
let missing = store.read_exact(&stored.identity).await;
assert!(
matches!(missing, Err(TargetError::NotFound)),
"expected not found after delete, got {missing:?}"
);
}
#[tokio::test]
async fn time_limited_exact_access_rejects_tamper_and_expiry() {
let Some(store) = store_from_env() else {
return;
};
let body = b"presign-body";
let key = attempt_key("prs");
let sibling_key = attempt_key("sib");
let stored = store
.put_create_only(&create_request(key.clone()), body.as_slice())
.await
.expect("put");
store
.put_create_only(&create_request(sibling_key.clone()), b"sibling".as_slice())
.await
.expect("sibling put");
let access = store
.presign_read(&stored.identity, Duration::from_secs(2))
.await
.expect("presign");
let client = reqwest::Client::new();
let ok = client
.get(&access.url)
.send()
.await
.expect("presign get")
.error_for_status()
.expect("presign status")
.bytes()
.await
.expect("presign body");
assert_eq!(ok.as_ref(), body);
let tampered = access.url.replacen(&key, &sibling_key, 1);
assert_ne!(tampered, access.url);
let tamper_status = client
.get(&tampered)
.send()
.await
.expect("tampered get")
.status()
.as_u16();
assert!(
matches!(tamper_status, 400 | 403),
"tampered URL must not authorize sibling, got {tamper_status}"
);
tokio::time::sleep(Duration::from_secs(3)).await;
let expired_status = client
.get(&access.url)
.send()
.await
.expect("expired get")
.status()
.as_u16();
assert!(
matches!(expired_status, 400 | 403),
"expired URL must fail, got {expired_status}"
);
}
#[tokio::test]
async fn version_addressed_and_stale_version_matrix() {
let Some(store) = store_from_env() else {
return;
};
if !store
.enable_bucket_versioning()
.await
.expect("enable versioning")
{
if require_env() {
let provider =
std::env::var("REMORA_TARGET_PROVIDER").unwrap_or_else(|_| "unknown".into());
if provider == "versitygw" {
panic!("versitygw must support bucket versioning for WP-08 matrix");
}
}
eprintln!(
"target provider {} cannot enable versioning; skipping version matrix",
std::env::var("REMORA_TARGET_PROVIDER").unwrap_or_else(|_| "unknown".into())
);
return;
}
let key = attempt_key("ver");
let first = store
.put_create_only(&create_request(key.clone()), b"one".as_slice())
.await
.expect("first version");
let first_version = first
.identity
.version_id
.clone()
.expect("version id after enable");
let second = store
.put_overwrite(&create_request(key.clone()), b"two".as_slice())
.await
.expect("second version");
let second_version = second
.identity
.version_id
.clone()
.expect("second version id");
assert_ne!(first_version, second_version);
let first_identity = ObjectIdentity {
target_store_id: store.config().target_store_id.clone(),
bucket: store.config().bucket.clone(),
key: key.clone(),
version_id: Some(first_version),
};
let second_identity = ObjectIdentity {
target_store_id: store.config().target_store_id.clone(),
bucket: store.config().bucket.clone(),
key: key.clone(),
version_id: Some(second_version),
};
assert_eq!(
store.read_exact(&first_identity).await.expect("v1"),
b"one".as_slice()
);
assert_eq!(
store.read_exact(&second_identity).await.expect("v2"),
b"two".as_slice()
);
let versioned_access = store
.presign_read(&first_identity, Duration::from_secs(60))
.await
.expect("versioned presign");
let client = reqwest::Client::new();
let body = client
.get(&versioned_access.url)
.send()
.await
.expect("versioned get")
.bytes()
.await
.expect("versioned body");
assert_eq!(body.as_ref(), b"one");
store
.delete_exact(&first_identity)
.await
.expect("delete v1");
assert!(matches!(
store.read_exact(&first_identity).await,
Err(TargetError::NotFound)
));
let stale_key = attempt_key("stv");
let keep = store
.put_create_only(&create_request(stale_key.clone()), b"keep".as_slice())
.await
.expect("stale put");
let version = keep.identity.version_id.clone().expect("stale version id");
let bogus = ObjectIdentity {
target_store_id: store.config().target_store_id.clone(),
bucket: store.config().bucket.clone(),
key: stale_key,
version_id: Some(format!("{version}-stale")),
};
let result = store.read_exact(&bogus).await;
assert!(
matches!(
result,
Err(TargetError::NotFound | TargetError::Internal(_))
),
"stale version must fail closed: {result:?}"
);
store
.purge_versions_with_prefix(RESERVED_NAMESPACE_SEGMENT)
.await
.expect("purge versions");
}