ms2k/remora
SHA256
Watch
0
Code Releases Activity

Implement crash-safe streaming worker runtime

Browse source
This commit is contained in:
ms2k 2026-07-17 18:51:44 +09:00
commit cdeef3cc1a
4 changed files with 7448 additions and 194 deletions
Download patch file Download diff file Expand all files Collapse all files

113
crates/remora-target/src/identity.rs
View file

@ -1,4 +1,4 @@
use remora_domain::OpaqueId;
use remora_domain::{OpaqueId, TargetPrefix};
use crate::error::TargetError;
@ -55,11 +55,21 @@ pub fn attempt_object_key(
attempt_id: &OpaqueId,
object_id: &OpaqueId,
relative_path: Option<&str>,
) -> String {
attempt_object_key_in("", attempt_id, object_id, relative_path)
}
#[must_use]
pub fn attempt_object_key_in(
target_prefix: &str,
attempt_id: &OpaqueId,
object_id: &OpaqueId,
relative_path: Option<&str>,
) -> String {
let mut key = format!(
"{RESERVED_NAMESPACE_SEGMENT}/a/{}/o/{}",
attempt_id.as_str(),
object_id.as_str()
"{}/o/{}",
attempt_staging_prefix(target_prefix, attempt_id).trim_end_matches('/'),
object_id.as_str(),
);
if let Some(path) = relative_path.filter(|value| !value.is_empty()) {
let trimmed = path.trim_matches('/');
@ -74,19 +84,46 @@ pub fn attempt_object_key(
/// Construct a reserved-namespace key for a manifest descriptor object.
#[must_use]
pub fn attempt_manifest_key(attempt_id: &OpaqueId, manifest_id: &OpaqueId) -> String {
attempt_manifest_key_in("", attempt_id, manifest_id)
}
#[must_use]
pub fn attempt_manifest_key_in(
target_prefix: &str,
attempt_id: &OpaqueId,
manifest_id: &OpaqueId,
) -> String {
format!(
"{RESERVED_NAMESPACE_SEGMENT}/a/{}/m/{}",
attempt_id.as_str(),
"{}/m/{}",
attempt_staging_prefix(target_prefix, attempt_id).trim_end_matches('/'),
manifest_id.as_str()
)
}
#[must_use]
pub fn attempt_staging_prefix(target_prefix: &str, attempt_id: &OpaqueId) -> String {
let prefix = target_prefix.trim_end_matches('/');
if prefix.is_empty() {
format!("{RESERVED_NAMESPACE_SEGMENT}/a/{attempt_id}/")
} else {
format!("{prefix}/{RESERVED_NAMESPACE_SEGMENT}/a/{attempt_id}/")
}
}
#[must_use]
pub fn attempt_object_prefix(target_prefix: &str, attempt_id: &OpaqueId) -> String {
format!(
"{}/o/",
attempt_staging_prefix(target_prefix, attempt_id).trim_end_matches('/')
)
}
/// Return true when `key` is under the reserved service namespace.
#[must_use]
pub fn is_reserved_key(key: &str) -> bool {
let trimmed = key.trim_start_matches('/');
trimmed == RESERVED_NAMESPACE_SEGMENT
|| trimmed.starts_with(&format!("{RESERVED_NAMESPACE_SEGMENT}/"))
key.trim_matches('/')
.split('/')
.any(|segment| segment == RESERVED_NAMESPACE_SEGMENT)
}
/// Validate a reserved attempt object or manifest identity layout.
@ -94,19 +131,27 @@ pub fn validate_attempt_key(key: &str) -> Result<(), TargetError> {
if key.is_empty() || key.starts_with('/') || key.contains("//") {
return Err(TargetError::validation("invalid object key"));
}
if !is_reserved_key(key) {
let segments = key.split('/').collect::<Vec<_>>();
let reserved = segments
.iter()
.enumerate()
.filter(|(_, segment)| **segment == RESERVED_NAMESPACE_SEGMENT)
.map(|(index, _)| index)
.collect::<Vec<_>>();
if reserved.len() != 1 {
return Err(TargetError::validation(
"object key must be under the reserved service namespace",
));
}
let segments = key.split('/').collect::<Vec<_>>();
if segments.len() < 5
|| segments[0] != RESERVED_NAMESPACE_SEGMENT
|| segments[1] != "a"
|| OpaqueId::parse(segments[2]).is_err()
|| OpaqueId::parse(segments[4]).is_err()
|| !matches!(segments[3], "o" | "m")
|| (segments[3] == "m" && segments.len() != 5)
let reserved = reserved[0];
let target_prefix = segments[..reserved].join("/");
if TargetPrefix::parse(&target_prefix, [RESERVED_NAMESPACE_SEGMENT]).is_err()
|| segments.len() < reserved + 5
|| segments[reserved + 1] != "a"
|| OpaqueId::parse(segments[reserved + 2]).is_err()
|| OpaqueId::parse(segments[reserved + 4]).is_err()
|| !matches!(segments[reserved + 3], "o" | "m")
|| (segments[reserved + 3] == "m" && segments.len() != reserved + 5)
{
return Err(TargetError::validation(
"object key must contain attempt and object identities",
@ -115,6 +160,16 @@ pub fn validate_attempt_key(key: &str) -> Result<(), TargetError> {
Ok(())
}
pub fn attempt_key_attempt_id(key: &str) -> Result<OpaqueId, TargetError> {
validate_attempt_key(key)?;
let segments = key.split('/').collect::<Vec<_>>();
let reserved = segments
.iter()
.position(|segment| *segment == RESERVED_NAMESPACE_SEGMENT)
.expect("validated key has a reserved namespace");
OpaqueId::parse(segments[reserved + 2]).map_err(|_| TargetError::validation("invalid attempt"))
}
#[cfg(test)]
mod tests {
use super::*;
@ -131,7 +186,29 @@ mod tests {
attempt_manifest_key(&attempt, &object),
"_remora/a/attempt01aaaaaaaa/m/object01aaaaaaaaa"
);
let prefixed = attempt_object_key_in("users/alice", &attempt, &object, None);
assert_eq!(
prefixed,
"users/alice/_remora/a/attempt01aaaaaaaa/o/object01aaaaaaaaa"
);
assert!(is_reserved_key(&prefixed));
assert!(
validate_attempt_key(&attempt_manifest_key_in("users/alice", &attempt, &object))
.is_ok()
);
assert!(validate_attempt_key("public/data").is_err());
assert!(validate_attempt_key("_remora/arbitrary").is_err());
}
#[test]
fn literal_null_version_is_retained_as_an_exact_s3_identity() {
let identity = ObjectIdentity {
target_store_id: "target".into(),
bucket: "bucket".into(),
key: "_remora/a/attempt01aaaaaaaa/o/object01aaaaaaaaa".into(),
version_id: None,
}
.with_version(Some("null".into()));
assert_eq!(identity.version_id.as_deref(), Some("null"));
}
}

9
crates/remora-target/src/lib.rs
View file

@ -18,10 +18,11 @@ pub use digest::{ContentSha256, hash_bytes, sha256_equal};
pub use error::TargetError;
pub use identity::{
CompletedPart, MultipartIdentity, ObjectIdentity, REMORA_SHA256_METADATA_KEY,
RESERVED_NAMESPACE_SEGMENT, attempt_manifest_key, attempt_object_key, is_reserved_key,
validate_attempt_key,
RESERVED_NAMESPACE_SEGMENT, attempt_key_attempt_id, attempt_manifest_key,
attempt_manifest_key_in, attempt_object_key, attempt_object_key_in, attempt_object_prefix,
attempt_staging_prefix, is_reserved_key, validate_attempt_key,
};
pub use store::{
CreateObjectRequest, MultipartSession, ObjectMetadata, StoredObject, TargetStore,
TimeLimitedAccess, finish_create_only,
CreateObjectRequest, MultipartSession, ObjectMetadata, StagingWriteDiscovery, StoredObject,
TargetStore, TimeLimitedAccess, finish_create_only,
};

171
crates/remora-target/src/multipart.rs
View file

@ -42,7 +42,6 @@ const PATH_SET: &AsciiSet = &CONTROLS
.add(b'}');
/// One version or delete marker listed under a key/prefix.
#[cfg(feature = "test-utils")]
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ObjectVersion {
pub key: String,
@ -50,6 +49,10 @@ pub struct ObjectVersion {
pub is_delete_marker: bool,
}
pub struct ObjectHead {
pub version_id: Option<String>,
}
/// Explicit multipart control plane for destination recovery.
///
/// OpenDAL streams multipart writes but does not expose list-parts recovery.
@ -79,6 +82,98 @@ impl MultipartClient {
Ok(Self { config, http })
}
pub async fn list_uploads_exact(
&self,
key: &str,
) -> Result<Vec<MultipartIdentity>, TargetError> {
let rooted_key = rooted_key(&self.config, key);
let mut uploads = Vec::new();
let mut key_marker = String::new();
let mut upload_id_marker = String::new();
loop {
let mut uri = format!(
"{}/{}/?uploads&prefix={}",
self.config.endpoint,
self.config.bucket,
encode_query(&rooted_key)
);
if !key_marker.is_empty() {
uri.push_str(&format!("&key-marker={}", encode_query(&key_marker)));
}
if !upload_id_marker.is_empty() {
uri.push_str(&format!(
"&upload-id-marker={}",
encode_query(&upload_id_marker)
));
}
let request = Request::builder()
.method(Method::GET)
.uri(uri)
.body(Bytes::new())
.map_err(|error| TargetError::internal(error.to_string()))?;
let response = self.send(request).await?;
let status = response.status().as_u16();
let body = response
.text()
.await
.map_err(|error| TargetError::transient(redact_sensitive(error.to_string())))?;
if status / 100 != 2 {
return Err(map_http_status(status, &body));
}
let parsed: ListMultipartUploadsResult = from_str(&body)
.map_err(|error| TargetError::internal(redact_sensitive(error.to_string())))?;
uploads.extend(
parsed
.uploads
.into_iter()
.filter(|upload| upload.key == rooted_key && !upload.upload_id.is_empty())
.map(|upload| MultipartIdentity {
target_store_id: self.config.target_store_id.clone(),
bucket: self.config.bucket.clone(),
key: key.to_owned(),
upload_id: upload.upload_id,
}),
);
if parsed.is_truncated {
key_marker = parsed.next_key_marker.unwrap_or_default();
upload_id_marker = parsed.next_upload_id_marker.unwrap_or_default();
if key_marker.is_empty() && upload_id_marker.is_empty() {
return Err(TargetError::internal(
"truncated multipart listing omitted continuation markers",
));
}
} else {
break;
}
}
Ok(uploads)
}
pub async fn head_object(&self, key: &str) -> Result<ObjectHead, TargetError> {
let path = object_path(&self.config, key);
let request = Request::builder()
.method(Method::HEAD)
.uri(format!("{}{}", self.config.endpoint, path))
.body(Bytes::new())
.map_err(|error| TargetError::internal(error.to_string()))?;
let response = self.send(request).await?;
let status = response.status().as_u16();
if status == 404 {
return Err(TargetError::NotFound);
}
if status / 100 != 2 {
let body = response.text().await.unwrap_or_default();
return Err(map_http_status(status, &body));
}
let version_id = response
.headers()
.get("x-amz-version-id")
.and_then(|value| value.to_str().ok())
.filter(|value| !value.is_empty())
.map(str::to_owned);
Ok(ObjectHead { version_id })
}
/// Enable bucket versioning when the peer supports it.
///
/// Peers that return NotImplemented are reported as `false` without error.
@ -343,7 +438,6 @@ impl MultipartClient {
}
/// List object versions and delete markers under an optional key prefix.
#[cfg(feature = "test-utils")]
pub async fn list_object_versions(
&self,
prefix: &str,
@ -353,8 +447,9 @@ impl MultipartClient {
let mut version_id_marker = String::new();
loop {
let mut uri = format!("{}/{}/?versions", self.config.endpoint, self.config.bucket);
if !prefix.is_empty() {
uri.push_str(&format!("&prefix={}", encode_query(prefix)));
let rooted_prefix = rooted_key(&self.config, prefix);
if !rooted_prefix.is_empty() {
uri.push_str(&format!("&prefix={}", encode_query(&rooted_prefix)));
}
if !key_marker.is_empty() {
uri.push_str(&format!("&key-marker={}", encode_query(&key_marker)));
@ -376,6 +471,13 @@ impl MultipartClient {
.text()
.await
.map_err(|error| TargetError::transient(redact_sensitive(error.to_string())))?;
if status == 501
&& (body.contains("NotImplemented")
|| body.contains("XNotImplemented")
|| body.contains("NotSupported"))
{
return Ok(Vec::new());
}
if status / 100 != 2 {
return Err(map_http_status(status, &body));
}
@ -384,7 +486,7 @@ impl MultipartClient {
for version in parsed.versions {
if !version.version_id.is_empty() {
versions.push(ObjectVersion {
key: version.key,
key: logical_key(&self.config, &version.key),
version_id: version.version_id,
is_delete_marker: false,
});
@ -393,7 +495,7 @@ impl MultipartClient {
for marker in parsed.delete_markers {
if !marker.version_id.is_empty() {
versions.push(ObjectVersion {
key: marker.key,
key: logical_key(&self.config, &marker.key),
version_id: marker.version_id,
is_delete_marker: true,
});
@ -454,16 +556,7 @@ impl MultipartClient {
fn object_path(config: &TargetStoreConfig, key: &str) -> String {
let key = key.trim_start_matches('/');
let rooted = if config.root.is_empty() {
key.to_owned()
} else {
let root = config.root.trim_matches('/');
if root.is_empty() {
key.to_owned()
} else {
format!("{root}/{key}")
}
};
let rooted = rooted_key(config, key);
let encoded = rooted
.split('/')
.map(|segment| utf8_percent_encode(segment, PATH_SET).to_string())
@ -472,6 +565,30 @@ fn object_path(config: &TargetStoreConfig, key: &str) -> String {
format!("/{}/{}", config.bucket, encoded)
}
fn rooted_key(config: &TargetStoreConfig, key: &str) -> String {
let key = key.trim_start_matches('/');
let root = config.root.trim_matches('/');
if root.is_empty() {
key.to_owned()
} else if key.is_empty() {
root.to_owned()
} else {
format!("{root}/{key}")
}
}
fn logical_key(config: &TargetStoreConfig, key: &str) -> String {
let root = config.root.trim_matches('/');
if root.is_empty() {
key.to_owned()
} else {
key.strip_prefix(root)
.and_then(|value| value.strip_prefix('/'))
.unwrap_or(key)
.to_owned()
}
}
fn encode_query(value: &str) -> String {
utf8_percent_encode(value, PATH_SET).to_string()
}
@ -525,6 +642,26 @@ struct InitiateMultipartUploadResult {
upload_id: String,
}
#[derive(Debug, Deserialize)]
struct ListMultipartUploadsResult {
#[serde(rename = "Upload", default)]
uploads: Vec<ListedMultipartUpload>,
#[serde(rename = "IsTruncated", default)]
is_truncated: bool,
#[serde(rename = "NextKeyMarker", default)]
next_key_marker: Option<String>,
#[serde(rename = "NextUploadIdMarker", default)]
next_upload_id_marker: Option<String>,
}
#[derive(Debug, Deserialize)]
struct ListedMultipartUpload {
#[serde(rename = "Key", default)]
key: String,
#[serde(rename = "UploadId", default)]
upload_id: String,
}
#[derive(Debug, Deserialize)]
struct ListPartsResult {
#[serde(rename = "Part", default)]
@ -560,7 +697,6 @@ struct CompletePart {
etag: String,
}
#[cfg(feature = "test-utils")]
#[derive(Debug, Deserialize)]
struct ListVersionsResult {
#[serde(rename = "Version", default)]
@ -575,7 +711,6 @@ struct ListVersionsResult {
next_version_id_marker: Option<String>,
}
#[cfg(feature = "test-utils")]
#[derive(Debug, Deserialize)]
struct ListedVersion {
#[serde(rename = "Key", default)]

62
crates/remora-target/src/store.rs
View file

@ -14,7 +14,7 @@ use crate::digest::{ContentHasher, ContentSha256, check_expected, hash_bytes};
use crate::error::{TargetError, map_opendal, redact_sensitive};
use crate::identity::{
CompletedPart, MultipartIdentity, ObjectIdentity, REMORA_SHA256_METADATA_KEY,
attempt_manifest_key, attempt_object_key, validate_attempt_key,
validate_attempt_key,
};
use crate::multipart::MultipartClient;
@ -42,9 +42,19 @@ impl CreateObjectRequest {
attempt_id: &OpaqueId,
object_id: &OpaqueId,
relative_path: Option<&str>,
) -> Self {
Self::for_target_attempt_object("", attempt_id, object_id, relative_path)
}
#[must_use]
pub fn for_target_attempt_object(
target_prefix: &str,
attempt_id: &OpaqueId,
object_id: &OpaqueId,
relative_path: Option<&str>,
) -> Self {
Self {
key: attempt_object_key(attempt_id, object_id, relative_path),
key: crate::attempt_object_key_in(target_prefix, attempt_id, object_id, relative_path),
expected_content_sha256: None,
chunk_size: None,
}
@ -52,8 +62,17 @@ impl CreateObjectRequest {
#[must_use]
pub fn for_attempt_manifest(attempt_id: &OpaqueId, manifest_id: &OpaqueId) -> Self {
Self::for_target_attempt_manifest("", attempt_id, manifest_id)
}
#[must_use]
pub fn for_target_attempt_manifest(
target_prefix: &str,
attempt_id: &OpaqueId,
manifest_id: &OpaqueId,
) -> Self {
Self {
key: attempt_manifest_key(attempt_id, manifest_id),
key: crate::attempt_manifest_key_in(target_prefix, attempt_id, manifest_id),
expected_content_sha256: None,
chunk_size: None,
}
@ -93,6 +112,12 @@ pub struct ObjectMetadata {
pub size: u64,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct StagingWriteDiscovery {
pub objects: Vec<ObjectIdentity>,
pub multiparts: Vec<MultipartIdentity>,
}
/// Time-limited exact-object access URL.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct TimeLimitedAccess {
@ -388,6 +413,36 @@ impl TargetStore {
self.multipart.abort(&session.identity).await
}
pub async fn discover_staging_write(
&self,
key: &str,
) -> Result<StagingWriteDiscovery, TargetError> {
validate_attempt_key(key)?;
let versions = self.multipart.list_object_versions(key).await?;
let mut objects = versions
.into_iter()
.filter(|version| version.key == key && !version.is_delete_marker)
.map(|version| ObjectIdentity {
target_store_id: self.config.target_store_id.clone(),
bucket: self.config.bucket.clone(),
key: key.to_owned(),
version_id: Some(version.version_id),
})
.collect::<Vec<_>>();
if objects.is_empty() {
match self.multipart.head_object(key).await {
Ok(head) => objects.push(self.new_object_identity(key, head.version_id)),
Err(TargetError::NotFound) => {}
Err(error) => return Err(error),
}
}
let multiparts = self.multipart.list_uploads_exact(key).await?;
Ok(StagingWriteDiscovery {
objects,
multiparts,
})
}
async fn open_writer(&self, request: &CreateObjectRequest) -> Result<Writer, TargetError> {
let chunk = request
.chunk_size
@ -429,6 +484,7 @@ impl TargetStore {
{
hasher.update(&chunk);
size = size.saturating_add(u64::try_from(chunk.len()).unwrap_or(u64::MAX));
tokio::task::yield_now().await;
}
Ok((hasher.finalize(), size))
}