ms2k/remora
SHA256
Watch
0
Code Releases Activity

Implement crash-safe streaming worker runtime

Browse source
This commit is contained in:
ms2k 2026-07-17 18:51:44 +09:00
commit cdeef3cc1a
1 changed files with 7448 additions and 194 deletions
Download patch file Download diff file Expand all files Collapse all files

171
crates/remora-target/src/multipart.rs
View file

@ -42,7 +42,6 @@ const PATH_SET: &AsciiSet = &CONTROLS
.add(b'}');
/// One version or delete marker listed under a key/prefix.
#[cfg(feature = "test-utils")]
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ObjectVersion {
pub key: String,
@ -50,6 +49,10 @@ pub struct ObjectVersion {
pub is_delete_marker: bool,
}
pub struct ObjectHead {
pub version_id: Option<String>,
}
/// Explicit multipart control plane for destination recovery.
///
/// OpenDAL streams multipart writes but does not expose list-parts recovery.
@ -79,6 +82,98 @@ impl MultipartClient {
Ok(Self { config, http })
}
pub async fn list_uploads_exact(
&self,
key: &str,
) -> Result<Vec<MultipartIdentity>, TargetError> {
let rooted_key = rooted_key(&self.config, key);
let mut uploads = Vec::new();
let mut key_marker = String::new();
let mut upload_id_marker = String::new();
loop {
let mut uri = format!(
"{}/{}/?uploads&prefix={}",
self.config.endpoint,
self.config.bucket,
encode_query(&rooted_key)
);
if !key_marker.is_empty() {
uri.push_str(&format!("&key-marker={}", encode_query(&key_marker)));
}
if !upload_id_marker.is_empty() {
uri.push_str(&format!(
"&upload-id-marker={}",
encode_query(&upload_id_marker)
));
}
let request = Request::builder()
.method(Method::GET)
.uri(uri)
.body(Bytes::new())
.map_err(|error| TargetError::internal(error.to_string()))?;
let response = self.send(request).await?;
let status = response.status().as_u16();
let body = response
.text()
.await
.map_err(|error| TargetError::transient(redact_sensitive(error.to_string())))?;
if status / 100 != 2 {
return Err(map_http_status(status, &body));
}
let parsed: ListMultipartUploadsResult = from_str(&body)
.map_err(|error| TargetError::internal(redact_sensitive(error.to_string())))?;
uploads.extend(
parsed
.uploads
.into_iter()
.filter(|upload| upload.key == rooted_key && !upload.upload_id.is_empty())
.map(|upload| MultipartIdentity {
target_store_id: self.config.target_store_id.clone(),
bucket: self.config.bucket.clone(),
key: key.to_owned(),
upload_id: upload.upload_id,
}),
);
if parsed.is_truncated {
key_marker = parsed.next_key_marker.unwrap_or_default();
upload_id_marker = parsed.next_upload_id_marker.unwrap_or_default();
if key_marker.is_empty() && upload_id_marker.is_empty() {
return Err(TargetError::internal(
"truncated multipart listing omitted continuation markers",
));
}
} else {
break;
}
}
Ok(uploads)
}
pub async fn head_object(&self, key: &str) -> Result<ObjectHead, TargetError> {
let path = object_path(&self.config, key);
let request = Request::builder()
.method(Method::HEAD)
.uri(format!("{}{}", self.config.endpoint, path))
.body(Bytes::new())
.map_err(|error| TargetError::internal(error.to_string()))?;
let response = self.send(request).await?;
let status = response.status().as_u16();
if status == 404 {
return Err(TargetError::NotFound);
}
if status / 100 != 2 {
let body = response.text().await.unwrap_or_default();
return Err(map_http_status(status, &body));
}
let version_id = response
.headers()
.get("x-amz-version-id")
.and_then(|value| value.to_str().ok())
.filter(|value| !value.is_empty())
.map(str::to_owned);
Ok(ObjectHead { version_id })
}
/// Enable bucket versioning when the peer supports it.
///
/// Peers that return NotImplemented are reported as `false` without error.
@ -343,7 +438,6 @@ impl MultipartClient {
}
/// List object versions and delete markers under an optional key prefix.
#[cfg(feature = "test-utils")]
pub async fn list_object_versions(
&self,
prefix: &str,
@ -353,8 +447,9 @@ impl MultipartClient {
let mut version_id_marker = String::new();
loop {
let mut uri = format!("{}/{}/?versions", self.config.endpoint, self.config.bucket);
if !prefix.is_empty() {
uri.push_str(&format!("&prefix={}", encode_query(prefix)));
let rooted_prefix = rooted_key(&self.config, prefix);
if !rooted_prefix.is_empty() {
uri.push_str(&format!("&prefix={}", encode_query(&rooted_prefix)));
}
if !key_marker.is_empty() {
uri.push_str(&format!("&key-marker={}", encode_query(&key_marker)));
@ -376,6 +471,13 @@ impl MultipartClient {
.text()
.await
.map_err(|error| TargetError::transient(redact_sensitive(error.to_string())))?;
if status == 501
&& (body.contains("NotImplemented")
|| body.contains("XNotImplemented")
|| body.contains("NotSupported"))
{
return Ok(Vec::new());
}
if status / 100 != 2 {
return Err(map_http_status(status, &body));
}
@ -384,7 +486,7 @@ impl MultipartClient {
for version in parsed.versions {
if !version.version_id.is_empty() {
versions.push(ObjectVersion {
key: version.key,
key: logical_key(&self.config, &version.key),
version_id: version.version_id,
is_delete_marker: false,
});
@ -393,7 +495,7 @@ impl MultipartClient {
for marker in parsed.delete_markers {
if !marker.version_id.is_empty() {
versions.push(ObjectVersion {
key: marker.key,
key: logical_key(&self.config, &marker.key),
version_id: marker.version_id,
is_delete_marker: true,
});
@ -454,16 +556,7 @@ impl MultipartClient {
fn object_path(config: &TargetStoreConfig, key: &str) -> String {
let key = key.trim_start_matches('/');
let rooted = if config.root.is_empty() {
key.to_owned()
} else {
let root = config.root.trim_matches('/');
if root.is_empty() {
key.to_owned()
} else {
format!("{root}/{key}")
}
};
let rooted = rooted_key(config, key);
let encoded = rooted
.split('/')
.map(|segment| utf8_percent_encode(segment, PATH_SET).to_string())
@ -472,6 +565,30 @@ fn object_path(config: &TargetStoreConfig, key: &str) -> String {
format!("/{}/{}", config.bucket, encoded)
}
fn rooted_key(config: &TargetStoreConfig, key: &str) -> String {
let key = key.trim_start_matches('/');
let root = config.root.trim_matches('/');
if root.is_empty() {
key.to_owned()
} else if key.is_empty() {
root.to_owned()
} else {
format!("{root}/{key}")
}
}
fn logical_key(config: &TargetStoreConfig, key: &str) -> String {
let root = config.root.trim_matches('/');
if root.is_empty() {
key.to_owned()
} else {
key.strip_prefix(root)
.and_then(|value| value.strip_prefix('/'))
.unwrap_or(key)
.to_owned()
}
}
fn encode_query(value: &str) -> String {
utf8_percent_encode(value, PATH_SET).to_string()
}
@ -525,6 +642,26 @@ struct InitiateMultipartUploadResult {
upload_id: String,
}
#[derive(Debug, Deserialize)]
struct ListMultipartUploadsResult {
#[serde(rename = "Upload", default)]
uploads: Vec<ListedMultipartUpload>,
#[serde(rename = "IsTruncated", default)]
is_truncated: bool,
#[serde(rename = "NextKeyMarker", default)]
next_key_marker: Option<String>,
#[serde(rename = "NextUploadIdMarker", default)]
next_upload_id_marker: Option<String>,
}
#[derive(Debug, Deserialize)]
struct ListedMultipartUpload {
#[serde(rename = "Key", default)]
key: String,
#[serde(rename = "UploadId", default)]
upload_id: String,
}
#[derive(Debug, Deserialize)]
struct ListPartsResult {
#[serde(rename = "Part", default)]
@ -560,7 +697,6 @@ struct CompletePart {
etag: String,
}
#[cfg(feature = "test-utils")]
#[derive(Debug, Deserialize)]
struct ListVersionsResult {
#[serde(rename = "Version", default)]
@ -575,7 +711,6 @@ struct ListVersionsResult {
next_version_id_marker: Option<String>,
}
#[cfg(feature = "test-utils")]
#[derive(Debug, Deserialize)]
struct ListedVersion {
#[serde(rename = "Key", default)]