Add exact-identity target storage adapter
This commit is contained in:
parent
f569707f97
commit
854e6e1c17
1 changed files with 3263 additions and 41 deletions
92
crates/remora-target/src/body.rs
Normal file
92
crates/remora-target/src/body.rs
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
use opendal::Writer;
|
||||
use remora_stream::{StreamError, TargetBody, TargetFinish};
|
||||
|
||||
use crate::digest::{ContentHasher, ContentSha256, check_expected};
|
||||
use crate::error::{TargetError, map_opendal};
|
||||
use crate::identity::ObjectIdentity;
|
||||
|
||||
/// Streaming create-only target body backed by an OpenDAL writer.
|
||||
///
|
||||
/// SHA-256 is accumulated over every written byte. The open writer cannot carry
|
||||
/// final digest metadata (unknown until finish); the independently computed
|
||||
/// digest is returned on finish for durable persistence.
|
||||
pub struct OpenDalTargetBody {
|
||||
writer: Writer,
|
||||
hasher: ContentHasher,
|
||||
bytes_written: u64,
|
||||
identity: ObjectIdentity,
|
||||
expected_sha256: Option<ContentSha256>,
|
||||
}
|
||||
|
||||
impl OpenDalTargetBody {
|
||||
pub(crate) fn new(
|
||||
writer: Writer,
|
||||
identity: ObjectIdentity,
|
||||
expected_sha256: Option<ContentSha256>,
|
||||
) -> Self {
|
||||
Self {
|
||||
writer,
|
||||
hasher: ContentHasher::new(),
|
||||
bytes_written: 0,
|
||||
identity,
|
||||
expected_sha256,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn identity(&self) -> &ObjectIdentity {
|
||||
&self.identity
|
||||
}
|
||||
|
||||
/// Finish the write and return identity plus independently computed digest.
|
||||
pub async fn finish_object(mut self) -> Result<StoredWrite, TargetError> {
|
||||
let metadata = self.writer.close().await.map_err(map_opendal)?;
|
||||
let version_id = metadata.version().map(str::to_owned);
|
||||
let content_sha256 = self.hasher.finalize();
|
||||
check_expected(self.expected_sha256.as_ref(), &content_sha256)?;
|
||||
let content_length = metadata.content_length();
|
||||
let bytes_written = if content_length > 0 {
|
||||
content_length
|
||||
} else {
|
||||
self.bytes_written
|
||||
};
|
||||
Ok(StoredWrite {
|
||||
identity: self.identity.with_version(version_id),
|
||||
bytes_written,
|
||||
content_sha256,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl TargetBody for OpenDalTargetBody {
|
||||
async fn write_all(&mut self, data: &[u8]) -> Result<(), StreamError> {
|
||||
if data.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
self.writer
|
||||
.write(data.to_vec())
|
||||
.await
|
||||
.map_err(|error| map_opendal(error).into_stream_error())?;
|
||||
self.hasher.update(data);
|
||||
self.bytes_written = self
|
||||
.bytes_written
|
||||
.saturating_add(u64::try_from(data.len()).unwrap_or(u64::MAX));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn finish(self) -> Result<TargetFinish, StreamError> {
|
||||
let stored = self.finish_object().await?;
|
||||
Ok(TargetFinish {
|
||||
bytes_written: stored.bytes_written,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of completing a create-only stream write.
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct StoredWrite {
|
||||
pub identity: ObjectIdentity,
|
||||
pub bytes_written: u64,
|
||||
/// Independently computed SHA-256 of written bytes.
|
||||
pub content_sha256: ContentSha256,
|
||||
}
|
||||
Loading…
Reference in a new issue