ms2k/remora
SHA256
Watch
0
Code Releases Activity

Stabilize result ordering and streaming evidence

Browse source
This commit is contained in:
ms2k 2026-07-17 03:36:41 +09:00
commit b2afdfe97a
3 changed files with 154 additions and 40 deletions
Download patch file Download diff file Expand all files Collapse all files

37
crates/remora-result/src/inventory.rs
View file

@ -10,8 +10,8 @@ use sqlx::PgPool;
use crate::ResultError;
use crate::types::{
CommittedResult, MANIFEST_SCHEMA_VERSION, PageSize, PublicationInput, ResultMember,
ResultMemberPage,
CommittedResult, MANIFEST_SCHEMA_VERSION, ManifestBuildMetrics, PageSize, PublicationInput,
ResultMember, ResultMemberPage,
};
#[derive(Clone)]
@ -131,10 +131,27 @@ impl ResultInventory {
manifest_id: &OpaqueId,
target: &TargetStore,
) -> Result<PublicationInput, ResultError> {
self.create_manifest_with_metrics(attempt_id, manifest_id, target)
.await
.map(|(input, _)| input)
}
pub async fn create_manifest_with_metrics(
&self,
attempt_id: &OpaqueId,
manifest_id: &OpaqueId,
target: &TargetStore,
) -> Result<(PublicationInput, ManifestBuildMetrics), ResultError> {
self.close(attempt_id).await?;
if let Some(existing) = self.publication_input(attempt_id).await? {
return if existing.manifest_id == *manifest_id {
Ok(existing)
Ok((
existing,
ManifestBuildMetrics {
members_streamed: 0,
max_encoded_member_bytes: 0,
},
))
} else {
Err(ResultError::NotSealed)
};
@ -151,6 +168,10 @@ impl ResultInventory {
.bind(attempt_id.as_str())
.fetch(&self.pool);
let mut first = true;
let mut metrics = ManifestBuildMetrics {
members_streamed: 0,
max_encoded_member_bytes: 0,
};
while let Some(row) = rows.try_next().await? {
if !first {
body.write_all(b",")
@ -160,6 +181,8 @@ impl ResultInventory {
first = false;
let member = row.into_member()?;
let encoded = serde_json::to_vec(&member).map_err(|_| ResultError::Integrity)?;
metrics.members_streamed = metrics.members_streamed.saturating_add(1);
metrics.max_encoded_member_bytes = metrics.max_encoded_member_bytes.max(encoded.len());
body.write_all(&encoded)
.await
.map_err(ResultError::ManifestWrite)?;
@ -182,7 +205,7 @@ impl ResultInventory {
total_bytes,
};
self.persist_sealed(&input).await?;
Ok(input)
Ok((input, metrics))
}
pub async fn validate_publication_input(
@ -343,11 +366,13 @@ const MEMBER_SELECT: &str = "SELECT member_id, path, size, object_store, object_
FROM result_inventory_members WHERE attempt_id = $1";
const MEMBER_SELECT_ORDERED: &str = "SELECT member_id, path, size, object_store, object_bucket,
object_key, object_version_id, object_sha256
FROM result_inventory_members WHERE attempt_id = $1 ORDER BY path, member_id";
FROM result_inventory_members WHERE attempt_id = $1
ORDER BY path COLLATE \"C\", member_id COLLATE \"C\"";
const MEMBER_PAGE_SELECT: &str = "SELECT member_id, path, size, object_store, object_bucket,
object_key, object_version_id, object_sha256
FROM result_inventory_members
WHERE attempt_id = $1 AND path > $2 ORDER BY path, member_id LIMIT $3";
WHERE attempt_id = $1 AND path COLLATE \"C\" > $2 COLLATE \"C\"
ORDER BY path COLLATE \"C\", member_id COLLATE \"C\" LIMIT $3";
const MEMBER_LOOKUP_SELECT: &str = "SELECT member_id, path, size, object_store, object_bucket,
object_key, object_version_id, object_sha256
FROM result_inventory_members WHERE attempt_id = $1 AND member_id = $2";

4
crates/remora-result/src/lib.rs
View file

@ -9,6 +9,6 @@ pub use error::ResultError;
pub use inventory::ResultInventory;
pub use migrate::{migrate, migrator};
pub use types::{
CommittedResult, MANIFEST_SCHEMA_VERSION, ManifestObjectIdentity, PageSize, PublicationInput,
ResultMember, ResultMemberPage,
CommittedResult, MANIFEST_SCHEMA_VERSION, ManifestBuildMetrics, ManifestObjectIdentity,
PageSize, PublicationInput, ResultMember, ResultMemberPage,
};

6
crates/remora-result/src/types.rs
View file

@ -138,6 +138,12 @@ pub struct PublicationInput {
pub type CommittedResult = PublicationInput;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct ManifestBuildMetrics {
pub members_streamed: u64,
pub max_encoded_member_bytes: usize,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ResultMemberPage {
pub items: Vec<ResultMember>,