ms2k/remora
SHA256
Watch
0
Code Releases Activity

Stabilize result ordering and streaming evidence

Browse source
This commit is contained in:
ms2k 2026-07-17 03:36:41 +09:00
commit b2afdfe97a
6 changed files with 154 additions and 40 deletions
Download patch file Download diff file Expand all files Collapse all files

11
Dockerfile.test
View file

@ -158,11 +158,20 @@ RUN cargo nextest archive \
--locked \
--archive-file /tmp/remora-result-tests.tar.zst
FROM kind-result-build AS kind-result-verified
RUN cargo clippy \
--package remora-result \
--all-targets \
--all-features \
--locked \
-- -D warnings \
&& cargo test --package remora-result --doc --locked
FROM rust-tools AS kind-result-runner
WORKDIR /workspace
COPY Cargo.toml ./
COPY crates crates
COPY --from=kind-result-build /tmp/remora-result-tests.tar.zst /tests/remora-result-tests.tar.zst
COPY --from=kind-result-verified /tmp/remora-result-tests.tar.zst /tests/remora-result-tests.tar.zst
CMD ["bash", "-ec", "test -s /run/remora-fixture/profile.env && test -n \"$DATABASE_URL\" && test -n \"$REMORA_TARGET_ENDPOINT\" && case $REMORA_TARGET_PROVIDER in garage|versitygw) ;; *) exit 1 ;; esac && export REMORA_RESULT_REQUIRE=1 && cargo-nextest nextest run --archive-file /tests/remora-result-tests.tar.zst --workspace-remap /workspace --test-threads=1"]
FROM ${NODE_IMAGE} AS node-runtime

4
crates/remora-result/migrations/0001_result_inventory.sql
View file

@ -24,8 +24,8 @@ CREATE TABLE result_inventories (
CREATE TABLE result_inventory_members (
attempt_id TEXT NOT NULL REFERENCES result_inventories(attempt_id) ON DELETE CASCADE,
member_id TEXT NOT NULL,
path TEXT NOT NULL,
member_id TEXT COLLATE "C" NOT NULL,
path TEXT COLLATE "C" NOT NULL,
size BIGINT NOT NULL CHECK (size >= 0),
object_store TEXT NOT NULL,
object_bucket TEXT NOT NULL,

37
crates/remora-result/src/inventory.rs
View file

@ -10,8 +10,8 @@ use sqlx::PgPool;
use crate::ResultError;
use crate::types::{
CommittedResult, MANIFEST_SCHEMA_VERSION, PageSize, PublicationInput, ResultMember,
ResultMemberPage,
CommittedResult, MANIFEST_SCHEMA_VERSION, ManifestBuildMetrics, PageSize, PublicationInput,
ResultMember, ResultMemberPage,
};
#[derive(Clone)]
@ -131,10 +131,27 @@ impl ResultInventory {
manifest_id: &OpaqueId,
target: &TargetStore,
) -> Result<PublicationInput, ResultError> {
self.create_manifest_with_metrics(attempt_id, manifest_id, target)
.await
.map(|(input, _)| input)
}
pub async fn create_manifest_with_metrics(
&self,
attempt_id: &OpaqueId,
manifest_id: &OpaqueId,
target: &TargetStore,
) -> Result<(PublicationInput, ManifestBuildMetrics), ResultError> {
self.close(attempt_id).await?;
if let Some(existing) = self.publication_input(attempt_id).await? {
return if existing.manifest_id == *manifest_id {
Ok(existing)
Ok((
existing,
ManifestBuildMetrics {
members_streamed: 0,
max_encoded_member_bytes: 0,
},
))
} else {
Err(ResultError::NotSealed)
};
@ -151,6 +168,10 @@ impl ResultInventory {
.bind(attempt_id.as_str())
.fetch(&self.pool);
let mut first = true;
let mut metrics = ManifestBuildMetrics {
members_streamed: 0,
max_encoded_member_bytes: 0,
};
while let Some(row) = rows.try_next().await? {
if !first {
body.write_all(b",")
@ -160,6 +181,8 @@ impl ResultInventory {
first = false;
let member = row.into_member()?;
let encoded = serde_json::to_vec(&member).map_err(|_| ResultError::Integrity)?;
metrics.members_streamed = metrics.members_streamed.saturating_add(1);
metrics.max_encoded_member_bytes = metrics.max_encoded_member_bytes.max(encoded.len());
body.write_all(&encoded)
.await
.map_err(ResultError::ManifestWrite)?;
@ -182,7 +205,7 @@ impl ResultInventory {
total_bytes,
};
self.persist_sealed(&input).await?;
Ok(input)
Ok((input, metrics))
}
pub async fn validate_publication_input(
@ -343,11 +366,13 @@ const MEMBER_SELECT: &str = "SELECT member_id, path, size, object_store, object_
FROM result_inventory_members WHERE attempt_id = $1";
const MEMBER_SELECT_ORDERED: &str = "SELECT member_id, path, size, object_store, object_bucket,
object_key, object_version_id, object_sha256
FROM result_inventory_members WHERE attempt_id = $1 ORDER BY path, member_id";
FROM result_inventory_members WHERE attempt_id = $1
ORDER BY path COLLATE \"C\", member_id COLLATE \"C\"";
const MEMBER_PAGE_SELECT: &str = "SELECT member_id, path, size, object_store, object_bucket,
object_key, object_version_id, object_sha256
FROM result_inventory_members
WHERE attempt_id = $1 AND path > $2 ORDER BY path, member_id LIMIT $3";
WHERE attempt_id = $1 AND path COLLATE \"C\" > $2 COLLATE \"C\"
ORDER BY path COLLATE \"C\", member_id COLLATE \"C\" LIMIT $3";
const MEMBER_LOOKUP_SELECT: &str = "SELECT member_id, path, size, object_store, object_bucket,
object_key, object_version_id, object_sha256
FROM result_inventory_members WHERE attempt_id = $1 AND member_id = $2";

4
crates/remora-result/src/lib.rs
View file

@ -9,6 +9,6 @@ pub use error::ResultError;
pub use inventory::ResultInventory;
pub use migrate::{migrate, migrator};
pub use types::{
CommittedResult, MANIFEST_SCHEMA_VERSION, ManifestObjectIdentity, PageSize, PublicationInput,
ResultMember, ResultMemberPage,
CommittedResult, MANIFEST_SCHEMA_VERSION, ManifestBuildMetrics, ManifestObjectIdentity,
PageSize, PublicationInput, ResultMember, ResultMemberPage,
};

6
crates/remora-result/src/types.rs
View file

@ -138,6 +138,12 @@ pub struct PublicationInput {
pub type CommittedResult = PublicationInput;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct ManifestBuildMetrics {
pub members_streamed: u64,
pub max_encoded_member_bytes: usize,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ResultMemberPage {
pub items: Vec<ResultMember>,

128
crates/remora-result/tests/substrate.rs
View file

@ -7,8 +7,10 @@ use remora_result::{
MANIFEST_SCHEMA_VERSION, PageSize, ResultError, ResultInventory, ResultMember,
migrate as result_migrate,
};
use remora_stream::TargetBody;
use remora_target::{
ContentSha256, CreateObjectRequest, ObjectIdentity, TargetStore, TargetStoreConfig, hash_bytes,
ContentSha256, CreateObjectRequest, ObjectIdentity, StoredObject, TargetStore,
TargetStoreConfig, hash_bytes,
};
use serde_json::Value;
use sqlx::PgPool;
@ -223,47 +225,38 @@ async fn integrity_stale_attempt_and_descriptor_version_fail_closed() {
#[tokio::test]
async fn pages_large_ordered_inventory_without_materializing_all_rows() {
let Some(pool) = test_pool().await else {
let (Some(pool), Some(target)) = (test_pool().await, target_store()) else {
return;
};
let inventory = ResultInventory::new(pool.clone());
let inventory = ResultInventory::new(pool);
let attempt = unique_id("attempt");
inventory.create(&attempt).await.unwrap();
let digest = hash_bytes(b"");
for index in (0..2_500_u32).rev() {
let member_id = OpaqueId::parse(format!("member-{index:08}")).unwrap();
let stored = target
.put_create_only(
&CreateObjectRequest::for_attempt_object(&attempt, &member_id, None),
Vec::new(),
)
.await
.unwrap();
let member = ResultMember::new(
member_id,
&format!("inventory/{index:08}"),
0,
member_identity(&attempt, &unique_id("object")),
digest.clone(),
stored.size,
stored.identity,
stored.content_sha256,
)
.unwrap();
inventory.add_member(&attempt, &member).await.unwrap();
}
inventory.close(&attempt).await.unwrap();
let mut committed = placeholder_publication(&attempt);
committed.member_count = 2_500;
sqlx::query(
"UPDATE result_inventories SET
state = 'sealed', manifest_schema_version = $2, manifest_id = $3,
manifest_store = $4, manifest_bucket = $5, manifest_key = $6,
manifest_version_id = $7, manifest_sha256 = $8, manifest_size = $9
WHERE attempt_id = $1",
)
.bind(attempt.as_str())
.bind(committed.schema_version as i32)
.bind(committed.manifest_id.as_str())
.bind(&committed.manifest.identity.target_store_id)
.bind(&committed.manifest.identity.bucket)
.bind(&committed.manifest.identity.key)
.bind(&committed.manifest.identity.version_id)
.bind(committed.manifest.content_sha256.as_str())
.bind(committed.manifest.size as i64)
.execute(&pool)
let (committed, metrics) = inventory
.create_manifest_with_metrics(&attempt, &unique_id("manifest"), &target)
.await
.unwrap();
assert_eq!(metrics.members_streamed, 2_500);
assert!(metrics.max_encoded_member_bytes < 1_024);
assert_eq!(committed.member_count, 2_500);
let mut after = None;
let mut observed = 0_u32;
@ -284,6 +277,87 @@ async fn pages_large_ordered_inventory_without_materializing_all_rows() {
assert_eq!(observed, 2_500);
}
#[tokio::test]
async fn binary_path_order_is_identical_in_manifest_and_pages() {
let (Some(pool), Some(target)) = (test_pool().await, target_store()) else {
return;
};
let inventory = ResultInventory::new(pool);
let attempt = unique_id("attempt");
inventory.create(&attempt).await.unwrap();
for path in ["한/file", "é/file", "a/file", "Z/file"] {
let member = stored_member(&target, &attempt, path, b"").await;
inventory.add_member(&attempt, &member).await.unwrap();
}
let committed = inventory
.create_manifest(&attempt, &unique_id("manifest"), &target)
.await
.unwrap();
let manifest: Value = serde_json::from_slice(
&target
.read_exact(&committed.manifest.identity)
.await
.unwrap(),
)
.unwrap();
let manifest_paths = manifest["members"]
.as_array()
.unwrap()
.iter()
.map(|member| member["path"].as_str().unwrap())
.collect::<Vec<_>>();
assert_eq!(manifest_paths, ["Z/file", "a/file", "é/file", "한/file"]);
let page = inventory
.page_members(&committed, None, PageSize::new(10).unwrap())
.await
.unwrap();
let page_paths = page
.items
.iter()
.map(ResultMember::path)
.collect::<Vec<_>>();
assert_eq!(page_paths, manifest_paths);
}
#[tokio::test]
async fn large_member_integrity_validation_keeps_manifest_allocation_member_bounded() {
let (Some(pool), Some(target)) = (test_pool().await, target_store()) else {
return;
};
let inventory = ResultInventory::new(pool);
let attempt = unique_id("attempt");
let member_id = unique_id("member");
inventory.create(&attempt).await.unwrap();
let request = CreateObjectRequest::for_attempt_object(&attempt, &member_id, None);
let mut body = target.open_create_only(&request).await.unwrap();
let chunk = vec![0x5a; 1024 * 1024];
for _ in 0..32 {
body.write_all(&chunk).await.unwrap();
}
let written = body.finish_object().await.unwrap();
let stored = StoredObject {
identity: written.identity,
content_sha256: written.content_sha256,
size: written.bytes_written,
};
let member = ResultMember::new(
member_id,
"large/member.bin",
stored.size,
stored.identity,
stored.content_sha256,
)
.unwrap();
inventory.add_member(&attempt, &member).await.unwrap();
let (committed, metrics) = inventory
.create_manifest_with_metrics(&attempt, &unique_id("manifest"), &target)
.await
.unwrap();
assert_eq!(committed.total_bytes, 32 * 1024 * 1024);
assert_eq!(metrics.members_streamed, 1);
assert!(metrics.max_encoded_member_bytes < 1_024);
}
fn member_identity(attempt: &OpaqueId, object: &OpaqueId) -> ObjectIdentity {
ObjectIdentity {
target_store_id: "fixture-store".into(),