Stabilize result ordering and streaming evidence

Dockerfile.test

@@ -158,11 +158,20 @@ RUN cargo nextest archive \
     --locked \
     --archive-file /tmp/remora-result-tests.tar.zst
 
+FROM kind-result-build AS kind-result-verified
+RUN cargo clippy \
+    --package remora-result \
+    --all-targets \
+    --all-features \
+    --locked \
+    -- -D warnings \
+    && cargo test --package remora-result --doc --locked
+
 FROM rust-tools AS kind-result-runner
 WORKDIR /workspace
 COPY Cargo.toml ./
 COPY crates crates
-COPY --from=kind-result-build /tmp/remora-result-tests.tar.zst /tests/remora-result-tests.tar.zst
+COPY --from=kind-result-verified /tmp/remora-result-tests.tar.zst /tests/remora-result-tests.tar.zst
 CMD ["bash", "-ec", "test -s /run/remora-fixture/profile.env && test -n \"$DATABASE_URL\" && test -n \"$REMORA_TARGET_ENDPOINT\" && case $REMORA_TARGET_PROVIDER in garage|versitygw) ;; *) exit 1 ;; esac && export REMORA_RESULT_REQUIRE=1 && cargo-nextest nextest run --archive-file /tests/remora-result-tests.tar.zst --workspace-remap /workspace --test-threads=1"]
 
 FROM ${NODE_IMAGE} AS node-runtime

crates/remora-result/migrations/0001_result_inventory.sql

@@ -24,8 +24,8 @@ CREATE TABLE result_inventories (
 
 CREATE TABLE result_inventory_members (
     attempt_id TEXT NOT NULL REFERENCES result_inventories(attempt_id) ON DELETE CASCADE,
-    member_id TEXT NOT NULL,
+    member_id TEXT COLLATE "C" NOT NULL,
-    path TEXT NOT NULL,
+    path TEXT COLLATE "C" NOT NULL,
     size BIGINT NOT NULL CHECK (size >= 0),
     object_store TEXT NOT NULL,
     object_bucket TEXT NOT NULL,

crates/remora-result/src/inventory.rs

@@ -10,8 +10,8 @@ use sqlx::PgPool;
 
 use crate::ResultError;
 use crate::types::{
-    CommittedResult, MANIFEST_SCHEMA_VERSION, PageSize, PublicationInput, ResultMember,
+    CommittedResult, MANIFEST_SCHEMA_VERSION, ManifestBuildMetrics, PageSize, PublicationInput,
-    ResultMemberPage,
+    ResultMember, ResultMemberPage,
 };
 
 #[derive(Clone)]
@@ -131,10 +131,27 @@ impl ResultInventory {
         manifest_id: &OpaqueId,
         target: &TargetStore,
     ) -> Result<PublicationInput, ResultError> {
+        self.create_manifest_with_metrics(attempt_id, manifest_id, target)
+            .await
+            .map(|(input, _)| input)
+    }
+
+    pub async fn create_manifest_with_metrics(
+        &self,
+        attempt_id: &OpaqueId,
+        manifest_id: &OpaqueId,
+        target: &TargetStore,
+    ) -> Result<(PublicationInput, ManifestBuildMetrics), ResultError> {
         self.close(attempt_id).await?;
         if let Some(existing) = self.publication_input(attempt_id).await? {
             return if existing.manifest_id == *manifest_id {
-                Ok(existing)
+                Ok((
+                    existing,
+                    ManifestBuildMetrics {
+                        members_streamed: 0,
+                        max_encoded_member_bytes: 0,
+                    },
+                ))
             } else {
                 Err(ResultError::NotSealed)
             };
@@ -151,6 +168,10 @@ impl ResultInventory {
             .bind(attempt_id.as_str())
             .fetch(&self.pool);
         let mut first = true;
+        let mut metrics = ManifestBuildMetrics {
+            members_streamed: 0,
+            max_encoded_member_bytes: 0,
+        };
         while let Some(row) = rows.try_next().await? {
             if !first {
                 body.write_all(b",")
@@ -160,6 +181,8 @@ impl ResultInventory {
             first = false;
             let member = row.into_member()?;
             let encoded = serde_json::to_vec(&member).map_err(|_| ResultError::Integrity)?;
+            metrics.members_streamed = metrics.members_streamed.saturating_add(1);
+            metrics.max_encoded_member_bytes = metrics.max_encoded_member_bytes.max(encoded.len());
             body.write_all(&encoded)
                 .await
                 .map_err(ResultError::ManifestWrite)?;
@@ -182,7 +205,7 @@ impl ResultInventory {
             total_bytes,
         };
         self.persist_sealed(&input).await?;
-        Ok(input)
+        Ok((input, metrics))
     }
 
     pub async fn validate_publication_input(
@@ -343,11 +366,13 @@ const MEMBER_SELECT: &str = "SELECT member_id, path, size, object_store, object_
     FROM result_inventory_members WHERE attempt_id = $1";
 const MEMBER_SELECT_ORDERED: &str = "SELECT member_id, path, size, object_store, object_bucket,
     object_key, object_version_id, object_sha256
-    FROM result_inventory_members WHERE attempt_id = $1 ORDER BY path, member_id";
+    FROM result_inventory_members WHERE attempt_id = $1
+    ORDER BY path COLLATE \"C\", member_id COLLATE \"C\"";
 const MEMBER_PAGE_SELECT: &str = "SELECT member_id, path, size, object_store, object_bucket,
     object_key, object_version_id, object_sha256
     FROM result_inventory_members
-    WHERE attempt_id = $1 AND path > $2 ORDER BY path, member_id LIMIT $3";
+    WHERE attempt_id = $1 AND path COLLATE \"C\" > $2 COLLATE \"C\"
+    ORDER BY path COLLATE \"C\", member_id COLLATE \"C\" LIMIT $3";
 const MEMBER_LOOKUP_SELECT: &str = "SELECT member_id, path, size, object_store, object_bucket,
     object_key, object_version_id, object_sha256
     FROM result_inventory_members WHERE attempt_id = $1 AND member_id = $2";

crates/remora-result/src/lib.rs

@@ -9,6 +9,6 @@ pub use error::ResultError;
 pub use inventory::ResultInventory;
 pub use migrate::{migrate, migrator};
 pub use types::{
-    CommittedResult, MANIFEST_SCHEMA_VERSION, ManifestObjectIdentity, PageSize, PublicationInput,
+    CommittedResult, MANIFEST_SCHEMA_VERSION, ManifestBuildMetrics, ManifestObjectIdentity,
-    ResultMember, ResultMemberPage,
+    PageSize, PublicationInput, ResultMember, ResultMemberPage,
 };

crates/remora-result/src/types.rs

@@ -138,6 +138,12 @@ pub struct PublicationInput {
 
 pub type CommittedResult = PublicationInput;
 
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub struct ManifestBuildMetrics {
+    pub members_streamed: u64,
+    pub max_encoded_member_bytes: usize,
+}
+
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub struct ResultMemberPage {
     pub items: Vec<ResultMember>,

crates/remora-result/tests/substrate.rs

@@ -7,8 +7,10 @@ use remora_result::{
     MANIFEST_SCHEMA_VERSION, PageSize, ResultError, ResultInventory, ResultMember,
     migrate as result_migrate,
 };
+use remora_stream::TargetBody;
 use remora_target::{
-    ContentSha256, CreateObjectRequest, ObjectIdentity, TargetStore, TargetStoreConfig, hash_bytes,
+    ContentSha256, CreateObjectRequest, ObjectIdentity, StoredObject, TargetStore,
+    TargetStoreConfig, hash_bytes,
 };
 use serde_json::Value;
 use sqlx::PgPool;
@@ -223,47 +225,38 @@ async fn integrity_stale_attempt_and_descriptor_version_fail_closed() {
 
 #[tokio::test]
 async fn pages_large_ordered_inventory_without_materializing_all_rows() {
-    let Some(pool) = test_pool().await else {
+    let (Some(pool), Some(target)) = (test_pool().await, target_store()) else {
         return;
     };
-    let inventory = ResultInventory::new(pool.clone());
+    let inventory = ResultInventory::new(pool);
     let attempt = unique_id("attempt");
     inventory.create(&attempt).await.unwrap();
-    let digest = hash_bytes(b"");
     for index in (0..2_500_u32).rev() {
         let member_id = OpaqueId::parse(format!("member-{index:08}")).unwrap();
+        let stored = target
+            .put_create_only(
+                &CreateObjectRequest::for_attempt_object(&attempt, &member_id, None),
+                Vec::new(),
+            )
+            .await
+            .unwrap();
         let member = ResultMember::new(
             member_id,
             &format!("inventory/{index:08}"),
-            0,
+            stored.size,
-            member_identity(&attempt, &unique_id("object")),
+            stored.identity,
-            digest.clone(),
+            stored.content_sha256,
         )
         .unwrap();
         inventory.add_member(&attempt, &member).await.unwrap();
     }
-    inventory.close(&attempt).await.unwrap();
+    let (committed, metrics) = inventory
-    let mut committed = placeholder_publication(&attempt);
+        .create_manifest_with_metrics(&attempt, &unique_id("manifest"), &target)
-    committed.member_count = 2_500;
-    sqlx::query(
-        "UPDATE result_inventories SET
-            state = 'sealed', manifest_schema_version = $2, manifest_id = $3,
-            manifest_store = $4, manifest_bucket = $5, manifest_key = $6,
-            manifest_version_id = $7, manifest_sha256 = $8, manifest_size = $9
-         WHERE attempt_id = $1",
-    )
-    .bind(attempt.as_str())
-    .bind(committed.schema_version as i32)
-    .bind(committed.manifest_id.as_str())
-    .bind(&committed.manifest.identity.target_store_id)
-    .bind(&committed.manifest.identity.bucket)
-    .bind(&committed.manifest.identity.key)
-    .bind(&committed.manifest.identity.version_id)
-    .bind(committed.manifest.content_sha256.as_str())
-    .bind(committed.manifest.size as i64)
-    .execute(&pool)
         .await
         .unwrap();
+    assert_eq!(metrics.members_streamed, 2_500);
+    assert!(metrics.max_encoded_member_bytes < 1_024);
+    assert_eq!(committed.member_count, 2_500);
 
     let mut after = None;
     let mut observed = 0_u32;
@@ -284,6 +277,87 @@ async fn pages_large_ordered_inventory_without_materializing_all_rows() {
     assert_eq!(observed, 2_500);
 }
 
+#[tokio::test]
+async fn binary_path_order_is_identical_in_manifest_and_pages() {
+    let (Some(pool), Some(target)) = (test_pool().await, target_store()) else {
+        return;
+    };
+    let inventory = ResultInventory::new(pool);
+    let attempt = unique_id("attempt");
+    inventory.create(&attempt).await.unwrap();
+    for path in ["한/file", "é/file", "a/file", "Z/file"] {
+        let member = stored_member(&target, &attempt, path, b"").await;
+        inventory.add_member(&attempt, &member).await.unwrap();
+    }
+    let committed = inventory
+        .create_manifest(&attempt, &unique_id("manifest"), &target)
+        .await
+        .unwrap();
+    let manifest: Value = serde_json::from_slice(
+        &target
+            .read_exact(&committed.manifest.identity)
+            .await
+            .unwrap(),
+    )
+    .unwrap();
+    let manifest_paths = manifest["members"]
+        .as_array()
+        .unwrap()
+        .iter()
+        .map(|member| member["path"].as_str().unwrap())
+        .collect::<Vec<_>>();
+    assert_eq!(manifest_paths, ["Z/file", "a/file", "é/file", "한/file"]);
+    let page = inventory
+        .page_members(&committed, None, PageSize::new(10).unwrap())
+        .await
+        .unwrap();
+    let page_paths = page
+        .items
+        .iter()
+        .map(ResultMember::path)
+        .collect::<Vec<_>>();
+    assert_eq!(page_paths, manifest_paths);
+}
+
+#[tokio::test]
+async fn large_member_integrity_validation_keeps_manifest_allocation_member_bounded() {
+    let (Some(pool), Some(target)) = (test_pool().await, target_store()) else {
+        return;
+    };
+    let inventory = ResultInventory::new(pool);
+    let attempt = unique_id("attempt");
+    let member_id = unique_id("member");
+    inventory.create(&attempt).await.unwrap();
+    let request = CreateObjectRequest::for_attempt_object(&attempt, &member_id, None);
+    let mut body = target.open_create_only(&request).await.unwrap();
+    let chunk = vec![0x5a; 1024 * 1024];
+    for _ in 0..32 {
+        body.write_all(&chunk).await.unwrap();
+    }
+    let written = body.finish_object().await.unwrap();
+    let stored = StoredObject {
+        identity: written.identity,
+        content_sha256: written.content_sha256,
+        size: written.bytes_written,
+    };
+    let member = ResultMember::new(
+        member_id,
+        "large/member.bin",
+        stored.size,
+        stored.identity,
+        stored.content_sha256,
+    )
+    .unwrap();
+    inventory.add_member(&attempt, &member).await.unwrap();
+    let (committed, metrics) = inventory
+        .create_manifest_with_metrics(&attempt, &unique_id("manifest"), &target)
+        .await
+        .unwrap();
+    assert_eq!(committed.total_bytes, 32 * 1024 * 1024);
+    assert_eq!(metrics.members_streamed, 1);
+    assert!(metrics.max_encoded_member_bytes < 1_024);
+}
+
 fn member_identity(attempt: &OpaqueId, object: &OpaqueId) -> ObjectIdentity {
     ObjectIdentity {
         target_store_id: "fixture-store".into(),