Bound browser upload identity memory

nuthatch.cabal

@@ -13,6 +13,7 @@ extra-source-files:
   web/src/transport.ts
   web/test/contract-integration.mjs
   web/test/fixtures/contract.json
+  web/test/fixtures/upload-content-sha256.json
 
 common shared
   default-language:   Haskell2010
@@ -417,6 +418,7 @@ test-suite nuthatch-unit
     Nuthatch.Test.Recovery
     Nuthatch.Test.Runtime
     Nuthatch.Test.Streaming
+    Nuthatch.Test.UploadIdentityFixture
   build-depends:
     , nuthatch
     , base

test/Nuthatch/Test/UploadIdentityFixture.hs

@@ -0,0 +1,77 @@
+{-# LANGUAGE DeriveAnyClass #-}
+{-# LANGUAGE DeriveGeneric #-}
+
+module Nuthatch.Test.UploadIdentityFixture
+  ( tests
+  ) where
+
+import           Crypto.Hash (Digest, SHA256, hash)
+import           Data.Aeson
+  ( FromJSON (..)
+  , eitherDecodeFileStrict
+  , withObject
+  , (.:)
+  )
+import qualified Data.ByteArray as BA
+import qualified Data.ByteString as BS
+import qualified Data.ByteString.Base16 as B16
+import qualified Data.Text as T
+import qualified Data.Text.Encoding as TE
+import           GHC.Generics (Generic)
+import           Test.Tasty (TestTree)
+import           Test.Tasty.HUnit (assertEqual, testCase)
+
+data Fixture = Fixture
+  { name    :: String
+  , source  :: FixtureSource
+  , sha256  :: T.Text
+  }
+  deriving stock (Generic)
+  deriving anyclass (FromJSON)
+
+data FixtureSource
+  = Utf8 T.Text
+  | Hex T.Text
+  | RepeatHex T.Text Int
+
+instance FromJSON FixtureSource where
+  parseJSON = withObject "FixtureSource" $ \object -> do
+    kind <- object .: "kind"
+    case (kind :: T.Text) of
+      "utf8" -> Utf8 <$> object .: "value"
+      "hex" -> Hex <$> object .: "value"
+      "repeat-hex" -> RepeatHex
+        <$> object .: "pattern"
+        <*> object .: "bytes"
+      _ -> fail ("unknown fixture source kind: " <> T.unpack kind)
+
+tests :: TestTree
+tests = testCase "Haskell SHA-256 matches browser upload identity fixtures" $ do
+  decoded <-
+    eitherDecodeFileStrict "web/test/fixtures/upload-content-sha256.json"
+      :: IO (Either String [Fixture])
+  fixtures <- either fail pure decoded
+  mapM_ verify fixtures
+
+verify :: Fixture -> IO ()
+verify Fixture {name, source, sha256} =
+  assertEqual name sha256 (digestHex (fixtureBytes source))
+
+digestHex :: BS.ByteString -> T.Text
+digestHex bytes =
+  TE.decodeUtf8 . B16.encode . BA.convert $
+    (hash bytes :: Digest SHA256)
+
+fixtureBytes :: FixtureSource -> BS.ByteString
+fixtureBytes = \case
+  Utf8 text -> TE.encodeUtf8 text
+  Hex text -> decodeHex text
+  RepeatHex patternText byteCount ->
+    let patternBytes = decodeHex patternText
+        repeats =
+          (byteCount + BS.length patternBytes - 1) `div` BS.length patternBytes
+    in BS.take byteCount (BS.concat (replicate repeats patternBytes))
+
+decodeHex :: T.Text -> BS.ByteString
+decodeHex text =
+  either error id (B16.decode (TE.encodeUtf8 text))

test/Spec.hs

@@ -10,6 +10,7 @@ import qualified Nuthatch.Test.Protocol as Protocol
 import qualified Nuthatch.Test.Recovery as Recovery
 import qualified Nuthatch.Test.Runtime as Runtime
 import qualified Nuthatch.Test.Streaming as Streaming
+import qualified Nuthatch.Test.UploadIdentityFixture as UploadIdentityFixture
 import           Test.Tasty (defaultMain, testGroup)
 
 main :: IO ()
@@ -20,6 +21,7 @@ main = defaultMain $
     , Protocol.tests
     , Recovery.tests
     , Streaming.tests
+    , UploadIdentityFixture.tests
     , Product.tests
     , Runtime.tests
     , Oidc.tests

web/package-lock.json

@@ -8,6 +8,7 @@
       "name": "nuthatch-web",
       "version": "0.0.0",
       "dependencies": {
+        "@noble/hashes": "2.2.0",
         "svelte": "5.56.6"
       },
       "devDependencies": {
@@ -119,6 +120,18 @@
         "@emnapi/runtime": "^1.7.1"
       }
     },
+    "node_modules/@noble/hashes": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-2.2.0.tgz",
+      "integrity": "sha512-IYqDGiTXab6FniAgnSdZwgWbomxpy9FtYvLKs7wCUs2a8RkITG+DFGO1DM9cr+E3/RgADRpFjrKVaJ1z6sjtEg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 20.19.0"
+      },
+      "funding": {
+        "url": "https://paulmillr.com/funding/"
+      }
+    },
     "node_modules/@oxc-project/types": {
       "version": "0.139.0",
       "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.139.0.tgz",

web/package.json

@@ -15,6 +15,7 @@
     "test": "node --test test/*.test.mjs"
   },
   "dependencies": {
+    "@noble/hashes": "2.2.0",
     "svelte": "5.56.6"
   },
   "devDependencies": {

web/src/workflows/upload-hash-core.ts

@@ -0,0 +1,23 @@
+import { sha256 } from '@noble/hashes/sha2.js';
+import { bytesToHex } from '@noble/hashes/utils.js';
+
+export type IncrementalSha256 = {
+  update(bytes: Uint8Array): void;
+  digest(): string;
+  destroy(): void;
+};
+
+export function createIncrementalSha256(): IncrementalSha256 {
+  const hash = sha256.create();
+  return {
+    update(bytes) {
+      hash.update(bytes);
+    },
+    digest() {
+      return bytesToHex(hash.digest());
+    },
+    destroy() {
+      hash.destroy();
+    }
+  };
+}

web/src/workflows/upload-hash-worker.ts

@@ -0,0 +1,28 @@
+import { createIncrementalSha256 } from './upload-hash-core.ts';
+
+type HashWorkerRequest =
+  | { kind: 'chunk'; bytes: ArrayBuffer }
+  | { kind: 'finish' };
+
+const hash = createIncrementalSha256();
+let finished = false;
+
+self.onmessage = (event: MessageEvent<HashWorkerRequest>) => {
+  try {
+    if (event.data.kind === 'chunk') {
+      hash.update(new Uint8Array(event.data.bytes));
+      self.postMessage({ kind: 'ready' });
+      return;
+    }
+    const digest = hash.digest();
+    finished = true;
+    self.postMessage({ kind: 'digest', digest });
+  } catch (error) {
+    self.postMessage({
+      kind: 'error',
+      message: error instanceof Error ? error.message : String(error)
+    });
+  } finally {
+    if (finished) hash.destroy();
+  }
+};

web/src/workflows/upload-identity.ts

@@ -1,6 +1,163 @@
-export async function uploadContentSha256(file: File): Promise<string> {
-  const digest = await crypto.subtle.digest('SHA-256', await file.arrayBuffer());
-  return Array.from(new Uint8Array(digest), (byte) =>
-    byte.toString(16).padStart(2, '0')
-  ).join('');
+export const UPLOAD_HASH_CHUNK_BYTES = 1024 * 1024;
+
+type HashWorkerResponse =
+  | { kind: 'ready' }
+  | { kind: 'digest'; digest: string }
+  | { kind: 'error'; message: string };
+
+export type HashWorker = {
+  onmessage: ((event: MessageEvent<HashWorkerResponse>) => void) | null;
+  onerror: ((event: ErrorEvent) => void) | null;
+  postMessage(message: unknown, transfer: Transferable[]): void;
+  terminate(): void;
+};
+
+export type UploadHashProgress = {
+  chunksRead: number;
+  liveChunks: number;
+  peakLiveChunks: number;
+  chunkBytes: number;
+};
+
+export type UploadHashOptions = {
+  signal?: AbortSignal;
+  chunkBytes?: number;
+  workerFactory?: () => HashWorker;
+  onProgress?: (progress: UploadHashProgress) => void | Promise<void>;
+};
+
+export async function uploadContentSha256(
+  file: Blob,
+  options: UploadHashOptions = {}
+): Promise<string> {
+  const chunkBytes = options.chunkBytes ?? UPLOAD_HASH_CHUNK_BYTES;
+  if (!Number.isSafeInteger(chunkBytes) || chunkBytes <= 0) {
+    throw new RangeError('Upload hash chunk size must be a positive integer.');
+  }
+  const worker =
+    options.workerFactory?.() ??
+    new Worker(new URL('./upload-hash-worker.ts', import.meta.url), {
+      type: 'module'
+    });
+  let chunksRead = 0;
+  let liveChunks = 0;
+  let peakLiveChunks = 0;
+
+  const report = (currentChunkBytes: number) =>
+    options.onProgress?.({
+      chunksRead,
+      liveChunks,
+      peakLiveChunks,
+      chunkBytes: currentChunkBytes
+    });
+
+  try {
+    throwIfAborted(options.signal);
+    for (let offset = 0; offset < file.size; offset += chunkBytes) {
+      const size = Math.min(chunkBytes, file.size - offset);
+      const bytes = await file.slice(offset, offset + size).arrayBuffer();
+      throwIfAborted(options.signal);
+      chunksRead += 1;
+      liveChunks += 1;
+      peakLiveChunks = Math.max(peakLiveChunks, liveChunks);
+      await waitForProgress(report(size), options.signal);
+      const response = await exchange(
+        worker,
+        { kind: 'chunk', bytes },
+        [bytes],
+        options.signal
+      );
+      liveChunks -= 1;
+      await waitForProgress(report(size), options.signal);
+      if (response.kind !== 'ready') {
+        throw new Error('Upload hash worker returned an unexpected response.');
+      }
+    }
+    const response = await exchange(
+      worker,
+      { kind: 'finish' },
+      [],
+      options.signal
+    );
+    if (response.kind !== 'digest') {
+      throw new Error('Upload hash worker returned an unexpected response.');
+    }
+    return response.digest;
+  } finally {
+    worker.terminate();
+  }
+}
+
+function exchange(
+  worker: HashWorker,
+  message: unknown,
+  transfer: Transferable[],
+  signal: AbortSignal | undefined
+): Promise<HashWorkerResponse> {
+  return new Promise((resolve, reject) => {
+    const cleanup = () => {
+      worker.onmessage = null;
+      worker.onerror = null;
+      signal?.removeEventListener('abort', aborted);
+    };
+    const aborted = () => {
+      cleanup();
+      reject(abortError());
+    };
+    worker.onmessage = (event) => {
+      cleanup();
+      if (event.data.kind === 'error') {
+        reject(new Error(event.data.message));
+      } else {
+        resolve(event.data);
+      }
+    };
+    worker.onerror = (event) => {
+      cleanup();
+      reject(new Error(event.message || 'Upload hash worker failed.'));
+    };
+    signal?.addEventListener('abort', aborted, { once: true });
+    if (signal?.aborted) {
+      aborted();
+      return;
+    }
+    worker.postMessage(message, transfer);
+  });
+}
+
+function throwIfAborted(signal: AbortSignal | undefined): void {
+  if (signal?.aborted) throw abortError();
+}
+
+function waitForProgress(
+  progress: void | Promise<void>,
+  signal: AbortSignal | undefined
+): Promise<void> {
+  if (progress === undefined) return Promise.resolve();
+  return new Promise((resolve, reject) => {
+    const cleanup = () => signal?.removeEventListener('abort', aborted);
+    const aborted = () => {
+      cleanup();
+      reject(abortError());
+    };
+    signal?.addEventListener('abort', aborted, { once: true });
+    if (signal?.aborted) {
+      aborted();
+      return;
+    }
+    Promise.resolve(progress).then(
+      () => {
+        cleanup();
+        resolve();
+      },
+      (error) => {
+        cleanup();
+        reject(error);
+      }
+    );
+  });
+}
+
+function abortError(): DOMException {
+  return new DOMException('Upload hashing was cancelled.', 'AbortError');
 }

web/test/browser-boundaries.test.mjs

@@ -24,6 +24,15 @@ test('Svelte components own neither HTTP nor browser persistence policy', async
   ]);
 });
 
+test('upload identity never buffers or digests a complete File', async () => {
+  const sources = await sourceFiles('src');
+  for (const file of sources) {
+    const source = await readFile(file, 'utf8');
+    assert.doesNotMatch(source, /\bfile\.arrayBuffer\s*\(/i, file);
+    assert.doesNotMatch(source, /\bcrypto\.subtle\.digest\s*\(/i, file);
+  }
+});
+
 async function sourceFiles(directory) {
   const entries = await readdir(directory, { withFileTypes: true });
   return (

web/test/fixtures/upload-content-sha256.json

@@ -0,0 +1,31 @@
+[
+  {
+    "name": "empty",
+    "source": { "kind": "hex", "value": "" },
+    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+  },
+  {
+    "name": "unicode",
+    "source": { "kind": "utf8", "value": "문서 α — upload 🐦" },
+    "sha256": "4f208359621d2df948b5590f2ca1dce5d9aa0a0cdfea99e2c1f14cd68ad6673d"
+  },
+  {
+    "name": "binary",
+    "source": { "kind": "hex", "value": "0001027f80feff00a55a" },
+    "sha256": "3503f46719b47bbc0705dfe864c012d47dd98f043e2e31e88c81ae8c02450378"
+  },
+  {
+    "name": "exact-chunk",
+    "source": { "kind": "repeat-hex", "pattern": "a5", "bytes": 1048576 },
+    "sha256": "16c7f1d8a38b4b84560e558ab03b13c82e2ff374d87eaacb4df22f03604e7a4f"
+  },
+  {
+    "name": "multi-chunk",
+    "source": {
+      "kind": "repeat-hex",
+      "pattern": "0123456789abcdef",
+      "bytes": 2097289
+    },
+    "sha256": "f5c3a40a9469303cefc467312c16b5728e7b242d60c7e2791e651ea219d4a2c0"
+  }
+]

web/test/upload-identity.test.mjs

@@ -0,0 +1,196 @@
+import assert from 'node:assert/strict';
+import { readFile } from 'node:fs/promises';
+import test from 'node:test';
+import { createIncrementalSha256 } from '../src/workflows/upload-hash-core.ts';
+import {
+  UPLOAD_HASH_CHUNK_BYTES,
+  uploadContentSha256
+} from '../src/workflows/upload-identity.ts';
+import {
+  captureUpload,
+  executeUpload,
+  initialUploadState
+} from '../src/workflows/upload.ts';
+
+const fixtures = JSON.parse(
+  await readFile(
+    new URL('./fixtures/upload-content-sha256.json', import.meta.url),
+    'utf8'
+  )
+);
+
+test('incremental browser SHA-256 matches shared fixtures', () => {
+  for (const fixture of fixtures) {
+    const hash = createIncrementalSha256();
+    const bytes = fixtureBytes(fixture.source);
+    for (let offset = 0; offset < bytes.length; offset += 7919) {
+      hash.update(bytes.subarray(offset, offset + 7919));
+    }
+    assert.equal(hash.digest(), fixture.sha256, fixture.name);
+    hash.destroy();
+  }
+});
+
+test('upload hashing retains at most one bounded chunk', async () => {
+  const size = UPLOAD_HASH_CHUNK_BYTES * 5 + 137;
+  const progress = [];
+  const worker = new FakeHashWorker();
+  const digest = await uploadContentSha256(new Blob([new Uint8Array(size)]), {
+    workerFactory: () => worker,
+    onProgress: (sample) => progress.push(sample)
+  });
+
+  assert.equal(
+    digest,
+    'ab58d3a8f7489bd83685df43c86a2b69b083dc86542f69603f57f7ae304939c2'
+  );
+  assert.equal(Math.max(...progress.map(({ liveChunks }) => liveChunks)), 1);
+  assert.equal(Math.max(...progress.map(({ peakLiveChunks }) => peakLiveChunks)), 1);
+  assert.ok(
+    progress.every(({ chunkBytes }) => chunkBytes <= UPLOAD_HASH_CHUNK_BYTES)
+  );
+  assert.equal(worker.terminateCount, 1);
+});
+
+test('cancellation terminates an in-flight worker without uploading', async () => {
+  let chunkPosted;
+  const posted = new Promise((resolve) => {
+    chunkPosted = resolve;
+  });
+  const worker = new FakeHashWorker({
+    holdChunks: true,
+    onChunk: chunkPosted
+  });
+  const controller = new AbortController();
+  const state = initialUploadState();
+  captureUpload(
+    state,
+    'Personal',
+    '/payload.bin',
+    new File([new Uint8Array(32)], 'payload.bin'),
+    'operation-1'
+  );
+  let uploads = 0;
+  const client = {
+    upload: async () => {
+      uploads += 1;
+      throw new Error('unexpected upload');
+    }
+  };
+  const hashing = executeUpload(client, { csrf: 'csrf' }, state, (file) =>
+    uploadContentSha256(file, {
+      signal: controller.signal,
+      workerFactory: () => worker
+    })
+  );
+
+  await posted;
+  controller.abort();
+  await assert.rejects(hashing, { name: 'AbortError' });
+  assert.equal(uploads, 0);
+  assert.equal(worker.terminateCount, 1);
+});
+
+test('cancellation interrupts stalled progress instrumentation', async () => {
+  let progressStarted;
+  const started = new Promise((resolve) => {
+    progressStarted = resolve;
+  });
+  const worker = new FakeHashWorker();
+  const controller = new AbortController();
+  const hashing = uploadContentSha256(new Blob([new Uint8Array(32)]), {
+    signal: controller.signal,
+    workerFactory: () => worker,
+    onProgress: () => {
+      progressStarted();
+      return new Promise(() => {});
+    }
+  });
+
+  await started;
+  controller.abort();
+  await assert.rejects(hashing, { name: 'AbortError' });
+  assert.equal(worker.terminateCount, 1);
+});
+
+test('worker failure prevents the upload request', async () => {
+  const file = new File([new Uint8Array(32)], 'payload.bin');
+  const state = initialUploadState();
+  captureUpload(state, 'Personal', '/payload.bin', file, 'operation-1');
+  let uploads = 0;
+  const client = {
+    upload: async () => {
+      uploads += 1;
+      throw new Error('unexpected upload');
+    }
+  };
+  const worker = new FakeHashWorker({ failure: 'worker unavailable' });
+
+  await assert.rejects(
+    executeUpload(client, { csrf: 'csrf' }, state, (pending) =>
+      uploadContentSha256(pending, { workerFactory: () => worker })
+    ),
+    /worker unavailable/
+  );
+  assert.equal(uploads, 0);
+  assert.equal(worker.terminateCount, 1);
+});
+
+class FakeHashWorker {
+  onmessage = null;
+  onerror = null;
+  terminateCount = 0;
+  #failure;
+  #hash = createIncrementalSha256();
+  #holdChunks;
+  #onChunk;
+
+  constructor({ failure, holdChunks = false, onChunk = () => {} } = {}) {
+    this.#failure = failure;
+    this.#holdChunks = holdChunks;
+    this.#onChunk = onChunk;
+  }
+
+  postMessage(message, transfer) {
+    const cloned = structuredClone(message, { transfer });
+    queueMicrotask(() => {
+      if (this.#failure !== undefined) {
+        this.onmessage?.({
+          data: { kind: 'error', message: this.#failure }
+        });
+        return;
+      }
+      if (cloned.kind === 'chunk') {
+        this.#onChunk();
+        if (this.#holdChunks) return;
+        this.#hash.update(new Uint8Array(cloned.bytes));
+        this.onmessage?.({ data: { kind: 'ready' } });
+        return;
+      }
+      const digest = this.#hash.digest();
+      this.onmessage?.({ data: { kind: 'digest', digest } });
+    });
+  }
+
+  terminate() {
+    this.terminateCount += 1;
+    this.#hash.destroy();
+  }
+}
+
+function fixtureBytes(source) {
+  if (source.kind === 'utf8') return new TextEncoder().encode(source.value);
+  if (source.kind === 'hex') return hexBytes(source.value);
+  const pattern = hexBytes(source.pattern);
+  return Uint8Array.from(
+    { length: source.bytes },
+    (_, index) => pattern[index % pattern.length]
+  );
+}
+
+function hexBytes(value) {
+  return Uint8Array.from(
+    value.match(/.{2}/g) ?? [],
+    (byte) => Number.parseInt(byte, 16)
+  );
+}