lazaret/tests/fixtures/ignored-workspace-v1/benchmark.json

166 lines
10 KiB
JSON
Raw Normal View History

{
"schema": "lazaret-ignored-workspace-benchmark-v1",
"qualification_environment": {
"architecture": "x86_64",
"kernel": "Linux 6.9 or later",
"cpu": {
"physical_cores_minimum": 8,
"allocation": "one hardware thread from each allocated physical core",
"governor": "performance"
},
"memory": {
"bytes_minimum": 17179869184,
"swap": "disabled-for-benchmark-cgroup"
},
"storage": {
"kind": "local-block-device",
"network_backed": false,
"copy_on_write": false
},
"filesystem": {
"type": "XFS",
"block_bytes": 4096,
"required_mount_options": [
"relatime",
"inode64"
],
"forbidden_features": [
"quota"
]
},
"qualification_result_fields": [
"kernel_release",
"cpu_model",
"cpu_microcode",
"allocated_cpu_topology",
"memory_bytes",
"storage_model",
"storage_transport",
"storage_stack",
"filesystem_type",
"filesystem_block_bytes",
"filesystem_mount_options",
"fuse_userspace_version",
"runsc_version",
"git_version",
"uv_version",
"rust_version"
],
"qualification_result_policy": "record exact observed values in the generated run result, not the source tree, without requiring literal model or version equality; every supported component must satisfy the release matrix"
},
"placements": {
"direct": "one native reference tree containing the same visible bytes and metadata on the same backing filesystem, without FUSE or runsc",
"routed": "the same backing filesystem through the production FUSE router and supported runsc"
},
"sampling": {
"independent_runs": 3,
"warmup_samples_per_cell": 2,
"measured_samples_per_cell": 15,
"order": "paired alternating direct-routed then routed-direct",
"clock": "CLOCK_MONOTONIC_RAW elapsed nanoseconds",
"statistic": "median",
"relative_statistic": "median of paired routed/direct elapsed-time ratios",
"direct_calibration_formula": "direct placement median <= absolute_median_ms / relative_median_ratio in every independent run and preparation state",
"median_definition": "sort numeric values ascending and select value at zero-based index floor(count/2); every measured count is odd",
"within_run_relative_mad_formula": "median(abs(sample - median(samples))) / median(samples), evaluated separately for direct elapsed times, routed elapsed times, and paired routed/direct ratios in each workload, preparation state, and independent run",
"between_run_median_spread_formula": "(max(run_medians) - min(run_medians)) / median(run_medians), evaluated across the three independent-run medians separately for direct elapsed time, routed elapsed time, and paired routed/direct ratio in each workload and preparation state",
"outlier_policy": "retain every completed measured sample",
"sample_timeout": "four times the workload absolute budget",
"within_run_relative_mad_limit": 0.10,
"between_run_median_spread_limit": 0.10,
"budget_tolerance": 0.0
},
"preparation_states": {
"unprimed": "fresh uniquely named materialization, empty workload output directories, and no fixture prewalk; operating-system cache dropping is not required",
"primed": "fresh materialization followed by one untimed complete workload invocation before measured samples"
},
"normalization": {
"environment": "reject a run that violates any qualification-environment constraint or direct-placement calibration limit; record every exact observed qualification-result field",
"background_load": "reject a sample when another process consumes more than five percent of one reference core or benchmark memory pressure causes reclaim",
"pass_rule": "every independent run and preparation state meets the direct-placement calibration, routed absolute median, paired median-ratio budget, and both variance limits"
},
"identity_encoding": {
"integer_encoding": "unsigned integers are fixed-width big-endian; the command exit status is signed two's-complement i32 big-endian",
"fixture": {
"context_hex": "6c617a617265742d69676e6f7265642d776f726b73706163652d666978747572652d763100",
"root_entry": "omitted",
"path": "raw relative pathname bytes with slash separators, no leading or trailing slash, and no NUL",
"ordering": "ascending lexicographic order of complete raw path byte strings",
"kind_tags": {
"directory": 1,
"regular": 2,
"symlink": 3
},
"mode": "u32 of st_mode masked with octal 7777",
"size": "u64 payload length for regular files and symlinks; zero for directories",
"payload": "regular file bytes, raw symlink target bytes, or empty bytes for a directory",
"record": "u32(path length) || path || u8(kind tag) || u32(mode) || u64(size) || SHA-256(payload)",
"stream": "context bytes || u32(record count) || records in declared ordering",
"identity": "SHA-256(stream)",
"unsupported_entry": "reject identity construction for any other file kind"
},
"result": {
"context_hex": "6c617a617265742d69676e6f7265642d776f726b73706163652d726573756c742d763100",
"workload": "u32(UTF-8 workload ID length) || UTF-8 workload ID",
"command_record": "i32(exit status) || u64(stdout length) || exact stdout bytes || u64(stderr length) || exact stderr bytes",
"ordering": "command records remain in command_sequence order",
"stream": "context bytes || framed workload || initial fixture identity || u32(command record count) || ordered command records || post-workload fixture identity",
"identity": "SHA-256(stream)",
"successful_sample": "every command exit status is zero and both fixture identities and every output byte are retained"
}
},
"workloads": [
{
"id": "metadata-10000",
"category": "metadata",
"fixture_seed": "b4e3a0d11bcd823e8ea7e9fc8a2e4ec4f4cb628e8df62df28ba369ecb2944dd7",
"fixture": "100 directories each containing 100 regular files of 64 bytes; half host-backed and half ignored isolated",
"fixture_algorithm": "directories d000 through d099; files f000 through f099; content is the 32-byte SHA-256 of fixture_seed, slash, and byte path repeated twice; even directories are host-backed and odd directories are ignored isolated",
"operation": "bytewise walk, lstat, open, read 64 bytes, close, and statx each file",
"command_sequence": [["lazaret-workspace-benchmark", "metadata", "--root", "ROOT", "--bytewise-order", "--statx-mask", "basic-stats"]],
"result_oracle": "exactly 10000 regular files, 640000 content bytes, and the canonical fixture identity",
"preparation_states": ["unprimed", "primed"],
"absolute_median_ms": 2000,
"relative_median_ratio": 25.0
},
{
"id": "git-2000-status-add-reset",
"category": "git",
"fixture_seed": "7ff9a3db8752e32d8d782c4c8d09d733be72f9dcba9d49d681b31193749f3ef4",
"fixture": "1000 tracked files, 500 ordinary untracked files, 500 ignored isolated files, and ignored host sentinels",
"fixture_algorithm": "tracked/t0000 through t0999 are committed; ordinary/u0000 through u0499 are untracked; ignored/i0000 through i0499 match one root ignore rule and have distinct host sentinel and isolated content derived from fixture_seed and byte path",
"operation": "porcelain-v2 status, add the ordinary untracked set, quiet mixed reset, and porcelain-v2 status",
"command_sequence": [["git", "status", "--porcelain=v2", "--untracked-files=all"], ["git", "add", "--", "ordinary"], ["git", "reset", "--quiet", "--mixed"], ["git", "status", "--porcelain=v2", "--untracked-files=all"]],
"result_oracle": "both status byte streams equal the locked direct placement streams, the index returns to its initial identity, and every ignored host sentinel is unchanged",
"preparation_states": ["unprimed", "primed"],
"absolute_median_ms": 2500,
"relative_median_ratio": 8.0
},
{
"id": "uv-offline-32-wheels",
"category": "package-manager",
"fixture_seed": "df5d5a7d287cc0a140d95c77a5f560db66929a138d5dfc854a6af4f90d609184",
"fixture": "one root wheel and 31 deterministic local dependency wheels with no build scripts or network access",
"fixture_algorithm": "packages lazaret-bench-pkg-00 through lazaret-bench-pkg-31 at version 1.0.0; package 00 depends exactly on packages 01 through 31; each pure-Python wheel contains one module whose VALUE is the SHA-256 of fixture_seed and normalized package name",
"operation": "uv pip install --offline --no-index into a fresh ignored virtual environment",
"command_sequence": [["uv", "pip", "install", "--offline", "--no-index", "--find-links", "WHEELHOUSE", "--python", "VENV/bin/python", "lazaret-bench-pkg-00==1.0.0"]],
"result_oracle": "all 32 exact distributions are installed and importing every module yields its fixture-derived VALUE",
"preparation_states": ["unprimed", "primed"],
"absolute_median_ms": 5000,
"relative_median_ratio": 6.0
},
{
"id": "cargo-offline-32-crates",
"category": "build",
"fixture_seed": "42324c58067ac0869fb4ec486f42fec74e0f3147a12aa2aee80af45c3bb8595d",
"fixture": "one binary and 31 path-library crates with deterministic generated Rust sources and an exact lockfile",
"fixture_algorithm": "workspace packages lazaret-bench-crate-00 through lazaret-bench-crate-31 at version 1.0.0; package 00 is a binary with exact path dependencies on libraries 01 through 31; each library returns the first eight bytes of SHA-256 of fixture_seed and package name as a u64 constant",
"operation": "cargo check --offline --locked --workspace with a fresh ignored target directory",
"command_sequence": [["cargo", "check", "--offline", "--locked", "--workspace", "--target-dir", "TARGET"]],
"result_oracle": "exit zero, no lockfile change, no source-tree change, and all output remains beneath the ignored target directory",
"preparation_states": ["unprimed", "primed"],
"absolute_median_ms": 8000,
"relative_median_ratio": 5.0
}
]
}