Generalize benchmark qualification environment

This commit is contained in:
ms2k 2026-07-15 02:49:39 +09:00
commit 2d4ec2ed8b
4 changed files with 91 additions and 52 deletions

View file

@ -203,7 +203,7 @@ fn repository_example_loads_with_ipv4_normalization_and_private_denies() {
resolved resolved
.effective_allow_prefixes() .effective_allow_prefixes()
.iter() .iter()
.any(|prefix| prefix.address() == [1, 1, 1, 1] && prefix.prefix_length() == 32) .any(|prefix| prefix.address() == [192, 0, 2, 53] && prefix.prefix_length() == 32)
); );
for (address, length) in [ for (address, length) in [
([10, 0, 0, 0], 8), ([10, 0, 0, 0], 8),
@ -281,9 +281,11 @@ fn selected_profile_loads_into_confined_nonserializable_authority() {
let resolved = load_trusted_configuration(request(&tree, config, environment)).unwrap(); let resolved = load_trusted_configuration(request(&tree, config, environment)).unwrap();
assert_eq!(resolved.selected_profile(), "default"); assert_eq!(resolved.selected_profile(), "default");
assert_eq!( assert!(
resolved.effective_allow_prefixes()[0].address(), resolved
[1, 2, 3, 0] .effective_allow_prefixes()
.iter()
.any(|prefix| prefix.address() == [192, 0, 2, 0] && prefix.prefix_length() == 24)
); );
assert!( assert!(
resolved resolved

View file

@ -736,13 +736,16 @@ changes, non-UTF-8 and case-folded matching, linked worktrees and nested
repositories, every operation class above, open-handle transitions, concurrent repositories, every operation class above, open-handle transitions, concurrent
host and sandbox mutation, router loss, and absence of silent loss or copy-back. host and sandbox mutation, router loss, and absence of silent loss or copy-back.
The same corpus defines the benchmark protocol. Qualification runs direct-host The same corpus defines the benchmark protocol. Qualification runs direct-host
and routed placements on the recorded reference environment, uses identical and routed placements within the recorded qualification class, uses identical
fresh fixture materialization for each sample, and reports both the routed fresh fixture materialization for each sample, and reports the direct-placement
absolute median and the paired routed/direct median ratio. Metadata, Git, offline calibration, routed absolute median, and paired routed/direct median ratio.
package installation, and offline build workloads must meet both limits. The Metadata, Git, offline package installation, and offline build workloads must
reference environment, preparation states, sampling order, variance tolerance, meet all three limits. Exact observed hardware, storage, filesystem, kernel, and
normalization, and numeric budgets are part of the corpus and may not be relaxed tool versions are generated run-result metadata rather than source-controlled
by a production implementation. eligibility identities. The qualification constraints, direct calibration,
preparation states, sampling order, variance tolerance, normalization, and
numeric budgets are part of the corpus and may not be relaxed by a production
implementation.
Ignored-path isolation is defense in depth and does not replace the mandatory Git Ignored-path isolation is defense in depth and does not replace the mandatory Git
Sentry protections below. Sentry protections below.

View file

@ -1,30 +1,52 @@
{ {
"schema": "lazaret-ignored-workspace-benchmark-v1", "schema": "lazaret-ignored-workspace-benchmark-v1",
"reference_environment": { "qualification_environment": {
"architecture": "x86_64", "architecture": "x86_64",
"kernel": "Linux 6.9 or later", "kernel": "Linux 6.9 or later",
"cpu": "reference x86_64 processor", "cpu": {
"cpu_set": "one hardware thread from each of eight physical cores", "physical_cores_minimum": 8,
"cpu_governor": "performance", "allocation": "one hardware thread from each allocated physical core",
"memory_bytes": 17179869184, "governor": "performance"
"swap": "disabled-for-benchmark-cgroup", },
"storage": "reference local NVMe-class block device", "memory": {
"storage_stack": "NVMe with dm-crypt and no network or copy-on-write layer", "bytes_minimum": 17179869184,
"filesystem": "XFS", "swap": "disabled-for-benchmark-cgroup"
"filesystem_block_bytes": 4096, },
"filesystem_mount_options": [ "storage": {
"rw", "kind": "local-block-device",
"nosuid", "network_backed": false,
"nodev", "copy_on_write": false
},
"filesystem": {
"type": "XFS",
"block_bytes": 4096,
"required_mount_options": [
"relatime", "relatime",
"inode64", "inode64"
"noquota"
], ],
"fuse_userspace": "qualification-result-version", "forbidden_features": [
"runsc": "qualification-result-version", "quota"
"git": "qualification-result-version", ]
"uv": "qualification-result-version", },
"rust": "qualification-result-version" "qualification_result_fields": [
"kernel_release",
"cpu_model",
"cpu_microcode",
"allocated_cpu_topology",
"memory_bytes",
"storage_model",
"storage_transport",
"storage_stack",
"filesystem_type",
"filesystem_block_bytes",
"filesystem_mount_options",
"fuse_userspace_version",
"runsc_version",
"git_version",
"uv_version",
"rust_version"
],
"qualification_result_policy": "record exact observed values in the generated run result, not the source tree, without requiring literal model or version equality; every supported component must satisfy the release matrix"
}, },
"placements": { "placements": {
"direct": "one native reference tree containing the same visible bytes and metadata on the same backing filesystem, without FUSE or runsc", "direct": "one native reference tree containing the same visible bytes and metadata on the same backing filesystem, without FUSE or runsc",
@ -38,6 +60,7 @@
"clock": "CLOCK_MONOTONIC_RAW elapsed nanoseconds", "clock": "CLOCK_MONOTONIC_RAW elapsed nanoseconds",
"statistic": "median", "statistic": "median",
"relative_statistic": "median of paired routed/direct elapsed-time ratios", "relative_statistic": "median of paired routed/direct elapsed-time ratios",
"direct_calibration_formula": "direct placement median <= absolute_median_ms / relative_median_ratio in every independent run and preparation state",
"median_definition": "sort numeric values ascending and select value at zero-based index floor(count/2); every measured count is odd", "median_definition": "sort numeric values ascending and select value at zero-based index floor(count/2); every measured count is odd",
"within_run_relative_mad_formula": "median(abs(sample - median(samples))) / median(samples), evaluated separately for direct elapsed times, routed elapsed times, and paired routed/direct ratios in each workload, preparation state, and independent run", "within_run_relative_mad_formula": "median(abs(sample - median(samples))) / median(samples), evaluated separately for direct elapsed times, routed elapsed times, and paired routed/direct ratios in each workload, preparation state, and independent run",
"between_run_median_spread_formula": "(max(run_medians) - min(run_medians)) / median(run_medians), evaluated across the three independent-run medians separately for direct elapsed time, routed elapsed time, and paired routed/direct ratio in each workload and preparation state", "between_run_median_spread_formula": "(max(run_medians) - min(run_medians)) / median(run_medians), evaluated across the three independent-run medians separately for direct elapsed time, routed elapsed time, and paired routed/direct ratio in each workload and preparation state",
@ -52,9 +75,9 @@
"primed": "fresh materialization followed by one untimed complete workload invocation before measured samples" "primed": "fresh materialization followed by one untimed complete workload invocation before measured samples"
}, },
"normalization": { "normalization": {
"environment": "reject a run that differs from any reference environment field", "environment": "reject a run that violates any qualification-environment constraint or direct-placement calibration limit; record every exact observed qualification-result field",
"background_load": "reject a sample when another process consumes more than five percent of one reference core or benchmark memory pressure causes reclaim", "background_load": "reject a sample when another process consumes more than five percent of one reference core or benchmark memory pressure causes reclaim",
"pass_rule": "every independent run and preparation state meets both the absolute median and paired median-ratio budget and both variance limits" "pass_rule": "every independent run and preparation state meets the direct-placement calibration, routed absolute median, paired median-ratio budget, and both variance limits"
}, },
"identity_encoding": { "identity_encoding": {
"integer_encoding": "unsigned integers are fixed-width big-endian; the command exit status is signed two's-complement i32 big-endian", "integer_encoding": "unsigned integers are fixed-width big-endian; the command exit status is signed two's-complement i32 big-endian",

View file

@ -406,30 +406,37 @@ class IgnoredWorkspaceContractTests(unittest.TestCase):
validate_semantic_consistency(contract, copied_force_add) validate_semantic_consistency(contract, copied_force_add)
def test_benchmark_reference_conditions_are_complete(self) -> None: def test_benchmark_reference_conditions_are_complete(self) -> None:
reference = load("benchmark.json")["reference_environment"] qualification = load("benchmark.json")["qualification_environment"]
required = { required = {
"architecture", "architecture",
"kernel", "kernel",
"cpu", "cpu",
"cpu_set", "memory",
"cpu_governor",
"memory_bytes",
"swap",
"storage", "storage",
"storage_stack",
"filesystem", "filesystem",
"filesystem_block_bytes", "qualification_result_fields",
"filesystem_mount_options", "qualification_result_policy",
"fuse_userspace",
"runsc",
"git",
"uv",
"rust",
} }
self.assertEqual(set(reference), required) self.assertEqual(set(qualification), required)
self.assertEqual(reference["architecture"], "x86_64") self.assertEqual(qualification["architecture"], "x86_64")
self.assertEqual(reference["filesystem"], "XFS") self.assertEqual(
self.assertGreaterEqual(reference["memory_bytes"], 16 * 1024**3) set(qualification["cpu"]),
{"physical_cores_minimum", "allocation", "governor"},
)
self.assertGreaterEqual(qualification["cpu"]["physical_cores_minimum"], 8)
self.assertGreaterEqual(qualification["memory"]["bytes_minimum"], 16 * 1024**3)
self.assertEqual(
qualification["storage"],
{
"kind": "local-block-device",
"network_backed": False,
"copy_on_write": False,
},
)
self.assertEqual(qualification["filesystem"]["type"], "XFS")
self.assertEqual(qualification["filesystem"]["block_bytes"], 4096)
self.assertIn("cpu_model", qualification["qualification_result_fields"])
self.assertIn("storage_model", qualification["qualification_result_fields"])
def test_sampling_protocol_fixes_statistics_and_variance(self) -> None: def test_sampling_protocol_fixes_statistics_and_variance(self) -> None:
sampling = load("benchmark.json")["sampling"] sampling = load("benchmark.json")["sampling"]
@ -441,6 +448,10 @@ class IgnoredWorkspaceContractTests(unittest.TestCase):
sampling["relative_statistic"], sampling["relative_statistic"],
"median of paired routed/direct elapsed-time ratios", "median of paired routed/direct elapsed-time ratios",
) )
self.assertEqual(
sampling["direct_calibration_formula"],
"direct placement median <= absolute_median_ms / relative_median_ratio in every independent run and preparation state",
)
self.assertEqual( self.assertEqual(
sampling["within_run_relative_mad_formula"], sampling["within_run_relative_mad_formula"],
"median(abs(sample - median(samples))) / median(samples), evaluated separately for direct elapsed times, routed elapsed times, and paired routed/direct ratios in each workload, preparation state, and independent run", "median(abs(sample - median(samples))) / median(samples), evaluated separately for direct elapsed times, routed elapsed times, and paired routed/direct ratios in each workload, preparation state, and independent run",