Generalize benchmark qualification environment
This commit is contained in:
parent
c1b40f923a
commit
2d4ec2ed8b
4 changed files with 91 additions and 52 deletions
|
|
@ -203,7 +203,7 @@ fn repository_example_loads_with_ipv4_normalization_and_private_denies() {
|
||||||
resolved
|
resolved
|
||||||
.effective_allow_prefixes()
|
.effective_allow_prefixes()
|
||||||
.iter()
|
.iter()
|
||||||
.any(|prefix| prefix.address() == [1, 1, 1, 1] && prefix.prefix_length() == 32)
|
.any(|prefix| prefix.address() == [192, 0, 2, 53] && prefix.prefix_length() == 32)
|
||||||
);
|
);
|
||||||
for (address, length) in [
|
for (address, length) in [
|
||||||
([10, 0, 0, 0], 8),
|
([10, 0, 0, 0], 8),
|
||||||
|
|
@ -281,9 +281,11 @@ fn selected_profile_loads_into_confined_nonserializable_authority() {
|
||||||
let resolved = load_trusted_configuration(request(&tree, config, environment)).unwrap();
|
let resolved = load_trusted_configuration(request(&tree, config, environment)).unwrap();
|
||||||
|
|
||||||
assert_eq!(resolved.selected_profile(), "default");
|
assert_eq!(resolved.selected_profile(), "default");
|
||||||
assert_eq!(
|
assert!(
|
||||||
resolved.effective_allow_prefixes()[0].address(),
|
resolved
|
||||||
[1, 2, 3, 0]
|
.effective_allow_prefixes()
|
||||||
|
.iter()
|
||||||
|
.any(|prefix| prefix.address() == [192, 0, 2, 0] && prefix.prefix_length() == 24)
|
||||||
);
|
);
|
||||||
assert!(
|
assert!(
|
||||||
resolved
|
resolved
|
||||||
|
|
|
||||||
|
|
@ -736,13 +736,16 @@ changes, non-UTF-8 and case-folded matching, linked worktrees and nested
|
||||||
repositories, every operation class above, open-handle transitions, concurrent
|
repositories, every operation class above, open-handle transitions, concurrent
|
||||||
host and sandbox mutation, router loss, and absence of silent loss or copy-back.
|
host and sandbox mutation, router loss, and absence of silent loss or copy-back.
|
||||||
The same corpus defines the benchmark protocol. Qualification runs direct-host
|
The same corpus defines the benchmark protocol. Qualification runs direct-host
|
||||||
and routed placements on the recorded reference environment, uses identical
|
and routed placements within the recorded qualification class, uses identical
|
||||||
fresh fixture materialization for each sample, and reports both the routed
|
fresh fixture materialization for each sample, and reports the direct-placement
|
||||||
absolute median and the paired routed/direct median ratio. Metadata, Git, offline
|
calibration, routed absolute median, and paired routed/direct median ratio.
|
||||||
package installation, and offline build workloads must meet both limits. The
|
Metadata, Git, offline package installation, and offline build workloads must
|
||||||
reference environment, preparation states, sampling order, variance tolerance,
|
meet all three limits. Exact observed hardware, storage, filesystem, kernel, and
|
||||||
normalization, and numeric budgets are part of the corpus and may not be relaxed
|
tool versions are generated run-result metadata rather than source-controlled
|
||||||
by a production implementation.
|
eligibility identities. The qualification constraints, direct calibration,
|
||||||
|
preparation states, sampling order, variance tolerance, normalization, and
|
||||||
|
numeric budgets are part of the corpus and may not be relaxed by a production
|
||||||
|
implementation.
|
||||||
|
|
||||||
Ignored-path isolation is defense in depth and does not replace the mandatory Git
|
Ignored-path isolation is defense in depth and does not replace the mandatory Git
|
||||||
Sentry protections below.
|
Sentry protections below.
|
||||||
|
|
|
||||||
|
|
@ -1,30 +1,52 @@
|
||||||
{
|
{
|
||||||
"schema": "lazaret-ignored-workspace-benchmark-v1",
|
"schema": "lazaret-ignored-workspace-benchmark-v1",
|
||||||
"reference_environment": {
|
"qualification_environment": {
|
||||||
"architecture": "x86_64",
|
"architecture": "x86_64",
|
||||||
"kernel": "Linux 6.9 or later",
|
"kernel": "Linux 6.9 or later",
|
||||||
"cpu": "reference x86_64 processor",
|
"cpu": {
|
||||||
"cpu_set": "one hardware thread from each of eight physical cores",
|
"physical_cores_minimum": 8,
|
||||||
"cpu_governor": "performance",
|
"allocation": "one hardware thread from each allocated physical core",
|
||||||
"memory_bytes": 17179869184,
|
"governor": "performance"
|
||||||
"swap": "disabled-for-benchmark-cgroup",
|
},
|
||||||
"storage": "reference local NVMe-class block device",
|
"memory": {
|
||||||
"storage_stack": "NVMe with dm-crypt and no network or copy-on-write layer",
|
"bytes_minimum": 17179869184,
|
||||||
"filesystem": "XFS",
|
"swap": "disabled-for-benchmark-cgroup"
|
||||||
"filesystem_block_bytes": 4096,
|
},
|
||||||
"filesystem_mount_options": [
|
"storage": {
|
||||||
"rw",
|
"kind": "local-block-device",
|
||||||
"nosuid",
|
"network_backed": false,
|
||||||
"nodev",
|
"copy_on_write": false
|
||||||
"relatime",
|
},
|
||||||
"inode64",
|
"filesystem": {
|
||||||
"noquota"
|
"type": "XFS",
|
||||||
|
"block_bytes": 4096,
|
||||||
|
"required_mount_options": [
|
||||||
|
"relatime",
|
||||||
|
"inode64"
|
||||||
|
],
|
||||||
|
"forbidden_features": [
|
||||||
|
"quota"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"qualification_result_fields": [
|
||||||
|
"kernel_release",
|
||||||
|
"cpu_model",
|
||||||
|
"cpu_microcode",
|
||||||
|
"allocated_cpu_topology",
|
||||||
|
"memory_bytes",
|
||||||
|
"storage_model",
|
||||||
|
"storage_transport",
|
||||||
|
"storage_stack",
|
||||||
|
"filesystem_type",
|
||||||
|
"filesystem_block_bytes",
|
||||||
|
"filesystem_mount_options",
|
||||||
|
"fuse_userspace_version",
|
||||||
|
"runsc_version",
|
||||||
|
"git_version",
|
||||||
|
"uv_version",
|
||||||
|
"rust_version"
|
||||||
],
|
],
|
||||||
"fuse_userspace": "qualification-result-version",
|
"qualification_result_policy": "record exact observed values in the generated run result, not the source tree, without requiring literal model or version equality; every supported component must satisfy the release matrix"
|
||||||
"runsc": "qualification-result-version",
|
|
||||||
"git": "qualification-result-version",
|
|
||||||
"uv": "qualification-result-version",
|
|
||||||
"rust": "qualification-result-version"
|
|
||||||
},
|
},
|
||||||
"placements": {
|
"placements": {
|
||||||
"direct": "one native reference tree containing the same visible bytes and metadata on the same backing filesystem, without FUSE or runsc",
|
"direct": "one native reference tree containing the same visible bytes and metadata on the same backing filesystem, without FUSE or runsc",
|
||||||
|
|
@ -38,6 +60,7 @@
|
||||||
"clock": "CLOCK_MONOTONIC_RAW elapsed nanoseconds",
|
"clock": "CLOCK_MONOTONIC_RAW elapsed nanoseconds",
|
||||||
"statistic": "median",
|
"statistic": "median",
|
||||||
"relative_statistic": "median of paired routed/direct elapsed-time ratios",
|
"relative_statistic": "median of paired routed/direct elapsed-time ratios",
|
||||||
|
"direct_calibration_formula": "direct placement median <= absolute_median_ms / relative_median_ratio in every independent run and preparation state",
|
||||||
"median_definition": "sort numeric values ascending and select value at zero-based index floor(count/2); every measured count is odd",
|
"median_definition": "sort numeric values ascending and select value at zero-based index floor(count/2); every measured count is odd",
|
||||||
"within_run_relative_mad_formula": "median(abs(sample - median(samples))) / median(samples), evaluated separately for direct elapsed times, routed elapsed times, and paired routed/direct ratios in each workload, preparation state, and independent run",
|
"within_run_relative_mad_formula": "median(abs(sample - median(samples))) / median(samples), evaluated separately for direct elapsed times, routed elapsed times, and paired routed/direct ratios in each workload, preparation state, and independent run",
|
||||||
"between_run_median_spread_formula": "(max(run_medians) - min(run_medians)) / median(run_medians), evaluated across the three independent-run medians separately for direct elapsed time, routed elapsed time, and paired routed/direct ratio in each workload and preparation state",
|
"between_run_median_spread_formula": "(max(run_medians) - min(run_medians)) / median(run_medians), evaluated across the three independent-run medians separately for direct elapsed time, routed elapsed time, and paired routed/direct ratio in each workload and preparation state",
|
||||||
|
|
@ -52,9 +75,9 @@
|
||||||
"primed": "fresh materialization followed by one untimed complete workload invocation before measured samples"
|
"primed": "fresh materialization followed by one untimed complete workload invocation before measured samples"
|
||||||
},
|
},
|
||||||
"normalization": {
|
"normalization": {
|
||||||
"environment": "reject a run that differs from any reference environment field",
|
"environment": "reject a run that violates any qualification-environment constraint or direct-placement calibration limit; record every exact observed qualification-result field",
|
||||||
"background_load": "reject a sample when another process consumes more than five percent of one reference core or benchmark memory pressure causes reclaim",
|
"background_load": "reject a sample when another process consumes more than five percent of one reference core or benchmark memory pressure causes reclaim",
|
||||||
"pass_rule": "every independent run and preparation state meets both the absolute median and paired median-ratio budget and both variance limits"
|
"pass_rule": "every independent run and preparation state meets the direct-placement calibration, routed absolute median, paired median-ratio budget, and both variance limits"
|
||||||
},
|
},
|
||||||
"identity_encoding": {
|
"identity_encoding": {
|
||||||
"integer_encoding": "unsigned integers are fixed-width big-endian; the command exit status is signed two's-complement i32 big-endian",
|
"integer_encoding": "unsigned integers are fixed-width big-endian; the command exit status is signed two's-complement i32 big-endian",
|
||||||
|
|
|
||||||
|
|
@ -406,30 +406,37 @@ class IgnoredWorkspaceContractTests(unittest.TestCase):
|
||||||
validate_semantic_consistency(contract, copied_force_add)
|
validate_semantic_consistency(contract, copied_force_add)
|
||||||
|
|
||||||
def test_benchmark_reference_conditions_are_complete(self) -> None:
|
def test_benchmark_reference_conditions_are_complete(self) -> None:
|
||||||
reference = load("benchmark.json")["reference_environment"]
|
qualification = load("benchmark.json")["qualification_environment"]
|
||||||
required = {
|
required = {
|
||||||
"architecture",
|
"architecture",
|
||||||
"kernel",
|
"kernel",
|
||||||
"cpu",
|
"cpu",
|
||||||
"cpu_set",
|
"memory",
|
||||||
"cpu_governor",
|
|
||||||
"memory_bytes",
|
|
||||||
"swap",
|
|
||||||
"storage",
|
"storage",
|
||||||
"storage_stack",
|
|
||||||
"filesystem",
|
"filesystem",
|
||||||
"filesystem_block_bytes",
|
"qualification_result_fields",
|
||||||
"filesystem_mount_options",
|
"qualification_result_policy",
|
||||||
"fuse_userspace",
|
|
||||||
"runsc",
|
|
||||||
"git",
|
|
||||||
"uv",
|
|
||||||
"rust",
|
|
||||||
}
|
}
|
||||||
self.assertEqual(set(reference), required)
|
self.assertEqual(set(qualification), required)
|
||||||
self.assertEqual(reference["architecture"], "x86_64")
|
self.assertEqual(qualification["architecture"], "x86_64")
|
||||||
self.assertEqual(reference["filesystem"], "XFS")
|
self.assertEqual(
|
||||||
self.assertGreaterEqual(reference["memory_bytes"], 16 * 1024**3)
|
set(qualification["cpu"]),
|
||||||
|
{"physical_cores_minimum", "allocation", "governor"},
|
||||||
|
)
|
||||||
|
self.assertGreaterEqual(qualification["cpu"]["physical_cores_minimum"], 8)
|
||||||
|
self.assertGreaterEqual(qualification["memory"]["bytes_minimum"], 16 * 1024**3)
|
||||||
|
self.assertEqual(
|
||||||
|
qualification["storage"],
|
||||||
|
{
|
||||||
|
"kind": "local-block-device",
|
||||||
|
"network_backed": False,
|
||||||
|
"copy_on_write": False,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
self.assertEqual(qualification["filesystem"]["type"], "XFS")
|
||||||
|
self.assertEqual(qualification["filesystem"]["block_bytes"], 4096)
|
||||||
|
self.assertIn("cpu_model", qualification["qualification_result_fields"])
|
||||||
|
self.assertIn("storage_model", qualification["qualification_result_fields"])
|
||||||
|
|
||||||
def test_sampling_protocol_fixes_statistics_and_variance(self) -> None:
|
def test_sampling_protocol_fixes_statistics_and_variance(self) -> None:
|
||||||
sampling = load("benchmark.json")["sampling"]
|
sampling = load("benchmark.json")["sampling"]
|
||||||
|
|
@ -441,6 +448,10 @@ class IgnoredWorkspaceContractTests(unittest.TestCase):
|
||||||
sampling["relative_statistic"],
|
sampling["relative_statistic"],
|
||||||
"median of paired routed/direct elapsed-time ratios",
|
"median of paired routed/direct elapsed-time ratios",
|
||||||
)
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
sampling["direct_calibration_formula"],
|
||||||
|
"direct placement median <= absolute_median_ms / relative_median_ratio in every independent run and preparation state",
|
||||||
|
)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
sampling["within_run_relative_mad_formula"],
|
sampling["within_run_relative_mad_formula"],
|
||||||
"median(abs(sample - median(samples))) / median(samples), evaluated separately for direct elapsed times, routed elapsed times, and paired routed/direct ratios in each workload, preparation state, and independent run",
|
"median(abs(sample - median(samples))) / median(samples), evaluated separately for direct elapsed times, routed elapsed times, and paired routed/direct ratios in each workload, preparation state, and independent run",
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue