ms2k/lazaret
SHA256
Watch
0
Code Releases 3 Activity

Generalize benchmark qualification environment

Browse source
This commit is contained in:
ms2k 2026-07-15 02:49:39 +09:00
commit 2d4ec2ed8b
4 changed files with 91 additions and 52 deletions
Download patch file Download diff file Expand all files Collapse all files

10
crates/lazaret-core/tests/configuration.rs
View file

@ -203,7 +203,7 @@ fn repository_example_loads_with_ipv4_normalization_and_private_denies() {
resolved
.effective_allow_prefixes()
.iter()
.any(|prefix| prefix.address() == [1, 1, 1, 1] && prefix.prefix_length() == 32)
.any(|prefix| prefix.address() == [192, 0, 2, 53] && prefix.prefix_length() == 32)
);
for (address, length) in [
([10, 0, 0, 0], 8),
@ -281,9 +281,11 @@ fn selected_profile_loads_into_confined_nonserializable_authority() {
let resolved = load_trusted_configuration(request(&tree, config, environment)).unwrap();
assert_eq!(resolved.selected_profile(), "default");
assert_eq!(
resolved.effective_allow_prefixes()[0].address(),
[1, 2, 3, 0]
assert!(
resolved
.effective_allow_prefixes()
.iter()
.any(|prefix| prefix.address() == [192, 0, 2, 0] && prefix.prefix_length() == 24)
);
assert!(
resolved

17
docs/specification.md
View file

@ -736,13 +736,16 @@ changes, non-UTF-8 and case-folded matching, linked worktrees and nested
repositories, every operation class above, open-handle transitions, concurrent
host and sandbox mutation, router loss, and absence of silent loss or copy-back.
The same corpus defines the benchmark protocol. Qualification runs direct-host
and routed placements on the recorded reference environment, uses identical
fresh fixture materialization for each sample, and reports both the routed
absolute median and the paired routed/direct median ratio. Metadata, Git, offline
package installation, and offline build workloads must meet both limits. The
reference environment, preparation states, sampling order, variance tolerance,
normalization, and numeric budgets are part of the corpus and may not be relaxed
by a production implementation.
and routed placements within the recorded qualification class, uses identical
fresh fixture materialization for each sample, and reports the direct-placement
calibration, routed absolute median, and paired routed/direct median ratio.
Metadata, Git, offline package installation, and offline build workloads must
meet all three limits. Exact observed hardware, storage, filesystem, kernel, and
tool versions are generated run-result metadata rather than source-controlled
eligibility identities. The qualification constraints, direct calibration,
preparation states, sampling order, variance tolerance, normalization, and
numeric budgets are part of the corpus and may not be relaxed by a production
implementation.
Ignored-path isolation is defense in depth and does not replace the mandatory Git
Sentry protections below.

71
tests/fixtures/ignored-workspace-v1/benchmark.json vendored
View file

@ -1,30 +1,52 @@
{
"schema": "lazaret-ignored-workspace-benchmark-v1",
"reference_environment": {
"qualification_environment": {
"architecture": "x86_64",
"kernel": "Linux 6.9 or later",
"cpu": "reference x86_64 processor",
"cpu_set": "one hardware thread from each of eight physical cores",
"cpu_governor": "performance",
"memory_bytes": 17179869184,
"swap": "disabled-for-benchmark-cgroup",
"storage": "reference local NVMe-class block device",
"storage_stack": "NVMe with dm-crypt and no network or copy-on-write layer",
"filesystem": "XFS",
"filesystem_block_bytes": 4096,
"filesystem_mount_options": [
"rw",
"nosuid",
"nodev",
"relatime",
"inode64",
"noquota"
"cpu": {
"physical_cores_minimum": 8,
"allocation": "one hardware thread from each allocated physical core",
"governor": "performance"
},
"memory": {
"bytes_minimum": 17179869184,
"swap": "disabled-for-benchmark-cgroup"
},
"storage": {
"kind": "local-block-device",
"network_backed": false,
"copy_on_write": false
},
"filesystem": {
"type": "XFS",
"block_bytes": 4096,
"required_mount_options": [
"relatime",
"inode64"
],
"forbidden_features": [
"quota"
]
},
"qualification_result_fields": [
"kernel_release",
"cpu_model",
"cpu_microcode",
"allocated_cpu_topology",
"memory_bytes",
"storage_model",
"storage_transport",
"storage_stack",
"filesystem_type",
"filesystem_block_bytes",
"filesystem_mount_options",
"fuse_userspace_version",
"runsc_version",
"git_version",
"uv_version",
"rust_version"
],
"fuse_userspace": "qualification-result-version",
"runsc": "qualification-result-version",
"git": "qualification-result-version",
"uv": "qualification-result-version",
"rust": "qualification-result-version"
"qualification_result_policy": "record exact observed values in the generated run result, not the source tree, without requiring literal model or version equality; every supported component must satisfy the release matrix"
},
"placements": {
"direct": "one native reference tree containing the same visible bytes and metadata on the same backing filesystem, without FUSE or runsc",
@ -38,6 +60,7 @@
"clock": "CLOCK_MONOTONIC_RAW elapsed nanoseconds",
"statistic": "median",
"relative_statistic": "median of paired routed/direct elapsed-time ratios",
"direct_calibration_formula": "direct placement median <= absolute_median_ms / relative_median_ratio in every independent run and preparation state",
"median_definition": "sort numeric values ascending and select value at zero-based index floor(count/2); every measured count is odd",
"within_run_relative_mad_formula": "median(abs(sample - median(samples))) / median(samples), evaluated separately for direct elapsed times, routed elapsed times, and paired routed/direct ratios in each workload, preparation state, and independent run",
"between_run_median_spread_formula": "(max(run_medians) - min(run_medians)) / median(run_medians), evaluated across the three independent-run medians separately for direct elapsed time, routed elapsed time, and paired routed/direct ratio in each workload and preparation state",
@ -52,9 +75,9 @@
"primed": "fresh materialization followed by one untimed complete workload invocation before measured samples"
},
"normalization": {
"environment": "reject a run that differs from any reference environment field",
"environment": "reject a run that violates any qualification-environment constraint or direct-placement calibration limit; record every exact observed qualification-result field",
"background_load": "reject a sample when another process consumes more than five percent of one reference core or benchmark memory pressure causes reclaim",
"pass_rule": "every independent run and preparation state meets both the absolute median and paired median-ratio budget and both variance limits"
"pass_rule": "every independent run and preparation state meets the direct-placement calibration, routed absolute median, paired median-ratio budget, and both variance limits"
},
"identity_encoding": {
"integer_encoding": "unsigned integers are fixed-width big-endian; the command exit status is signed two's-complement i32 big-endian",

45
tests/tooling/test_ignored_workspace_contract.py
View file

@ -406,30 +406,37 @@ class IgnoredWorkspaceContractTests(unittest.TestCase):
validate_semantic_consistency(contract, copied_force_add)
def test_benchmark_reference_conditions_are_complete(self) -> None:
reference = load("benchmark.json")["reference_environment"]
qualification = load("benchmark.json")["qualification_environment"]
required = {
"architecture",
"kernel",
"cpu",
"cpu_set",
"cpu_governor",
"memory_bytes",
"swap",
"memory",
"storage",
"storage_stack",
"filesystem",
"filesystem_block_bytes",
"filesystem_mount_options",
"fuse_userspace",
"runsc",
"git",
"uv",
"rust",
"qualification_result_fields",
"qualification_result_policy",
}
self.assertEqual(set(reference), required)
self.assertEqual(reference["architecture"], "x86_64")
self.assertEqual(reference["filesystem"], "XFS")
self.assertGreaterEqual(reference["memory_bytes"], 16 * 1024**3)
self.assertEqual(set(qualification), required)
self.assertEqual(qualification["architecture"], "x86_64")
self.assertEqual(
set(qualification["cpu"]),
{"physical_cores_minimum", "allocation", "governor"},
)
self.assertGreaterEqual(qualification["cpu"]["physical_cores_minimum"], 8)
self.assertGreaterEqual(qualification["memory"]["bytes_minimum"], 16 * 1024**3)
self.assertEqual(
qualification["storage"],
{
"kind": "local-block-device",
"network_backed": False,
"copy_on_write": False,
},
)
self.assertEqual(qualification["filesystem"]["type"], "XFS")
self.assertEqual(qualification["filesystem"]["block_bytes"], 4096)
self.assertIn("cpu_model", qualification["qualification_result_fields"])
self.assertIn("storage_model", qualification["qualification_result_fields"])
def test_sampling_protocol_fixes_statistics_and_variance(self) -> None:
sampling = load("benchmark.json")["sampling"]
@ -441,6 +448,10 @@ class IgnoredWorkspaceContractTests(unittest.TestCase):
sampling["relative_statistic"],
"median of paired routed/direct elapsed-time ratios",
)
self.assertEqual(
sampling["direct_calibration_formula"],
"direct placement median <= absolute_median_ms / relative_median_ratio in every independent run and preparation state",
)
self.assertEqual(
sampling["within_run_relative_mad_formula"],
"median(abs(sample - median(samples))) / median(samples), evaluated separately for direct elapsed times, routed elapsed times, and paired routed/direct ratios in each workload, preparation state, and independent run",