feat(bench): add matrix runner, gamescope and per-session controls

- bench.py: gamescope-headless display (default, 1920x1080; ~2% cost vs
  desktop for isolation from the shared Wayland desktop), --shader,
  --dh, --disable-mods/--add-mods, --world-zip, crash-safe restore
  journal, spectator mode in the plan (elevated cameras fell to the
  sea floor and S4 measured underwater darkness)
- matrix.toml/matrix.py: declarative, resumable B0-B14 runner keyed on
  mods hash (warns when B0 must be re-run)
- build.py: bench-only resource-pack variants (C1-fa, C1-fa-ext,
  emf-lod*-fa, C1-bl); every bench variant pins resourcePacks so packs
  don't leak between sessions; shipped tier output unchanged
- analyze.py: tagged labels use their own block's baseline
This commit is contained in:
logan committed 2026-10-16 22:14:45 -04:00
1 parent 8f56e2f780
commit a15c912ad0
10 files changed
+2196 -58

No files matched your search

+52 -10
View File
@@ -7,8 +7,10 @@
Groups windows by (label, variant, scene). Each session contributes its median p99 and
median 1 % low (frametime-based); a group's value is the median of its session medians
and its spread is max - min of those. With --baseline/--noise, every non-baseline group
is compared with the baseline variant on the same scene (same label if present, else
all labels pooled): win = median p99 improves by more than the noise label's p99 spread
is compared with the baseline variant on the same scene: same label if present, else the
same block (a tagged label 'B3.dh-off' looks in 'B3', 'B3.baseline', then all 'B3.*'
pooled), else all labels pooled. A tagged session of the baseline variant itself is
compared with 'B3' / 'B3.baseline' when one exists: win = median p99 improves by more than the noise label's p99 spread
AND 1 % low does not worsen by more than its 1 % low spread; loss = p99 worsens beyond
the spread or 1 % low drops beyond it; otherwise tie (ties keep upstream defaults).
Rows flagged crashed / timeout / no_data / no_markers / low_n / vsync_suspect are
@@ -79,18 +81,57 @@ def verdict(base, cand, spread_p99, spread_low):
return "tie"
def baseline_for(summary, rows, label, scene, baseline):
if (label, baseline, scene) in summary:
return summary[(label, baseline, scene)], label
pooled = [r for r in rows if r["variant"] == baseline and r["scene"] == scene]
def block_of(label):
"""'B3.dh-off' -> 'B3'; untagged labels (and None) are their own block."""
return label.split(".", 1)[0] if label else label
def _pooled(rows, labels, baseline, scene, name):
pooled = [r for r in rows if r["variant"] == baseline and r["scene"] == scene
and (labels is None or r.get("label") in labels)]
if not pooled:
return None
return summarise([dict(r, label=name) for r in pooled]).get((name, baseline, scene))
def baseline_for(summary, rows, label, scene, baseline):
"""(baseline stats, where-label) for one candidate group, or (None, None).
Order: the candidate's own label; then its block - the untagged block label
('B3', for tagged candidates), '<block>.baseline', then every '<block>.*' label pooled; then all labels
pooled. A tagged session of the baseline variant itself (e.g. 'B3.dh-off' running
C1 against baseline C1) is compared only with 'B3' or 'B3.baseline'."""
block = block_of(label)
tagged = bool(label) and block != label
order = [label] + ([block] if tagged else []) + ([f"{block}.baseline"] if block else [])
for lab in order:
if (lab, baseline, scene) in summary:
return summary[(lab, baseline, scene)], lab
if block:
labels = {r.get("label") for r in rows
if r.get("label") == block or str(r.get("label")).startswith(block + ".")}
b = _pooled(rows, labels, baseline, scene, f"{block}.*")
if b is not None:
return b, f"{block}.*"
b = _pooled(rows, None, baseline, scene, "*")
return (b, "*") if b is not None else (None, None)
def self_baseline(summary, label, scene, baseline):
"""Baseline for a tagged group that runs the baseline variant itself, or None
(then the group *is* the baseline)."""
block = block_of(label)
if not label or block == label:
return None, None
s = summarise([dict(r, label="*") for r in pooled])
return s.get(("*", baseline, scene)), "*"
for lab in (block, f"{block}.baseline"):
if lab != label and (lab, baseline, scene) in summary:
return summary[(lab, baseline, scene)], lab
return None, None
def table(summary, rows, baseline=None, noise=None):
spreads = noise_spread(summary, noise) if noise else None
full = summarise(rows) # baselines may live in labels filtered out of the table
hdr = ["label", "variant", "scene", "sess", "win#", "p99 ms", "p99 spr", "1%low",
"1%low spr", "avg fps"]
if baseline:
@@ -103,10 +144,11 @@ def table(summary, rows, baseline=None, noise=None):
f"{s['p99']:.2f}", f"{s['p99_spread']:.2f}", f"{s['low']:.1f}",
f"{s['low_spread']:.1f}", f"{s['fps']:.0f}"]
if baseline:
if var == baseline:
b, blab = (self_baseline(full, lab, scene, baseline) if var == baseline
else baseline_for(full, rows, lab, scene, baseline))
if var == baseline and b is None:
cells += ["—", "", "", "baseline"]
else:
b, blab = baseline_for(summary, rows, lab, scene, baseline)
if b is None:
cells += ["—", "", "", "no baseline"]
elif spreads is None: