feat(bench): add matrix runner, gamescope and per-session controls
- bench.py: gamescope-headless display (default, 1920x1080; ~2% cost vs desktop for isolation from the shared Wayland desktop), --shader, --dh, --disable-mods/--add-mods, --world-zip, crash-safe restore journal, spectator mode in the plan (elevated cameras fell to the sea floor and S4 measured underwater darkness) - matrix.toml/matrix.py: declarative, resumable B0-B14 runner keyed on mods hash (warns when B0 must be re-run) - build.py: bench-only resource-pack variants (C1-fa, C1-fa-ext, emf-lod*-fa, C1-bl); every bench variant pins resourcePacks so packs don't leak between sessions; shipped tier output unchanged - analyze.py: tagged labels use their own block's baseline
This commit is contained in:
1 parent
8f56e2f780
commit
a15c912ad0
10 files changed
+2196
-58
No files matched your search
@@ -7,8 +7,10 @@
|
||||
Groups windows by (label, variant, scene). Each session contributes its median p99 and
|
||||
median 1 % low (frametime-based); a group's value is the median of its session medians
|
||||
and its spread is max - min of those. With --baseline/--noise, every non-baseline group
|
||||
is compared with the baseline variant on the same scene (same label if present, else
|
||||
all labels pooled): win = median p99 improves by more than the noise label's p99 spread
|
||||
is compared with the baseline variant on the same scene: same label if present, else the
|
||||
same block (a tagged label 'B3.dh-off' looks in 'B3', 'B3.baseline', then all 'B3.*'
|
||||
pooled), else all labels pooled. A tagged session of the baseline variant itself is
|
||||
compared with 'B3' / 'B3.baseline' when one exists: win = median p99 improves by more than the noise label's p99 spread
|
||||
AND 1 % low does not worsen by more than its 1 % low spread; loss = p99 worsens beyond
|
||||
the spread or 1 % low drops beyond it; otherwise tie (ties keep upstream defaults).
|
||||
Rows flagged crashed / timeout / no_data / no_markers / low_n / vsync_suspect are
|
||||
@@ -79,18 +81,57 @@ def verdict(base, cand, spread_p99, spread_low):
|
||||
return "tie"
|
||||
|
||||
|
||||
def baseline_for(summary, rows, label, scene, baseline):
|
||||
if (label, baseline, scene) in summary:
|
||||
return summary[(label, baseline, scene)], label
|
||||
pooled = [r for r in rows if r["variant"] == baseline and r["scene"] == scene]
|
||||
def block_of(label):
|
||||
"""'B3.dh-off' -> 'B3'; untagged labels (and None) are their own block."""
|
||||
return label.split(".", 1)[0] if label else label
|
||||
|
||||
|
||||
def _pooled(rows, labels, baseline, scene, name):
|
||||
pooled = [r for r in rows if r["variant"] == baseline and r["scene"] == scene
|
||||
and (labels is None or r.get("label") in labels)]
|
||||
if not pooled:
|
||||
return None
|
||||
return summarise([dict(r, label=name) for r in pooled]).get((name, baseline, scene))
|
||||
|
||||
|
||||
def baseline_for(summary, rows, label, scene, baseline):
|
||||
"""(baseline stats, where-label) for one candidate group, or (None, None).
|
||||
|
||||
Order: the candidate's own label; then its block - the untagged block label
|
||||
('B3', for tagged candidates), '<block>.baseline', then every '<block>.*' label pooled; then all labels
|
||||
pooled. A tagged session of the baseline variant itself (e.g. 'B3.dh-off' running
|
||||
C1 against baseline C1) is compared only with 'B3' or 'B3.baseline'."""
|
||||
block = block_of(label)
|
||||
tagged = bool(label) and block != label
|
||||
order = [label] + ([block] if tagged else []) + ([f"{block}.baseline"] if block else [])
|
||||
for lab in order:
|
||||
if (lab, baseline, scene) in summary:
|
||||
return summary[(lab, baseline, scene)], lab
|
||||
if block:
|
||||
labels = {r.get("label") for r in rows
|
||||
if r.get("label") == block or str(r.get("label")).startswith(block + ".")}
|
||||
b = _pooled(rows, labels, baseline, scene, f"{block}.*")
|
||||
if b is not None:
|
||||
return b, f"{block}.*"
|
||||
b = _pooled(rows, None, baseline, scene, "*")
|
||||
return (b, "*") if b is not None else (None, None)
|
||||
|
||||
|
||||
def self_baseline(summary, label, scene, baseline):
|
||||
"""Baseline for a tagged group that runs the baseline variant itself, or None
|
||||
(then the group *is* the baseline)."""
|
||||
block = block_of(label)
|
||||
if not label or block == label:
|
||||
return None, None
|
||||
s = summarise([dict(r, label="*") for r in pooled])
|
||||
return s.get(("*", baseline, scene)), "*"
|
||||
for lab in (block, f"{block}.baseline"):
|
||||
if lab != label and (lab, baseline, scene) in summary:
|
||||
return summary[(lab, baseline, scene)], lab
|
||||
return None, None
|
||||
|
||||
|
||||
def table(summary, rows, baseline=None, noise=None):
|
||||
spreads = noise_spread(summary, noise) if noise else None
|
||||
full = summarise(rows) # baselines may live in labels filtered out of the table
|
||||
hdr = ["label", "variant", "scene", "sess", "win#", "p99 ms", "p99 spr", "1%low",
|
||||
"1%low spr", "avg fps"]
|
||||
if baseline:
|
||||
@@ -103,10 +144,11 @@ def table(summary, rows, baseline=None, noise=None):
|
||||
f"{s['p99']:.2f}", f"{s['p99_spread']:.2f}", f"{s['low']:.1f}",
|
||||
f"{s['low_spread']:.1f}", f"{s['fps']:.0f}"]
|
||||
if baseline:
|
||||
if var == baseline:
|
||||
b, blab = (self_baseline(full, lab, scene, baseline) if var == baseline
|
||||
else baseline_for(full, rows, lab, scene, baseline))
|
||||
if var == baseline and b is None:
|
||||
cells += ["—", "", "", "baseline"]
|
||||
else:
|
||||
b, blab = baseline_for(summary, rows, lab, scene, baseline)
|
||||
if b is None:
|
||||
cells += ["—", "", "", "no baseline"]
|
||||
elif spreads is None:
|
||||
|
||||
Reference in new issue
Block a user