feat(bench): add matrix runner, gamescope and per-session controls
- bench.py: gamescope-headless display (default, 1920x1080; ~2% cost vs desktop for isolation from the shared Wayland desktop), --shader, --dh, --disable-mods/--add-mods, --world-zip, crash-safe restore journal, spectator mode in the plan (elevated cameras fell to the sea floor and S4 measured underwater darkness) - matrix.toml/matrix.py: declarative, resumable B0-B14 runner keyed on mods hash (warns when B0 must be re-run) - build.py: bench-only resource-pack variants (C1-fa, C1-fa-ext, emf-lod*-fa, C1-bl); every bench variant pins resourcePacks so packs don't leak between sessions; shipped tier output unchanged - analyze.py: tagged labels use their own block's baseline
This commit is contained in:
1 parent
8f56e2f780
commit
a15c912ad0
10 files changed
+2196
-58
No files matched your search
@@ -0,0 +1,97 @@
|
||||
"""analyze.py: tagged labels ('B3.dh-off') take their baseline from the same block.
|
||||
|
||||
python3 -m unittest discover -s analysis/client-perf/harness/tests
|
||||
"""
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE.parent))
|
||||
|
||||
import analyze # noqa: E402
|
||||
|
||||
|
||||
def row(label, variant, scene, session, p99, low=100.0, fps=200.0):
|
||||
return {"kind": "window", "label": label, "variant": variant, "scene": scene,
|
||||
"session": session, "flags": [],
|
||||
"metrics": {"p99_ms": p99, "one_low_fps": low, "avg_fps": fps}}
|
||||
|
||||
|
||||
def cell(table, label, variant, scene):
|
||||
for line in table.splitlines():
|
||||
cells = [c.strip() for c in line.strip("|").split("|")]
|
||||
if cells[:3] == [label, variant, scene]:
|
||||
return cells
|
||||
raise AssertionError(f"no row {label}/{variant}/{scene} in\n{table}")
|
||||
|
||||
|
||||
class BlockBaseline(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.rows = [
|
||||
# noise: spread 0.2 ms / 2 fps
|
||||
row("B0", "C0", "s1", "n1", 10.0, 100), row("B0", "C0", "s1", "n2", 10.2, 102),
|
||||
# global C1 reference elsewhere (B2) is much slower than the B3 one
|
||||
row("B2", "C1", "s1", "g1", 20.0),
|
||||
row("B3", "C1", "s1", "b1", 10.0),
|
||||
row("B3", "dh-r64", "s1", "b2", 8.0),
|
||||
row("B3.dh-off", "C1", "s1", "b3", 6.0),
|
||||
]
|
||||
|
||||
def test_tagged_self_variant_uses_untagged_block_label(self):
|
||||
s = analyze.summarise(self.rows)
|
||||
b, where = analyze.self_baseline(s, "B3.dh-off", "s1", "C1")
|
||||
self.assertEqual(where, "B3")
|
||||
self.assertEqual(b["p99"], 10.0)
|
||||
|
||||
def test_untagged_block_label_is_its_own_baseline(self):
|
||||
s = analyze.summarise(self.rows)
|
||||
self.assertEqual(analyze.self_baseline(s, "B3", "s1", "C1"), (None, None))
|
||||
|
||||
def test_tagged_other_variant_prefers_block_over_global(self):
|
||||
rows = self.rows + [row("B12a.no-x", "X", "s1", "t1", 9.0)]
|
||||
rows += [row("B12a", "C1", "s1", "t0", 12.0)]
|
||||
s = analyze.summarise(rows)
|
||||
b, where = analyze.baseline_for(s, rows, "B12a.no-x", "s1", "C1")
|
||||
self.assertEqual((where, b["p99"]), ("B12a", 12.0))
|
||||
|
||||
def test_block_baseline_tag(self):
|
||||
rows = self.rows + [row("B11.baseline", "C1", "s5", "x0", 12.0),
|
||||
row("B11", "sod-thr3", "s5", "x1", 11.0),
|
||||
row("B2", "C1", "s5", "x2", 30.0)]
|
||||
s = analyze.summarise(rows)
|
||||
b, where = analyze.baseline_for(s, rows, "B11", "s5", "C1")
|
||||
# untagged candidate: own label has no C1 -> '<block>.baseline', not global
|
||||
self.assertEqual((where, b["p99"]), ("B11.baseline", 12.0))
|
||||
rows.append(row("B11.x", "opt-mip4", "s5", "x3", 11.0))
|
||||
s = analyze.summarise(rows)
|
||||
b, where = analyze.baseline_for(s, rows, "B11.x", "s5", "C1")
|
||||
self.assertEqual((where, b["p99"]), ("B11.baseline", 12.0))
|
||||
|
||||
def test_block_pool_then_global(self):
|
||||
rows = self.rows + [row("B4.J0", "C1", "s2", "j0", 10.0),
|
||||
row("B4.J1", "C1", "s2", "j1", 12.0),
|
||||
row("B4.x", "Y", "s2", "j2", 9.0),
|
||||
row("B2", "C1", "s2", "g2", 40.0)]
|
||||
s = analyze.summarise(rows)
|
||||
b, where = analyze.baseline_for(s, rows, "B4.x", "s2", "C1")
|
||||
self.assertEqual((where, b["p99"]), ("B4.*", 11.0))
|
||||
b, where = analyze.baseline_for(s, rows, "B9.z", "s2", "C1")
|
||||
self.assertEqual(where, "*")
|
||||
self.assertIsNone(analyze.baseline_for(s, rows, "B9.z", "s9", "C1")[0])
|
||||
|
||||
def test_table_verdict_against_block(self):
|
||||
t = analyze.table(analyze.summarise(self.rows), self.rows, "C1", "B0")
|
||||
c = cell(t, "B3.dh-off", "C1", "s1")
|
||||
self.assertEqual(c[-4:], ["C1@B3", "+4.00", "+0.0", "win"])
|
||||
self.assertEqual(cell(t, "B3", "C1", "s1")[-1], "baseline")
|
||||
self.assertEqual(cell(t, "B3", "dh-r64", "s1")[-4], "C1@B3")
|
||||
|
||||
def test_table_label_filter_still_finds_block_baseline(self):
|
||||
shown = [r for r in self.rows if r["label"] in ("B3.dh-off", "B0")]
|
||||
t = analyze.table(analyze.summarise(shown), self.rows, "C1", "B0")
|
||||
self.assertEqual(cell(t, "B3.dh-off", "C1", "s1")[-4], "C1@B3")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user