logan
a15c912ad0
- bench.py: gamescope-headless display (default, 1920x1080; ~2% cost vs desktop for isolation from the shared Wayland desktop), --shader, --dh, --disable-mods/--add-mods, --world-zip, crash-safe restore journal, spectator mode in the plan (elevated cameras fell to the sea floor and S4 measured underwater darkness) - matrix.toml/matrix.py: declarative, resumable B0-B14 runner keyed on mods hash (warns when B0 must be re-run) - build.py: bench-only resource-pack variants (C1-fa, C1-fa-ext, emf-lod*-fa, C1-bl); every bench variant pins resourcePacks so packs don't leak between sessions; shipped tier output unchanged - analyze.py: tagged labels use their own block's baseline
98 lines
4.2 KiB
Python
98 lines
4.2 KiB
Python
"""analyze.py: tagged labels ('B3.dh-off') take their baseline from the same block.
|
|
|
|
python3 -m unittest discover -s analysis/client-perf/harness/tests
|
|
"""
|
|
import sys
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
sys.path.insert(0, str(HERE.parent))
|
|
|
|
import analyze # noqa: E402
|
|
|
|
|
|
def row(label, variant, scene, session, p99, low=100.0, fps=200.0):
|
|
return {"kind": "window", "label": label, "variant": variant, "scene": scene,
|
|
"session": session, "flags": [],
|
|
"metrics": {"p99_ms": p99, "one_low_fps": low, "avg_fps": fps}}
|
|
|
|
|
|
def cell(table, label, variant, scene):
|
|
for line in table.splitlines():
|
|
cells = [c.strip() for c in line.strip("|").split("|")]
|
|
if cells[:3] == [label, variant, scene]:
|
|
return cells
|
|
raise AssertionError(f"no row {label}/{variant}/{scene} in\n{table}")
|
|
|
|
|
|
class BlockBaseline(unittest.TestCase):
|
|
def setUp(self):
|
|
self.rows = [
|
|
# noise: spread 0.2 ms / 2 fps
|
|
row("B0", "C0", "s1", "n1", 10.0, 100), row("B0", "C0", "s1", "n2", 10.2, 102),
|
|
# global C1 reference elsewhere (B2) is much slower than the B3 one
|
|
row("B2", "C1", "s1", "g1", 20.0),
|
|
row("B3", "C1", "s1", "b1", 10.0),
|
|
row("B3", "dh-r64", "s1", "b2", 8.0),
|
|
row("B3.dh-off", "C1", "s1", "b3", 6.0),
|
|
]
|
|
|
|
def test_tagged_self_variant_uses_untagged_block_label(self):
|
|
s = analyze.summarise(self.rows)
|
|
b, where = analyze.self_baseline(s, "B3.dh-off", "s1", "C1")
|
|
self.assertEqual(where, "B3")
|
|
self.assertEqual(b["p99"], 10.0)
|
|
|
|
def test_untagged_block_label_is_its_own_baseline(self):
|
|
s = analyze.summarise(self.rows)
|
|
self.assertEqual(analyze.self_baseline(s, "B3", "s1", "C1"), (None, None))
|
|
|
|
def test_tagged_other_variant_prefers_block_over_global(self):
|
|
rows = self.rows + [row("B12a.no-x", "X", "s1", "t1", 9.0)]
|
|
rows += [row("B12a", "C1", "s1", "t0", 12.0)]
|
|
s = analyze.summarise(rows)
|
|
b, where = analyze.baseline_for(s, rows, "B12a.no-x", "s1", "C1")
|
|
self.assertEqual((where, b["p99"]), ("B12a", 12.0))
|
|
|
|
def test_block_baseline_tag(self):
|
|
rows = self.rows + [row("B11.baseline", "C1", "s5", "x0", 12.0),
|
|
row("B11", "sod-thr3", "s5", "x1", 11.0),
|
|
row("B2", "C1", "s5", "x2", 30.0)]
|
|
s = analyze.summarise(rows)
|
|
b, where = analyze.baseline_for(s, rows, "B11", "s5", "C1")
|
|
# untagged candidate: own label has no C1 -> '<block>.baseline', not global
|
|
self.assertEqual((where, b["p99"]), ("B11.baseline", 12.0))
|
|
rows.append(row("B11.x", "opt-mip4", "s5", "x3", 11.0))
|
|
s = analyze.summarise(rows)
|
|
b, where = analyze.baseline_for(s, rows, "B11.x", "s5", "C1")
|
|
self.assertEqual((where, b["p99"]), ("B11.baseline", 12.0))
|
|
|
|
def test_block_pool_then_global(self):
|
|
rows = self.rows + [row("B4.J0", "C1", "s2", "j0", 10.0),
|
|
row("B4.J1", "C1", "s2", "j1", 12.0),
|
|
row("B4.x", "Y", "s2", "j2", 9.0),
|
|
row("B2", "C1", "s2", "g2", 40.0)]
|
|
s = analyze.summarise(rows)
|
|
b, where = analyze.baseline_for(s, rows, "B4.x", "s2", "C1")
|
|
self.assertEqual((where, b["p99"]), ("B4.*", 11.0))
|
|
b, where = analyze.baseline_for(s, rows, "B9.z", "s2", "C1")
|
|
self.assertEqual(where, "*")
|
|
self.assertIsNone(analyze.baseline_for(s, rows, "B9.z", "s9", "C1")[0])
|
|
|
|
def test_table_verdict_against_block(self):
|
|
t = analyze.table(analyze.summarise(self.rows), self.rows, "C1", "B0")
|
|
c = cell(t, "B3.dh-off", "C1", "s1")
|
|
self.assertEqual(c[-4:], ["C1@B3", "+4.00", "+0.0", "win"])
|
|
self.assertEqual(cell(t, "B3", "C1", "s1")[-1], "baseline")
|
|
self.assertEqual(cell(t, "B3", "dh-r64", "s1")[-4], "C1@B3")
|
|
|
|
def test_table_label_filter_still_finds_block_baseline(self):
|
|
shown = [r for r in self.rows if r["label"] in ("B3.dh-off", "B0")]
|
|
t = analyze.table(analyze.summarise(shown), self.rows, "C1", "B0")
|
|
self.assertEqual(cell(t, "B3.dh-off", "C1", "s1")[-4], "C1@B3")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|