feat(bench): add matrix runner, gamescope and per-session controls
- bench.py: gamescope-headless display (default, 1920x1080; ~2% cost vs desktop for isolation from the shared Wayland desktop), --shader, --dh, --disable-mods/--add-mods, --world-zip, crash-safe restore journal, spectator mode in the plan (elevated cameras fell to the sea floor and S4 measured underwater darkness) - matrix.toml/matrix.py: declarative, resumable B0-B14 runner keyed on mods hash (warns when B0 must be re-run) - build.py: bench-only resource-pack variants (C1-fa, C1-fa-ext, emf-lod*-fa, C1-bl); every bench variant pins resourcePacks so packs don't leak between sessions; shipped tier output unchanged - analyze.py: tagged labels use their own block's baseline
This commit is contained in:
logan committed
1 parent
8f56e2f780
commit
a15c912ad0
10 files changed
+2196
-58
No files matched your search
@@ -0,0 +1,342 @@
|
||||
"""Offline tests for matrix.py (no game; the bench.py runner is faked).
|
||||
|
||||
python3 -m unittest discover -s analysis/client-perf/harness/tests
|
||||
"""
|
||||
import contextlib
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import tomllib
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE.parent))
|
||||
|
||||
import matrix # noqa: E402
|
||||
|
||||
MINI = """
|
||||
[defaults]
|
||||
reps = 3
|
||||
warmup = 15
|
||||
capture = 30
|
||||
display = "gamescope"
|
||||
res = "1280x720"
|
||||
world = "v1"
|
||||
shader = "off"
|
||||
dh = "on"
|
||||
launch_overhead_s = 60
|
||||
|
||||
[worlds]
|
||||
v1 = "{tmp}/worlds/v1.zip"
|
||||
s4 = "{tmp}/worlds/s4-*.zip"
|
||||
|
||||
[trials]
|
||||
gnetum = "{tmp}/trials/gnetum-*.jar"
|
||||
|
||||
[[missing_variants]]
|
||||
id = "C1-fa"
|
||||
blocks = ["B8"]
|
||||
contents = "C1 + Fresh Animations"
|
||||
|
||||
[[block]]
|
||||
id = "B0"
|
||||
rule = "noise"
|
||||
sessions = [{{ variant = "C0", plan = "s1", repeat = 3 }}]
|
||||
|
||||
[[block]]
|
||||
id = "B3"
|
||||
rule = "dh"
|
||||
sessions = [
|
||||
{{ variant = "C1", plan = "s1,s5" }},
|
||||
{{ variant = "C1", plan = "s1,s5", dh = "off", tag = "dh-off" }},
|
||||
{{ variant = "C1", plan = "s2", jvm = "J2", heap = 6144, world = "s4", tag = "x" }},
|
||||
]
|
||||
|
||||
[[block]]
|
||||
id = "B6"
|
||||
rule = "mf"
|
||||
interleave = true
|
||||
sessions = [
|
||||
{{ variant = "C1", plan = "s1", repeat = 2 }},
|
||||
{{ variant = "mf", plan = "s1", repeat = 2 }},
|
||||
]
|
||||
|
||||
[[block]]
|
||||
id = "B8"
|
||||
rule = "entities"
|
||||
sessions = [{{ variant = "C1-fa", plan = "s3" }}]
|
||||
|
||||
[[block]]
|
||||
id = "B12b"
|
||||
rule = "trials"
|
||||
sessions = [{{ variant = "C1", plan = "s1", add_mods = ["gnetum"], disable_mods = ["a-*.jar"], tag = "g" }}]
|
||||
|
||||
[[block]]
|
||||
id = "B4h"
|
||||
rule = "heap"
|
||||
params = ["gc"]
|
||||
sessions = [{{ variant = "C1", plan = "s5", jvm = "${{gc}}", heap = 4096 }}]
|
||||
|
||||
[[block]]
|
||||
id = "B13"
|
||||
rule = "profile"
|
||||
sessions = [{{ variant = "C1", plan = "s1", manual = true }}]
|
||||
|
||||
[[block]]
|
||||
id = "B5"
|
||||
rule = "sable"
|
||||
deferred = true
|
||||
sessions = [{{ variant = "C1", plan = "s4" }}]
|
||||
|
||||
[[block]]
|
||||
id = "B14"
|
||||
rule = "spawn"
|
||||
manual = true
|
||||
sessions = []
|
||||
"""
|
||||
|
||||
|
||||
class Fixture(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = Path(tempfile.mkdtemp())
|
||||
self.out = self.tmp / "out"
|
||||
for v in ("C0", "C1", "mf"):
|
||||
(self.out / v).mkdir(parents=True)
|
||||
self.mods = self.tmp / "mods"
|
||||
self.mods.mkdir()
|
||||
(self.mods / "a-1.jar").write_bytes(b"x")
|
||||
(self.tmp / "worlds").mkdir()
|
||||
(self.tmp / "worlds" / "v1.zip").write_bytes(b"z")
|
||||
(self.tmp / "worlds" / "s4-4.zip").write_bytes(b"z")
|
||||
(self.tmp / "trials").mkdir()
|
||||
(self.tmp / "trials" / "gnetum-1.0.jar").write_bytes(b"j")
|
||||
self.path = self.tmp / "matrix.toml"
|
||||
self.path.write_text(MINI.format(tmp=self.tmp))
|
||||
self.m = matrix.load(self.path, self.out)
|
||||
self.state_path = self.tmp / "state.json"
|
||||
self.calls = []
|
||||
|
||||
def runner(self, codes=None):
|
||||
codes = list(codes or [])
|
||||
|
||||
def fake(argv):
|
||||
self.calls.append(argv)
|
||||
return codes.pop(0) if codes else 0
|
||||
return fake
|
||||
|
||||
def run_blocks(self, ids, codes=None, **kw):
|
||||
state = matrix.State(self.state_path)
|
||||
with contextlib.redirect_stdout(io.StringIO()), \
|
||||
contextlib.redirect_stderr(io.StringIO()) as err:
|
||||
rc = matrix.run_blocks(self.m, ids, state, self.mods, runner=self.runner(codes),
|
||||
out_dir=self.out, **kw)
|
||||
return rc, state, err.getvalue()
|
||||
|
||||
|
||||
class Loading(Fixture):
|
||||
def test_real_matrix_validates(self):
|
||||
m = matrix.load() # the shipped matrix.toml against the real out/ dir
|
||||
ids = [b["id"] for b in m["block"]]
|
||||
for want in ["B0", "B1", "B2", "B3", "B4", "B5", "B6", "B7", "B8", "B9", "B10",
|
||||
"B11", "B12a", "B12b", "B13", "B14"]:
|
||||
self.assertIn(want, ids)
|
||||
bm = matrix.block_map(m)
|
||||
self.assertTrue(bm["B5"]["deferred"])
|
||||
self.assertTrue(bm["B14"]["manual"])
|
||||
b0 = matrix.expand(m, bm["B0"])
|
||||
self.assertEqual([(s["variant"], s["plan"]) for s in b0], [("C0", ["s1"])] * 3)
|
||||
|
||||
def test_real_matrix_emits_only_bench_flags(self):
|
||||
m = matrix.load()
|
||||
self.assertEqual((m["defaults"]["display"], m["defaults"]["res"]),
|
||||
("gamescope-headless", "1920x1080"))
|
||||
params = {"gc": "J1", "scene": "s2"}
|
||||
n = 0
|
||||
for b in m["block"]:
|
||||
for s in matrix.expand(m, b, params):
|
||||
argv, _ = matrix.bench_argv(m, s)
|
||||
self.assertEqual(argv[0], "session")
|
||||
flags = {a for a in argv[1:] if a.startswith("--")}
|
||||
self.assertLessEqual(flags, matrix.BENCH_SESSION_FLAGS, (b["id"], argv))
|
||||
n += 1
|
||||
self.assertGreater(n, 50)
|
||||
|
||||
def test_fa_and_leaves_variants_generated(self):
|
||||
m = matrix.load()
|
||||
missing = matrix.missing_variant_ids(m)
|
||||
for v in ("C1-fa", "C1-fa-ext", "emf-lod20-fa", "emf-lod8-fa", "C1-bl"):
|
||||
self.assertNotIn(v, missing)
|
||||
self.assertTrue((matrix.OUT_DIR / v / "options.txt").is_file(), v)
|
||||
|
||||
def test_validation_errors(self):
|
||||
d = tomllib.loads(MINI.format(tmp=self.tmp))
|
||||
d["block"][0]["sessions"][0]["plan"] = "s5,s1"
|
||||
d["block"][0]["sessions"][0]["typo"] = 1
|
||||
d["block"][1]["sessions"].append({"variant": "nope", "plan": "s6"})
|
||||
d["block"][1]["sessions"].append({"variant": "C1", "plan": "s1", "dh": "off",
|
||||
"tag": "dh-off"})
|
||||
d["block"][4]["sessions"][0]["add_mods"] = ["unknown"]
|
||||
d["block"][5]["sessions"][0]["jvm"] = "${other}"
|
||||
d["block"][0]["sessions"][0]["world"] = "nowhere"
|
||||
d["defaults"]["display"] = "tv"
|
||||
errs = "\n".join(matrix.validate(d, self.out))
|
||||
for frag in ["S1->S5 order", "unknown key 'typo'", "variant 'nope'",
|
||||
"unknown scene 's6'", "distinct tag", "not in [trials]",
|
||||
"${other} not declared", "world 'nowhere'", "display 'tv'"]:
|
||||
self.assertIn(frag, errs)
|
||||
|
||||
def test_missing_variant_is_valid_but_blocked(self):
|
||||
s = matrix.expand(self.m, matrix.block_map(self.m)["B8"])[0]
|
||||
self.assertTrue(matrix.variant_missing(s, self.out))
|
||||
|
||||
|
||||
class Commands(Fixture):
|
||||
def test_argv(self):
|
||||
b3 = matrix.expand(self.m, matrix.block_map(self.m)["B3"])
|
||||
argv, miss = matrix.bench_argv(self.m, b3[1])
|
||||
self.assertEqual(miss, [])
|
||||
self.assertEqual(argv[:9], ["session", "--variant", "C1", "--plan", "s1,s5",
|
||||
"--reps", "3", "--label", "B3.dh-off"])
|
||||
self.assertIn("--dh", argv)
|
||||
self.assertEqual(argv[argv.index("--dh") + 1], "off")
|
||||
self.assertEqual(argv[argv.index("--world-zip") + 1], str(self.tmp / "worlds/v1.zip"))
|
||||
argv, _ = matrix.bench_argv(self.m, b3[2])
|
||||
for flag, val in [("--jvm", "J2"), ("--heap", "6144"), ("--display", "gamescope"),
|
||||
("--res", "1280x720"), ("--shader", "off"),
|
||||
("--world-zip", str(self.tmp / "worlds/s4-4.zip"))]:
|
||||
self.assertEqual(argv[argv.index(flag) + 1], val)
|
||||
|
||||
def test_trials_and_disable(self):
|
||||
s = matrix.expand(self.m, matrix.block_map(self.m)["B12b"])[0]
|
||||
argv, miss = matrix.bench_argv(self.m, s)
|
||||
self.assertEqual(argv[argv.index("--add-mods") + 1],
|
||||
str(self.tmp / "trials/gnetum-1.0.jar"))
|
||||
self.assertEqual(argv[argv.index("--disable-mods") + 1], "a-*.jar")
|
||||
(self.tmp / "trials" / "gnetum-1.0.jar").unlink()
|
||||
_, miss = matrix.bench_argv(self.m, s)
|
||||
self.assertTrue(any("gnetum" in x for x in miss))
|
||||
|
||||
def test_live_world_and_overrides(self):
|
||||
s = matrix.expand(self.m, matrix.block_map(self.m)["B0"],
|
||||
overrides={"display": "desktop", "res": None})[0]
|
||||
argv, _ = matrix.bench_argv(self.m, s, live_world=True)
|
||||
self.assertNotIn("--world-zip", argv)
|
||||
self.assertEqual(argv[argv.index("--display") + 1], "desktop")
|
||||
self.assertEqual(argv[argv.index("--res") + 1], "1280x720")
|
||||
|
||||
def test_interleave_and_params(self):
|
||||
bm = matrix.block_map(self.m)
|
||||
self.assertEqual([s["variant"] for s in matrix.expand(self.m, bm["B6"])],
|
||||
["C1", "mf", "C1", "mf"])
|
||||
s = matrix.expand(self.m, bm["B4h"])[0]
|
||||
self.assertEqual(matrix.unresolved(s), ["gc"])
|
||||
s = matrix.expand(self.m, bm["B4h"], params={"gc": "J2"})[0]
|
||||
self.assertEqual(matrix.unresolved(s), [])
|
||||
self.assertEqual(s["jvm"], "J2")
|
||||
|
||||
def test_seconds(self):
|
||||
b3 = matrix.expand(self.m, matrix.block_map(self.m)["B3"])
|
||||
self.assertEqual(matrix.session_seconds(b3[0], 60), 60 + 3 * 45 + 3 * 130)
|
||||
|
||||
|
||||
class Resume(Fixture):
|
||||
def test_runs_then_skips(self):
|
||||
rc, state, _ = self.run_blocks(["B0"])
|
||||
self.assertEqual(rc, 0)
|
||||
self.assertEqual(len(self.calls), 3)
|
||||
self.assertEqual(state.data["b0"]["mods_hash"], matrix.mods_hash(self.mods))
|
||||
saved = json.loads(self.state_path.read_text())
|
||||
self.assertEqual({e["status"] for e in saved["sessions"].values()}, {"ok"})
|
||||
self.calls.clear()
|
||||
rc, _, _ = self.run_blocks(["B0"])
|
||||
self.assertEqual((rc, self.calls), (0, []))
|
||||
|
||||
def test_mod_change_invalidates_and_warns(self):
|
||||
self.run_blocks(["B0"])
|
||||
(self.mods / "new-mod.jar").write_bytes(b"y")
|
||||
self.calls.clear()
|
||||
rc, _, err = self.run_blocks(["B3"])
|
||||
self.assertEqual(rc, 0)
|
||||
self.assertIn("B0 MUST BE RE-RUN", err)
|
||||
self.calls.clear()
|
||||
rc, _, _ = self.run_blocks(["B0"])
|
||||
self.assertEqual(len(self.calls), 3) # B0 sessions are stale under the new set
|
||||
|
||||
def test_warns_without_b0(self):
|
||||
_, _, err = self.run_blocks(["B3"])
|
||||
self.assertIn("B0 (noise floor) has not been completed", err)
|
||||
|
||||
def test_spec_change_reruns(self):
|
||||
self.run_blocks(["B0"])
|
||||
self.calls.clear()
|
||||
self.run_blocks(["B0"], live_world=True) # different command -> not done
|
||||
self.assertEqual(len(self.calls), 3)
|
||||
|
||||
def test_retry_once_then_ok(self):
|
||||
rc, state, _ = self.run_blocks(["B0"], codes=[1, 0])
|
||||
self.assertEqual(rc, 0)
|
||||
self.assertEqual(len(self.calls), 4)
|
||||
self.assertEqual(state.data["sessions"]["B0#0"]["attempts"], 2)
|
||||
|
||||
def test_stop_after_two_failures(self):
|
||||
rc, state, _ = self.run_blocks(["B0"], codes=[1, 1])
|
||||
self.assertEqual(rc, 1)
|
||||
self.assertEqual(len(self.calls), 2)
|
||||
self.assertEqual(state.data["sessions"]["B0#0"]["status"], "failed")
|
||||
self.assertNotIn("B0#1", state.data["sessions"])
|
||||
self.calls.clear()
|
||||
rc, _, _ = self.run_blocks(["B0"]) # resume retries the failed one
|
||||
self.assertEqual((rc, len(self.calls)), (0, 3))
|
||||
|
||||
def test_dry_run_runs_nothing(self):
|
||||
rc, _, _ = self.run_blocks(["B0", "B3"], dry_run=True)
|
||||
self.assertEqual((rc, self.calls), (0, []))
|
||||
self.assertFalse(self.state_path.exists())
|
||||
|
||||
def test_blocked_aborts_or_skips(self):
|
||||
rc, _, _ = self.run_blocks(["B8"])
|
||||
self.assertEqual((rc, self.calls), (2, []))
|
||||
rc, _, _ = self.run_blocks(["B4h"]) # unresolved ${gc}
|
||||
self.assertEqual((rc, self.calls), (2, []))
|
||||
rc, _, _ = self.run_blocks(["B8", "B6"], skip_blocked=True)
|
||||
self.assertEqual((rc, len(self.calls)), (0, 4))
|
||||
|
||||
def test_manual_deferred_skipped(self):
|
||||
rc, _, _ = self.run_blocks(["B13", "B5", "B14"])
|
||||
self.assertEqual((rc, self.calls), (0, []))
|
||||
rc, _, _ = self.run_blocks(["B13", "B5"], include_manual=True, include_deferred=True)
|
||||
self.assertEqual(len(self.calls), 2)
|
||||
|
||||
def test_mods_restored_check(self):
|
||||
def mutating(argv):
|
||||
self.calls.append(argv)
|
||||
(self.mods / f"leak{len(self.calls)}.jar").write_bytes(b"l")
|
||||
return 0
|
||||
state = matrix.State(self.state_path)
|
||||
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
|
||||
rc = matrix.run_blocks(self.m, ["B0"], state, self.mods, runner=mutating,
|
||||
out_dir=self.out)
|
||||
self.assertEqual((rc, len(self.calls)), (1, 1))
|
||||
|
||||
def test_unknown_block(self):
|
||||
with self.assertRaises(matrix.MatrixError):
|
||||
self.run_blocks(["B99"])
|
||||
|
||||
|
||||
class Cli(unittest.TestCase):
|
||||
def test_list_and_plan(self):
|
||||
buf = io.StringIO()
|
||||
with contextlib.redirect_stdout(buf):
|
||||
self.assertEqual(matrix.main(["list"]), 0)
|
||||
self.assertEqual(matrix.main(["plan", "B4h", "--param", "gc=J1",
|
||||
"--display", "desktop"]), 0)
|
||||
out = buf.getvalue()
|
||||
self.assertIn("| B0 | noise floor | 3 |", out)
|
||||
self.assertIn("--jvm J1 --heap 4096", out)
|
||||
self.assertIn("--display desktop", out)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user