"""Offline tests for matrix.py (no game; the bench.py runner is faked). python3 -m unittest discover -s analysis/client-perf/harness/tests """ import contextlib import io import json import sys import tempfile import tomllib import unittest from pathlib import Path HERE = Path(__file__).resolve().parent sys.path.insert(0, str(HERE.parent)) import matrix # noqa: E402 MINI = """ [defaults] reps = 3 warmup = 15 capture = 30 display = "gamescope" res = "1280x720" world = "v1" shader = "off" dh = "on" launch_overhead_s = 60 [worlds] v1 = "{tmp}/worlds/v1.zip" s4 = "{tmp}/worlds/s4-*.zip" [trials] gnetum = "{tmp}/trials/gnetum-*.jar" [[missing_variants]] id = "C1-fa" blocks = ["B8"] contents = "C1 + Fresh Animations" [[block]] id = "B0" rule = "noise" sessions = [{{ variant = "C0", plan = "s1", repeat = 3 }}] [[block]] id = "B3" rule = "dh" sessions = [ {{ variant = "C1", plan = "s1,s5" }}, {{ variant = "C1", plan = "s1,s5", dh = "off", tag = "dh-off" }}, {{ variant = "C1", plan = "s2", jvm = "J2", heap = 6144, world = "s4", tag = "x" }}, ] [[block]] id = "B6" rule = "mf" interleave = true sessions = [ {{ variant = "C1", plan = "s1", repeat = 2 }}, {{ variant = "mf", plan = "s1", repeat = 2 }}, ] [[block]] id = "B8" rule = "entities" sessions = [{{ variant = "C1-fa", plan = "s3" }}] [[block]] id = "B12b" rule = "trials" sessions = [{{ variant = "C1", plan = "s1", add_mods = ["gnetum"], disable_mods = ["a-*.jar"], tag = "g" }}] [[block]] id = "B4h" rule = "heap" params = ["gc"] sessions = [{{ variant = "C1", plan = "s5", jvm = "${{gc}}", heap = 4096 }}] [[block]] id = "B13" rule = "profile" sessions = [{{ variant = "C1", plan = "s1", manual = true }}] [[block]] id = "B5" rule = "sable" deferred = true sessions = [{{ variant = "C1", plan = "s4" }}] [[block]] id = "B14" rule = "spawn" manual = true sessions = [] """ class Fixture(unittest.TestCase): def setUp(self): self.tmp = Path(tempfile.mkdtemp()) self.out = self.tmp / "out" for v in ("C0", "C1", "mf"): (self.out / v).mkdir(parents=True) self.mods = self.tmp / "mods" self.mods.mkdir() (self.mods / "a-1.jar").write_bytes(b"x") (self.tmp / "worlds").mkdir() (self.tmp / "worlds" / "v1.zip").write_bytes(b"z") (self.tmp / "worlds" / "s4-4.zip").write_bytes(b"z") (self.tmp / "trials").mkdir() (self.tmp / "trials" / "gnetum-1.0.jar").write_bytes(b"j") self.path = self.tmp / "matrix.toml" self.path.write_text(MINI.format(tmp=self.tmp)) self.m = matrix.load(self.path, self.out) self.state_path = self.tmp / "state.json" self.calls = [] def runner(self, codes=None): codes = list(codes or []) def fake(argv): self.calls.append(argv) return codes.pop(0) if codes else 0 return fake def run_blocks(self, ids, codes=None, **kw): state = matrix.State(self.state_path) with contextlib.redirect_stdout(io.StringIO()), \ contextlib.redirect_stderr(io.StringIO()) as err: rc = matrix.run_blocks(self.m, ids, state, self.mods, runner=self.runner(codes), out_dir=self.out, **kw) return rc, state, err.getvalue() class Loading(Fixture): def test_real_matrix_validates(self): m = matrix.load() # the shipped matrix.toml against the real out/ dir ids = [b["id"] for b in m["block"]] for want in ["B0", "B1", "B2", "B3", "B4", "B5", "B6", "B7", "B8", "B9", "B10", "B11", "B12a", "B12b", "B13", "B14"]: self.assertIn(want, ids) bm = matrix.block_map(m) self.assertTrue(bm["B5"]["deferred"]) self.assertTrue(bm["B14"]["manual"]) b0 = matrix.expand(m, bm["B0"]) self.assertEqual([(s["variant"], s["plan"]) for s in b0], [("C0", ["s1"])] * 3) def test_real_matrix_emits_only_bench_flags(self): m = matrix.load() self.assertEqual((m["defaults"]["display"], m["defaults"]["res"]), ("gamescope-headless", "1920x1080")) params = {"gc": "J1", "scene": "s2"} n = 0 for b in m["block"]: for s in matrix.expand(m, b, params): argv, _ = matrix.bench_argv(m, s) self.assertEqual(argv[0], "session") flags = {a for a in argv[1:] if a.startswith("--")} self.assertLessEqual(flags, matrix.BENCH_SESSION_FLAGS, (b["id"], argv)) n += 1 self.assertGreater(n, 50) def test_fa_and_leaves_variants_generated(self): m = matrix.load() missing = matrix.missing_variant_ids(m) for v in ("C1-fa", "C1-fa-ext", "emf-lod20-fa", "emf-lod8-fa", "C1-bl"): self.assertNotIn(v, missing) self.assertTrue((matrix.OUT_DIR / v / "options.txt").is_file(), v) def test_validation_errors(self): d = tomllib.loads(MINI.format(tmp=self.tmp)) d["block"][0]["sessions"][0]["plan"] = "s5,s1" d["block"][0]["sessions"][0]["typo"] = 1 d["block"][1]["sessions"].append({"variant": "nope", "plan": "s6"}) d["block"][1]["sessions"].append({"variant": "C1", "plan": "s1", "dh": "off", "tag": "dh-off"}) d["block"][4]["sessions"][0]["add_mods"] = ["unknown"] d["block"][5]["sessions"][0]["jvm"] = "${other}" d["block"][0]["sessions"][0]["world"] = "nowhere" d["defaults"]["display"] = "tv" errs = "\n".join(matrix.validate(d, self.out)) for frag in ["S1->S5 order", "unknown key 'typo'", "variant 'nope'", "unknown scene 's6'", "distinct tag", "not in [trials]", "${other} not declared", "world 'nowhere'", "display 'tv'"]: self.assertIn(frag, errs) def test_missing_variant_is_valid_but_blocked(self): s = matrix.expand(self.m, matrix.block_map(self.m)["B8"])[0] self.assertTrue(matrix.variant_missing(s, self.out)) class Commands(Fixture): def test_argv(self): b3 = matrix.expand(self.m, matrix.block_map(self.m)["B3"]) argv, miss = matrix.bench_argv(self.m, b3[1]) self.assertEqual(miss, []) self.assertEqual(argv[:9], ["session", "--variant", "C1", "--plan", "s1,s5", "--reps", "3", "--label", "B3.dh-off"]) self.assertIn("--dh", argv) self.assertEqual(argv[argv.index("--dh") + 1], "off") self.assertEqual(argv[argv.index("--world-zip") + 1], str(self.tmp / "worlds/v1.zip")) argv, _ = matrix.bench_argv(self.m, b3[2]) for flag, val in [("--jvm", "J2"), ("--heap", "6144"), ("--display", "gamescope"), ("--res", "1280x720"), ("--shader", "off"), ("--world-zip", str(self.tmp / "worlds/s4-4.zip"))]: self.assertEqual(argv[argv.index(flag) + 1], val) def test_trials_and_disable(self): s = matrix.expand(self.m, matrix.block_map(self.m)["B12b"])[0] argv, miss = matrix.bench_argv(self.m, s) self.assertEqual(argv[argv.index("--add-mods") + 1], str(self.tmp / "trials/gnetum-1.0.jar")) self.assertEqual(argv[argv.index("--disable-mods") + 1], "a-*.jar") (self.tmp / "trials" / "gnetum-1.0.jar").unlink() _, miss = matrix.bench_argv(self.m, s) self.assertTrue(any("gnetum" in x for x in miss)) def test_live_world_and_overrides(self): s = matrix.expand(self.m, matrix.block_map(self.m)["B0"], overrides={"display": "desktop", "res": None})[0] argv, _ = matrix.bench_argv(self.m, s, live_world=True) self.assertNotIn("--world-zip", argv) self.assertEqual(argv[argv.index("--display") + 1], "desktop") self.assertEqual(argv[argv.index("--res") + 1], "1280x720") def test_interleave_and_params(self): bm = matrix.block_map(self.m) self.assertEqual([s["variant"] for s in matrix.expand(self.m, bm["B6"])], ["C1", "mf", "C1", "mf"]) s = matrix.expand(self.m, bm["B4h"])[0] self.assertEqual(matrix.unresolved(s), ["gc"]) s = matrix.expand(self.m, bm["B4h"], params={"gc": "J2"})[0] self.assertEqual(matrix.unresolved(s), []) self.assertEqual(s["jvm"], "J2") def test_seconds(self): b3 = matrix.expand(self.m, matrix.block_map(self.m)["B3"]) self.assertEqual(matrix.session_seconds(b3[0], 60), 60 + 10 + 3 * 45 + 3 * 130) class ShaderOptions(Fixture): def b7(self, **extra): d = tomllib.loads(MINI.format(tmp=self.tmp)) sess = {"variant": "C1", "plan": "s1,s4", "shader": "Pack.zip", "shader_profile": "low", "tag": "p-low", **extra} d["block"].append({"id": "B7", "rule": "shaders", "sessions": [sess]}) return d def test_emitted(self): f = self.tmp / "Pack.low.txt" f.write_text("A=1\n") d = self.b7(shader_options=str(f)) self.assertEqual(matrix.validate(d, self.out), []) s = matrix.expand(d, matrix.block_map(d)["B7"])[0] argv, miss = matrix.bench_argv(d, s) self.assertEqual(miss, []) self.assertEqual(argv[argv.index("--shader") + 1], "Pack.zip") self.assertEqual(argv[argv.index("--shader-options") + 1], str(f)) self.assertIn("--shader-options", matrix.BENCH_SESSION_FLAGS) def test_missing_file_and_relative_path(self): d = self.b7(shader_options="shader-profiles/does-not-exist.txt") s = matrix.expand(d, matrix.block_map(d)["B7"])[0] argv, miss = matrix.bench_argv(d, s) self.assertEqual(argv[argv.index("--shader-options") + 1], str(matrix.HERE / "shader-profiles/does-not-exist.txt")) self.assertTrue(any("shader options" in x for x in miss)) def test_needs_shader(self): d = self.b7(shader_options="x.txt", shader="off") self.assertIn("shader_options needs shader", "\n".join(matrix.validate(d, self.out))) def test_spec_hash_tracks_file_content(self): f = self.tmp / "Pack.low.txt" f.write_text("A=1\n") d = self.b7(shader_options=str(f)) s = matrix.expand(d, matrix.block_map(d)["B7"])[0] argv, _ = matrix.bench_argv(d, s) h1 = matrix.spec_hash(argv, s) f.write_text("A=2\n") self.assertNotEqual(h1, matrix.spec_hash(argv, s)) # sessions without shader_options keep the original hash formula (resume state) b0 = matrix.expand(self.m, matrix.block_map(self.m)["B0"])[0] a0, _ = matrix.bench_argv(self.m, b0) blob = json.dumps({"argv": a0, "manual": False}, sort_keys=True) import hashlib self.assertEqual(matrix.spec_hash(a0, b0), hashlib.sha256(blob.encode()).hexdigest()[:16]) def test_real_b7_automated(self): m = matrix.load() rows = matrix.expand(m, matrix.block_map(m)["B7"]) shaded = [s for s in rows if s["shader"] != "off"] self.assertEqual(len(shaded), 4) for s in rows: self.assertFalse(s.get("manual"), s["label"]) for s in shaded: argv, miss = matrix.bench_argv(m, s) self.assertEqual([x for x in miss if "shader options" in x], [], s["label"]) self.assertTrue(Path(argv[argv.index("--shader-options") + 1]).is_file()) class Resume(Fixture): def test_runs_then_skips(self): rc, state, _ = self.run_blocks(["B0"]) self.assertEqual(rc, 0) self.assertEqual(len(self.calls), 3) self.assertEqual(state.data["b0"]["mods_hash"], matrix.mods_hash(self.mods)) saved = json.loads(self.state_path.read_text()) self.assertEqual({e["status"] for e in saved["sessions"].values()}, {"ok"}) self.calls.clear() rc, _, _ = self.run_blocks(["B0"]) self.assertEqual((rc, self.calls), (0, [])) def test_mod_change_invalidates_and_warns(self): self.run_blocks(["B0"]) (self.mods / "new-mod.jar").write_bytes(b"y") self.calls.clear() rc, _, err = self.run_blocks(["B3"]) self.assertEqual(rc, 0) self.assertIn("B0 MUST BE RE-RUN", err) self.calls.clear() rc, _, _ = self.run_blocks(["B0"]) self.assertEqual(len(self.calls), 3) # B0 sessions are stale under the new set def test_warns_without_b0(self): _, _, err = self.run_blocks(["B3"]) self.assertIn("B0 (noise floor) has not been completed", err) def test_spec_change_reruns(self): self.run_blocks(["B0"]) self.calls.clear() self.run_blocks(["B0"], live_world=True) # different command -> not done self.assertEqual(len(self.calls), 3) def test_retry_once_then_ok(self): rc, state, _ = self.run_blocks(["B0"], codes=[1, 0]) self.assertEqual(rc, 0) self.assertEqual(len(self.calls), 4) self.assertEqual(state.data["sessions"]["B0#0"]["attempts"], 2) def test_stop_after_two_failures(self): rc, state, _ = self.run_blocks(["B0"], codes=[1, 1]) self.assertEqual(rc, 1) self.assertEqual(len(self.calls), 2) self.assertEqual(state.data["sessions"]["B0#0"]["status"], "failed") self.assertNotIn("B0#1", state.data["sessions"]) self.calls.clear() rc, _, _ = self.run_blocks(["B0"]) # resume retries the failed one self.assertEqual((rc, len(self.calls)), (0, 3)) def test_dry_run_runs_nothing(self): rc, _, _ = self.run_blocks(["B0", "B3"], dry_run=True) self.assertEqual((rc, self.calls), (0, [])) self.assertFalse(self.state_path.exists()) def test_blocked_aborts_or_skips(self): rc, _, _ = self.run_blocks(["B8"]) self.assertEqual((rc, self.calls), (2, [])) rc, _, _ = self.run_blocks(["B4h"]) # unresolved ${gc} self.assertEqual((rc, self.calls), (2, [])) rc, _, _ = self.run_blocks(["B8", "B6"], skip_blocked=True) self.assertEqual((rc, len(self.calls)), (0, 4)) def test_manual_deferred_skipped(self): rc, _, _ = self.run_blocks(["B13", "B5", "B14"]) self.assertEqual((rc, self.calls), (0, [])) rc, _, _ = self.run_blocks(["B13", "B5"], include_manual=True, include_deferred=True) self.assertEqual(len(self.calls), 2) def test_mods_restored_check(self): def mutating(argv): self.calls.append(argv) (self.mods / f"leak{len(self.calls)}.jar").write_bytes(b"l") return 0 state = matrix.State(self.state_path) with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): rc = matrix.run_blocks(self.m, ["B0"], state, self.mods, runner=mutating, out_dir=self.out) self.assertEqual((rc, len(self.calls)), (1, 1)) def test_unknown_block(self): with self.assertRaises(matrix.MatrixError): self.run_blocks(["B99"]) class Cli(unittest.TestCase): def test_list_and_plan(self): buf = io.StringIO() with contextlib.redirect_stdout(buf): self.assertEqual(matrix.main(["list"]), 0) self.assertEqual(matrix.main(["plan", "B4h", "--param", "gc=J1", "--display", "desktop"]), 0) out = buf.getvalue() self.assertIn("| B0 | noise floor | 3 |", out) self.assertIn("--jvm J1 --heap 4096", out) self.assertIn("--display desktop", out) if __name__ == "__main__": unittest.main()