logan
4fa9a6ab81
- gen_shader_profiles.py + shader-profiles/: Iris per-pack option files resolved from each pack's profile.* lines (Iris has no profile key) - bench.py: flag rows shader_not_loaded when Iris falls back to vanilla rendering (analyze.py treats it as invalid) - matrix: shader_options session key, trial jar globs pinned to the downloaded Modrinth builds, post-join settle 45 s (B0 v1 showed rep 1 still warming at 10 s)
405 lines
15 KiB
Python
405 lines
15 KiB
Python
"""Offline tests for matrix.py (no game; the bench.py runner is faked).
|
|
|
|
python3 -m unittest discover -s analysis/client-perf/harness/tests
|
|
"""
|
|
import contextlib
|
|
import io
|
|
import json
|
|
import sys
|
|
import tempfile
|
|
import tomllib
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
sys.path.insert(0, str(HERE.parent))
|
|
|
|
import matrix # noqa: E402
|
|
|
|
MINI = """
|
|
[defaults]
|
|
reps = 3
|
|
warmup = 15
|
|
capture = 30
|
|
display = "gamescope"
|
|
res = "1280x720"
|
|
world = "v1"
|
|
shader = "off"
|
|
dh = "on"
|
|
launch_overhead_s = 60
|
|
|
|
[worlds]
|
|
v1 = "{tmp}/worlds/v1.zip"
|
|
s4 = "{tmp}/worlds/s4-*.zip"
|
|
|
|
[trials]
|
|
gnetum = "{tmp}/trials/gnetum-*.jar"
|
|
|
|
[[missing_variants]]
|
|
id = "C1-fa"
|
|
blocks = ["B8"]
|
|
contents = "C1 + Fresh Animations"
|
|
|
|
[[block]]
|
|
id = "B0"
|
|
rule = "noise"
|
|
sessions = [{{ variant = "C0", plan = "s1", repeat = 3 }}]
|
|
|
|
[[block]]
|
|
id = "B3"
|
|
rule = "dh"
|
|
sessions = [
|
|
{{ variant = "C1", plan = "s1,s5" }},
|
|
{{ variant = "C1", plan = "s1,s5", dh = "off", tag = "dh-off" }},
|
|
{{ variant = "C1", plan = "s2", jvm = "J2", heap = 6144, world = "s4", tag = "x" }},
|
|
]
|
|
|
|
[[block]]
|
|
id = "B6"
|
|
rule = "mf"
|
|
interleave = true
|
|
sessions = [
|
|
{{ variant = "C1", plan = "s1", repeat = 2 }},
|
|
{{ variant = "mf", plan = "s1", repeat = 2 }},
|
|
]
|
|
|
|
[[block]]
|
|
id = "B8"
|
|
rule = "entities"
|
|
sessions = [{{ variant = "C1-fa", plan = "s3" }}]
|
|
|
|
[[block]]
|
|
id = "B12b"
|
|
rule = "trials"
|
|
sessions = [{{ variant = "C1", plan = "s1", add_mods = ["gnetum"], disable_mods = ["a-*.jar"], tag = "g" }}]
|
|
|
|
[[block]]
|
|
id = "B4h"
|
|
rule = "heap"
|
|
params = ["gc"]
|
|
sessions = [{{ variant = "C1", plan = "s5", jvm = "${{gc}}", heap = 4096 }}]
|
|
|
|
[[block]]
|
|
id = "B13"
|
|
rule = "profile"
|
|
sessions = [{{ variant = "C1", plan = "s1", manual = true }}]
|
|
|
|
[[block]]
|
|
id = "B5"
|
|
rule = "sable"
|
|
deferred = true
|
|
sessions = [{{ variant = "C1", plan = "s4" }}]
|
|
|
|
[[block]]
|
|
id = "B14"
|
|
rule = "spawn"
|
|
manual = true
|
|
sessions = []
|
|
"""
|
|
|
|
|
|
class Fixture(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = Path(tempfile.mkdtemp())
|
|
self.out = self.tmp / "out"
|
|
for v in ("C0", "C1", "mf"):
|
|
(self.out / v).mkdir(parents=True)
|
|
self.mods = self.tmp / "mods"
|
|
self.mods.mkdir()
|
|
(self.mods / "a-1.jar").write_bytes(b"x")
|
|
(self.tmp / "worlds").mkdir()
|
|
(self.tmp / "worlds" / "v1.zip").write_bytes(b"z")
|
|
(self.tmp / "worlds" / "s4-4.zip").write_bytes(b"z")
|
|
(self.tmp / "trials").mkdir()
|
|
(self.tmp / "trials" / "gnetum-1.0.jar").write_bytes(b"j")
|
|
self.path = self.tmp / "matrix.toml"
|
|
self.path.write_text(MINI.format(tmp=self.tmp))
|
|
self.m = matrix.load(self.path, self.out)
|
|
self.state_path = self.tmp / "state.json"
|
|
self.calls = []
|
|
|
|
def runner(self, codes=None):
|
|
codes = list(codes or [])
|
|
|
|
def fake(argv):
|
|
self.calls.append(argv)
|
|
return codes.pop(0) if codes else 0
|
|
return fake
|
|
|
|
def run_blocks(self, ids, codes=None, **kw):
|
|
state = matrix.State(self.state_path)
|
|
with contextlib.redirect_stdout(io.StringIO()), \
|
|
contextlib.redirect_stderr(io.StringIO()) as err:
|
|
rc = matrix.run_blocks(self.m, ids, state, self.mods, runner=self.runner(codes),
|
|
out_dir=self.out, **kw)
|
|
return rc, state, err.getvalue()
|
|
|
|
|
|
class Loading(Fixture):
|
|
def test_real_matrix_validates(self):
|
|
m = matrix.load() # the shipped matrix.toml against the real out/ dir
|
|
ids = [b["id"] for b in m["block"]]
|
|
for want in ["B0", "B1", "B2", "B3", "B4", "B5", "B6", "B7", "B8", "B9", "B10",
|
|
"B11", "B12a", "B12b", "B13", "B14"]:
|
|
self.assertIn(want, ids)
|
|
bm = matrix.block_map(m)
|
|
self.assertTrue(bm["B5"]["deferred"])
|
|
self.assertTrue(bm["B14"]["manual"])
|
|
b0 = matrix.expand(m, bm["B0"])
|
|
self.assertEqual([(s["variant"], s["plan"]) for s in b0], [("C0", ["s1"])] * 3)
|
|
|
|
def test_real_matrix_emits_only_bench_flags(self):
|
|
m = matrix.load()
|
|
self.assertEqual((m["defaults"]["display"], m["defaults"]["res"]),
|
|
("gamescope-headless", "1920x1080"))
|
|
params = {"gc": "J1", "scene": "s2"}
|
|
n = 0
|
|
for b in m["block"]:
|
|
for s in matrix.expand(m, b, params):
|
|
argv, _ = matrix.bench_argv(m, s)
|
|
self.assertEqual(argv[0], "session")
|
|
flags = {a for a in argv[1:] if a.startswith("--")}
|
|
self.assertLessEqual(flags, matrix.BENCH_SESSION_FLAGS, (b["id"], argv))
|
|
n += 1
|
|
self.assertGreater(n, 50)
|
|
|
|
def test_fa_and_leaves_variants_generated(self):
|
|
m = matrix.load()
|
|
missing = matrix.missing_variant_ids(m)
|
|
for v in ("C1-fa", "C1-fa-ext", "emf-lod20-fa", "emf-lod8-fa", "C1-bl"):
|
|
self.assertNotIn(v, missing)
|
|
self.assertTrue((matrix.OUT_DIR / v / "options.txt").is_file(), v)
|
|
|
|
def test_validation_errors(self):
|
|
d = tomllib.loads(MINI.format(tmp=self.tmp))
|
|
d["block"][0]["sessions"][0]["plan"] = "s5,s1"
|
|
d["block"][0]["sessions"][0]["typo"] = 1
|
|
d["block"][1]["sessions"].append({"variant": "nope", "plan": "s6"})
|
|
d["block"][1]["sessions"].append({"variant": "C1", "plan": "s1", "dh": "off",
|
|
"tag": "dh-off"})
|
|
d["block"][4]["sessions"][0]["add_mods"] = ["unknown"]
|
|
d["block"][5]["sessions"][0]["jvm"] = "${other}"
|
|
d["block"][0]["sessions"][0]["world"] = "nowhere"
|
|
d["defaults"]["display"] = "tv"
|
|
errs = "\n".join(matrix.validate(d, self.out))
|
|
for frag in ["S1->S5 order", "unknown key 'typo'", "variant 'nope'",
|
|
"unknown scene 's6'", "distinct tag", "not in [trials]",
|
|
"${other} not declared", "world 'nowhere'", "display 'tv'"]:
|
|
self.assertIn(frag, errs)
|
|
|
|
def test_missing_variant_is_valid_but_blocked(self):
|
|
s = matrix.expand(self.m, matrix.block_map(self.m)["B8"])[0]
|
|
self.assertTrue(matrix.variant_missing(s, self.out))
|
|
|
|
|
|
class Commands(Fixture):
|
|
def test_argv(self):
|
|
b3 = matrix.expand(self.m, matrix.block_map(self.m)["B3"])
|
|
argv, miss = matrix.bench_argv(self.m, b3[1])
|
|
self.assertEqual(miss, [])
|
|
self.assertEqual(argv[:9], ["session", "--variant", "C1", "--plan", "s1,s5",
|
|
"--reps", "3", "--label", "B3.dh-off"])
|
|
self.assertIn("--dh", argv)
|
|
self.assertEqual(argv[argv.index("--dh") + 1], "off")
|
|
self.assertEqual(argv[argv.index("--world-zip") + 1], str(self.tmp / "worlds/v1.zip"))
|
|
argv, _ = matrix.bench_argv(self.m, b3[2])
|
|
for flag, val in [("--jvm", "J2"), ("--heap", "6144"), ("--display", "gamescope"),
|
|
("--res", "1280x720"), ("--shader", "off"),
|
|
("--world-zip", str(self.tmp / "worlds/s4-4.zip"))]:
|
|
self.assertEqual(argv[argv.index(flag) + 1], val)
|
|
|
|
def test_trials_and_disable(self):
|
|
s = matrix.expand(self.m, matrix.block_map(self.m)["B12b"])[0]
|
|
argv, miss = matrix.bench_argv(self.m, s)
|
|
self.assertEqual(argv[argv.index("--add-mods") + 1],
|
|
str(self.tmp / "trials/gnetum-1.0.jar"))
|
|
self.assertEqual(argv[argv.index("--disable-mods") + 1], "a-*.jar")
|
|
(self.tmp / "trials" / "gnetum-1.0.jar").unlink()
|
|
_, miss = matrix.bench_argv(self.m, s)
|
|
self.assertTrue(any("gnetum" in x for x in miss))
|
|
|
|
def test_live_world_and_overrides(self):
|
|
s = matrix.expand(self.m, matrix.block_map(self.m)["B0"],
|
|
overrides={"display": "desktop", "res": None})[0]
|
|
argv, _ = matrix.bench_argv(self.m, s, live_world=True)
|
|
self.assertNotIn("--world-zip", argv)
|
|
self.assertEqual(argv[argv.index("--display") + 1], "desktop")
|
|
self.assertEqual(argv[argv.index("--res") + 1], "1280x720")
|
|
|
|
def test_interleave_and_params(self):
|
|
bm = matrix.block_map(self.m)
|
|
self.assertEqual([s["variant"] for s in matrix.expand(self.m, bm["B6"])],
|
|
["C1", "mf", "C1", "mf"])
|
|
s = matrix.expand(self.m, bm["B4h"])[0]
|
|
self.assertEqual(matrix.unresolved(s), ["gc"])
|
|
s = matrix.expand(self.m, bm["B4h"], params={"gc": "J2"})[0]
|
|
self.assertEqual(matrix.unresolved(s), [])
|
|
self.assertEqual(s["jvm"], "J2")
|
|
|
|
def test_seconds(self):
|
|
b3 = matrix.expand(self.m, matrix.block_map(self.m)["B3"])
|
|
self.assertEqual(matrix.session_seconds(b3[0], 60), 60 + 10 + 3 * 45 + 3 * 130)
|
|
|
|
|
|
class ShaderOptions(Fixture):
|
|
def b7(self, **extra):
|
|
d = tomllib.loads(MINI.format(tmp=self.tmp))
|
|
sess = {"variant": "C1", "plan": "s1,s4", "shader": "Pack.zip",
|
|
"shader_profile": "low", "tag": "p-low", **extra}
|
|
d["block"].append({"id": "B7", "rule": "shaders", "sessions": [sess]})
|
|
return d
|
|
|
|
def test_emitted(self):
|
|
f = self.tmp / "Pack.low.txt"
|
|
f.write_text("A=1\n")
|
|
d = self.b7(shader_options=str(f))
|
|
self.assertEqual(matrix.validate(d, self.out), [])
|
|
s = matrix.expand(d, matrix.block_map(d)["B7"])[0]
|
|
argv, miss = matrix.bench_argv(d, s)
|
|
self.assertEqual(miss, [])
|
|
self.assertEqual(argv[argv.index("--shader") + 1], "Pack.zip")
|
|
self.assertEqual(argv[argv.index("--shader-options") + 1], str(f))
|
|
self.assertIn("--shader-options", matrix.BENCH_SESSION_FLAGS)
|
|
|
|
def test_missing_file_and_relative_path(self):
|
|
d = self.b7(shader_options="shader-profiles/does-not-exist.txt")
|
|
s = matrix.expand(d, matrix.block_map(d)["B7"])[0]
|
|
argv, miss = matrix.bench_argv(d, s)
|
|
self.assertEqual(argv[argv.index("--shader-options") + 1],
|
|
str(matrix.HERE / "shader-profiles/does-not-exist.txt"))
|
|
self.assertTrue(any("shader options" in x for x in miss))
|
|
|
|
def test_needs_shader(self):
|
|
d = self.b7(shader_options="x.txt", shader="off")
|
|
self.assertIn("shader_options needs shader", "\n".join(matrix.validate(d, self.out)))
|
|
|
|
def test_spec_hash_tracks_file_content(self):
|
|
f = self.tmp / "Pack.low.txt"
|
|
f.write_text("A=1\n")
|
|
d = self.b7(shader_options=str(f))
|
|
s = matrix.expand(d, matrix.block_map(d)["B7"])[0]
|
|
argv, _ = matrix.bench_argv(d, s)
|
|
h1 = matrix.spec_hash(argv, s)
|
|
f.write_text("A=2\n")
|
|
self.assertNotEqual(h1, matrix.spec_hash(argv, s))
|
|
# sessions without shader_options keep the original hash formula (resume state)
|
|
b0 = matrix.expand(self.m, matrix.block_map(self.m)["B0"])[0]
|
|
a0, _ = matrix.bench_argv(self.m, b0)
|
|
blob = json.dumps({"argv": a0, "manual": False}, sort_keys=True)
|
|
import hashlib
|
|
self.assertEqual(matrix.spec_hash(a0, b0),
|
|
hashlib.sha256(blob.encode()).hexdigest()[:16])
|
|
|
|
def test_real_b7_automated(self):
|
|
m = matrix.load()
|
|
rows = matrix.expand(m, matrix.block_map(m)["B7"])
|
|
shaded = [s for s in rows if s["shader"] != "off"]
|
|
self.assertEqual(len(shaded), 4)
|
|
for s in rows:
|
|
self.assertFalse(s.get("manual"), s["label"])
|
|
for s in shaded:
|
|
argv, miss = matrix.bench_argv(m, s)
|
|
self.assertEqual([x for x in miss if "shader options" in x], [], s["label"])
|
|
self.assertTrue(Path(argv[argv.index("--shader-options") + 1]).is_file())
|
|
|
|
|
|
class Resume(Fixture):
|
|
def test_runs_then_skips(self):
|
|
rc, state, _ = self.run_blocks(["B0"])
|
|
self.assertEqual(rc, 0)
|
|
self.assertEqual(len(self.calls), 3)
|
|
self.assertEqual(state.data["b0"]["mods_hash"], matrix.mods_hash(self.mods))
|
|
saved = json.loads(self.state_path.read_text())
|
|
self.assertEqual({e["status"] for e in saved["sessions"].values()}, {"ok"})
|
|
self.calls.clear()
|
|
rc, _, _ = self.run_blocks(["B0"])
|
|
self.assertEqual((rc, self.calls), (0, []))
|
|
|
|
def test_mod_change_invalidates_and_warns(self):
|
|
self.run_blocks(["B0"])
|
|
(self.mods / "new-mod.jar").write_bytes(b"y")
|
|
self.calls.clear()
|
|
rc, _, err = self.run_blocks(["B3"])
|
|
self.assertEqual(rc, 0)
|
|
self.assertIn("B0 MUST BE RE-RUN", err)
|
|
self.calls.clear()
|
|
rc, _, _ = self.run_blocks(["B0"])
|
|
self.assertEqual(len(self.calls), 3) # B0 sessions are stale under the new set
|
|
|
|
def test_warns_without_b0(self):
|
|
_, _, err = self.run_blocks(["B3"])
|
|
self.assertIn("B0 (noise floor) has not been completed", err)
|
|
|
|
def test_spec_change_reruns(self):
|
|
self.run_blocks(["B0"])
|
|
self.calls.clear()
|
|
self.run_blocks(["B0"], live_world=True) # different command -> not done
|
|
self.assertEqual(len(self.calls), 3)
|
|
|
|
def test_retry_once_then_ok(self):
|
|
rc, state, _ = self.run_blocks(["B0"], codes=[1, 0])
|
|
self.assertEqual(rc, 0)
|
|
self.assertEqual(len(self.calls), 4)
|
|
self.assertEqual(state.data["sessions"]["B0#0"]["attempts"], 2)
|
|
|
|
def test_stop_after_two_failures(self):
|
|
rc, state, _ = self.run_blocks(["B0"], codes=[1, 1])
|
|
self.assertEqual(rc, 1)
|
|
self.assertEqual(len(self.calls), 2)
|
|
self.assertEqual(state.data["sessions"]["B0#0"]["status"], "failed")
|
|
self.assertNotIn("B0#1", state.data["sessions"])
|
|
self.calls.clear()
|
|
rc, _, _ = self.run_blocks(["B0"]) # resume retries the failed one
|
|
self.assertEqual((rc, len(self.calls)), (0, 3))
|
|
|
|
def test_dry_run_runs_nothing(self):
|
|
rc, _, _ = self.run_blocks(["B0", "B3"], dry_run=True)
|
|
self.assertEqual((rc, self.calls), (0, []))
|
|
self.assertFalse(self.state_path.exists())
|
|
|
|
def test_blocked_aborts_or_skips(self):
|
|
rc, _, _ = self.run_blocks(["B8"])
|
|
self.assertEqual((rc, self.calls), (2, []))
|
|
rc, _, _ = self.run_blocks(["B4h"]) # unresolved ${gc}
|
|
self.assertEqual((rc, self.calls), (2, []))
|
|
rc, _, _ = self.run_blocks(["B8", "B6"], skip_blocked=True)
|
|
self.assertEqual((rc, len(self.calls)), (0, 4))
|
|
|
|
def test_manual_deferred_skipped(self):
|
|
rc, _, _ = self.run_blocks(["B13", "B5", "B14"])
|
|
self.assertEqual((rc, self.calls), (0, []))
|
|
rc, _, _ = self.run_blocks(["B13", "B5"], include_manual=True, include_deferred=True)
|
|
self.assertEqual(len(self.calls), 2)
|
|
|
|
def test_mods_restored_check(self):
|
|
def mutating(argv):
|
|
self.calls.append(argv)
|
|
(self.mods / f"leak{len(self.calls)}.jar").write_bytes(b"l")
|
|
return 0
|
|
state = matrix.State(self.state_path)
|
|
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
|
|
rc = matrix.run_blocks(self.m, ["B0"], state, self.mods, runner=mutating,
|
|
out_dir=self.out)
|
|
self.assertEqual((rc, len(self.calls)), (1, 1))
|
|
|
|
def test_unknown_block(self):
|
|
with self.assertRaises(matrix.MatrixError):
|
|
self.run_blocks(["B99"])
|
|
|
|
|
|
class Cli(unittest.TestCase):
|
|
def test_list_and_plan(self):
|
|
buf = io.StringIO()
|
|
with contextlib.redirect_stdout(buf):
|
|
self.assertEqual(matrix.main(["list"]), 0)
|
|
self.assertEqual(matrix.main(["plan", "B4h", "--param", "gc=J1",
|
|
"--display", "desktop"]), 0)
|
|
out = buf.getvalue()
|
|
self.assertIn("| B0 | noise floor | 3 |", out)
|
|
self.assertIn("--jvm J1 --heap 4096", out)
|
|
self.assertIn("--display desktop", out)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|