feat(bench): add matrix runner, gamescope and per-session controls
- bench.py: gamescope-headless display (default, 1920x1080; ~2% cost vs desktop for isolation from the shared Wayland desktop), --shader, --dh, --disable-mods/--add-mods, --world-zip, crash-safe restore journal, spectator mode in the plan (elevated cameras fell to the sea floor and S4 measured underwater darkness) - matrix.toml/matrix.py: declarative, resumable B0-B14 runner keyed on mods hash (warns when B0 must be re-run) - build.py: bench-only resource-pack variants (C1-fa, C1-fa-ext, emf-lod*-fa, C1-bl); every bench variant pins resourcePacks so packs don't leak between sessions; shipped tier output unchanged - analyze.py: tagged labels use their own block's baseline
This commit is contained in:
logan committed
1 parent
8f56e2f780
commit
a15c912ad0
10 files changed
+2196
-58
No files matched your search
@@ -0,0 +1,97 @@
|
||||
"""analyze.py: tagged labels ('B3.dh-off') take their baseline from the same block.
|
||||
|
||||
python3 -m unittest discover -s analysis/client-perf/harness/tests
|
||||
"""
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE.parent))
|
||||
|
||||
import analyze # noqa: E402
|
||||
|
||||
|
||||
def row(label, variant, scene, session, p99, low=100.0, fps=200.0):
|
||||
return {"kind": "window", "label": label, "variant": variant, "scene": scene,
|
||||
"session": session, "flags": [],
|
||||
"metrics": {"p99_ms": p99, "one_low_fps": low, "avg_fps": fps}}
|
||||
|
||||
|
||||
def cell(table, label, variant, scene):
|
||||
for line in table.splitlines():
|
||||
cells = [c.strip() for c in line.strip("|").split("|")]
|
||||
if cells[:3] == [label, variant, scene]:
|
||||
return cells
|
||||
raise AssertionError(f"no row {label}/{variant}/{scene} in\n{table}")
|
||||
|
||||
|
||||
class BlockBaseline(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.rows = [
|
||||
# noise: spread 0.2 ms / 2 fps
|
||||
row("B0", "C0", "s1", "n1", 10.0, 100), row("B0", "C0", "s1", "n2", 10.2, 102),
|
||||
# global C1 reference elsewhere (B2) is much slower than the B3 one
|
||||
row("B2", "C1", "s1", "g1", 20.0),
|
||||
row("B3", "C1", "s1", "b1", 10.0),
|
||||
row("B3", "dh-r64", "s1", "b2", 8.0),
|
||||
row("B3.dh-off", "C1", "s1", "b3", 6.0),
|
||||
]
|
||||
|
||||
def test_tagged_self_variant_uses_untagged_block_label(self):
|
||||
s = analyze.summarise(self.rows)
|
||||
b, where = analyze.self_baseline(s, "B3.dh-off", "s1", "C1")
|
||||
self.assertEqual(where, "B3")
|
||||
self.assertEqual(b["p99"], 10.0)
|
||||
|
||||
def test_untagged_block_label_is_its_own_baseline(self):
|
||||
s = analyze.summarise(self.rows)
|
||||
self.assertEqual(analyze.self_baseline(s, "B3", "s1", "C1"), (None, None))
|
||||
|
||||
def test_tagged_other_variant_prefers_block_over_global(self):
|
||||
rows = self.rows + [row("B12a.no-x", "X", "s1", "t1", 9.0)]
|
||||
rows += [row("B12a", "C1", "s1", "t0", 12.0)]
|
||||
s = analyze.summarise(rows)
|
||||
b, where = analyze.baseline_for(s, rows, "B12a.no-x", "s1", "C1")
|
||||
self.assertEqual((where, b["p99"]), ("B12a", 12.0))
|
||||
|
||||
def test_block_baseline_tag(self):
|
||||
rows = self.rows + [row("B11.baseline", "C1", "s5", "x0", 12.0),
|
||||
row("B11", "sod-thr3", "s5", "x1", 11.0),
|
||||
row("B2", "C1", "s5", "x2", 30.0)]
|
||||
s = analyze.summarise(rows)
|
||||
b, where = analyze.baseline_for(s, rows, "B11", "s5", "C1")
|
||||
# untagged candidate: own label has no C1 -> '<block>.baseline', not global
|
||||
self.assertEqual((where, b["p99"]), ("B11.baseline", 12.0))
|
||||
rows.append(row("B11.x", "opt-mip4", "s5", "x3", 11.0))
|
||||
s = analyze.summarise(rows)
|
||||
b, where = analyze.baseline_for(s, rows, "B11.x", "s5", "C1")
|
||||
self.assertEqual((where, b["p99"]), ("B11.baseline", 12.0))
|
||||
|
||||
def test_block_pool_then_global(self):
|
||||
rows = self.rows + [row("B4.J0", "C1", "s2", "j0", 10.0),
|
||||
row("B4.J1", "C1", "s2", "j1", 12.0),
|
||||
row("B4.x", "Y", "s2", "j2", 9.0),
|
||||
row("B2", "C1", "s2", "g2", 40.0)]
|
||||
s = analyze.summarise(rows)
|
||||
b, where = analyze.baseline_for(s, rows, "B4.x", "s2", "C1")
|
||||
self.assertEqual((where, b["p99"]), ("B4.*", 11.0))
|
||||
b, where = analyze.baseline_for(s, rows, "B9.z", "s2", "C1")
|
||||
self.assertEqual(where, "*")
|
||||
self.assertIsNone(analyze.baseline_for(s, rows, "B9.z", "s9", "C1")[0])
|
||||
|
||||
def test_table_verdict_against_block(self):
|
||||
t = analyze.table(analyze.summarise(self.rows), self.rows, "C1", "B0")
|
||||
c = cell(t, "B3.dh-off", "C1", "s1")
|
||||
self.assertEqual(c[-4:], ["C1@B3", "+4.00", "+0.0", "win"])
|
||||
self.assertEqual(cell(t, "B3", "C1", "s1")[-1], "baseline")
|
||||
self.assertEqual(cell(t, "B3", "dh-r64", "s1")[-4], "C1@B3")
|
||||
|
||||
def test_table_label_filter_still_finds_block_baseline(self):
|
||||
shown = [r for r in self.rows if r["label"] in ("B3.dh-off", "B0")]
|
||||
t = analyze.table(analyze.summarise(shown), self.rows, "C1", "B0")
|
||||
self.assertEqual(cell(t, "B3.dh-off", "C1", "s1")[-4], "C1@B3")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,273 @@
|
||||
"""Offline tests for bench.py session instance changes (temp dirs only, no game).
|
||||
|
||||
python3 -m unittest discover -s analysis/client-perf/harness/tests
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE.parent))
|
||||
|
||||
import bench # noqa: E402
|
||||
|
||||
IRIS = """#This file stores configuration options for Iris, such as the currently active shaderpack
|
||||
#Fri Oct 16 19:37:17 EDT 2026
|
||||
allowUnknownShaders=false
|
||||
enableShaders=true
|
||||
maxShadowRenderDistance=32
|
||||
shaderPack=
|
||||
"""
|
||||
|
||||
DH = """[server]
|
||||
\tenableRealTimeUpdates = true
|
||||
|
||||
[client]
|
||||
\tshowDhOptionsButtonInMinecraftUi = true
|
||||
|
||||
\t\t[client.advanced.debugging]
|
||||
\t\t\tlodOnlyMode = false
|
||||
\t\t\t# DISABLED: Disable rendering
|
||||
\t\t\trendererMode = "DEFAULT"
|
||||
|
||||
\t\t\t[client.advanced.debugging.debugWireframe]
|
||||
\t\t\t\tenableRendering = false
|
||||
\t\t\t\trendererMode = "DEFAULT"
|
||||
"""
|
||||
|
||||
|
||||
class GameDir(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self._td = tempfile.TemporaryDirectory()
|
||||
self.game = Path(self._td.name)
|
||||
(self.game / "config").mkdir()
|
||||
(self.game / "mods").mkdir()
|
||||
(self.game / "shaderpacks").mkdir()
|
||||
(self.game / "saves" / "onionbench").mkdir(parents=True)
|
||||
(self.game / "saves" / "onionbench" / "level.dat").write_bytes(b"orig")
|
||||
(self.game / "config" / "iris.properties").write_text(IRIS)
|
||||
(self.game / "config" / "DistantHorizons.toml").write_text(DH)
|
||||
(self.game / "shaderpacks" / "Pack A.zip").write_bytes(b"zip")
|
||||
for n in ("BadOptimizations-2.4.1.jar", "particle_core-0.3.3.jar", "sodium-0.8.jar"):
|
||||
(self.game / "mods" / n).write_bytes(n.encode())
|
||||
self.j = bench.Journal.begin(self.game, "ptest")
|
||||
|
||||
def tearDown(self):
|
||||
self._td.cleanup()
|
||||
|
||||
def snapshot(self):
|
||||
"""Instance contents, excluding the journal itself (setUp opened one)."""
|
||||
return {p.relative_to(self.game).as_posix(): p.read_bytes()
|
||||
for p in sorted(self.game.rglob("*")) if p.is_file()
|
||||
and p.relative_to(self.game).parts[0] not in (bench.PENDING, bench.BACKUP_DIR)}
|
||||
|
||||
|
||||
class IrisAndDh(GameDir):
|
||||
def test_properties_rewrite(self):
|
||||
out = bench.set_properties(IRIS, {"enableShaders": "false", "shaderPack": "a:b=c.zip"})
|
||||
self.assertIn("enableShaders=false\n", out)
|
||||
self.assertIn("shaderPack=a\\:b\\=c.zip\n", out)
|
||||
self.assertIn("maxShadowRenderDistance=32", out)
|
||||
self.assertTrue(out.startswith("#This file"))
|
||||
self.assertIn("newKey=1", bench.set_properties(IRIS, {"newKey": "1"}))
|
||||
|
||||
def test_shader_on_off_restore(self):
|
||||
before = self.snapshot()
|
||||
opts = self.game.parent / f"{self.game.name}-opts.txt"
|
||||
opts.write_text("SHADOW=off\n")
|
||||
try:
|
||||
bench.apply_shader(self.game, self.j, "Pack A.zip", opts)
|
||||
finally:
|
||||
text = opts.read_text()
|
||||
opts.unlink()
|
||||
cfg = (self.game / "config" / "iris.properties").read_text()
|
||||
self.assertIn("enableShaders=true", cfg)
|
||||
self.assertIn("shaderPack=Pack A.zip", cfg)
|
||||
self.assertEqual((self.game / "shaderpacks" / "Pack A.zip.txt").read_text(), text)
|
||||
self.assertEqual(bench.restore_pending(self.game), [])
|
||||
self.assertEqual(self.snapshot(), before)
|
||||
|
||||
j = bench.Journal.begin(self.game, "p2")
|
||||
bench.apply_shader(self.game, j, "off")
|
||||
cfg = (self.game / "config" / "iris.properties").read_text()
|
||||
self.assertIn("enableShaders=false", cfg)
|
||||
self.assertIn("shaderPack=\n", cfg)
|
||||
bench.restore_pending(self.game)
|
||||
self.assertEqual(self.snapshot(), before)
|
||||
|
||||
def test_dh_rewrite_only_debugging_section(self):
|
||||
off = bench.set_dh_rendering(DH, False)
|
||||
self.assertIn('\t\t\trendererMode = "DISABLED"\n\n', off)
|
||||
self.assertIn('\t\t\t\trendererMode = "DEFAULT"', off) # wireframe section untouched
|
||||
self.assertIn("enableRendering = false", off)
|
||||
self.assertEqual(bench.set_dh_rendering(off, True), DH)
|
||||
with self.assertRaises(ValueError):
|
||||
bench.set_dh_rendering("[client]\n\trendererMode = \"DEFAULT\"\n", False)
|
||||
|
||||
def test_apply_dh_restore(self):
|
||||
bench.apply_dh(self.game, self.j, False)
|
||||
self.assertIn('"DISABLED"', (self.game / "config" / "DistantHorizons.toml").read_text())
|
||||
bench.restore_pending(self.game)
|
||||
self.assertEqual((self.game / "config" / "DistantHorizons.toml").read_text(), DH)
|
||||
|
||||
|
||||
class Mods(GameDir):
|
||||
def test_disable_add_restore(self):
|
||||
before = self.snapshot()
|
||||
names = bench.resolve_disable(self.game, ["badopt*", "PARTICLE_*"])
|
||||
self.assertEqual(names, ["BadOptimizations-2.4.1.jar", "particle_core-0.3.3.jar"])
|
||||
with self.assertRaises(ValueError):
|
||||
bench.resolve_disable(self.game, ["badopt*", "nomatch*"])
|
||||
extra = self.game.parent / f"{self.game.name}-trial.jar"
|
||||
extra.write_bytes(b"trial")
|
||||
try:
|
||||
jars = bench.check_add_mods(self.game, [str(extra)])
|
||||
h0 = bench.mods_hash(self.game / "mods")
|
||||
bench.disable_mods(self.game, self.j, names)
|
||||
bench.add_mods(self.game, self.j, jars)
|
||||
finally:
|
||||
extra.unlink()
|
||||
mods = sorted(p.name for p in (self.game / "mods").iterdir())
|
||||
self.assertEqual(mods, sorted(["BadOptimizations-2.4.1.jar.disabled",
|
||||
f"{self.game.name}-trial.jar",
|
||||
"particle_core-0.3.3.jar.disabled", "sodium-0.8.jar"]))
|
||||
self.assertNotEqual(bench.mods_hash(self.game / "mods"), h0)
|
||||
self.assertEqual(bench.restore_pending(self.game), [])
|
||||
self.assertEqual(self.snapshot(), before)
|
||||
self.assertEqual(bench.mods_hash(self.game / "mods"), h0)
|
||||
|
||||
def test_add_mods_validation(self):
|
||||
with self.assertRaises(ValueError):
|
||||
bench.check_add_mods(self.game, [str(self.game / "nope.jar")])
|
||||
with self.assertRaises(ValueError): # name collision
|
||||
bench.check_add_mods(self.game, [str(self.game / "mods" / "sodium-0.8.jar")])
|
||||
|
||||
|
||||
class WorldZip(GameDir):
|
||||
def make_zip(self, prefix):
|
||||
z = self.game.parent / f"{self.game.name}-{prefix.strip('/') or 'flat'}.zip"
|
||||
with zipfile.ZipFile(z, "w") as zf:
|
||||
zf.writestr(prefix + "level.dat", b"zipped")
|
||||
zf.writestr(prefix + "region/r.0.0.mca", b"region")
|
||||
self.addCleanup(z.unlink)
|
||||
return z
|
||||
|
||||
def check_swap(self, prefix):
|
||||
before = self.snapshot()
|
||||
z = self.make_zip(prefix)
|
||||
self.assertEqual(bench.zip_world_root(z), prefix)
|
||||
sha = bench.swap_world(self.game, self.j, z)
|
||||
self.assertEqual(sha, bench.sha256_file(z))
|
||||
w = self.game / "saves" / "onionbench"
|
||||
self.assertEqual((w / "level.dat").read_bytes(), b"zipped")
|
||||
self.assertTrue((w / "region" / "r.0.0.mca").exists())
|
||||
self.assertTrue((self.game / "saves" / "onionbench.harness-bak" / "level.dat").exists())
|
||||
self.assertEqual(sorted(p.name for p in (self.game / "saves").iterdir()),
|
||||
["onionbench", "onionbench.harness-bak"])
|
||||
self.assertEqual(bench.restore_pending(self.game), [])
|
||||
self.assertEqual(self.snapshot(), before)
|
||||
self.assertEqual([p.name for p in (self.game / "saves").iterdir()], ["onionbench"])
|
||||
|
||||
def test_flat_layout(self):
|
||||
self.check_swap("")
|
||||
|
||||
def test_single_top_dir_layout(self):
|
||||
self.check_swap("bench-world-v1/")
|
||||
|
||||
def test_bad_zips(self):
|
||||
bad = self.game.parent / f"{self.game.name}-bad.zip"
|
||||
self.addCleanup(lambda: bad.unlink(missing_ok=True))
|
||||
with zipfile.ZipFile(bad, "w") as zf:
|
||||
zf.writestr("a/level.dat", b"x")
|
||||
zf.writestr("b/level.dat", b"x")
|
||||
with self.assertRaises(ValueError):
|
||||
bench.zip_world_root(bad)
|
||||
with zipfile.ZipFile(bad, "w") as zf:
|
||||
zf.writestr("../evil/level.dat", b"x")
|
||||
with self.assertRaises(ValueError):
|
||||
bench.zip_world_root(bad)
|
||||
|
||||
|
||||
class PendingJournal(GameDir):
|
||||
def test_crash_then_restore_from_disk(self):
|
||||
before = self.snapshot()
|
||||
z = WorldZip.make_zip(self, "w/")
|
||||
bench.apply_shader(self.game, self.j, "Pack A.zip")
|
||||
bench.apply_dh(self.game, self.j, False)
|
||||
bench.disable_mods(self.game, self.j, ["sodium-0.8.jar"])
|
||||
bench.swap_world(self.game, self.j, z)
|
||||
plan = self.game / "saves" / "onionbench" / "datapacks" / "autoplan"
|
||||
self.j.record({"op": "tree", "path": plan.relative_to(self.game).as_posix()})
|
||||
plan.mkdir(parents=True)
|
||||
(plan / "pack.mcmeta").write_text("{}")
|
||||
# simulate a hard kill: only the on-disk journal survives
|
||||
del self.j
|
||||
self.assertTrue(bench.pending_exists(self.game))
|
||||
data = json.loads((self.game / bench.PENDING).read_text())
|
||||
self.assertEqual([e["op"] for e in data["entries"]],
|
||||
["file", "file", "file", "rename", "world", "tree"])
|
||||
with self.assertRaises(RuntimeError):
|
||||
bench.Journal.begin(self.game, "again")
|
||||
self.assertEqual(bench.restore_pending(self.game), [])
|
||||
self.assertFalse(bench.pending_exists(self.game))
|
||||
self.assertFalse((self.game / bench.BACKUP_DIR).exists())
|
||||
self.assertEqual(self.snapshot(), before)
|
||||
|
||||
def test_entry_written_before_mutation_is_tolerated(self):
|
||||
# journal says "renamed", but the kill came before os.rename ran
|
||||
self.j.record({"op": "rename", "src": "mods/sodium-0.8.jar",
|
||||
"dst": "mods/sodium-0.8.jar.disabled"})
|
||||
self.j.record({"op": "added", "path": "mods/never-copied.jar"})
|
||||
before = self.snapshot()
|
||||
self.assertEqual(bench.restore_pending(self.game), [])
|
||||
self.assertEqual(self.snapshot(), before)
|
||||
|
||||
def test_failed_entry_stays_journaled(self):
|
||||
self.j.record({"op": "bogus"})
|
||||
failed = bench.restore_pending(self.game)
|
||||
self.assertEqual(len(failed), 1)
|
||||
data = json.loads((self.game / bench.PENDING).read_text())
|
||||
self.assertEqual(data["entries"], [{"op": "bogus"}])
|
||||
|
||||
def test_nothing_pending(self):
|
||||
bench.restore_pending(self.game)
|
||||
self.assertEqual(bench.restore_pending(self.game), [])
|
||||
|
||||
|
||||
class OptionsAndFlags(unittest.TestCase):
|
||||
def test_resource_pack_reset(self):
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
g = Path(td)
|
||||
(g / "options.txt").write_text('version:3955\nresourcePacks:["vanilla","x.zip"]\n'
|
||||
"maxFps:260\n")
|
||||
bench.reset_resource_packs(g)
|
||||
vals = bench.options_values(g, {"resourcePacks", "incompatibleResourcePacks"})
|
||||
self.assertEqual(vals, {"resourcePacks": '["vanilla","mod_resources"]',
|
||||
"incompatibleResourcePacks": "[]"})
|
||||
text = (g / "options.txt").read_text()
|
||||
self.assertTrue(text.startswith("version:3955\n"))
|
||||
self.assertIn("maxFps:260", text)
|
||||
(g / "options.txt").unlink()
|
||||
bench.reset_resource_packs(g) # absent file -> just the baseline
|
||||
self.assertEqual(len((g / "options.txt").read_text().splitlines()), 2)
|
||||
|
||||
def test_env_flags(self):
|
||||
gpus = [{"device": "0x13c0", "dpm_level": "auto"},
|
||||
{"device": bench.DGPU_DEVICE, "dpm_level": "auto"}]
|
||||
self.assertEqual(bench.env_flags(gpus, True), ["gpu_dpm_auto", "server_running"])
|
||||
gpus[1]["dpm_level"] = "high"
|
||||
self.assertEqual(bench.env_flags(gpus, False), [])
|
||||
|
||||
def test_build_parser_session_defaults(self):
|
||||
a = bench.build_parser().parse_args(["session", "--variant", "C1"])
|
||||
self.assertEqual((a.shader, a.dh, a.disable_mods, a.add_mods, a.world_zip),
|
||||
("off", "on", None, None, None))
|
||||
a = bench.build_parser().parse_args(["restore"])
|
||||
self.assertIs(a.fn, bench.cmd_restore)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,342 @@
|
||||
"""Offline tests for matrix.py (no game; the bench.py runner is faked).
|
||||
|
||||
python3 -m unittest discover -s analysis/client-perf/harness/tests
|
||||
"""
|
||||
import contextlib
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import tomllib
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE.parent))
|
||||
|
||||
import matrix # noqa: E402
|
||||
|
||||
MINI = """
|
||||
[defaults]
|
||||
reps = 3
|
||||
warmup = 15
|
||||
capture = 30
|
||||
display = "gamescope"
|
||||
res = "1280x720"
|
||||
world = "v1"
|
||||
shader = "off"
|
||||
dh = "on"
|
||||
launch_overhead_s = 60
|
||||
|
||||
[worlds]
|
||||
v1 = "{tmp}/worlds/v1.zip"
|
||||
s4 = "{tmp}/worlds/s4-*.zip"
|
||||
|
||||
[trials]
|
||||
gnetum = "{tmp}/trials/gnetum-*.jar"
|
||||
|
||||
[[missing_variants]]
|
||||
id = "C1-fa"
|
||||
blocks = ["B8"]
|
||||
contents = "C1 + Fresh Animations"
|
||||
|
||||
[[block]]
|
||||
id = "B0"
|
||||
rule = "noise"
|
||||
sessions = [{{ variant = "C0", plan = "s1", repeat = 3 }}]
|
||||
|
||||
[[block]]
|
||||
id = "B3"
|
||||
rule = "dh"
|
||||
sessions = [
|
||||
{{ variant = "C1", plan = "s1,s5" }},
|
||||
{{ variant = "C1", plan = "s1,s5", dh = "off", tag = "dh-off" }},
|
||||
{{ variant = "C1", plan = "s2", jvm = "J2", heap = 6144, world = "s4", tag = "x" }},
|
||||
]
|
||||
|
||||
[[block]]
|
||||
id = "B6"
|
||||
rule = "mf"
|
||||
interleave = true
|
||||
sessions = [
|
||||
{{ variant = "C1", plan = "s1", repeat = 2 }},
|
||||
{{ variant = "mf", plan = "s1", repeat = 2 }},
|
||||
]
|
||||
|
||||
[[block]]
|
||||
id = "B8"
|
||||
rule = "entities"
|
||||
sessions = [{{ variant = "C1-fa", plan = "s3" }}]
|
||||
|
||||
[[block]]
|
||||
id = "B12b"
|
||||
rule = "trials"
|
||||
sessions = [{{ variant = "C1", plan = "s1", add_mods = ["gnetum"], disable_mods = ["a-*.jar"], tag = "g" }}]
|
||||
|
||||
[[block]]
|
||||
id = "B4h"
|
||||
rule = "heap"
|
||||
params = ["gc"]
|
||||
sessions = [{{ variant = "C1", plan = "s5", jvm = "${{gc}}", heap = 4096 }}]
|
||||
|
||||
[[block]]
|
||||
id = "B13"
|
||||
rule = "profile"
|
||||
sessions = [{{ variant = "C1", plan = "s1", manual = true }}]
|
||||
|
||||
[[block]]
|
||||
id = "B5"
|
||||
rule = "sable"
|
||||
deferred = true
|
||||
sessions = [{{ variant = "C1", plan = "s4" }}]
|
||||
|
||||
[[block]]
|
||||
id = "B14"
|
||||
rule = "spawn"
|
||||
manual = true
|
||||
sessions = []
|
||||
"""
|
||||
|
||||
|
||||
class Fixture(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = Path(tempfile.mkdtemp())
|
||||
self.out = self.tmp / "out"
|
||||
for v in ("C0", "C1", "mf"):
|
||||
(self.out / v).mkdir(parents=True)
|
||||
self.mods = self.tmp / "mods"
|
||||
self.mods.mkdir()
|
||||
(self.mods / "a-1.jar").write_bytes(b"x")
|
||||
(self.tmp / "worlds").mkdir()
|
||||
(self.tmp / "worlds" / "v1.zip").write_bytes(b"z")
|
||||
(self.tmp / "worlds" / "s4-4.zip").write_bytes(b"z")
|
||||
(self.tmp / "trials").mkdir()
|
||||
(self.tmp / "trials" / "gnetum-1.0.jar").write_bytes(b"j")
|
||||
self.path = self.tmp / "matrix.toml"
|
||||
self.path.write_text(MINI.format(tmp=self.tmp))
|
||||
self.m = matrix.load(self.path, self.out)
|
||||
self.state_path = self.tmp / "state.json"
|
||||
self.calls = []
|
||||
|
||||
def runner(self, codes=None):
|
||||
codes = list(codes or [])
|
||||
|
||||
def fake(argv):
|
||||
self.calls.append(argv)
|
||||
return codes.pop(0) if codes else 0
|
||||
return fake
|
||||
|
||||
def run_blocks(self, ids, codes=None, **kw):
|
||||
state = matrix.State(self.state_path)
|
||||
with contextlib.redirect_stdout(io.StringIO()), \
|
||||
contextlib.redirect_stderr(io.StringIO()) as err:
|
||||
rc = matrix.run_blocks(self.m, ids, state, self.mods, runner=self.runner(codes),
|
||||
out_dir=self.out, **kw)
|
||||
return rc, state, err.getvalue()
|
||||
|
||||
|
||||
class Loading(Fixture):
|
||||
def test_real_matrix_validates(self):
|
||||
m = matrix.load() # the shipped matrix.toml against the real out/ dir
|
||||
ids = [b["id"] for b in m["block"]]
|
||||
for want in ["B0", "B1", "B2", "B3", "B4", "B5", "B6", "B7", "B8", "B9", "B10",
|
||||
"B11", "B12a", "B12b", "B13", "B14"]:
|
||||
self.assertIn(want, ids)
|
||||
bm = matrix.block_map(m)
|
||||
self.assertTrue(bm["B5"]["deferred"])
|
||||
self.assertTrue(bm["B14"]["manual"])
|
||||
b0 = matrix.expand(m, bm["B0"])
|
||||
self.assertEqual([(s["variant"], s["plan"]) for s in b0], [("C0", ["s1"])] * 3)
|
||||
|
||||
def test_real_matrix_emits_only_bench_flags(self):
|
||||
m = matrix.load()
|
||||
self.assertEqual((m["defaults"]["display"], m["defaults"]["res"]),
|
||||
("gamescope-headless", "1920x1080"))
|
||||
params = {"gc": "J1", "scene": "s2"}
|
||||
n = 0
|
||||
for b in m["block"]:
|
||||
for s in matrix.expand(m, b, params):
|
||||
argv, _ = matrix.bench_argv(m, s)
|
||||
self.assertEqual(argv[0], "session")
|
||||
flags = {a for a in argv[1:] if a.startswith("--")}
|
||||
self.assertLessEqual(flags, matrix.BENCH_SESSION_FLAGS, (b["id"], argv))
|
||||
n += 1
|
||||
self.assertGreater(n, 50)
|
||||
|
||||
def test_fa_and_leaves_variants_generated(self):
|
||||
m = matrix.load()
|
||||
missing = matrix.missing_variant_ids(m)
|
||||
for v in ("C1-fa", "C1-fa-ext", "emf-lod20-fa", "emf-lod8-fa", "C1-bl"):
|
||||
self.assertNotIn(v, missing)
|
||||
self.assertTrue((matrix.OUT_DIR / v / "options.txt").is_file(), v)
|
||||
|
||||
def test_validation_errors(self):
|
||||
d = tomllib.loads(MINI.format(tmp=self.tmp))
|
||||
d["block"][0]["sessions"][0]["plan"] = "s5,s1"
|
||||
d["block"][0]["sessions"][0]["typo"] = 1
|
||||
d["block"][1]["sessions"].append({"variant": "nope", "plan": "s6"})
|
||||
d["block"][1]["sessions"].append({"variant": "C1", "plan": "s1", "dh": "off",
|
||||
"tag": "dh-off"})
|
||||
d["block"][4]["sessions"][0]["add_mods"] = ["unknown"]
|
||||
d["block"][5]["sessions"][0]["jvm"] = "${other}"
|
||||
d["block"][0]["sessions"][0]["world"] = "nowhere"
|
||||
d["defaults"]["display"] = "tv"
|
||||
errs = "\n".join(matrix.validate(d, self.out))
|
||||
for frag in ["S1->S5 order", "unknown key 'typo'", "variant 'nope'",
|
||||
"unknown scene 's6'", "distinct tag", "not in [trials]",
|
||||
"${other} not declared", "world 'nowhere'", "display 'tv'"]:
|
||||
self.assertIn(frag, errs)
|
||||
|
||||
def test_missing_variant_is_valid_but_blocked(self):
|
||||
s = matrix.expand(self.m, matrix.block_map(self.m)["B8"])[0]
|
||||
self.assertTrue(matrix.variant_missing(s, self.out))
|
||||
|
||||
|
||||
class Commands(Fixture):
|
||||
def test_argv(self):
|
||||
b3 = matrix.expand(self.m, matrix.block_map(self.m)["B3"])
|
||||
argv, miss = matrix.bench_argv(self.m, b3[1])
|
||||
self.assertEqual(miss, [])
|
||||
self.assertEqual(argv[:9], ["session", "--variant", "C1", "--plan", "s1,s5",
|
||||
"--reps", "3", "--label", "B3.dh-off"])
|
||||
self.assertIn("--dh", argv)
|
||||
self.assertEqual(argv[argv.index("--dh") + 1], "off")
|
||||
self.assertEqual(argv[argv.index("--world-zip") + 1], str(self.tmp / "worlds/v1.zip"))
|
||||
argv, _ = matrix.bench_argv(self.m, b3[2])
|
||||
for flag, val in [("--jvm", "J2"), ("--heap", "6144"), ("--display", "gamescope"),
|
||||
("--res", "1280x720"), ("--shader", "off"),
|
||||
("--world-zip", str(self.tmp / "worlds/s4-4.zip"))]:
|
||||
self.assertEqual(argv[argv.index(flag) + 1], val)
|
||||
|
||||
def test_trials_and_disable(self):
|
||||
s = matrix.expand(self.m, matrix.block_map(self.m)["B12b"])[0]
|
||||
argv, miss = matrix.bench_argv(self.m, s)
|
||||
self.assertEqual(argv[argv.index("--add-mods") + 1],
|
||||
str(self.tmp / "trials/gnetum-1.0.jar"))
|
||||
self.assertEqual(argv[argv.index("--disable-mods") + 1], "a-*.jar")
|
||||
(self.tmp / "trials" / "gnetum-1.0.jar").unlink()
|
||||
_, miss = matrix.bench_argv(self.m, s)
|
||||
self.assertTrue(any("gnetum" in x for x in miss))
|
||||
|
||||
def test_live_world_and_overrides(self):
|
||||
s = matrix.expand(self.m, matrix.block_map(self.m)["B0"],
|
||||
overrides={"display": "desktop", "res": None})[0]
|
||||
argv, _ = matrix.bench_argv(self.m, s, live_world=True)
|
||||
self.assertNotIn("--world-zip", argv)
|
||||
self.assertEqual(argv[argv.index("--display") + 1], "desktop")
|
||||
self.assertEqual(argv[argv.index("--res") + 1], "1280x720")
|
||||
|
||||
def test_interleave_and_params(self):
|
||||
bm = matrix.block_map(self.m)
|
||||
self.assertEqual([s["variant"] for s in matrix.expand(self.m, bm["B6"])],
|
||||
["C1", "mf", "C1", "mf"])
|
||||
s = matrix.expand(self.m, bm["B4h"])[0]
|
||||
self.assertEqual(matrix.unresolved(s), ["gc"])
|
||||
s = matrix.expand(self.m, bm["B4h"], params={"gc": "J2"})[0]
|
||||
self.assertEqual(matrix.unresolved(s), [])
|
||||
self.assertEqual(s["jvm"], "J2")
|
||||
|
||||
def test_seconds(self):
|
||||
b3 = matrix.expand(self.m, matrix.block_map(self.m)["B3"])
|
||||
self.assertEqual(matrix.session_seconds(b3[0], 60), 60 + 3 * 45 + 3 * 130)
|
||||
|
||||
|
||||
class Resume(Fixture):
|
||||
def test_runs_then_skips(self):
|
||||
rc, state, _ = self.run_blocks(["B0"])
|
||||
self.assertEqual(rc, 0)
|
||||
self.assertEqual(len(self.calls), 3)
|
||||
self.assertEqual(state.data["b0"]["mods_hash"], matrix.mods_hash(self.mods))
|
||||
saved = json.loads(self.state_path.read_text())
|
||||
self.assertEqual({e["status"] for e in saved["sessions"].values()}, {"ok"})
|
||||
self.calls.clear()
|
||||
rc, _, _ = self.run_blocks(["B0"])
|
||||
self.assertEqual((rc, self.calls), (0, []))
|
||||
|
||||
def test_mod_change_invalidates_and_warns(self):
|
||||
self.run_blocks(["B0"])
|
||||
(self.mods / "new-mod.jar").write_bytes(b"y")
|
||||
self.calls.clear()
|
||||
rc, _, err = self.run_blocks(["B3"])
|
||||
self.assertEqual(rc, 0)
|
||||
self.assertIn("B0 MUST BE RE-RUN", err)
|
||||
self.calls.clear()
|
||||
rc, _, _ = self.run_blocks(["B0"])
|
||||
self.assertEqual(len(self.calls), 3) # B0 sessions are stale under the new set
|
||||
|
||||
def test_warns_without_b0(self):
|
||||
_, _, err = self.run_blocks(["B3"])
|
||||
self.assertIn("B0 (noise floor) has not been completed", err)
|
||||
|
||||
def test_spec_change_reruns(self):
|
||||
self.run_blocks(["B0"])
|
||||
self.calls.clear()
|
||||
self.run_blocks(["B0"], live_world=True) # different command -> not done
|
||||
self.assertEqual(len(self.calls), 3)
|
||||
|
||||
def test_retry_once_then_ok(self):
|
||||
rc, state, _ = self.run_blocks(["B0"], codes=[1, 0])
|
||||
self.assertEqual(rc, 0)
|
||||
self.assertEqual(len(self.calls), 4)
|
||||
self.assertEqual(state.data["sessions"]["B0#0"]["attempts"], 2)
|
||||
|
||||
def test_stop_after_two_failures(self):
|
||||
rc, state, _ = self.run_blocks(["B0"], codes=[1, 1])
|
||||
self.assertEqual(rc, 1)
|
||||
self.assertEqual(len(self.calls), 2)
|
||||
self.assertEqual(state.data["sessions"]["B0#0"]["status"], "failed")
|
||||
self.assertNotIn("B0#1", state.data["sessions"])
|
||||
self.calls.clear()
|
||||
rc, _, _ = self.run_blocks(["B0"]) # resume retries the failed one
|
||||
self.assertEqual((rc, len(self.calls)), (0, 3))
|
||||
|
||||
def test_dry_run_runs_nothing(self):
|
||||
rc, _, _ = self.run_blocks(["B0", "B3"], dry_run=True)
|
||||
self.assertEqual((rc, self.calls), (0, []))
|
||||
self.assertFalse(self.state_path.exists())
|
||||
|
||||
def test_blocked_aborts_or_skips(self):
|
||||
rc, _, _ = self.run_blocks(["B8"])
|
||||
self.assertEqual((rc, self.calls), (2, []))
|
||||
rc, _, _ = self.run_blocks(["B4h"]) # unresolved ${gc}
|
||||
self.assertEqual((rc, self.calls), (2, []))
|
||||
rc, _, _ = self.run_blocks(["B8", "B6"], skip_blocked=True)
|
||||
self.assertEqual((rc, len(self.calls)), (0, 4))
|
||||
|
||||
def test_manual_deferred_skipped(self):
|
||||
rc, _, _ = self.run_blocks(["B13", "B5", "B14"])
|
||||
self.assertEqual((rc, self.calls), (0, []))
|
||||
rc, _, _ = self.run_blocks(["B13", "B5"], include_manual=True, include_deferred=True)
|
||||
self.assertEqual(len(self.calls), 2)
|
||||
|
||||
def test_mods_restored_check(self):
|
||||
def mutating(argv):
|
||||
self.calls.append(argv)
|
||||
(self.mods / f"leak{len(self.calls)}.jar").write_bytes(b"l")
|
||||
return 0
|
||||
state = matrix.State(self.state_path)
|
||||
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
|
||||
rc = matrix.run_blocks(self.m, ["B0"], state, self.mods, runner=mutating,
|
||||
out_dir=self.out)
|
||||
self.assertEqual((rc, len(self.calls)), (1, 1))
|
||||
|
||||
def test_unknown_block(self):
|
||||
with self.assertRaises(matrix.MatrixError):
|
||||
self.run_blocks(["B99"])
|
||||
|
||||
|
||||
class Cli(unittest.TestCase):
|
||||
def test_list_and_plan(self):
|
||||
buf = io.StringIO()
|
||||
with contextlib.redirect_stdout(buf):
|
||||
self.assertEqual(matrix.main(["list"]), 0)
|
||||
self.assertEqual(matrix.main(["plan", "B4h", "--param", "gc=J1",
|
||||
"--display", "desktop"]), 0)
|
||||
out = buf.getvalue()
|
||||
self.assertIn("| B0 | noise floor | 3 |", out)
|
||||
self.assertIn("--jvm J1 --heap 4096", out)
|
||||
self.assertIn("--display desktop", out)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user