From a15c912ad019c28528619ac1304cb7581a92d244 Mon Sep 17 00:00:00 2001 From: Logan Date: Fri, 16 Oct 2026 22:14:42 -0400 Subject: [PATCH] feat(bench): add matrix runner, gamescope and per-session controls - bench.py: gamescope-headless display (default, 1920x1080; ~2% cost vs desktop for isolation from the shared Wayland desktop), --shader, --dh, --disable-mods/--add-mods, --world-zip, crash-safe restore journal, spectator mode in the plan (elevated cameras fell to the sea floor and S4 measured underwater darkness) - matrix.toml/matrix.py: declarative, resumable B0-B14 runner keyed on mods hash (warns when B0 must be re-run) - build.py: bench-only resource-pack variants (C1-fa, C1-fa-ext, emf-lod*-fa, C1-bl); every bench variant pins resourcePacks so packs don't leak between sessions; shipped tier output unchanged - analyze.py: tagged labels use their own block's baseline --- .gitignore | 1 + analysis/client-perf/harness/README.md | 53 +- analysis/client-perf/harness/analyze.py | 62 +- analysis/client-perf/harness/bench.py | 485 +++++++++++++- analysis/client-perf/harness/matrix.py | 602 ++++++++++++++++++ analysis/client-perf/harness/matrix.toml | 294 +++++++++ .../harness/tests/test_analyze_block.py | 97 +++ .../harness/tests/test_bench_flags.py | 273 ++++++++ .../client-perf/harness/tests/test_matrix.py | 342 ++++++++++ .../client-perf/proposed-configs/build.py | 45 +- 10 files changed, 2196 insertions(+), 58 deletions(-) create mode 100644 analysis/client-perf/harness/matrix.py create mode 100644 analysis/client-perf/harness/matrix.toml create mode 100644 analysis/client-perf/harness/tests/test_analyze_block.py create mode 100644 analysis/client-perf/harness/tests/test_bench_flags.py create mode 100644 analysis/client-perf/harness/tests/test_matrix.py diff --git a/.gitignore b/.gitignore index aa8634f..9c5ccbe 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ immersive_paintings_cache/ # Large or regenerable local artifacts *.tar.gz analysis/client-perf/proposed-configs/out/ +__pycache__/ diff --git a/analysis/client-perf/harness/README.md b/analysis/client-perf/harness/README.md index 02745c8..a0f313a 100644 --- a/analysis/client-perf/harness/README.md +++ b/analysis/client-perf/harness/README.md @@ -21,6 +21,41 @@ python3 -m unittest discover -s tests # offline tests (from this d `session --dry-run` prints the datapack it would write and changes nothing. `bench.py meta` prints the environment snapshot; `bench.py disarm` removes a leftover plan pack. +## Per-session instance changes (undone after every session) + +```sh +python3 bench.py session --variant C1 --plan s1,s2,s4 --shader ComplementaryReimagined_r5.9.3.zip \ + [--shader-options profile.txt] --label B7 +python3 bench.py session --variant C1 --plan s1 --dh off +python3 bench.py session --variant C1 --plan s1 --disable-mods 'badoptimizations*,particle_core*' +python3 bench.py session --variant C1 --plan s2 --add-mods ~/trials/gnetum.jar +python3 bench.py session --variant C1 --plan s4 --world-zip ~/onion-bench/worlds/bench-world-v1-s4-1.zip +python3 bench.py restore # after a hard kill: replay .harness-pending.json +``` + +- `--shader NAME|off` (default `off`): `config/iris.properties` `enableShaders` / + `shaderPack`; NAME must exist in `shaderpacks/`. `--shader-options FILE` is copied to + `shaderpacks/.txt` (Iris' per-pack options file; journaled whenever a shader is set). +- `--dh on|off` (default `on`): `[client.advanced.debugging] rendererMode` = + `DEFAULT` / `DISABLED` in `config/DistantHorizons.toml`, set after the variant (DH's + `quickEnableRendering` is GUI-only and maps to this key). +- `--disable-mods GLOB,..`: case-insensitive globs over `mods/*.jar`, renamed to + `.jar.disabled`; a glob that matches nothing is an error. `--add-mods PATH,..` copies jars + in (name collision = error). `mods_hash` is taken after these changes. +- `--world-zip PATH`: `saves/onionbench` -> `saves/onionbench.harness-bak`, zip extracted + (world at the zip root or under one top directory, needs `level.dat`); sha256 recorded. +- Every session first resets `options.txt` `resourcePacks` / `incompatibleResourcePacks` + to `["vanilla","mod_resources"]` / `[]` (variants may override); not undone. +- Crash safety: before the first change `session` writes `.minecraft/.harness-pending.json` + (write-ahead: each entry precedes its change; file backups in `.minecraft/.harness-backup/`). + The normal `finally` and `bench.py restore` run the same restore, newest entry first; + failed entries stay in the journal. `session` and `boot` refuse to start while it exists. + SIGTERM unwinds through the restore; SIGKILL leaves the journal for `bench.py restore`. +- Rows record `shader`, `shader_options(_sha256)`, `dh`, `disabled_mods`, `added_mods`, + `world_zip(_sha256)`, `resource_packs`, `restore_errors`; warn flags `gpu_dpm_auto` (RX + 7800 XT `power_dpm_force_performance_level` != `high`), `server_running`, + `restore_failed`. + - **boot**: N launches; waits for `Game took N seconds` (or `joined the game` with `--join`); outcome `ok` / `crash` / `timeout`. Crash = new file in `crash-reports/` or `hs_err_pid*.log`, client java exited, or a log line with `Failed to create mod @@ -84,8 +119,9 @@ join seconds, block-atlas size, `level.dat` hash. only reaches Xwayland windows, `windowactivate --sync` hangs under kwin, focus-steal prevention can drop keystrokes, and typed chat lands in whatever has focus. Hence no input injection at all in v2. -- Still manual: **S6** (JEI/GUI overlay needs a key press), **B7** shader profiles (set in - Iris' GUI), `dh-off`-style checks needing F3 reads, `/sparkc` heap/GC reads (B1/B4). +- Still manual: **S6** (JEI/GUI overlay needs a key press), producing B7 shader option + profiles (export once from Iris' GUI, then pass `--shader-options`), checks needing F3 + reads, `/sparkc` heap/GC reads (B1/B4). - MangoHud's `log_duration=1800` in `~/.local/bin/mangohud-local` caps one CSV at 30 min; `session` warns when a plan is longer (B5's 45-min decay run needs a split or a larger value). @@ -94,9 +130,10 @@ join seconds, block-atlas size, `level.dat` hash. the start of each capture (identical across variants). - **FTB Backups 3** backs up the world right after join (its index `ftbbackups3/backups.json` doesn't parse, so it treats the world as never backed up) - and every `backup_timer` = 120 min. For the bench instance only, set `auto: false` in - `saves/onionbench/serverconfig/ftbbackups3-server.snbt` (copy of - `config/ftbbackups3-server.snbt`; the world override wins and is never touched by pack - updates). Not done by the harness. Never edit the repo's `config/` for this. -- The dedicated server in tmux `onion-mc` keeps running during benches and was observed - using ~78 % of a core while idle; it is recorded per row as a noise covariate. + and every `backup_timer` = 120 min. **Done (bench instance only):** `auto: false` in + the instance's `config/ftbbackups3-server.snbt`. Never edit the repo's `config/` for this. +- Stop the dedicated server (tmux `onion-mc`) while benching (owner-approved); it used up + to ~78 % of a core idle. Rows are flagged `server_running` if it is up. +- amdgpu DPM is `auto` (no root); rows are flagged `gpu_dpm_auto`. For tighter noise: + `echo high | sudo tee /sys/class/drm/card1/device/power_dpm_force_performance_level`. +- Scenes run in **spectator** mode (set by the plan) so elevated cameras don't fall. diff --git a/analysis/client-perf/harness/analyze.py b/analysis/client-perf/harness/analyze.py index 2f10d1e..b5251c5 100644 --- a/analysis/client-perf/harness/analyze.py +++ b/analysis/client-perf/harness/analyze.py @@ -7,8 +7,10 @@ Groups windows by (label, variant, scene). Each session contributes its median p99 and median 1 % low (frametime-based); a group's value is the median of its session medians and its spread is max - min of those. With --baseline/--noise, every non-baseline group -is compared with the baseline variant on the same scene (same label if present, else -all labels pooled): win = median p99 improves by more than the noise label's p99 spread +is compared with the baseline variant on the same scene: same label if present, else the +same block (a tagged label 'B3.dh-off' looks in 'B3', 'B3.baseline', then all 'B3.*' +pooled), else all labels pooled. A tagged session of the baseline variant itself is +compared with 'B3' / 'B3.baseline' when one exists: win = median p99 improves by more than the noise label's p99 spread AND 1 % low does not worsen by more than its 1 % low spread; loss = p99 worsens beyond the spread or 1 % low drops beyond it; otherwise tie (ties keep upstream defaults). Rows flagged crashed / timeout / no_data / no_markers / low_n / vsync_suspect are @@ -79,18 +81,57 @@ def verdict(base, cand, spread_p99, spread_low): return "tie" -def baseline_for(summary, rows, label, scene, baseline): - if (label, baseline, scene) in summary: - return summary[(label, baseline, scene)], label - pooled = [r for r in rows if r["variant"] == baseline and r["scene"] == scene] +def block_of(label): + """'B3.dh-off' -> 'B3'; untagged labels (and None) are their own block.""" + return label.split(".", 1)[0] if label else label + + +def _pooled(rows, labels, baseline, scene, name): + pooled = [r for r in rows if r["variant"] == baseline and r["scene"] == scene + and (labels is None or r.get("label") in labels)] if not pooled: + return None + return summarise([dict(r, label=name) for r in pooled]).get((name, baseline, scene)) + + +def baseline_for(summary, rows, label, scene, baseline): + """(baseline stats, where-label) for one candidate group, or (None, None). + + Order: the candidate's own label; then its block - the untagged block label + ('B3', for tagged candidates), '.baseline', then every '.*' label pooled; then all labels + pooled. A tagged session of the baseline variant itself (e.g. 'B3.dh-off' running + C1 against baseline C1) is compared only with 'B3' or 'B3.baseline'.""" + block = block_of(label) + tagged = bool(label) and block != label + order = [label] + ([block] if tagged else []) + ([f"{block}.baseline"] if block else []) + for lab in order: + if (lab, baseline, scene) in summary: + return summary[(lab, baseline, scene)], lab + if block: + labels = {r.get("label") for r in rows + if r.get("label") == block or str(r.get("label")).startswith(block + ".")} + b = _pooled(rows, labels, baseline, scene, f"{block}.*") + if b is not None: + return b, f"{block}.*" + b = _pooled(rows, None, baseline, scene, "*") + return (b, "*") if b is not None else (None, None) + + +def self_baseline(summary, label, scene, baseline): + """Baseline for a tagged group that runs the baseline variant itself, or None + (then the group *is* the baseline).""" + block = block_of(label) + if not label or block == label: return None, None - s = summarise([dict(r, label="*") for r in pooled]) - return s.get(("*", baseline, scene)), "*" + for lab in (block, f"{block}.baseline"): + if lab != label and (lab, baseline, scene) in summary: + return summary[(lab, baseline, scene)], lab + return None, None def table(summary, rows, baseline=None, noise=None): spreads = noise_spread(summary, noise) if noise else None + full = summarise(rows) # baselines may live in labels filtered out of the table hdr = ["label", "variant", "scene", "sess", "win#", "p99 ms", "p99 spr", "1%low", "1%low spr", "avg fps"] if baseline: @@ -103,10 +144,11 @@ def table(summary, rows, baseline=None, noise=None): f"{s['p99']:.2f}", f"{s['p99_spread']:.2f}", f"{s['low']:.1f}", f"{s['low_spread']:.1f}", f"{s['fps']:.0f}"] if baseline: - if var == baseline: + b, blab = (self_baseline(full, lab, scene, baseline) if var == baseline + else baseline_for(full, rows, lab, scene, baseline)) + if var == baseline and b is None: cells += ["—", "", "", "baseline"] else: - b, blab = baseline_for(summary, rows, lab, scene, baseline) if b is None: cells += ["—", "", "", "no baseline"] elif spreads is None: diff --git a/analysis/client-perf/harness/bench.py b/analysis/client-perf/harness/bench.py index eef8750..f56cc4d 100644 --- a/analysis/client-perf/harness/bench.py +++ b/analysis/client-perf/harness/bench.py @@ -4,6 +4,7 @@ bench.py boot --n 3 [--variant C1] [--join] # cold-launch reliability + startup time bench.py session --variant C1 --plan s1,s2,s3 --reps 3 [--label B2] [--dry-run] bench.py disarm # remove a leftover autoplan datapack + bench.py restore # undo a crashed session's changes bench.py meta # print the environment snapshot `session` applies the variant, pins options.txt, writes a per-session datapack whose @@ -15,6 +16,7 @@ done, slices the MangoHud CSV per window and appends one JSON row per window to """ import argparse import datetime as dt +import fnmatch import hashlib import json import os @@ -24,6 +26,7 @@ import signal import subprocess import sys import time +import zipfile from pathlib import Path HERE = Path(__file__).resolve().parent @@ -51,6 +54,18 @@ PINS = [ ("enableVsync", "false"), ] +# Reset before every variant apply so resource-pack state never leaks between sessions. +# options.txt had neither key on 2026-10-16, so this is the NeoForge default. +RESOURCE_PACK_BASELINE = [ + ("resourcePacks", '["vanilla","mod_resources"]'), + ("incompatibleResourcePacks", "[]"), +] + +PENDING = ".harness-pending.json" # crash-safe restore journal, in GAME +BACKUP_DIR = ".harness-backup" # file backups referenced by the journal, in GAME +DGPU_DEVICE = "0x747e" # RX 7800 XT +DH_RENDER_SECTION, DH_RENDER_KEY = "client.advanced.debugging", "rendererMode" + # BENCH-MATRIX §3: one session runs its scenes in this order (S1 first). SCENE_ORDER = ["s1", "s2", "s2s", "s3", "s4", "s5"] SCENE_FN = {"s1": "bench:s1", "s2": "bench:s2", "s2s": "bench:s2", "s3": "bench:s3", @@ -335,6 +350,10 @@ def generate_datapack(sid, plan, reps, warmup_s, capture_s, settle_s): f"execute unless entity @a run schedule function {ns}/poll 20t\n", f"data/bench/function/autoplan/{sid}/begin.mcfunction": f"scoreboard players set {holder} bench_plan 1\n" + # spectator: elevated scene cameras must not fall (S4 measured the sea floor) + "gamemode spectator @a\n" + "time set 6000\n" + "weather clear\n" "function bench:storm_off\n" "function bench:tour_stop\n" f"say BENCH_PLAN {sid} begin\n" @@ -567,6 +586,8 @@ def snapshot(env, fast=False): lvl = GAME / "saves" / WORLD / "level.dat" return { "machine": MACHINE, + "display": DISPLAY_MODE["mode"], + "res": "%dx%d" % DISPLAY_MODE["res"], "git_sha": sha, "git_dirty": dirty, "harness_sha": sha256_file(Path(__file__))[:16], "mods_hash": mh, "mods_count": mc, @@ -619,10 +640,29 @@ def crash_files(): return names +DISPLAY_MODE = {"mode": "gamescope-headless", "res": (1920, 1080)} + + +def display_prefix(mode, res): + """gamescope wrapper argv for a display mode (empty for the desktop).""" + if mode == "desktop": + return [] + w, h = res + cmd = ["gamescope", "-W", str(w), "-H", str(h), "-w", str(w), "-h", str(h), + "--force-windows-fullscreen", "--immediate-flips", "-r", "1000", "-o", "1000"] + if mode == "gamescope-headless": + cmd += ["--backend", "headless"] + return cmd + ["--"] + + def launch(env, world): if (GAME / "logs" / "latest.log").exists(): (GAME / "logs" / "latest.log").unlink() - cmd = ["setsid", "prismlauncher", "-d", str(PRISM), "--launch", INSTANCE] + env = dict(env) + if DISPLAY_MODE["mode"] != "desktop": + env.setdefault("WAYLAND_DISPLAY", "wayland-0") + cmd = ["setsid", *display_prefix(DISPLAY_MODE["mode"], DISPLAY_MODE["res"]), + "prismlauncher", "-d", str(PRISM), "--launch", INSTANCE] if world: cmd += ["--world", world] return subprocess.Popen(cmd, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) @@ -759,6 +799,283 @@ def write_datapack(files): p.write_text(text) +# ---------------------------------------------------------------- journaled session mutations + +class Journal: + """Write-ahead restore journal (GAME/.harness-pending.json). Every entry is written + before the mutation it undoes, so restore_pending() works after a hard kill.""" + + def __init__(self, game, sid=None): + self.game, self.sid, self.entries = Path(game), sid, [] + self.path, self.bdir = self.game / PENDING, self.game / BACKUP_DIR + + @classmethod + def begin(cls, game, sid): + j = cls(game, sid) + if j.path.exists(): + raise RuntimeError(f"{j.path} exists: run `bench.py restore` first") + j._write() + return j + + def _write(self): + tmp = self.path.with_name(self.path.name + ".tmp") + tmp.write_text(json.dumps({"session": self.sid, "entries": self.entries}, indent=1)) + os.replace(tmp, self.path) + + def record(self, entry): + self.entries.append(entry) + self._write() + + def backup_file(self, rel): + """Journal the current content (or absence) of GAME/rel; first backup wins.""" + rel = Path(rel).as_posix() + if any(e["op"] == "file" and e["path"] == rel for e in self.entries): + return + src, backup = self.game / rel, None + if src.is_file(): + self.bdir.mkdir(exist_ok=True) + backup = f"{len(self.entries)}-{src.name}" + shutil.copy2(src, self.bdir / backup) + self.record({"op": "file", "path": rel, "backup": backup}) + + +def pending_exists(game=None): + return (Path(game or GAME) / PENDING).exists() + + +def _undo(game, e, bdir): + op = e["op"] + if op == "file": + dst = game / e["path"] + if e["backup"] is None: + if dst.exists(): + dst.unlink() + else: + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(bdir / e["backup"], dst) + elif op == "rename": + src, dst = game / e["src"], game / e["dst"] + if dst.exists() and not src.exists(): + os.rename(dst, src) + elif op in ("added", "tree"): + p = game / e["path"] + if p.is_dir(): + shutil.rmtree(p) + elif p.exists(): + p.unlink() + elif op == "world": + path, bak, ext = game / e["path"], game / e["backup"], game / e["extract"] + if ext.exists(): + shutil.rmtree(ext) + if bak.exists(): + if path.exists(): + shutil.rmtree(path) + os.rename(bak, path) + elif not e["had_world"] and path.exists(): + shutil.rmtree(path) + else: + raise ValueError(f"unknown journal op {op!r}") + + +def restore_pending(game=None): + """Undo every journaled mutation (newest first). Returns [(entry, error)] for the + entries that failed; those stay in the journal, otherwise journal + backups go.""" + game = Path(game or GAME) + path, bdir = game / PENDING, game / BACKUP_DIR + if not path.exists(): + return [] + data = json.loads(path.read_text()) + failed = [] + for e in reversed(data.get("entries", [])): + try: + _undo(game, e, bdir) + except (OSError, ValueError, KeyError) as exc: + failed.append((e, f"{type(exc).__name__}: {exc}")) + if failed: + data["entries"] = [e for e, _ in reversed(failed)] + path.write_text(json.dumps(data, indent=1)) + else: + path.unlink() + if bdir.exists(): + shutil.rmtree(bdir) + return failed + + +def prop_escape(v): + """java.util.Properties value escaping (what Iris' store() writes).""" + out = "".join("\\" + c if c in "\\=:#!" else c for c in v) + return "\\" + out if out.startswith(" ") else out + + +def set_properties(text, want): + """Set key=value pairs in .properties text, keeping comments and other keys.""" + out, seen = [], set() + for line in text.splitlines(): + s = line.lstrip() + if s and s[0] not in "#!": + k = re.split(r"\s*[=:]\s*|\s+", s, maxsplit=1)[0] + if k in want: + line = f"{k}={prop_escape(want[k])}" + seen.add(k) + out.append(line) + out += [f"{k}={prop_escape(v)}" for k, v in want.items() if k not in seen] + return "\n".join(out) + "\n" + + +def apply_shader(game, journal, shader, options_file=None): + """Iris on with shaderpacks/, or off. Iris' per-pack options file + shaderpacks/.txt is journaled too, so edits made during the run revert.""" + game = Path(game) + cfg = game / "config" / "iris.properties" + journal.backup_file("config/iris.properties") + on = shader != "off" + text = cfg.read_text() if cfg.exists() else "" + cfg.parent.mkdir(parents=True, exist_ok=True) + cfg.write_text(set_properties(text, {"enableShaders": "true" if on else "false", + "shaderPack": shader if on else ""})) + if on: + rel = f"shaderpacks/{shader}.txt" + journal.backup_file(rel) + if options_file: + shutil.copyfile(options_file, game / rel) + + +def set_dh_rendering(text, on): + """Set [client.advanced.debugging] rendererMode (what DH's GUI-only + quickEnableRendering toggles): DEFAULT or DISABLED.""" + out, section, hit = [], None, False + for line in text.splitlines(): + m = re.match(r"^\s*\[([^\]]+)\]\s*$", line) + if m: + section = m.group(1).strip() + elif section == DH_RENDER_SECTION: + m = re.match(rf"^(\s*){DH_RENDER_KEY}\s*=", line) + if m: + line = f'{m.group(1)}{DH_RENDER_KEY} = "{"DEFAULT" if on else "DISABLED"}"' + hit = True + out.append(line) + if not hit: + raise ValueError(f"no {DH_RENDER_KEY} under [{DH_RENDER_SECTION}] in DistantHorizons.toml") + return "\n".join(out) + "\n" + + +def apply_dh(game, journal, on): + p = Path(game) / "config" / "DistantHorizons.toml" + new = set_dh_rendering(p.read_text(), on) # raises before journaling if the key is gone + journal.backup_file("config/DistantHorizons.toml") + p.write_text(new) + + +def split_list(s): + return [x.strip() for x in (s or "").split(",") if x.strip()] + + +def resolve_disable(game, globs): + """Jar names in GAME/mods matching any glob (case-insensitive). Every glob must hit.""" + jars = sorted(p.name for p in (Path(game) / "mods").glob("*.jar")) + hits, empty = [], [] + for g in globs: + m = [j for j in jars if fnmatch.fnmatchcase(j.lower(), g.lower())] + if not m: + empty.append(g) + hits += [j for j in m if j not in hits] + if empty: + raise ValueError(f"--disable-mods glob(s) matched no jar in mods/: {empty}") + return hits + + +def check_add_mods(game, paths): + out = [] + for s in paths: + p = Path(s).expanduser() + if not p.is_file() or p.suffix != ".jar": + raise ValueError(f"--add-mods: {p} is not a .jar file") + if (Path(game) / "mods" / p.name).exists(): + raise ValueError(f"--add-mods: mods/{p.name} already exists") + out.append(p) + return out + + +def disable_mods(game, journal, names): + for n in names: + src, dst = f"mods/{n}", f"mods/{n}.disabled" + if (Path(game) / dst).exists(): + raise ValueError(f"{dst} already exists") + journal.record({"op": "rename", "src": src, "dst": dst}) + os.rename(Path(game) / src, Path(game) / dst) + + +def add_mods(game, journal, paths): + for p in paths: + journal.record({"op": "added", "path": f"mods/{p.name}"}) + shutil.copy2(p, Path(game) / "mods" / p.name) + + +def zip_world_root(zip_path): + """Member prefix of the world inside the zip ('' or 'top/'); rejects unsafe paths.""" + with zipfile.ZipFile(zip_path) as z: + names = z.namelist() + for n in names: + if n.startswith(("/", "\\")) or ".." in Path(n).parts: + raise ValueError(f"unsafe member path in {zip_path}: {n}") + if "level.dat" in names: + return "" + tops = {n.split("/", 1)[0] for n in names if n.strip("/")} + if len(tops) == 1: + top = tops.pop() + if f"{top}/level.dat" in names: + return top + "/" + raise ValueError(f"{zip_path}: no level.dat at the root or under a single top directory") + + +def swap_world(game, journal, zip_path, world=WORLD): + """Replace saves/ with the zip's world (original kept as + .harness-bak until restore). Returns the zip's sha256.""" + prefix = zip_world_root(zip_path) + saves = Path(game) / "saves" + path, bak, ext = saves / world, saves / f"{world}.harness-bak", saves / f".{world}.harness-extract" + for p in (bak, ext): + if p.exists(): + raise ValueError(f"{p} exists (unrestored earlier swap?)") + digest = sha256_file(zip_path) + journal.record({"op": "world", "path": f"saves/{world}", "backup": f"saves/{bak.name}", + "extract": f"saves/{ext.name}", "had_world": path.exists()}) + if path.exists(): + os.rename(path, bak) + with zipfile.ZipFile(zip_path) as z: + z.extractall(ext) + os.rename(ext / prefix if prefix else ext, path) + if ext.exists(): + shutil.rmtree(ext) + return digest + + +def reset_resource_packs(game): + opts = Path(game) / "options.txt" + opts.write_text(merge_pins(opts.read_text() if opts.exists() else "", + RESOURCE_PACK_BASELINE)) + + +def options_values(game, keys): + opts = Path(game) / "options.txt" + vals = {} + for line in (opts.read_text().splitlines() if opts.exists() else []): + k, _, v = line.partition(":") + if k in keys: + vals[k] = v + return vals + + +def env_flags(gpus, server_running): + """Warn flags for uncontrolled noise sources.""" + flags = [] + if any(g.get("device") == DGPU_DEVICE and g.get("dpm_level") != "high" for g in gpus): + flags.append("gpu_dpm_auto") + if server_running: + flags.append("server_running") + return flags + + def effective_jvm_flags(pids): for pid in pids: args = [a.decode(errors="replace") for a in proc_cmdline(pid) if a] @@ -780,7 +1097,14 @@ def pick_csvs(t_launch): # ---------------------------------------------------------------- subcommands +def refuse_pending(): + if pending_exists(): + sys.exit(f"{GAME / PENDING} exists: a previous session did not restore the " + f"instance. Run `bench.py restore` first.") + + def cmd_boot(a): + refuse_pending() env = bench_env() preflight(a.kill_stale) disarm() @@ -851,12 +1175,29 @@ def cmd_session(a): timeout = a.timeout or int(a.boot_timeout + a.settle + budget * 1.25 + 120) if not (PROPOSED / "out" / a.variant).is_dir(): sys.exit(f"variant {a.variant} not in {PROPOSED / 'out'} - run build.py first") + # validate every instance change before touching anything + try: + if a.shader != "off" and not (GAME / "shaderpacks" / a.shader).exists(): + raise ValueError(f"--shader {a.shader}: not in {GAME / 'shaderpacks'}") + if a.shader_options and (a.shader == "off" or not Path(a.shader_options).is_file()): + raise ValueError("--shader-options needs --shader NAME and an existing file") + to_disable = resolve_disable(GAME, split_list(a.disable_mods)) + to_add = check_add_mods(GAME, split_list(a.add_mods)) + if a.world_zip: + zip_world_root(a.world_zip) + except (ValueError, OSError, zipfile.BadZipFile) as e: + sys.exit(str(e)) if a.dry_run: log(f"dry run: session {sid}, variant {a.variant}, plan {plan} x{a.reps}, " f"est. scene time {budget}s, timeout {timeout}s") log(f"would pin options.txt: {', '.join(f'{k}:{v}' for k, v in PINS)}") if a.jvm or a.heap: log(f"would set instance.cfg: jvm={a.jvm} heap={a.heap}") + log(f"would reset options.txt: {dict(RESOURCE_PACK_BASELINE)}") + log(f"would set shader={a.shader} options={a.shader_options} dh={a.dh} " + f"disable={to_disable} add={[p.name for p in to_add]} world_zip={a.world_zip}") + if pending_exists(): + log(f"WARNING: {GAME / PENDING} exists; a real run would refuse to start") log(f"would write datapack {plan_dir()}:") for rel, text in files.items(): print(f"--- {PLAN_PACK}/{rel}") @@ -864,22 +1205,53 @@ def cmd_session(a): return 0 if budget + a.settle + 120 > 1700: log("WARNING: plan exceeds ~28 min; mangohud-local log_duration=1800 will truncate the CSV") + refuse_pending() env = bench_env() preflight(a.kill_stale) - apply_out = apply_variant(a.variant) - apply_pins() - if a.jvm or a.heap: - apply_jvm(a.jvm, a.heap) - cfg_hash, cfg_n = tree_hash(GAME / "config", config_skip) - opt_hash = sha256_file(GAME / "options.txt") - meta = snapshot(env) - write_datapack(files) - before = crash_files() - t_launch = time.time() - log(f"session {sid}: {a.variant} {plan} x{a.reps}, timeout {timeout}s") - prism = launch(env, WORLD) - tail = LogTail(GAME / "logs" / "latest.log") - state = {"done": False, "last": None} + # SIGTERM (e.g. a matrix runner timeout) unwinds through the restore below + signal.signal(signal.SIGTERM, lambda *_: sys.exit(143)) + journal = Journal.begin(GAME, sid) + prism = None + state = {"done": False} + try: + reset_resource_packs(GAME) + apply_out = apply_variant(a.variant) + apply_pins() + if a.jvm or a.heap: + apply_jvm(a.jvm, a.heap) + apply_shader(GAME, journal, a.shader, a.shader_options) + apply_dh(GAME, journal, a.dh == "on") + disable_mods(GAME, journal, to_disable) + add_mods(GAME, journal, to_add) + world_sha = swap_world(GAME, journal, a.world_zip) if a.world_zip else None + cfg_hash, cfg_n = tree_hash(GAME / "config", config_skip) + opt_hash = sha256_file(GAME / "options.txt") + packs = options_values(GAME, dict(RESOURCE_PACK_BASELINE)) + meta = snapshot(env) + journal.record({"op": "tree", "path": plan_dir().relative_to(GAME).as_posix()}) + write_datapack(files) + before = crash_files() + t_launch = time.time() + log(f"session {sid}: {a.variant} {plan} x{a.reps}, timeout {timeout}s") + prism = launch(env, WORLD) + tail = LogTail(GAME / "logs" / "latest.log") + outcome, detail = _run_session(a, tail, prism, before, timeout, state) + finally: + if prism is not None: + kill_client(prism) + failed = restore_pending(GAME) + for e, err in failed: + log(f"RESTORE FAILED {e}: {err}") + if failed: + log(f"journal kept at {GAME / PENDING}; fix and run `bench.py restore`") + jvm_flags = state.get("jvm_flags") + return _record_session(a, sid, plan, tail, t_launch, outcome, detail, jvm_flags, + cfg_hash, cfg_n, opt_hash, apply_out, meta, packs, world_sha, + to_disable, to_add, failed) + + +def _run_session(a, tail, prism, before, timeout, state): + """Monitor the running client until BENCH_DONE / crash / exit / timeout.""" def on_line(line): m = MARK_RE.search(line) @@ -888,19 +1260,21 @@ def cmd_session(a): if DONE_RE.search(line): state["done"] = True - jvm_flags = None - try: - def until(): - nonlocal jvm_flags - if jvm_flags is None: - jvm_flags = effective_jvm_flags(client_pids()) - return state["done"] - outcome, detail, _ = monitor(tail, prism, before, until, timeout, on_line) - time.sleep(2) # let MangoHud flush the tail of the capture - tail.poll() - finally: - kill_client(prism) - disarm() + state["jvm_flags"] = None + + def until(): + if state["jvm_flags"] is None: + state["jvm_flags"] = effective_jvm_flags(client_pids()) + return state["done"] + outcome, detail, _ = monitor(tail, prism, before, until, timeout, on_line) + time.sleep(2) # let MangoHud flush the tail of the capture + tail.poll() + return outcome, detail + + +def _record_session(a, sid, plan, tail, t_launch, outcome, detail, jvm_flags, cfg_hash, + cfg_n, opt_hash, apply_out, meta, packs, world_sha, to_disable, to_add, + restore_failed): mem_after = meminfo_available_mb() lines = tail.lines ev = scan_events(lines) @@ -925,11 +1299,21 @@ def cmd_session(a): "jvm_set": a.jvm, "heap_mb": a.heap, "jvm_flags": jvm_flags, "config_hash": cfg_hash[:16], "config_files": cfg_n, "options_hash": opt_hash[:16], "apply_output": apply_out, + "shader": a.shader, "shader_options": a.shader_options, + "shader_options_sha256": sha256_file(a.shader_options) if a.shader_options else None, + "dh": a.dh, "disabled_mods": to_disable, "added_mods": [p.name for p in to_add], + "world_zip": a.world_zip, "world_zip_sha256": world_sha, + "resource_packs": packs.get("resourcePacks"), + "incompatible_resource_packs": packs.get("incompatibleResourcePacks"), + "restore_errors": [err for _, err in restore_failed], "session_outcome": outcome, "session_detail": detail, "startup_s": ev["game_took_s"], "join_s": round(ev["joined_t"] - t_launch, 2) if ev["joined_t"] else None, "atlas": ev["atlas"], "log": log_path, **meta, "mem_available_after_mb": mem_after, } + noise = env_flags(meta["gpus"], meta["server_running"]) + if restore_failed: + noise.append("restore_failed") n_ok = 0 for scene in plan: warmup = 0 if scene == "s5" else a.warmup @@ -940,7 +1324,7 @@ def cmd_session(a): t_start=ts, t_end=te) if ts is None or te is None: row.update(metrics=None, flags=[{"crash": "crashed", "timeout": "timeout"} - .get(outcome, "no_markers")]) + .get(outcome, "no_markers"), *noise]) append_jsonl(RESULTS / "session.jsonl", row) log(f"{scene} rep {rep}: no complete window ({outcome})") continue @@ -955,6 +1339,7 @@ def cmd_session(a): flags = validity_flags(m, te - ts, meta["refresh_rates"], backups, ts, te) if outcome != "ok": flags.append("session_" + outcome) # warn: window itself completed + flags += noise row.update(metrics=m, flags=flags) if src: row.update(csv=src[2].name, align=src[1], csv_gpu=src[3].get("gpu", "").strip(), @@ -965,7 +1350,7 @@ def cmd_session(a): log(f"{scene} rep {rep}: n={m['n']} p50={m['p50_ms']} p99={m['p99_ms']} " f"1%low={m['one_low_fps']} avg={m['avg_fps']} {','.join(flags)}") log(f"session {sid} {outcome}: {n_ok} windows recorded -> {RESULTS / 'session.jsonl'}") - return 0 if outcome == "ok" else 1 + return 0 if outcome == "ok" and not restore_failed else 1 def cmd_disarm(a): @@ -973,12 +1358,27 @@ def cmd_disarm(a): return 0 +def cmd_restore(a): + if client_pids(): + sys.exit("bench client is running; close it before restoring") + if not pending_exists(): + log("nothing pending" + ("; removed leftover plan pack" if disarm() else "")) + return 0 + failed = restore_pending(GAME) + for e, err in failed: + log(f"FAILED {e}: {err}") + disarm() + log(f"restore: {len(failed)} entr{'y' if len(failed) == 1 else 'ies'} failed" + if failed else "restored and cleared " + str(GAME / PENDING)) + return 1 if failed else 0 + + def cmd_meta(a): print(json.dumps(snapshot(bench_env()), indent=2)) return 0 -def main(argv=None): +def build_parser(): ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0], formatter_class=argparse.RawDescriptionHelpFormatter) sub = ap.add_subparsers(dest="cmd", required=True) @@ -1009,14 +1409,35 @@ def main(argv=None): s.add_argument("--any-order", action="store_true") s.add_argument("--kill-stale", action="store_true") s.add_argument("--dry-run", action="store_true", help="print the generated datapack only") + s.add_argument("--shader", default="off", + help="shaderpack file/dir name in GAME/shaderpacks, or off (Iris disabled)") + s.add_argument("--shader-options", help="file copied to shaderpacks/.txt for the session") + s.add_argument("--dh", choices=["on", "off"], default="on", + help="DH rendering via [client.advanced.debugging] rendererMode") + s.add_argument("--disable-mods", help="comma list of globs (case-insensitive) of mods/*.jar " + "renamed to .jar.disabled for the session") + s.add_argument("--add-mods", help="comma list of jar paths copied into mods/ for the session") + s.add_argument("--world-zip", help="world zip swapped in for saves/onionbench for the session") + for p in (b, s): + p.add_argument("--display", choices=["desktop", "gamescope", "gamescope-headless"], + default="gamescope-headless") + p.add_argument("--res", default="1920x1080", help="WxH (gamescope modes)") s.set_defaults(fn=cmd_session) d = sub.add_parser("disarm", help="remove a leftover autoplan datapack from the world") d.set_defaults(fn=cmd_disarm) + r = sub.add_parser("restore", help="replay and clear a leftover .harness-pending.json") + r.set_defaults(fn=cmd_restore) m = sub.add_parser("meta", help="print the environment/metadata snapshot") m.set_defaults(fn=cmd_meta) + return ap - a = ap.parse_args(argv) + +def main(argv=None): + a = build_parser().parse_args(argv) + if getattr(a, "display", None): + w, h = (int(x) for x in a.res.lower().split("x")) + DISPLAY_MODE.update(mode=a.display, res=(w, h)) return a.fn(a) diff --git a/analysis/client-perf/harness/matrix.py b/analysis/client-perf/harness/matrix.py new file mode 100644 index 0000000..a33e2f6 --- /dev/null +++ b/analysis/client-perf/harness/matrix.py @@ -0,0 +1,602 @@ +#!/usr/bin/env python3 +"""Run the BENCH-MATRIX blocks declared in matrix.toml through `bench.py session`. + + python3 matrix.py list # blocks, session counts, est. time + python3 matrix.py plan B3 [--param k=v] # the bench.py commands a block runs + python3 matrix.py run B0,B1 [--dry-run] # sequential, resumable + python3 matrix.py status # progress for the current mod set + +Resume: ~/onion-bench/results/matrix-state.json records each session (key +"#") with a hash of its exact command and the instance mods/ hash; a +session is skipped only when both still match. A failed session (non-zero exit: crash, +timeout, no join) is retried once, then the run stops. Running any block other than B0 +warns loudly when B0 was not run under the current mod set. Stdlib only. +""" +import argparse +import glob +import hashlib +import json +import os +import re +import shlex +import subprocess +import sys +import time +import tomllib +from pathlib import Path + +HERE = Path(__file__).resolve().parent +MATRIX = HERE / "matrix.toml" +OUT_DIR = HERE.parent / "proposed-configs" / "out" +GAME = Path.home() / "onion-bench" / "prism" / "instances" / "OnionBench" / ".minecraft" +STATE = Path.home() / "onion-bench" / "results" / "matrix-state.json" + +SCENE_ORDER = ["s1", "s2", "s2s", "s3", "s4", "s5"] +S5_SECONDS = 10 + 120 # bench.py: at S1, wait 10 s, then the 120 s tour (all captured) +JVM_SETS = {"J0", "J1", "J2", "J3"} +DISPLAYS = {"desktop", "gamescope", "gamescope-headless"} +PARAM_RE = re.compile(r"\$\{(\w+)\}") + +TOP_KEYS = {"defaults", "worlds", "trials", "missing_variants", "block"} +DEFAULT_KEYS = {"reps", "warmup", "capture", "display", "res", "world", "shader", "dh", + "launch_overhead_s"} +BLOCK_KEYS = {"id", "title", "rule", "proxy_note", "manual_checks", "sessions", "params", + "deferred", "manual", "interleave"} +SESSION_KEYS = {"variant", "plan", "reps", "repeat", "tag", "jvm", "heap", "shader", + "shader_profile", "dh", "disable_mods", "add_mods", "world", "warmup", + "capture", "manual", "note", "display", "res"} +MISSING_KEYS = {"id", "blocks", "contents"} +# Flags `bench.py session` accepts; bench_argv must emit only these. +BENCH_SESSION_FLAGS = { + "--variant", "--plan", "--reps", "--warmup", "--capture", "--settle", "--label", "--jvm", + "--heap", "--boot-timeout", "--timeout", "--any-order", "--kill-stale", "--dry-run", + "--display", "--res", "--shader", "--shader-options", "--dh", "--disable-mods", + "--add-mods", "--world-zip", +} + + +class MatrixError(Exception): + pass + + +def log(msg): + print(f"[matrix] {msg}", flush=True) + + +# ------------------------------------------------------------------ loading / validation + +def load(path=MATRIX, out_dir=OUT_DIR): + """Parse + validate matrix.toml. Raises MatrixError listing every problem.""" + with open(path, "rb") as f: + m = tomllib.load(f) + errors = validate(m, out_dir) + if errors: + raise MatrixError("matrix.toml invalid:\n " + "\n ".join(errors)) + return m + + +def split_plan(plan): + if isinstance(plan, str): + return [s.strip() for s in plan.split(",") if s.strip()] + return list(plan) + + +def _pos_int(v): + return isinstance(v, int) and not isinstance(v, bool) and v > 0 + + +def validate(m, out_dir=OUT_DIR): + err = [] + for k in m: + if k not in TOP_KEYS: + err.append(f"unknown top-level key {k!r}") + d = m.get("defaults", {}) + for k in d: + if k not in DEFAULT_KEYS: + err.append(f"defaults: unknown key {k!r}") + worlds, trials = m.get("worlds", {}), m.get("trials", {}) + missing_ids = set() + for mv in m.get("missing_variants", []): + for k in mv: + if k not in MISSING_KEYS: + err.append(f"missing_variants {mv.get('id')}: unknown key {k!r}") + if not mv.get("id") or not mv.get("contents"): + err.append("missing_variants entry needs id and contents") + missing_ids.add(mv.get("id")) + _check_common(d, "defaults", worlds, trials, err) + + seen = set() + for b in m.get("block", []): + bid = b.get("id") + where = f"block {bid}" + if not bid: + err.append("block without id") + continue + if bid in seen: + err.append(f"{where}: duplicate id") + seen.add(bid) + for k in b: + if k not in BLOCK_KEYS: + err.append(f"{where}: unknown key {k!r}") + if not b.get("rule"): + err.append(f"{where}: missing decision rule") + params = set(b.get("params", [])) + sessions = b.get("sessions", []) + if not sessions and not (b.get("manual") or b.get("deferred")): + err.append(f"{where}: no sessions (mark manual or deferred)") + groups = set() + for i, s in enumerate(sessions): + sw = f"{where} session {i}" + for k in s: + if k not in SESSION_KEYS: + err.append(f"{sw}: unknown key {k!r}") + for k in ("variant", "jvm", "plan"): + v = s.get(k) + for p in PARAM_RE.findall(v if isinstance(v, str) else ",".join(v or [])): + if p not in params: + err.append(f"{sw}: ${{{p}}} not declared in params") + var = s.get("variant") + if not var: + err.append(f"{sw}: missing variant") + elif not PARAM_RE.search(var) and not (Path(out_dir) / var).is_dir() \ + and var not in missing_ids: + err.append(f"{sw}: variant {var!r} not in {out_dir} nor in missing_variants") + plan = split_plan(s.get("plan", [])) + if not plan: + err.append(f"{sw}: empty plan") + concrete = [p for p in plan if not PARAM_RE.search(p)] + for p in concrete: + if p not in SCENE_ORDER: + err.append(f"{sw}: unknown scene {p!r} (S6 is manual-only)") + idx = [SCENE_ORDER.index(p) for p in concrete if p in SCENE_ORDER] + if idx != sorted(set(idx)): + err.append(f"{sw}: plan must be in S1->S5 order without duplicates") + if "repeat" in s and not _pos_int(s["repeat"]): + err.append(f"{sw}: repeat must be a positive int") + if "heap" in s and not _pos_int(s["heap"]): + err.append(f"{sw}: heap must be a positive int (MB)") + jvm = s.get("jvm") + if jvm is not None and jvm not in JVM_SETS and not PARAM_RE.search(jvm): + err.append(f"{sw}: jvm {jvm!r} not in {sorted(JVM_SETS)}") + for t in s.get("add_mods", []): + if t not in trials: + err.append(f"{sw}: add_mods {t!r} not in [trials]") + if not isinstance(s.get("disable_mods", []), list): + err.append(f"{sw}: disable_mods must be a list") + _check_common(s, sw, worlds, trials, err) + g = (var, s.get("tag")) + if g in groups: + err.append(f"{sw}: variant {var!r} repeated without a distinct tag " + f"(analyze.py would merge them); use repeat or tag") + groups.add(g) + return err + + +def _check_common(s, where, worlds, trials, err): + for k in ("reps", "warmup", "capture"): + if k in s and not _pos_int(s[k]): + err.append(f"{where}: {k} must be a positive int") + if "display" in s and s["display"] not in DISPLAYS: + err.append(f"{where}: display {s['display']!r} not in {sorted(DISPLAYS)}") + if "res" in s and not re.fullmatch(r"\d+x\d+", str(s["res"])): + err.append(f"{where}: res must be WxH") + if "dh" in s and s["dh"] not in ("on", "off"): + err.append(f"{where}: dh must be on/off") + if "world" in s and s["world"] not in worlds: + err.append(f"{where}: world {s['world']!r} not in [worlds]") + + +def block_map(m): + return {b["id"]: b for b in m.get("block", [])} + + +def missing_variant_ids(m): + return {mv["id"] for mv in m.get("missing_variants", [])} + + +# ------------------------------------------------------------------ expansion / commands + +def _subst(v, params): + if not isinstance(v, str): + return v + return PARAM_RE.sub(lambda mo: params.get(mo.group(1), mo.group(0)), v) + + +def expand(m, block, params=None, overrides=None): + """Flat list of concrete sessions for a block (repeat / interleave applied).""" + params, overrides = params or {}, overrides or {} + d = dict(m.get("defaults", {})) + d.update({k: v for k, v in overrides.items() if v is not None}) + groups = [] + for si, s in enumerate(block.get("sessions", [])): + merged = {k: v for k, v in d.items() if k != "launch_overhead_s"} + merged.update(s) + for k in ("display", "res"): # CLI overrides beat per-session values too + if overrides.get(k) is not None: + merged[k] = overrides[k] + merged["variant"] = _subst(merged["variant"], params) + if merged.get("jvm") is not None: + merged["jvm"] = _subst(merged["jvm"], params) + merged["plan"] = [_subst(p, params) for p in split_plan(merged["plan"])] + tag = s.get("tag") + merged["label"] = f"{block['id']}.{tag}" if tag else block["id"] + merged["source"] = si + groups.append([dict(merged, run=r + 1) for r in range(s.get("repeat", 1))]) + out = [] + if block.get("interleave"): + for r in range(max((len(g) for g in groups), default=0)): + out += [g[r] for g in groups if r < len(g)] + else: + for g in groups: + out += g + for i, s in enumerate(out): + s["index"] = i + s["block"] = block["id"] + return out + + +def unresolved(sess): + """Placeholder names still present in a session (missing --param).""" + vals = [sess["variant"], sess.get("jvm") or "", *sess["plan"]] + return sorted({p for v in vals for p in PARAM_RE.findall(v)}) + + +def resolve_path(p): + p = os.path.expanduser(p) + hits = sorted(glob.glob(p)) + return hits[-1] if hits else None + + +def bench_argv(m, sess, live_world=False): + """`bench.py session` argv for one session, plus a list of missing inputs.""" + missing = [] + a = ["session", "--variant", sess["variant"], "--plan", ",".join(sess["plan"]), + "--reps", str(sess.get("reps", 3)), "--label", sess["label"]] + for k in ("warmup", "capture"): + if k in sess: + a += [f"--{k}", str(sess[k])] + for k in ("display", "res", "shader", "dh", "jvm"): + if sess.get(k) is not None: + a += [f"--{k}", str(sess[k])] + if sess.get("heap"): + a += ["--heap", str(sess["heap"])] + if sess.get("disable_mods"): + a += ["--disable-mods", ",".join(sess["disable_mods"])] + if sess.get("add_mods"): + jars = [] + for t in sess["add_mods"]: + pat = m["trials"][t] + hit = resolve_path(pat) + if hit is None: + missing.append(f"trial jar {t}: {pat}") + hit = os.path.expanduser(pat) + jars.append(hit) + a += ["--add-mods", ",".join(jars)] + if sess.get("world") and not live_world: + pat = m["worlds"][sess["world"]] + hit = resolve_path(pat) + if hit is None: + missing.append(f"world {sess['world']}: {pat} (or run with --live-world)") + hit = os.path.expanduser(pat) + a += ["--world-zip", hit] + return a, missing + + +def variant_missing(sess, out_dir=OUT_DIR): + return not (Path(out_dir) / sess["variant"]).is_dir() + + +def session_seconds(sess, overhead): + reps, w, c = sess.get("reps", 3), sess.get("warmup", 15), sess.get("capture", 30) + scenes = sum(S5_SECONDS if p == "s5" else w + c for p in sess["plan"]) * reps + return overhead + scenes + + +def spec_hash(argv, sess): + blob = json.dumps({"argv": argv, "manual": bool(sess.get("manual"))}, sort_keys=True) + return hashlib.sha256(blob.encode()).hexdigest()[:16] + + +def fmt_cmd(argv): + return "python3 bench.py " + shlex.join(argv) + + +# ------------------------------------------------------------------ state + +def mods_hash(mods_dir): + """Same algorithm as bench.py mods_hash (name + size of every file in mods/).""" + h = hashlib.sha256() + files = sorted(p for p in Path(mods_dir).glob("*") if p.is_file()) + for p in files: + h.update(f"{p.name}\0{p.stat().st_size}\n".encode()) + return h.hexdigest() + + +class State: + def __init__(self, path): + self.path = Path(path) + self.data = {"version": 1, "sessions": {}, "b0": None} + if self.path.exists(): + self.data.update(json.loads(self.path.read_text())) + + def save(self): + self.path.parent.mkdir(parents=True, exist_ok=True) + tmp = self.path.with_suffix(".tmp") + tmp.write_text(json.dumps(self.data, indent=1, sort_keys=True)) + tmp.replace(self.path) + + @staticmethod + def key(sess): + return f"{sess['block']}#{sess['index']}" + + def entry(self, sess): + return self.data["sessions"].get(self.key(sess)) + + def is_done(self, sess, spec, mods): + e = self.entry(sess) + return bool(e and e["status"] == "ok" and e["spec"] == spec and e["mods_hash"] == mods) + + def record(self, sess, spec, mods, status, rc, attempts): + self.data["sessions"][self.key(sess)] = { + "spec": spec, "mods_hash": mods, "status": status, "rc": rc, + "attempts": attempts, "label": sess["label"], "variant": sess["variant"], + "ts": time.strftime("%Y-%m-%dT%H:%M:%S")} + + +def b0_warning(state, current): + b0 = state.data.get("b0") + if not b0: + return "B0 (noise floor) has not been completed - win thresholds are undefined" + if b0["mods_hash"] != current: + return (f"MOD SET CHANGED since B0 (B0 under {b0['mods_hash'][:12]}, now " + f"{current[:12]}): B0 MUST BE RE-RUN before results are comparable") + return None + + +def banner(msg): + bar = "!" * 78 + print(f"{bar}\n!! {msg}\n{bar}", file=sys.stderr, flush=True) + + +# ------------------------------------------------------------------ run + +def default_runner(argv): + return subprocess.run([sys.executable, str(HERE / "bench.py"), *argv], cwd=HERE).returncode + + +def run_blocks(m, block_ids, state, mods_dir, runner=default_runner, params=None, + overrides=None, dry_run=False, include_manual=False, include_deferred=False, + live_world=False, skip_blocked=False, out_dir=OUT_DIR): + """Run blocks in order. Returns 0 ok, 1 session failed, 2 preflight problem.""" + bm = block_map(m) + for bid in block_ids: + if bid not in bm: + raise MatrixError(f"unknown block {bid}; known: {', '.join(bm)}") + # preflight: build every session's command, collect what can't run + todo, problems = [], [] + for bid in block_ids: + b = bm[bid] + if b.get("deferred") and not include_deferred: + log(f"{bid}: deferred ({b.get('proxy_note', 'fleet-only')}) - skipped") + continue + if b.get("manual"): + log(f"{bid}: manual block - run by hand ({b.get('proxy_note', '')})") + continue + for s in expand(m, b, params, overrides): + if s.get("manual") and not include_manual: + log(f"{bid}#{s['index']} {s['label']}: manual - skipped") + continue + miss = [f"param ${{{p}}} (use --param {p}=...)" for p in unresolved(s)] + if not miss and variant_missing(s, out_dir): + miss.append(f"variant {s['variant']} not generated (see missing_variants)") + argv, inputs = bench_argv(m, s, live_world) + miss += inputs + if miss: + problems.append((s, miss)) + continue + todo.append((s, argv)) + for s, miss in problems: + log(f"BLOCKED {s['block']}#{s['index']} {s['label']}: " + "; ".join(miss)) + if problems and not (skip_blocked or dry_run): + log("fix the above, or pass --skip-blocked (and/or --live-world)") + return 2 + mods0 = mods_hash(mods_dir) + if "B0" not in block_ids and any(s["block"] != "B0" for s, _ in todo): + w = b0_warning(state, mods0) + if w: + banner(w) + total = len(todo) + for n, (s, argv) in enumerate(todo, 1): + spec = spec_hash(argv, s) + tagline = f"[{n}/{total}] {s['block']}#{s['index']} {s['label']}" + if state.is_done(s, spec, mods0): + log(f"{tagline}: done - skip") + continue + if dry_run: + log(f"{tagline}: {fmt_cmd(argv)}") + continue + now = mods_hash(mods_dir) + if now != mods0: + banner(f"instance mods/ changed during the run ({mods0[:12]} -> {now[:12]}); " + "a session did not restore it - stopping") + return 1 + rc = None + for attempt in (1, 2): + log(f"{tagline}: attempt {attempt}: {fmt_cmd(argv)}") + rc = runner(argv) + if rc == 0: + break + log(f"{tagline}: exit {rc}" + (" - retrying once" if attempt == 1 else "")) + status = "ok" if rc == 0 else "failed" + state.record(s, spec, mods0, status, rc, attempt) + if s["block"] == "B0": + _maybe_close_b0(m, state, mods0, params, overrides, live_world) + state.save() + if status != "ok": + log(f"{tagline}: failed twice - stopping the run") + return 1 + return 0 + + +def _maybe_close_b0(m, state, mods, params, overrides, live_world): + b = block_map(m).get("B0") + if not b: + return + for s in expand(m, b, params, overrides): + argv, _ = bench_argv(m, s, live_world) + if not state.is_done(s, spec_hash(argv, s), mods): + return + state.data["b0"] = {"mods_hash": mods, "ts": time.strftime("%Y-%m-%dT%H:%M:%S")} + + +# ------------------------------------------------------------------ list / plan / status + +def block_summary(m, b, overhead, out_dir=OUT_DIR): + ss = expand(m, b) + auto = [s for s in ss if not s.get("manual") and not b.get("manual")] + blocked = [s for s in auto if not unresolved(s) and variant_missing(s, out_dir)] + secs = sum(session_seconds(s, overhead) for s in auto) + return {"sessions": len(ss), "auto": len(auto), "manual": len(ss) - len(auto), + "blocked": len(blocked), "seconds": secs} + + +def cmd_list(m, a): + overhead = m.get("defaults", {}).get("launch_overhead_s", 60) + print(f"| block | title | sessions | auto | manual | blocked (variant) | est. min | notes |") + print("| --- | --- | --- | --- | --- | --- | --- | --- |") + tot = {"sessions": 0, "auto": 0, "seconds": 0} + for b in m.get("block", []): + s = block_summary(m, b, overhead) + notes = [] + if b.get("deferred"): + notes.append("deferred") + if b.get("manual"): + notes.append("manual block") + if b.get("params"): + notes.append("params: " + ",".join(b["params"])) + print(f"| {b['id']} | {b.get('title', '')} | {s['sessions']} | {s['auto']} | " + f"{s['manual']} | {s['blocked']} | {s['seconds'] / 60:.0f} | {'; '.join(notes)} |") + if not b.get("deferred"): + for k in tot: + tot[k] += s[k] + print(f"\nnot deferred: {tot['sessions']} sessions ({tot['auto']} unattended), " + f"est. {tot['seconds'] / 3600:.1f} h unattended") + return 0 + + +def cmd_plan(m, a): + bm = block_map(m) + for bid in a.blocks.split(","): + if bid not in bm: + raise MatrixError(f"unknown block {bid}") + b = bm[bid] + print(f"## {bid} {b.get('title', '')}" + (" (DEFERRED)" if b.get("deferred") else "") + + (" (MANUAL)" if b.get("manual") else "")) + print(f"rule: {b['rule']}") + if b.get("proxy_note"): + print(f"proxy: {b['proxy_note']}") + for c in b.get("manual_checks", []): + print(f"manual check: {c}") + for s in expand(m, b, a.params, overrides(a)): + argv, miss = bench_argv(m, s, a.live_world) + pre = "# MANUAL: " if s.get("manual") else "" + print(f"{pre}{fmt_cmd(argv)}") + if s.get("shader_profile"): + print(f"# set Iris profile {s['shader_profile']} for {s['shader']} first") + if s.get("note"): + print(f"# note: {s['note']}") + if unresolved(s): + print(f"# needs --param {','.join(unresolved(s))}") + if variant_missing(s) and not unresolved(s): + print(f"# BLOCKED: variant {s['variant']} not generated") + for x in miss: + print(f"# missing: {x}") + print() + return 0 + + +def cmd_status(m, a): + state = State(a.state) + mods = mods_hash(a.mods_dir) if Path(a.mods_dir).is_dir() else "" + w = b0_warning(state, mods) + if w: + banner(w) + print(f"mods hash {mods[:12]} | state {a.state}") + print("| block | done | failed | pending | manual/deferred |") + print("| --- | --- | --- | --- | --- |") + for b in m.get("block", []): + done = failed = pending = skipped = 0 + for s in expand(m, b, a.params, overrides(a)): + if s.get("manual") or b.get("manual") or b.get("deferred"): + skipped += 1 + continue + argv, _ = bench_argv(m, s, a.live_world) + e = state.entry(s) + if state.is_done(s, spec_hash(argv, s), mods): + done += 1 + elif e and e["status"] == "failed": + failed += 1 + else: + pending += 1 + print(f"| {b['id']} | {done} | {failed} | {pending} | {skipped} |") + return 0 + + +def cmd_run(m, a): + state = State(a.state) + return run_blocks(m, a.blocks.split(","), state, a.mods_dir, params=a.params, + overrides=overrides(a), dry_run=a.dry_run, + include_manual=a.include_manual, include_deferred=a.include_deferred, + live_world=a.live_world, skip_blocked=a.skip_blocked) + + +def overrides(a): + return {"display": getattr(a, "display", None), "res": getattr(a, "res", None)} + + +def parse_params(items): + out = {} + for it in items or []: + k, sep, v = it.partition("=") + if not sep: + raise MatrixError(f"--param needs name=value, got {it!r}") + out[k] = v + return out + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("--matrix", default=str(MATRIX)) + ap.add_argument("--state", default=str(STATE)) + ap.add_argument("--mods-dir", default=str(GAME / "mods")) + sub = ap.add_subparsers(dest="cmd", required=True) + sub.add_parser("list", help="blocks, session counts, estimated time") + common = argparse.ArgumentParser(add_help=False) + common.add_argument("--param", action="append", default=[], help="name=value") + common.add_argument("--display", choices=sorted(DISPLAYS)) + common.add_argument("--res") + common.add_argument("--live-world", action="store_true", + help="don't restore world zips; use saves/onionbench as-is") + p = sub.add_parser("plan", parents=[common], help="print the bench.py commands") + p.add_argument("blocks") + r = sub.add_parser("run", parents=[common], help="run blocks sequentially (resumable)") + r.add_argument("blocks", help="comma list, e.g. B0,B1") + r.add_argument("--dry-run", action="store_true") + r.add_argument("--include-manual", action="store_true") + r.add_argument("--include-deferred", action="store_true") + r.add_argument("--skip-blocked", action="store_true", + help="skip sessions with missing variants/inputs instead of aborting") + sub.add_parser("status", parents=[common], help="progress for the current mod set") + a = ap.parse_args(argv) + try: + a.params = parse_params(getattr(a, "param", [])) + m = load(a.matrix) + return {"list": cmd_list, "plan": cmd_plan, "run": cmd_run, + "status": cmd_status}[a.cmd](m, a) + except MatrixError as e: + print(f"matrix.py: {e}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/analysis/client-perf/harness/matrix.toml b/analysis/client-perf/harness/matrix.toml new file mode 100644 index 0000000..45bb5b7 --- /dev/null +++ b/analysis/client-perf/harness/matrix.toml @@ -0,0 +1,294 @@ +# Declarative encoding of BENCH-MATRIX.md §4 (blocks B0-B14) for the proxy bench machine +# (Ryzen 9 9900X / RX 7800 XT). Driven by matrix.py; every session becomes one +# `python3 bench.py session ...` launch. BENCH-MATRIX.md is authoritative - if this file +# disagrees with it, fix this file. +# +# Session keys: +# variant proposed-configs/out/ (or an entry in [[missing_variants]] -> blocked) +# plan scenes in S1->S5 order: s1,s2,s2s,s3,s4,s5 (S6 is not automatable) +# reps windows per scene inside the session (phase0: 3 runs) +# repeat separate sessions (= cold launches) of the same config (B0: 3) +# tag distinguishes sessions that share a variant but differ by flags; the +# bench label becomes "." so analyze.py groups them apart +# jvm/heap BENCH-MATRIX JVM set / MaxMemAlloc MB +# shader shaderpack file name or "off"; shader_profile = Iris profile (manual today) +# dh "on" | "off" (dh-off row of B3) +# disable_mods globs matched in the instance mods/ (B12a) +# add_mods jar paths/globs added for the session only (B12b) +# world key of [worlds]; restored from the zip before the session +# warmup/capture per-window seconds (bench.py defaults 15 / 30; S5 is fixed 10 + 120) +# manual true = cannot run unattended; `run` skips it unless --include-manual +# note free text shown by `plan` +# ${name} in variant/jvm/plan is a block parameter, supplied with `--param name=value`. + +[defaults] +reps = 3 +warmup = 15 +capture = 30 +display = "gamescope-headless" +res = "1920x1080" +world = "v1" +shader = "off" # pin: a shader/DH state must never leak from one session to the next +dh = "on" +launch_overhead_s = 60 # launch->join (~28 s measured) + settle 10 s + teardown/restore + +[worlds] +# BENCH-MATRIX §2. None of these exist yet: S4 (airships) is unbuilt. Until they do, +# `run --live-world` uses the instance's saves/onionbench as-is (no restore). +v1 = "~/onion-bench/worlds/bench-world-v1.zip" +s4-0 = "~/onion-bench/worlds/bench-world-v1-s4-0.zip" +s4-1 = "~/onion-bench/worlds/bench-world-v1-s4-1.zip" +s4-4 = "~/onion-bench/worlds/bench-world-v1.zip" # the main zip carries all 4 ships + +[trials] +# B12b trial jars: download the NeoForge 1.21.1 build from Modrinth into this directory. +gnetum = "~/onion-bench/trials/gnetum-*.jar" +better_block_entities = "~/onion-bench/trials/betterblockentities-*.jar" +flerovium = "~/onion-bench/trials/flerovium-*.jar" +super_resolution = "~/onion-bench/trials/superresolution-*.jar" +ixeris = "~/onion-bench/trials/ixeris-*.jar" +iris_flywheel_compat = "~/onion-bench/trials/irisflw-*.jar" +cull_leaves = "~/onion-bench/trials/cull-leaves-*.jar" + +# Variants a block needs that build.py does not generate. Sessions using them are +# reported as blocked until they exist in proposed-configs/out/. (C1-fa, C1-fa-ext, +# emf-lod20-fa, emf-lod8-fa and C1-bl are now generated by build.py; every bench +# variant pins options.txt resourcePacks.) +[[missing_variants]] +id = "dh-off" +blocks = [] +contents = """Not needed as a directory: BENCH-MATRIX defines dh-off as C1 with DH rendering \ +toggled off in the GUI. Encoded as variant C1 + `dh = "off"` (bench.py --dh off).""" + +# --------------------------------------------------------------------------------------- + +[[block]] +id = "B0" +title = "noise floor" +rule = "Defines each machine's threshold. P2 also checks `MemAvailable` drift (phase0 Step 4)." +proxy_note = "Single proxy machine. Re-run whenever the mod set changes (matrix.py warns)." +manual_checks = ["MemAvailable drift: compare mem_available_mb before/after across the 3 sessions (recorded per row)"] +sessions = [ + { variant = "C0", plan = "s1", repeat = 3 }, +] + +[[block]] +id = "B1" +title = "baseline" +rule = "Reference numbers for everything below" +manual_checks = [ + "S6 (GUI/JEI overlay) needs a key press - run by hand", + "/sparkc heapsummary after S1 loads", + "VRAM peak", +] +sessions = [ + { variant = "C0", plan = "s1,s2,s3,s4,s5" }, +] + +[[block]] +id = "B2" +title = "composite" +rule = "If C1 isn't a win on ≥ 2 machines, stop and bisect with B3–B11 before shipping anything" +proxy_note = "Proxy runs all three composites; one machine = single-machine evidence, fleet confirmation pending. Baseline is C0 (pooled from B1)." +manual_checks = ["S6 by hand", "/sparkc heapsummary after S1 loads", "VRAM peak"] +sessions = [ + { variant = "C1", plan = "s1,s2,s3,s4,s5" }, + { variant = "C1-deck", plan = "s1,s2,s3,s4,s5" }, + { variant = "C1-high", plan = "s1,s2,s3,s4,s5" }, +] + +[[block]] +id = "B3" +title = "Distant Horizons" +rule = """**Radius per tier** = the largest radius meeting the FPS floor on S1 *and* S5: P0 → deck, P1 → default, P2 → high. \ +| Adopt SSAO-off / LOW-quality for that tier on a win. \ +| Keep 2 threads unless 4 is a win on S5 *and* not a loss on S1""" +proxy_note = "Absolute floors are fleet values; on the proxy use relative wins only. P0/P1-only rows (dh-ssao, dh-qlow) run here too." +manual_checks = ["DH memory in F3"] +sessions = [ + { variant = "dh-r256", plan = "s1,s5" }, + { variant = "dh-r192", plan = "s1,s5" }, + { variant = "C1", plan = "s1,s5" }, + { variant = "dh-r64", plan = "s1,s5" }, + { variant = "C1", plan = "s1,s5", dh = "off", tag = "dh-off" }, + { variant = "dh-ssao", plan = "s1" }, + { variant = "dh-qlow", plan = "s1" }, + { variant = "dh-thr4", plan = "s1,s5" }, +] + +[[block]] +id = "B4" +title = "heap & GC: GC" +rule = "GC = best S5 1 % low. Heap = the smallest size within the spread of the best. Feeds `PLAYER-TUNING.md` §2. P1 isn't run (32 GB, not a constraint)." +proxy_note = "P2 values (-Xmx6G) on the proxy; the heap ladder (B4h) adds 8 G as the top rung." +manual_checks = ["/sparkc gc after S5 (pause count / max / total)", "swap delta"] +sessions = [ + { variant = "C1", plan = "s2,s5", jvm = "J0", heap = 6144, tag = "J0" }, + { variant = "C1", plan = "s2,s5", jvm = "J1", heap = 6144, tag = "J1" }, + { variant = "C1", plan = "s2,s5", jvm = "J2", heap = 6144, tag = "J2" }, + { variant = "C1", plan = "s2,s5", jvm = "J3", heap = 6144, tag = "J3" }, +] + +[[block]] +id = "B4h" +title = "heap & GC: heap ladder for the GC winner" +rule = "GC = best S5 1 % low. Heap = the smallest size within the spread of the best. Feeds `PLAYER-TUNING.md` §2. P1 isn't run (32 GB, not a constraint)." +params = ["gc"] +proxy_note = "Run after B4 with --param gc=." +sessions = [ + { variant = "C1", plan = "s2,s5", jvm = "${gc}", heap = 4096, tag = "heap4" }, + { variant = "C1", plan = "s2,s5", jvm = "${gc}", heap = 5120, tag = "heap5" }, + { variant = "C1", plan = "s2,s5", jvm = "${gc}", heap = 6144, tag = "heap6" }, + { variant = "C1", plan = "s2,s5", jvm = "${gc}", heap = 8192, tag = "heap8" }, +] + +[[block]] +id = "B5" +title = "Sable / SSRD" +deferred = true +rule = """SSRD per tier = the largest value meeting the floor with 4 ships. sable-noshade ships only on a win *and* if it looks acceptable. \ +| Decay > spread is a Sable #1011-class finding: report it, no config fix""" +proxy_note = "Fleet-only (P0/P1) per BENCH-MATRIX; deferred on the proxy. Needs the S4 world zips. The 45-min decay run exceeds mangohud-local log_duration=1800 and needs FPS at 0/15/30/45 min + heapsummary by hand." +manual_checks = ["sub-level count", "sable-noshade visual acceptability", "/sparkc heapsummary at 0/15/30/45 min"] +sessions = [ + { variant = "C1", plan = "s4", world = "s4-0", tag = "ships0" }, + { variant = "C1", plan = "s4", world = "s4-1", tag = "ships1" }, + { variant = "C1", plan = "s4", world = "s4-4", tag = "ships4" }, + { variant = "ssrd-32", plan = "s4", world = "s4-4" }, + { variant = "ssrd-64", plan = "s4", world = "s4-4" }, + { variant = "ssrd-128", plan = "s4", world = "s4-4" }, + { variant = "sable-noshade", plan = "s4", world = "s4-4" }, + { variant = "C1", plan = "s4", world = "s4-4", jvm = "J0", reps = 1, capture = 2700, repeat = 2, tag = "decay", manual = true, note = "45-min decay; CSV capped at 30 min by mangohud-local" }, +] + +[[block]] +id = "B6" +title = "ModernFix dynamic_resources" +rule = "Ship `dynamic_resources=true` only if: heap −300 MB or more *or* startup faster beyond spread, **and** zero missing textures, **and** no crash in 3 launches" +interleave = true +manual_checks = [ + "S6 by hand", + "heap after load (/sparkc heapsummary)", + "Missing-texture check: JEI @chipped/@rechiseled/@handcrafted, 4x4 Rechiseled CTM wall", +] +sessions = [ + { variant = "C1", plan = "s1,s2", repeat = 3 }, + { variant = "mf-dynres-off", plan = "s1,s2", repeat = 3 }, +] + +[[block]] +id = "B7" +title = "shaders" +rule = "Data for the owner's shader policy, no auto-decision. Report the FPS cost per profile and whether ship corruption appears with shadow profiles" +proxy_note = "Iris profiles are set in Iris' GUI; bench.py --shader only selects the pack, so profile rows stay manual until a profile mechanism exists." +manual_checks = ["Flywheel backend line in F3 (expect off with shaders)", "visual check for Iris #3164 on ships"] +sessions = [ + { variant = "C1", plan = "s1,s2,s4", world = "s4-4", shader = "off", tag = "noshader" }, + { variant = "C1", plan = "s1,s2,s4", world = "s4-4", shader = "MakeUp-UltraFast-9.5f.zip", shader_profile = "shadowless_medium", tag = "makeup-shadowless", manual = true }, + { variant = "C1", plan = "s1,s2,s4", world = "s4-4", shader = "MakeUp-UltraFast-9.5f.zip", shader_profile = "medium", tag = "makeup-medium", manual = true }, + { variant = "C1", plan = "s1,s2,s4", world = "s4-4", shader = "ComplementaryReimagined_r5.9.3.zip", shader_profile = "LOW", tag = "compl-low", manual = true }, + { variant = "C1", plan = "s1,s2,s4", world = "s4-4", shader = "ComplementaryReimagined_r5.9.3.zip", shader_profile = "MEDIUM", tag = "compl-medium", manual = true }, +] + +[[block]] +id = "B8" +title = "entities" +rule = "EMF: adopt the smallest LOD that's a win with FA on. LDL: per tier, the cheapest mode that's a win. FA guidance → `PLAYER-TUNING.md` §4" +sessions = [ + { variant = "C1", plan = "s3" }, + { variant = "C1-fa", plan = "s3" }, + { variant = "C1-fa-ext", plan = "s3" }, + { variant = "emf-lod20-fa", plan = "s3" }, + { variant = "emf-lod8-fa", plan = "s3" }, + { variant = "ldl-fancy", plan = "s3" }, + { variant = "ldl-fastest", plan = "s3" }, +] + +[[block]] +id = "B9" +title = "Flywheel" +rule = "Per tier: the smoothest value meeting the floor" +sessions = [ + { variant = "fw-flat", plan = "s2", shader = "off" }, + { variant = "C1", plan = "s2", shader = "off" }, + { variant = "fw-smooth", plan = "s2", shader = "off" }, +] + +[[block]] +id = "B10" +title = "Sound Physics" +rule = "Adopt each change that's a win. Listen for obvious audio breakage." +manual_checks = ["listen for audio breakage"] +sessions = [ + { variant = "spr-up", plan = "s2s" }, + { variant = "C1", plan = "s2s" }, + { variant = "spr-strict", plan = "s2s" }, + { variant = "spr-nocap", plan = "s2s" }, +] + +[[block]] +id = "B11" +title = "vanilla / Sodium" +rule = """3 threads only if S5 wins and S1 doesn't lose \ +| Lowest blend radius that's a win, per tier \ +| Mip 4 for mid/high only if VRAM peak ≤ 5.5 GB and no loss""" +proxy_note = "C1 baseline session added in-block so analyze.py compares within the label." +manual_checks = ["VRAM peak for opt-mip4"] +sessions = [ + { variant = "C1", plan = "s1,s2,s5", tag = "baseline" }, + { variant = "sod-thr3", plan = "s1,s5" }, + { variant = "opt-blend2", plan = "s5" }, + { variant = "opt-blend0", plan = "s5" }, + { variant = "opt-mip4", plan = "s1,s2" }, +] + +[[block]] +id = "B12a" +title = "added-mod ablation" +rule = "Keep the mod unless removing it is a win (it costs FPS) or it breaks something visible. Sodium Extra + RSO stay regardless for their per-machine options." +sessions = [ + { variant = "C1", plan = "s1,s2s,s3" }, + { variant = "C1", plan = "s1,s2s,s3", disable_mods = ["BadOptimizations-*.jar"], tag = "no-badopt" }, + { variant = "C1", plan = "s1,s2s,s3", disable_mods = ["particle_core-*.jar"], tag = "no-particlecore" }, + { variant = "C1", plan = "s1,s2s,s3", disable_mods = ["sodium-extra-*.jar"], tag = "no-sodiumextra" }, +] + +[[block]] +id = "B12b" +title = "trial mods" +rule = "Owner pre-approved helpful adds. Add with `packwiz modrinth add` only on a win with no breakage. **Not More Culling**: its NeoForge build declares `minecraft [1.21,1.21.1)`. A new mod set means re-running B0 first." +proxy_note = "add_mods entries name [trials] keys. Trials are temporary for the session; a permanent add changes the mod set and invalidates B0." +manual_checks = ["breakage check per trial", "S6 for Gnetum/Flerovium (GUI paths) by hand"] +sessions = [ + { variant = "C1", plan = "s1,s2,s3,s5", tag = "baseline" }, + { variant = "C1", plan = "s1,s2,s3,s5", add_mods = ["gnetum"], tag = "gnetum" }, + { variant = "C1", plan = "s2,s3", add_mods = ["better_block_entities"], tag = "bbe" }, + { variant = "C1-fa", plan = "s3", tag = "fa-baseline" }, + { variant = "C1-fa", plan = "s3", add_mods = ["better_block_entities"], tag = "bbe-fa" }, + { variant = "C1", plan = "s1,s2,s3,s5", add_mods = ["flerovium"], tag = "flerovium" }, + { variant = "C1", plan = "s1,s2,s5", add_mods = ["super_resolution"], tag = "superres" }, + { variant = "C1", plan = "s1,s2,s5", add_mods = ["ixeris"], tag = "ixeris" }, + { variant = "C1", plan = "s1,s2,s4", world = "s4-4", shader = "MakeUp-UltraFast-9.5f.zip", tag = "shader-baseline" }, + { variant = "C1", plan = "s1,s2,s4", world = "s4-4", shader = "MakeUp-UltraFast-9.5f.zip", add_mods = ["iris_flywheel_compat"], tag = "irisflw" }, + { variant = "C1-bl", plan = "s1,s5", tag = "bl-baseline" }, + { variant = "C1-bl", plan = "s1,s5", add_mods = ["cull_leaves"], tag = "cull-leaves", note = "MidnightLib is already in the pack (1.10.0)" }, +] + +[[block]] +id = "B13" +title = "profile" +rule = "Flag any mod over 5 % of render-thread time that isn't Sodium, Iris, DH or Flywheel" +params = ["scene"] +proxy_note = "Scene = the worst scene from B2 under C1 (--param scene=sN)." +manual_checks = ["/sparkc profiler start --timeout 60 during the capture; save the report"] +sessions = [ + { variant = "C1", plan = "${scene}", reps = 1, capture = 75, manual = true, note = "/sparkc is a client command; start it by hand at BENCH_MARK start" }, +] + +[[block]] +id = "B14" +title = "spawn tuning" +manual = true +rule = "Procedure and keep/back-off rule: `SPAWN-TUNING.md` §B14" +proxy_note = "Singleplayer fresh world `onionspawn`; river, jungle, warm ocean 10 min each; per-species and total entity counts, defaults vs shipped spawn files. Not a frametime session - run by hand." +sessions = [] diff --git a/analysis/client-perf/harness/tests/test_analyze_block.py b/analysis/client-perf/harness/tests/test_analyze_block.py new file mode 100644 index 0000000..d201499 --- /dev/null +++ b/analysis/client-perf/harness/tests/test_analyze_block.py @@ -0,0 +1,97 @@ +"""analyze.py: tagged labels ('B3.dh-off') take their baseline from the same block. + + python3 -m unittest discover -s analysis/client-perf/harness/tests +""" +import sys +import unittest +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE.parent)) + +import analyze # noqa: E402 + + +def row(label, variant, scene, session, p99, low=100.0, fps=200.0): + return {"kind": "window", "label": label, "variant": variant, "scene": scene, + "session": session, "flags": [], + "metrics": {"p99_ms": p99, "one_low_fps": low, "avg_fps": fps}} + + +def cell(table, label, variant, scene): + for line in table.splitlines(): + cells = [c.strip() for c in line.strip("|").split("|")] + if cells[:3] == [label, variant, scene]: + return cells + raise AssertionError(f"no row {label}/{variant}/{scene} in\n{table}") + + +class BlockBaseline(unittest.TestCase): + def setUp(self): + self.rows = [ + # noise: spread 0.2 ms / 2 fps + row("B0", "C0", "s1", "n1", 10.0, 100), row("B0", "C0", "s1", "n2", 10.2, 102), + # global C1 reference elsewhere (B2) is much slower than the B3 one + row("B2", "C1", "s1", "g1", 20.0), + row("B3", "C1", "s1", "b1", 10.0), + row("B3", "dh-r64", "s1", "b2", 8.0), + row("B3.dh-off", "C1", "s1", "b3", 6.0), + ] + + def test_tagged_self_variant_uses_untagged_block_label(self): + s = analyze.summarise(self.rows) + b, where = analyze.self_baseline(s, "B3.dh-off", "s1", "C1") + self.assertEqual(where, "B3") + self.assertEqual(b["p99"], 10.0) + + def test_untagged_block_label_is_its_own_baseline(self): + s = analyze.summarise(self.rows) + self.assertEqual(analyze.self_baseline(s, "B3", "s1", "C1"), (None, None)) + + def test_tagged_other_variant_prefers_block_over_global(self): + rows = self.rows + [row("B12a.no-x", "X", "s1", "t1", 9.0)] + rows += [row("B12a", "C1", "s1", "t0", 12.0)] + s = analyze.summarise(rows) + b, where = analyze.baseline_for(s, rows, "B12a.no-x", "s1", "C1") + self.assertEqual((where, b["p99"]), ("B12a", 12.0)) + + def test_block_baseline_tag(self): + rows = self.rows + [row("B11.baseline", "C1", "s5", "x0", 12.0), + row("B11", "sod-thr3", "s5", "x1", 11.0), + row("B2", "C1", "s5", "x2", 30.0)] + s = analyze.summarise(rows) + b, where = analyze.baseline_for(s, rows, "B11", "s5", "C1") + # untagged candidate: own label has no C1 -> '.baseline', not global + self.assertEqual((where, b["p99"]), ("B11.baseline", 12.0)) + rows.append(row("B11.x", "opt-mip4", "s5", "x3", 11.0)) + s = analyze.summarise(rows) + b, where = analyze.baseline_for(s, rows, "B11.x", "s5", "C1") + self.assertEqual((where, b["p99"]), ("B11.baseline", 12.0)) + + def test_block_pool_then_global(self): + rows = self.rows + [row("B4.J0", "C1", "s2", "j0", 10.0), + row("B4.J1", "C1", "s2", "j1", 12.0), + row("B4.x", "Y", "s2", "j2", 9.0), + row("B2", "C1", "s2", "g2", 40.0)] + s = analyze.summarise(rows) + b, where = analyze.baseline_for(s, rows, "B4.x", "s2", "C1") + self.assertEqual((where, b["p99"]), ("B4.*", 11.0)) + b, where = analyze.baseline_for(s, rows, "B9.z", "s2", "C1") + self.assertEqual(where, "*") + self.assertIsNone(analyze.baseline_for(s, rows, "B9.z", "s9", "C1")[0]) + + def test_table_verdict_against_block(self): + t = analyze.table(analyze.summarise(self.rows), self.rows, "C1", "B0") + c = cell(t, "B3.dh-off", "C1", "s1") + self.assertEqual(c[-4:], ["C1@B3", "+4.00", "+0.0", "win"]) + self.assertEqual(cell(t, "B3", "C1", "s1")[-1], "baseline") + self.assertEqual(cell(t, "B3", "dh-r64", "s1")[-4], "C1@B3") + + def test_table_label_filter_still_finds_block_baseline(self): + shown = [r for r in self.rows if r["label"] in ("B3.dh-off", "B0")] + t = analyze.table(analyze.summarise(shown), self.rows, "C1", "B0") + self.assertEqual(cell(t, "B3.dh-off", "C1", "s1")[-4], "C1@B3") + + +if __name__ == "__main__": + unittest.main() diff --git a/analysis/client-perf/harness/tests/test_bench_flags.py b/analysis/client-perf/harness/tests/test_bench_flags.py new file mode 100644 index 0000000..3a301a7 --- /dev/null +++ b/analysis/client-perf/harness/tests/test_bench_flags.py @@ -0,0 +1,273 @@ +"""Offline tests for bench.py session instance changes (temp dirs only, no game). + + python3 -m unittest discover -s analysis/client-perf/harness/tests +""" +import json +import sys +import tempfile +import unittest +import zipfile +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE.parent)) + +import bench # noqa: E402 + +IRIS = """#This file stores configuration options for Iris, such as the currently active shaderpack +#Fri Oct 16 19:37:17 EDT 2026 +allowUnknownShaders=false +enableShaders=true +maxShadowRenderDistance=32 +shaderPack= +""" + +DH = """[server] +\tenableRealTimeUpdates = true + +[client] +\tshowDhOptionsButtonInMinecraftUi = true + +\t\t[client.advanced.debugging] +\t\t\tlodOnlyMode = false +\t\t\t# DISABLED: Disable rendering +\t\t\trendererMode = "DEFAULT" + +\t\t\t[client.advanced.debugging.debugWireframe] +\t\t\t\tenableRendering = false +\t\t\t\trendererMode = "DEFAULT" +""" + + +class GameDir(unittest.TestCase): + def setUp(self): + self._td = tempfile.TemporaryDirectory() + self.game = Path(self._td.name) + (self.game / "config").mkdir() + (self.game / "mods").mkdir() + (self.game / "shaderpacks").mkdir() + (self.game / "saves" / "onionbench").mkdir(parents=True) + (self.game / "saves" / "onionbench" / "level.dat").write_bytes(b"orig") + (self.game / "config" / "iris.properties").write_text(IRIS) + (self.game / "config" / "DistantHorizons.toml").write_text(DH) + (self.game / "shaderpacks" / "Pack A.zip").write_bytes(b"zip") + for n in ("BadOptimizations-2.4.1.jar", "particle_core-0.3.3.jar", "sodium-0.8.jar"): + (self.game / "mods" / n).write_bytes(n.encode()) + self.j = bench.Journal.begin(self.game, "ptest") + + def tearDown(self): + self._td.cleanup() + + def snapshot(self): + """Instance contents, excluding the journal itself (setUp opened one).""" + return {p.relative_to(self.game).as_posix(): p.read_bytes() + for p in sorted(self.game.rglob("*")) if p.is_file() + and p.relative_to(self.game).parts[0] not in (bench.PENDING, bench.BACKUP_DIR)} + + +class IrisAndDh(GameDir): + def test_properties_rewrite(self): + out = bench.set_properties(IRIS, {"enableShaders": "false", "shaderPack": "a:b=c.zip"}) + self.assertIn("enableShaders=false\n", out) + self.assertIn("shaderPack=a\\:b\\=c.zip\n", out) + self.assertIn("maxShadowRenderDistance=32", out) + self.assertTrue(out.startswith("#This file")) + self.assertIn("newKey=1", bench.set_properties(IRIS, {"newKey": "1"})) + + def test_shader_on_off_restore(self): + before = self.snapshot() + opts = self.game.parent / f"{self.game.name}-opts.txt" + opts.write_text("SHADOW=off\n") + try: + bench.apply_shader(self.game, self.j, "Pack A.zip", opts) + finally: + text = opts.read_text() + opts.unlink() + cfg = (self.game / "config" / "iris.properties").read_text() + self.assertIn("enableShaders=true", cfg) + self.assertIn("shaderPack=Pack A.zip", cfg) + self.assertEqual((self.game / "shaderpacks" / "Pack A.zip.txt").read_text(), text) + self.assertEqual(bench.restore_pending(self.game), []) + self.assertEqual(self.snapshot(), before) + + j = bench.Journal.begin(self.game, "p2") + bench.apply_shader(self.game, j, "off") + cfg = (self.game / "config" / "iris.properties").read_text() + self.assertIn("enableShaders=false", cfg) + self.assertIn("shaderPack=\n", cfg) + bench.restore_pending(self.game) + self.assertEqual(self.snapshot(), before) + + def test_dh_rewrite_only_debugging_section(self): + off = bench.set_dh_rendering(DH, False) + self.assertIn('\t\t\trendererMode = "DISABLED"\n\n', off) + self.assertIn('\t\t\t\trendererMode = "DEFAULT"', off) # wireframe section untouched + self.assertIn("enableRendering = false", off) + self.assertEqual(bench.set_dh_rendering(off, True), DH) + with self.assertRaises(ValueError): + bench.set_dh_rendering("[client]\n\trendererMode = \"DEFAULT\"\n", False) + + def test_apply_dh_restore(self): + bench.apply_dh(self.game, self.j, False) + self.assertIn('"DISABLED"', (self.game / "config" / "DistantHorizons.toml").read_text()) + bench.restore_pending(self.game) + self.assertEqual((self.game / "config" / "DistantHorizons.toml").read_text(), DH) + + +class Mods(GameDir): + def test_disable_add_restore(self): + before = self.snapshot() + names = bench.resolve_disable(self.game, ["badopt*", "PARTICLE_*"]) + self.assertEqual(names, ["BadOptimizations-2.4.1.jar", "particle_core-0.3.3.jar"]) + with self.assertRaises(ValueError): + bench.resolve_disable(self.game, ["badopt*", "nomatch*"]) + extra = self.game.parent / f"{self.game.name}-trial.jar" + extra.write_bytes(b"trial") + try: + jars = bench.check_add_mods(self.game, [str(extra)]) + h0 = bench.mods_hash(self.game / "mods") + bench.disable_mods(self.game, self.j, names) + bench.add_mods(self.game, self.j, jars) + finally: + extra.unlink() + mods = sorted(p.name for p in (self.game / "mods").iterdir()) + self.assertEqual(mods, sorted(["BadOptimizations-2.4.1.jar.disabled", + f"{self.game.name}-trial.jar", + "particle_core-0.3.3.jar.disabled", "sodium-0.8.jar"])) + self.assertNotEqual(bench.mods_hash(self.game / "mods"), h0) + self.assertEqual(bench.restore_pending(self.game), []) + self.assertEqual(self.snapshot(), before) + self.assertEqual(bench.mods_hash(self.game / "mods"), h0) + + def test_add_mods_validation(self): + with self.assertRaises(ValueError): + bench.check_add_mods(self.game, [str(self.game / "nope.jar")]) + with self.assertRaises(ValueError): # name collision + bench.check_add_mods(self.game, [str(self.game / "mods" / "sodium-0.8.jar")]) + + +class WorldZip(GameDir): + def make_zip(self, prefix): + z = self.game.parent / f"{self.game.name}-{prefix.strip('/') or 'flat'}.zip" + with zipfile.ZipFile(z, "w") as zf: + zf.writestr(prefix + "level.dat", b"zipped") + zf.writestr(prefix + "region/r.0.0.mca", b"region") + self.addCleanup(z.unlink) + return z + + def check_swap(self, prefix): + before = self.snapshot() + z = self.make_zip(prefix) + self.assertEqual(bench.zip_world_root(z), prefix) + sha = bench.swap_world(self.game, self.j, z) + self.assertEqual(sha, bench.sha256_file(z)) + w = self.game / "saves" / "onionbench" + self.assertEqual((w / "level.dat").read_bytes(), b"zipped") + self.assertTrue((w / "region" / "r.0.0.mca").exists()) + self.assertTrue((self.game / "saves" / "onionbench.harness-bak" / "level.dat").exists()) + self.assertEqual(sorted(p.name for p in (self.game / "saves").iterdir()), + ["onionbench", "onionbench.harness-bak"]) + self.assertEqual(bench.restore_pending(self.game), []) + self.assertEqual(self.snapshot(), before) + self.assertEqual([p.name for p in (self.game / "saves").iterdir()], ["onionbench"]) + + def test_flat_layout(self): + self.check_swap("") + + def test_single_top_dir_layout(self): + self.check_swap("bench-world-v1/") + + def test_bad_zips(self): + bad = self.game.parent / f"{self.game.name}-bad.zip" + self.addCleanup(lambda: bad.unlink(missing_ok=True)) + with zipfile.ZipFile(bad, "w") as zf: + zf.writestr("a/level.dat", b"x") + zf.writestr("b/level.dat", b"x") + with self.assertRaises(ValueError): + bench.zip_world_root(bad) + with zipfile.ZipFile(bad, "w") as zf: + zf.writestr("../evil/level.dat", b"x") + with self.assertRaises(ValueError): + bench.zip_world_root(bad) + + +class PendingJournal(GameDir): + def test_crash_then_restore_from_disk(self): + before = self.snapshot() + z = WorldZip.make_zip(self, "w/") + bench.apply_shader(self.game, self.j, "Pack A.zip") + bench.apply_dh(self.game, self.j, False) + bench.disable_mods(self.game, self.j, ["sodium-0.8.jar"]) + bench.swap_world(self.game, self.j, z) + plan = self.game / "saves" / "onionbench" / "datapacks" / "autoplan" + self.j.record({"op": "tree", "path": plan.relative_to(self.game).as_posix()}) + plan.mkdir(parents=True) + (plan / "pack.mcmeta").write_text("{}") + # simulate a hard kill: only the on-disk journal survives + del self.j + self.assertTrue(bench.pending_exists(self.game)) + data = json.loads((self.game / bench.PENDING).read_text()) + self.assertEqual([e["op"] for e in data["entries"]], + ["file", "file", "file", "rename", "world", "tree"]) + with self.assertRaises(RuntimeError): + bench.Journal.begin(self.game, "again") + self.assertEqual(bench.restore_pending(self.game), []) + self.assertFalse(bench.pending_exists(self.game)) + self.assertFalse((self.game / bench.BACKUP_DIR).exists()) + self.assertEqual(self.snapshot(), before) + + def test_entry_written_before_mutation_is_tolerated(self): + # journal says "renamed", but the kill came before os.rename ran + self.j.record({"op": "rename", "src": "mods/sodium-0.8.jar", + "dst": "mods/sodium-0.8.jar.disabled"}) + self.j.record({"op": "added", "path": "mods/never-copied.jar"}) + before = self.snapshot() + self.assertEqual(bench.restore_pending(self.game), []) + self.assertEqual(self.snapshot(), before) + + def test_failed_entry_stays_journaled(self): + self.j.record({"op": "bogus"}) + failed = bench.restore_pending(self.game) + self.assertEqual(len(failed), 1) + data = json.loads((self.game / bench.PENDING).read_text()) + self.assertEqual(data["entries"], [{"op": "bogus"}]) + + def test_nothing_pending(self): + bench.restore_pending(self.game) + self.assertEqual(bench.restore_pending(self.game), []) + + +class OptionsAndFlags(unittest.TestCase): + def test_resource_pack_reset(self): + with tempfile.TemporaryDirectory() as td: + g = Path(td) + (g / "options.txt").write_text('version:3955\nresourcePacks:["vanilla","x.zip"]\n' + "maxFps:260\n") + bench.reset_resource_packs(g) + vals = bench.options_values(g, {"resourcePacks", "incompatibleResourcePacks"}) + self.assertEqual(vals, {"resourcePacks": '["vanilla","mod_resources"]', + "incompatibleResourcePacks": "[]"}) + text = (g / "options.txt").read_text() + self.assertTrue(text.startswith("version:3955\n")) + self.assertIn("maxFps:260", text) + (g / "options.txt").unlink() + bench.reset_resource_packs(g) # absent file -> just the baseline + self.assertEqual(len((g / "options.txt").read_text().splitlines()), 2) + + def test_env_flags(self): + gpus = [{"device": "0x13c0", "dpm_level": "auto"}, + {"device": bench.DGPU_DEVICE, "dpm_level": "auto"}] + self.assertEqual(bench.env_flags(gpus, True), ["gpu_dpm_auto", "server_running"]) + gpus[1]["dpm_level"] = "high" + self.assertEqual(bench.env_flags(gpus, False), []) + + def test_build_parser_session_defaults(self): + a = bench.build_parser().parse_args(["session", "--variant", "C1"]) + self.assertEqual((a.shader, a.dh, a.disable_mods, a.add_mods, a.world_zip), + ("off", "on", None, None, None)) + a = bench.build_parser().parse_args(["restore"]) + self.assertIs(a.fn, bench.cmd_restore) + + +if __name__ == "__main__": + unittest.main() diff --git a/analysis/client-perf/harness/tests/test_matrix.py b/analysis/client-perf/harness/tests/test_matrix.py new file mode 100644 index 0000000..76a0dc2 --- /dev/null +++ b/analysis/client-perf/harness/tests/test_matrix.py @@ -0,0 +1,342 @@ +"""Offline tests for matrix.py (no game; the bench.py runner is faked). + + python3 -m unittest discover -s analysis/client-perf/harness/tests +""" +import contextlib +import io +import json +import sys +import tempfile +import tomllib +import unittest +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE.parent)) + +import matrix # noqa: E402 + +MINI = """ +[defaults] +reps = 3 +warmup = 15 +capture = 30 +display = "gamescope" +res = "1280x720" +world = "v1" +shader = "off" +dh = "on" +launch_overhead_s = 60 + +[worlds] +v1 = "{tmp}/worlds/v1.zip" +s4 = "{tmp}/worlds/s4-*.zip" + +[trials] +gnetum = "{tmp}/trials/gnetum-*.jar" + +[[missing_variants]] +id = "C1-fa" +blocks = ["B8"] +contents = "C1 + Fresh Animations" + +[[block]] +id = "B0" +rule = "noise" +sessions = [{{ variant = "C0", plan = "s1", repeat = 3 }}] + +[[block]] +id = "B3" +rule = "dh" +sessions = [ + {{ variant = "C1", plan = "s1,s5" }}, + {{ variant = "C1", plan = "s1,s5", dh = "off", tag = "dh-off" }}, + {{ variant = "C1", plan = "s2", jvm = "J2", heap = 6144, world = "s4", tag = "x" }}, +] + +[[block]] +id = "B6" +rule = "mf" +interleave = true +sessions = [ + {{ variant = "C1", plan = "s1", repeat = 2 }}, + {{ variant = "mf", plan = "s1", repeat = 2 }}, +] + +[[block]] +id = "B8" +rule = "entities" +sessions = [{{ variant = "C1-fa", plan = "s3" }}] + +[[block]] +id = "B12b" +rule = "trials" +sessions = [{{ variant = "C1", plan = "s1", add_mods = ["gnetum"], disable_mods = ["a-*.jar"], tag = "g" }}] + +[[block]] +id = "B4h" +rule = "heap" +params = ["gc"] +sessions = [{{ variant = "C1", plan = "s5", jvm = "${{gc}}", heap = 4096 }}] + +[[block]] +id = "B13" +rule = "profile" +sessions = [{{ variant = "C1", plan = "s1", manual = true }}] + +[[block]] +id = "B5" +rule = "sable" +deferred = true +sessions = [{{ variant = "C1", plan = "s4" }}] + +[[block]] +id = "B14" +rule = "spawn" +manual = true +sessions = [] +""" + + +class Fixture(unittest.TestCase): + def setUp(self): + self.tmp = Path(tempfile.mkdtemp()) + self.out = self.tmp / "out" + for v in ("C0", "C1", "mf"): + (self.out / v).mkdir(parents=True) + self.mods = self.tmp / "mods" + self.mods.mkdir() + (self.mods / "a-1.jar").write_bytes(b"x") + (self.tmp / "worlds").mkdir() + (self.tmp / "worlds" / "v1.zip").write_bytes(b"z") + (self.tmp / "worlds" / "s4-4.zip").write_bytes(b"z") + (self.tmp / "trials").mkdir() + (self.tmp / "trials" / "gnetum-1.0.jar").write_bytes(b"j") + self.path = self.tmp / "matrix.toml" + self.path.write_text(MINI.format(tmp=self.tmp)) + self.m = matrix.load(self.path, self.out) + self.state_path = self.tmp / "state.json" + self.calls = [] + + def runner(self, codes=None): + codes = list(codes or []) + + def fake(argv): + self.calls.append(argv) + return codes.pop(0) if codes else 0 + return fake + + def run_blocks(self, ids, codes=None, **kw): + state = matrix.State(self.state_path) + with contextlib.redirect_stdout(io.StringIO()), \ + contextlib.redirect_stderr(io.StringIO()) as err: + rc = matrix.run_blocks(self.m, ids, state, self.mods, runner=self.runner(codes), + out_dir=self.out, **kw) + return rc, state, err.getvalue() + + +class Loading(Fixture): + def test_real_matrix_validates(self): + m = matrix.load() # the shipped matrix.toml against the real out/ dir + ids = [b["id"] for b in m["block"]] + for want in ["B0", "B1", "B2", "B3", "B4", "B5", "B6", "B7", "B8", "B9", "B10", + "B11", "B12a", "B12b", "B13", "B14"]: + self.assertIn(want, ids) + bm = matrix.block_map(m) + self.assertTrue(bm["B5"]["deferred"]) + self.assertTrue(bm["B14"]["manual"]) + b0 = matrix.expand(m, bm["B0"]) + self.assertEqual([(s["variant"], s["plan"]) for s in b0], [("C0", ["s1"])] * 3) + + def test_real_matrix_emits_only_bench_flags(self): + m = matrix.load() + self.assertEqual((m["defaults"]["display"], m["defaults"]["res"]), + ("gamescope-headless", "1920x1080")) + params = {"gc": "J1", "scene": "s2"} + n = 0 + for b in m["block"]: + for s in matrix.expand(m, b, params): + argv, _ = matrix.bench_argv(m, s) + self.assertEqual(argv[0], "session") + flags = {a for a in argv[1:] if a.startswith("--")} + self.assertLessEqual(flags, matrix.BENCH_SESSION_FLAGS, (b["id"], argv)) + n += 1 + self.assertGreater(n, 50) + + def test_fa_and_leaves_variants_generated(self): + m = matrix.load() + missing = matrix.missing_variant_ids(m) + for v in ("C1-fa", "C1-fa-ext", "emf-lod20-fa", "emf-lod8-fa", "C1-bl"): + self.assertNotIn(v, missing) + self.assertTrue((matrix.OUT_DIR / v / "options.txt").is_file(), v) + + def test_validation_errors(self): + d = tomllib.loads(MINI.format(tmp=self.tmp)) + d["block"][0]["sessions"][0]["plan"] = "s5,s1" + d["block"][0]["sessions"][0]["typo"] = 1 + d["block"][1]["sessions"].append({"variant": "nope", "plan": "s6"}) + d["block"][1]["sessions"].append({"variant": "C1", "plan": "s1", "dh": "off", + "tag": "dh-off"}) + d["block"][4]["sessions"][0]["add_mods"] = ["unknown"] + d["block"][5]["sessions"][0]["jvm"] = "${other}" + d["block"][0]["sessions"][0]["world"] = "nowhere" + d["defaults"]["display"] = "tv" + errs = "\n".join(matrix.validate(d, self.out)) + for frag in ["S1->S5 order", "unknown key 'typo'", "variant 'nope'", + "unknown scene 's6'", "distinct tag", "not in [trials]", + "${other} not declared", "world 'nowhere'", "display 'tv'"]: + self.assertIn(frag, errs) + + def test_missing_variant_is_valid_but_blocked(self): + s = matrix.expand(self.m, matrix.block_map(self.m)["B8"])[0] + self.assertTrue(matrix.variant_missing(s, self.out)) + + +class Commands(Fixture): + def test_argv(self): + b3 = matrix.expand(self.m, matrix.block_map(self.m)["B3"]) + argv, miss = matrix.bench_argv(self.m, b3[1]) + self.assertEqual(miss, []) + self.assertEqual(argv[:9], ["session", "--variant", "C1", "--plan", "s1,s5", + "--reps", "3", "--label", "B3.dh-off"]) + self.assertIn("--dh", argv) + self.assertEqual(argv[argv.index("--dh") + 1], "off") + self.assertEqual(argv[argv.index("--world-zip") + 1], str(self.tmp / "worlds/v1.zip")) + argv, _ = matrix.bench_argv(self.m, b3[2]) + for flag, val in [("--jvm", "J2"), ("--heap", "6144"), ("--display", "gamescope"), + ("--res", "1280x720"), ("--shader", "off"), + ("--world-zip", str(self.tmp / "worlds/s4-4.zip"))]: + self.assertEqual(argv[argv.index(flag) + 1], val) + + def test_trials_and_disable(self): + s = matrix.expand(self.m, matrix.block_map(self.m)["B12b"])[0] + argv, miss = matrix.bench_argv(self.m, s) + self.assertEqual(argv[argv.index("--add-mods") + 1], + str(self.tmp / "trials/gnetum-1.0.jar")) + self.assertEqual(argv[argv.index("--disable-mods") + 1], "a-*.jar") + (self.tmp / "trials" / "gnetum-1.0.jar").unlink() + _, miss = matrix.bench_argv(self.m, s) + self.assertTrue(any("gnetum" in x for x in miss)) + + def test_live_world_and_overrides(self): + s = matrix.expand(self.m, matrix.block_map(self.m)["B0"], + overrides={"display": "desktop", "res": None})[0] + argv, _ = matrix.bench_argv(self.m, s, live_world=True) + self.assertNotIn("--world-zip", argv) + self.assertEqual(argv[argv.index("--display") + 1], "desktop") + self.assertEqual(argv[argv.index("--res") + 1], "1280x720") + + def test_interleave_and_params(self): + bm = matrix.block_map(self.m) + self.assertEqual([s["variant"] for s in matrix.expand(self.m, bm["B6"])], + ["C1", "mf", "C1", "mf"]) + s = matrix.expand(self.m, bm["B4h"])[0] + self.assertEqual(matrix.unresolved(s), ["gc"]) + s = matrix.expand(self.m, bm["B4h"], params={"gc": "J2"})[0] + self.assertEqual(matrix.unresolved(s), []) + self.assertEqual(s["jvm"], "J2") + + def test_seconds(self): + b3 = matrix.expand(self.m, matrix.block_map(self.m)["B3"]) + self.assertEqual(matrix.session_seconds(b3[0], 60), 60 + 3 * 45 + 3 * 130) + + +class Resume(Fixture): + def test_runs_then_skips(self): + rc, state, _ = self.run_blocks(["B0"]) + self.assertEqual(rc, 0) + self.assertEqual(len(self.calls), 3) + self.assertEqual(state.data["b0"]["mods_hash"], matrix.mods_hash(self.mods)) + saved = json.loads(self.state_path.read_text()) + self.assertEqual({e["status"] for e in saved["sessions"].values()}, {"ok"}) + self.calls.clear() + rc, _, _ = self.run_blocks(["B0"]) + self.assertEqual((rc, self.calls), (0, [])) + + def test_mod_change_invalidates_and_warns(self): + self.run_blocks(["B0"]) + (self.mods / "new-mod.jar").write_bytes(b"y") + self.calls.clear() + rc, _, err = self.run_blocks(["B3"]) + self.assertEqual(rc, 0) + self.assertIn("B0 MUST BE RE-RUN", err) + self.calls.clear() + rc, _, _ = self.run_blocks(["B0"]) + self.assertEqual(len(self.calls), 3) # B0 sessions are stale under the new set + + def test_warns_without_b0(self): + _, _, err = self.run_blocks(["B3"]) + self.assertIn("B0 (noise floor) has not been completed", err) + + def test_spec_change_reruns(self): + self.run_blocks(["B0"]) + self.calls.clear() + self.run_blocks(["B0"], live_world=True) # different command -> not done + self.assertEqual(len(self.calls), 3) + + def test_retry_once_then_ok(self): + rc, state, _ = self.run_blocks(["B0"], codes=[1, 0]) + self.assertEqual(rc, 0) + self.assertEqual(len(self.calls), 4) + self.assertEqual(state.data["sessions"]["B0#0"]["attempts"], 2) + + def test_stop_after_two_failures(self): + rc, state, _ = self.run_blocks(["B0"], codes=[1, 1]) + self.assertEqual(rc, 1) + self.assertEqual(len(self.calls), 2) + self.assertEqual(state.data["sessions"]["B0#0"]["status"], "failed") + self.assertNotIn("B0#1", state.data["sessions"]) + self.calls.clear() + rc, _, _ = self.run_blocks(["B0"]) # resume retries the failed one + self.assertEqual((rc, len(self.calls)), (0, 3)) + + def test_dry_run_runs_nothing(self): + rc, _, _ = self.run_blocks(["B0", "B3"], dry_run=True) + self.assertEqual((rc, self.calls), (0, [])) + self.assertFalse(self.state_path.exists()) + + def test_blocked_aborts_or_skips(self): + rc, _, _ = self.run_blocks(["B8"]) + self.assertEqual((rc, self.calls), (2, [])) + rc, _, _ = self.run_blocks(["B4h"]) # unresolved ${gc} + self.assertEqual((rc, self.calls), (2, [])) + rc, _, _ = self.run_blocks(["B8", "B6"], skip_blocked=True) + self.assertEqual((rc, len(self.calls)), (0, 4)) + + def test_manual_deferred_skipped(self): + rc, _, _ = self.run_blocks(["B13", "B5", "B14"]) + self.assertEqual((rc, self.calls), (0, [])) + rc, _, _ = self.run_blocks(["B13", "B5"], include_manual=True, include_deferred=True) + self.assertEqual(len(self.calls), 2) + + def test_mods_restored_check(self): + def mutating(argv): + self.calls.append(argv) + (self.mods / f"leak{len(self.calls)}.jar").write_bytes(b"l") + return 0 + state = matrix.State(self.state_path) + with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + rc = matrix.run_blocks(self.m, ["B0"], state, self.mods, runner=mutating, + out_dir=self.out) + self.assertEqual((rc, len(self.calls)), (1, 1)) + + def test_unknown_block(self): + with self.assertRaises(matrix.MatrixError): + self.run_blocks(["B99"]) + + +class Cli(unittest.TestCase): + def test_list_and_plan(self): + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + self.assertEqual(matrix.main(["list"]), 0) + self.assertEqual(matrix.main(["plan", "B4h", "--param", "gc=J1", + "--display", "desktop"]), 0) + out = buf.getvalue() + self.assertIn("| B0 | noise floor | 3 |", out) + self.assertIn("--jvm J1 --heap 4096", out) + self.assertIn("--display desktop", out) + + +if __name__ == "__main__": + unittest.main() diff --git a/analysis/client-perf/proposed-configs/build.py b/analysis/client-perf/proposed-configs/build.py index 20d0532..f7e379e 100644 --- a/analysis/client-perf/proposed-configs/build.py +++ b/analysis/client-perf/proposed-configs/build.py @@ -190,8 +190,27 @@ BENCH = { } -def c1(extra): - return [DEFAULT, extra] +# Resource packs (names = zips in the bench instance's resourcepacks/). Every bench +# variant pins the list: apply_variant.py merges options.txt key by key, so a variant +# without the key would keep the previous session's packs. Later entries win. +FA = "file/FreshAnimations_v1.10.4.zip" +FA_EXT = "file/FA+All_Extensions-v1.8.1.zip" +BETTER_LEAVES = "file/Better-Leaves-9.6.zip" + + +def packs(*extra): + return {"options.txt": {"resourcePacks": json.dumps(["vanilla", "mod_resources", *extra], + separators=(",", ":"))}} + + +PACKS_BASE = packs() +# Keys the bench layers may add to a base that lacks them (the options.txt template +# has no resourcePacks line; tiers never set it, so shipped output is unaffected). +ALLOW_NEW = {"options.txt": {"resourcePacks"}} + + +def c1(*extra): + return [DEFAULT, PACKS_BASE, *extra] VARIANTS = { @@ -201,10 +220,10 @@ VARIANTS = { "T-mid": ([DEFAULT], False), "T-high": ([DEFAULT, HIGH], False), # bench: composites - "C0": ([], True), - "C1": ([DEFAULT], True), - "C1-deck": ([DEFAULT, DECK], True), - "C1-high": ([DEFAULT, HIGH], True), + "C0": ([PACKS_BASE], True), + "C1": (c1(), True), + "C1-deck": (c1(DECK), True), + "C1-high": (c1(HIGH), True), # bench: Distant Horizons "dh-r256": (c1({"config/DistantHorizons.toml": {(DH_Q, "lodChunkRenderDistanceRadius"): 256}}), True), "dh-r192": (c1({"config/DistantHorizons.toml": {(DH_Q, "lodChunkRenderDistanceRadius"): 192}}), True), @@ -226,6 +245,15 @@ VARIANTS = { "modelUpdateFrequency": "Average"}}), True), "emf-lod8": (c1({"config/entity_model_features.json": {"animationLODDistance": 8, "retainDetailOnLowFps": False}}), True), + "C1-fa": (c1(packs(FA)), True), + "C1-fa-ext": (c1(packs(FA, FA_EXT)), True), + "emf-lod20-fa": (c1({"config/entity_model_features.json": {"animationLODDistance": 20, + "modelUpdateFrequency": "Average"}}, + packs(FA)), True), + "emf-lod8-fa": (c1({"config/entity_model_features.json": {"animationLODDistance": 8, + "retainDetailOnLowFps": False}}, + packs(FA)), True), + "C1-bl": (c1(packs(BETTER_LEAVES)), True), "ldl-fancy": (c1({"config/lambdynlights.toml": {("", "mode"): "fancy"}}), True), "ldl-fastest": (c1({"config/lambdynlights.toml": {("", "mode"): "fastest", ("light_sources", "entities"): False}}), True), @@ -290,7 +318,8 @@ def apply_kv(text, ov, name, sep, append): sval = toml_val(val) if isinstance(val, bool) else str(val) rx = re.compile(rf"^{re.escape(key)}{re.escape(sep)}") hits = [i for i, ln in enumerate(lines) if rx.match(ln)] - if len(hits) > 1 or (not hits and not append): + may_add = append if isinstance(append, bool) else key in append + if len(hits) > 1 or (not hits and not may_add): raise SystemExit(f"{name}: {key}: {len(hits)} matches in base") if hits: lines[hits[0]] = f"{key}{sep}{sval}" @@ -327,7 +356,7 @@ def render(layers, name): if fmt == "json": return apply_json(text, merged, name), merged sep = ":" if fmt == "options" else "=" - return apply_kv(text, merged, name, sep, append), merged + return apply_kv(text, merged, name, sep, append or ALLOW_NEW.get(name, set())), merged def main():