feat(bench): add matrix runner, gamescope and per-session controls
- bench.py: gamescope-headless display (default, 1920x1080; ~2% cost vs desktop for isolation from the shared Wayland desktop), --shader, --dh, --disable-mods/--add-mods, --world-zip, crash-safe restore journal, spectator mode in the plan (elevated cameras fell to the sea floor and S4 measured underwater darkness) - matrix.toml/matrix.py: declarative, resumable B0-B14 runner keyed on mods hash (warns when B0 must be re-run) - build.py: bench-only resource-pack variants (C1-fa, C1-fa-ext, emf-lod*-fa, C1-bl); every bench variant pins resourcePacks so packs don't leak between sessions; shipped tier output unchanged - analyze.py: tagged labels use their own block's baseline
This commit is contained in:
1 parent
8f56e2f780
commit
a15c912ad0
10 files changed
+2196
-58
No files matched your search
@@ -4,6 +4,7 @@
|
||||
bench.py boot --n 3 [--variant C1] [--join] # cold-launch reliability + startup time
|
||||
bench.py session --variant C1 --plan s1,s2,s3 --reps 3 [--label B2] [--dry-run]
|
||||
bench.py disarm # remove a leftover autoplan datapack
|
||||
bench.py restore # undo a crashed session's changes
|
||||
bench.py meta # print the environment snapshot
|
||||
|
||||
`session` applies the variant, pins options.txt, writes a per-session datapack whose
|
||||
@@ -15,6 +16,7 @@ done, slices the MangoHud CSV per window and appends one JSON row per window to
|
||||
"""
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import fnmatch
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
@@ -24,6 +26,7 @@ import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
@@ -51,6 +54,18 @@ PINS = [
|
||||
("enableVsync", "false"),
|
||||
]
|
||||
|
||||
# Reset before every variant apply so resource-pack state never leaks between sessions.
|
||||
# options.txt had neither key on 2026-10-16, so this is the NeoForge default.
|
||||
RESOURCE_PACK_BASELINE = [
|
||||
("resourcePacks", '["vanilla","mod_resources"]'),
|
||||
("incompatibleResourcePacks", "[]"),
|
||||
]
|
||||
|
||||
PENDING = ".harness-pending.json" # crash-safe restore journal, in GAME
|
||||
BACKUP_DIR = ".harness-backup" # file backups referenced by the journal, in GAME
|
||||
DGPU_DEVICE = "0x747e" # RX 7800 XT
|
||||
DH_RENDER_SECTION, DH_RENDER_KEY = "client.advanced.debugging", "rendererMode"
|
||||
|
||||
# BENCH-MATRIX §3: one session runs its scenes in this order (S1 first).
|
||||
SCENE_ORDER = ["s1", "s2", "s2s", "s3", "s4", "s5"]
|
||||
SCENE_FN = {"s1": "bench:s1", "s2": "bench:s2", "s2s": "bench:s2", "s3": "bench:s3",
|
||||
@@ -335,6 +350,10 @@ def generate_datapack(sid, plan, reps, warmup_s, capture_s, settle_s):
|
||||
f"execute unless entity @a run schedule function {ns}/poll 20t\n",
|
||||
f"data/bench/function/autoplan/{sid}/begin.mcfunction":
|
||||
f"scoreboard players set {holder} bench_plan 1\n"
|
||||
# spectator: elevated scene cameras must not fall (S4 measured the sea floor)
|
||||
"gamemode spectator @a\n"
|
||||
"time set 6000\n"
|
||||
"weather clear\n"
|
||||
"function bench:storm_off\n"
|
||||
"function bench:tour_stop\n"
|
||||
f"say BENCH_PLAN {sid} begin\n"
|
||||
@@ -567,6 +586,8 @@ def snapshot(env, fast=False):
|
||||
lvl = GAME / "saves" / WORLD / "level.dat"
|
||||
return {
|
||||
"machine": MACHINE,
|
||||
"display": DISPLAY_MODE["mode"],
|
||||
"res": "%dx%d" % DISPLAY_MODE["res"],
|
||||
"git_sha": sha, "git_dirty": dirty,
|
||||
"harness_sha": sha256_file(Path(__file__))[:16],
|
||||
"mods_hash": mh, "mods_count": mc,
|
||||
@@ -619,10 +640,29 @@ def crash_files():
|
||||
return names
|
||||
|
||||
|
||||
DISPLAY_MODE = {"mode": "gamescope-headless", "res": (1920, 1080)}
|
||||
|
||||
|
||||
def display_prefix(mode, res):
|
||||
"""gamescope wrapper argv for a display mode (empty for the desktop)."""
|
||||
if mode == "desktop":
|
||||
return []
|
||||
w, h = res
|
||||
cmd = ["gamescope", "-W", str(w), "-H", str(h), "-w", str(w), "-h", str(h),
|
||||
"--force-windows-fullscreen", "--immediate-flips", "-r", "1000", "-o", "1000"]
|
||||
if mode == "gamescope-headless":
|
||||
cmd += ["--backend", "headless"]
|
||||
return cmd + ["--"]
|
||||
|
||||
|
||||
def launch(env, world):
|
||||
if (GAME / "logs" / "latest.log").exists():
|
||||
(GAME / "logs" / "latest.log").unlink()
|
||||
cmd = ["setsid", "prismlauncher", "-d", str(PRISM), "--launch", INSTANCE]
|
||||
env = dict(env)
|
||||
if DISPLAY_MODE["mode"] != "desktop":
|
||||
env.setdefault("WAYLAND_DISPLAY", "wayland-0")
|
||||
cmd = ["setsid", *display_prefix(DISPLAY_MODE["mode"], DISPLAY_MODE["res"]),
|
||||
"prismlauncher", "-d", str(PRISM), "--launch", INSTANCE]
|
||||
if world:
|
||||
cmd += ["--world", world]
|
||||
return subprocess.Popen(cmd, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
@@ -759,6 +799,283 @@ def write_datapack(files):
|
||||
p.write_text(text)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- journaled session mutations
|
||||
|
||||
class Journal:
|
||||
"""Write-ahead restore journal (GAME/.harness-pending.json). Every entry is written
|
||||
before the mutation it undoes, so restore_pending() works after a hard kill."""
|
||||
|
||||
def __init__(self, game, sid=None):
|
||||
self.game, self.sid, self.entries = Path(game), sid, []
|
||||
self.path, self.bdir = self.game / PENDING, self.game / BACKUP_DIR
|
||||
|
||||
@classmethod
|
||||
def begin(cls, game, sid):
|
||||
j = cls(game, sid)
|
||||
if j.path.exists():
|
||||
raise RuntimeError(f"{j.path} exists: run `bench.py restore` first")
|
||||
j._write()
|
||||
return j
|
||||
|
||||
def _write(self):
|
||||
tmp = self.path.with_name(self.path.name + ".tmp")
|
||||
tmp.write_text(json.dumps({"session": self.sid, "entries": self.entries}, indent=1))
|
||||
os.replace(tmp, self.path)
|
||||
|
||||
def record(self, entry):
|
||||
self.entries.append(entry)
|
||||
self._write()
|
||||
|
||||
def backup_file(self, rel):
|
||||
"""Journal the current content (or absence) of GAME/rel; first backup wins."""
|
||||
rel = Path(rel).as_posix()
|
||||
if any(e["op"] == "file" and e["path"] == rel for e in self.entries):
|
||||
return
|
||||
src, backup = self.game / rel, None
|
||||
if src.is_file():
|
||||
self.bdir.mkdir(exist_ok=True)
|
||||
backup = f"{len(self.entries)}-{src.name}"
|
||||
shutil.copy2(src, self.bdir / backup)
|
||||
self.record({"op": "file", "path": rel, "backup": backup})
|
||||
|
||||
|
||||
def pending_exists(game=None):
|
||||
return (Path(game or GAME) / PENDING).exists()
|
||||
|
||||
|
||||
def _undo(game, e, bdir):
|
||||
op = e["op"]
|
||||
if op == "file":
|
||||
dst = game / e["path"]
|
||||
if e["backup"] is None:
|
||||
if dst.exists():
|
||||
dst.unlink()
|
||||
else:
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(bdir / e["backup"], dst)
|
||||
elif op == "rename":
|
||||
src, dst = game / e["src"], game / e["dst"]
|
||||
if dst.exists() and not src.exists():
|
||||
os.rename(dst, src)
|
||||
elif op in ("added", "tree"):
|
||||
p = game / e["path"]
|
||||
if p.is_dir():
|
||||
shutil.rmtree(p)
|
||||
elif p.exists():
|
||||
p.unlink()
|
||||
elif op == "world":
|
||||
path, bak, ext = game / e["path"], game / e["backup"], game / e["extract"]
|
||||
if ext.exists():
|
||||
shutil.rmtree(ext)
|
||||
if bak.exists():
|
||||
if path.exists():
|
||||
shutil.rmtree(path)
|
||||
os.rename(bak, path)
|
||||
elif not e["had_world"] and path.exists():
|
||||
shutil.rmtree(path)
|
||||
else:
|
||||
raise ValueError(f"unknown journal op {op!r}")
|
||||
|
||||
|
||||
def restore_pending(game=None):
|
||||
"""Undo every journaled mutation (newest first). Returns [(entry, error)] for the
|
||||
entries that failed; those stay in the journal, otherwise journal + backups go."""
|
||||
game = Path(game or GAME)
|
||||
path, bdir = game / PENDING, game / BACKUP_DIR
|
||||
if not path.exists():
|
||||
return []
|
||||
data = json.loads(path.read_text())
|
||||
failed = []
|
||||
for e in reversed(data.get("entries", [])):
|
||||
try:
|
||||
_undo(game, e, bdir)
|
||||
except (OSError, ValueError, KeyError) as exc:
|
||||
failed.append((e, f"{type(exc).__name__}: {exc}"))
|
||||
if failed:
|
||||
data["entries"] = [e for e, _ in reversed(failed)]
|
||||
path.write_text(json.dumps(data, indent=1))
|
||||
else:
|
||||
path.unlink()
|
||||
if bdir.exists():
|
||||
shutil.rmtree(bdir)
|
||||
return failed
|
||||
|
||||
|
||||
def prop_escape(v):
|
||||
"""java.util.Properties value escaping (what Iris' store() writes)."""
|
||||
out = "".join("\\" + c if c in "\\=:#!" else c for c in v)
|
||||
return "\\" + out if out.startswith(" ") else out
|
||||
|
||||
|
||||
def set_properties(text, want):
|
||||
"""Set key=value pairs in .properties text, keeping comments and other keys."""
|
||||
out, seen = [], set()
|
||||
for line in text.splitlines():
|
||||
s = line.lstrip()
|
||||
if s and s[0] not in "#!":
|
||||
k = re.split(r"\s*[=:]\s*|\s+", s, maxsplit=1)[0]
|
||||
if k in want:
|
||||
line = f"{k}={prop_escape(want[k])}"
|
||||
seen.add(k)
|
||||
out.append(line)
|
||||
out += [f"{k}={prop_escape(v)}" for k, v in want.items() if k not in seen]
|
||||
return "\n".join(out) + "\n"
|
||||
|
||||
|
||||
def apply_shader(game, journal, shader, options_file=None):
|
||||
"""Iris on with shaderpacks/<shader>, or off. Iris' per-pack options file
|
||||
shaderpacks/<shader>.txt is journaled too, so edits made during the run revert."""
|
||||
game = Path(game)
|
||||
cfg = game / "config" / "iris.properties"
|
||||
journal.backup_file("config/iris.properties")
|
||||
on = shader != "off"
|
||||
text = cfg.read_text() if cfg.exists() else ""
|
||||
cfg.parent.mkdir(parents=True, exist_ok=True)
|
||||
cfg.write_text(set_properties(text, {"enableShaders": "true" if on else "false",
|
||||
"shaderPack": shader if on else ""}))
|
||||
if on:
|
||||
rel = f"shaderpacks/{shader}.txt"
|
||||
journal.backup_file(rel)
|
||||
if options_file:
|
||||
shutil.copyfile(options_file, game / rel)
|
||||
|
||||
|
||||
def set_dh_rendering(text, on):
|
||||
"""Set [client.advanced.debugging] rendererMode (what DH's GUI-only
|
||||
quickEnableRendering toggles): DEFAULT or DISABLED."""
|
||||
out, section, hit = [], None, False
|
||||
for line in text.splitlines():
|
||||
m = re.match(r"^\s*\[([^\]]+)\]\s*$", line)
|
||||
if m:
|
||||
section = m.group(1).strip()
|
||||
elif section == DH_RENDER_SECTION:
|
||||
m = re.match(rf"^(\s*){DH_RENDER_KEY}\s*=", line)
|
||||
if m:
|
||||
line = f'{m.group(1)}{DH_RENDER_KEY} = "{"DEFAULT" if on else "DISABLED"}"'
|
||||
hit = True
|
||||
out.append(line)
|
||||
if not hit:
|
||||
raise ValueError(f"no {DH_RENDER_KEY} under [{DH_RENDER_SECTION}] in DistantHorizons.toml")
|
||||
return "\n".join(out) + "\n"
|
||||
|
||||
|
||||
def apply_dh(game, journal, on):
|
||||
p = Path(game) / "config" / "DistantHorizons.toml"
|
||||
new = set_dh_rendering(p.read_text(), on) # raises before journaling if the key is gone
|
||||
journal.backup_file("config/DistantHorizons.toml")
|
||||
p.write_text(new)
|
||||
|
||||
|
||||
def split_list(s):
|
||||
return [x.strip() for x in (s or "").split(",") if x.strip()]
|
||||
|
||||
|
||||
def resolve_disable(game, globs):
|
||||
"""Jar names in GAME/mods matching any glob (case-insensitive). Every glob must hit."""
|
||||
jars = sorted(p.name for p in (Path(game) / "mods").glob("*.jar"))
|
||||
hits, empty = [], []
|
||||
for g in globs:
|
||||
m = [j for j in jars if fnmatch.fnmatchcase(j.lower(), g.lower())]
|
||||
if not m:
|
||||
empty.append(g)
|
||||
hits += [j for j in m if j not in hits]
|
||||
if empty:
|
||||
raise ValueError(f"--disable-mods glob(s) matched no jar in mods/: {empty}")
|
||||
return hits
|
||||
|
||||
|
||||
def check_add_mods(game, paths):
|
||||
out = []
|
||||
for s in paths:
|
||||
p = Path(s).expanduser()
|
||||
if not p.is_file() or p.suffix != ".jar":
|
||||
raise ValueError(f"--add-mods: {p} is not a .jar file")
|
||||
if (Path(game) / "mods" / p.name).exists():
|
||||
raise ValueError(f"--add-mods: mods/{p.name} already exists")
|
||||
out.append(p)
|
||||
return out
|
||||
|
||||
|
||||
def disable_mods(game, journal, names):
|
||||
for n in names:
|
||||
src, dst = f"mods/{n}", f"mods/{n}.disabled"
|
||||
if (Path(game) / dst).exists():
|
||||
raise ValueError(f"{dst} already exists")
|
||||
journal.record({"op": "rename", "src": src, "dst": dst})
|
||||
os.rename(Path(game) / src, Path(game) / dst)
|
||||
|
||||
|
||||
def add_mods(game, journal, paths):
|
||||
for p in paths:
|
||||
journal.record({"op": "added", "path": f"mods/{p.name}"})
|
||||
shutil.copy2(p, Path(game) / "mods" / p.name)
|
||||
|
||||
|
||||
def zip_world_root(zip_path):
|
||||
"""Member prefix of the world inside the zip ('' or 'top/'); rejects unsafe paths."""
|
||||
with zipfile.ZipFile(zip_path) as z:
|
||||
names = z.namelist()
|
||||
for n in names:
|
||||
if n.startswith(("/", "\\")) or ".." in Path(n).parts:
|
||||
raise ValueError(f"unsafe member path in {zip_path}: {n}")
|
||||
if "level.dat" in names:
|
||||
return ""
|
||||
tops = {n.split("/", 1)[0] for n in names if n.strip("/")}
|
||||
if len(tops) == 1:
|
||||
top = tops.pop()
|
||||
if f"{top}/level.dat" in names:
|
||||
return top + "/"
|
||||
raise ValueError(f"{zip_path}: no level.dat at the root or under a single top directory")
|
||||
|
||||
|
||||
def swap_world(game, journal, zip_path, world=WORLD):
|
||||
"""Replace saves/<world> with the zip's world (original kept as
|
||||
<world>.harness-bak until restore). Returns the zip's sha256."""
|
||||
prefix = zip_world_root(zip_path)
|
||||
saves = Path(game) / "saves"
|
||||
path, bak, ext = saves / world, saves / f"{world}.harness-bak", saves / f".{world}.harness-extract"
|
||||
for p in (bak, ext):
|
||||
if p.exists():
|
||||
raise ValueError(f"{p} exists (unrestored earlier swap?)")
|
||||
digest = sha256_file(zip_path)
|
||||
journal.record({"op": "world", "path": f"saves/{world}", "backup": f"saves/{bak.name}",
|
||||
"extract": f"saves/{ext.name}", "had_world": path.exists()})
|
||||
if path.exists():
|
||||
os.rename(path, bak)
|
||||
with zipfile.ZipFile(zip_path) as z:
|
||||
z.extractall(ext)
|
||||
os.rename(ext / prefix if prefix else ext, path)
|
||||
if ext.exists():
|
||||
shutil.rmtree(ext)
|
||||
return digest
|
||||
|
||||
|
||||
def reset_resource_packs(game):
|
||||
opts = Path(game) / "options.txt"
|
||||
opts.write_text(merge_pins(opts.read_text() if opts.exists() else "",
|
||||
RESOURCE_PACK_BASELINE))
|
||||
|
||||
|
||||
def options_values(game, keys):
|
||||
opts = Path(game) / "options.txt"
|
||||
vals = {}
|
||||
for line in (opts.read_text().splitlines() if opts.exists() else []):
|
||||
k, _, v = line.partition(":")
|
||||
if k in keys:
|
||||
vals[k] = v
|
||||
return vals
|
||||
|
||||
|
||||
def env_flags(gpus, server_running):
|
||||
"""Warn flags for uncontrolled noise sources."""
|
||||
flags = []
|
||||
if any(g.get("device") == DGPU_DEVICE and g.get("dpm_level") != "high" for g in gpus):
|
||||
flags.append("gpu_dpm_auto")
|
||||
if server_running:
|
||||
flags.append("server_running")
|
||||
return flags
|
||||
|
||||
|
||||
def effective_jvm_flags(pids):
|
||||
for pid in pids:
|
||||
args = [a.decode(errors="replace") for a in proc_cmdline(pid) if a]
|
||||
@@ -780,7 +1097,14 @@ def pick_csvs(t_launch):
|
||||
|
||||
# ---------------------------------------------------------------- subcommands
|
||||
|
||||
def refuse_pending():
|
||||
if pending_exists():
|
||||
sys.exit(f"{GAME / PENDING} exists: a previous session did not restore the "
|
||||
f"instance. Run `bench.py restore` first.")
|
||||
|
||||
|
||||
def cmd_boot(a):
|
||||
refuse_pending()
|
||||
env = bench_env()
|
||||
preflight(a.kill_stale)
|
||||
disarm()
|
||||
@@ -851,12 +1175,29 @@ def cmd_session(a):
|
||||
timeout = a.timeout or int(a.boot_timeout + a.settle + budget * 1.25 + 120)
|
||||
if not (PROPOSED / "out" / a.variant).is_dir():
|
||||
sys.exit(f"variant {a.variant} not in {PROPOSED / 'out'} - run build.py first")
|
||||
# validate every instance change before touching anything
|
||||
try:
|
||||
if a.shader != "off" and not (GAME / "shaderpacks" / a.shader).exists():
|
||||
raise ValueError(f"--shader {a.shader}: not in {GAME / 'shaderpacks'}")
|
||||
if a.shader_options and (a.shader == "off" or not Path(a.shader_options).is_file()):
|
||||
raise ValueError("--shader-options needs --shader NAME and an existing file")
|
||||
to_disable = resolve_disable(GAME, split_list(a.disable_mods))
|
||||
to_add = check_add_mods(GAME, split_list(a.add_mods))
|
||||
if a.world_zip:
|
||||
zip_world_root(a.world_zip)
|
||||
except (ValueError, OSError, zipfile.BadZipFile) as e:
|
||||
sys.exit(str(e))
|
||||
if a.dry_run:
|
||||
log(f"dry run: session {sid}, variant {a.variant}, plan {plan} x{a.reps}, "
|
||||
f"est. scene time {budget}s, timeout {timeout}s")
|
||||
log(f"would pin options.txt: {', '.join(f'{k}:{v}' for k, v in PINS)}")
|
||||
if a.jvm or a.heap:
|
||||
log(f"would set instance.cfg: jvm={a.jvm} heap={a.heap}")
|
||||
log(f"would reset options.txt: {dict(RESOURCE_PACK_BASELINE)}")
|
||||
log(f"would set shader={a.shader} options={a.shader_options} dh={a.dh} "
|
||||
f"disable={to_disable} add={[p.name for p in to_add]} world_zip={a.world_zip}")
|
||||
if pending_exists():
|
||||
log(f"WARNING: {GAME / PENDING} exists; a real run would refuse to start")
|
||||
log(f"would write datapack {plan_dir()}:")
|
||||
for rel, text in files.items():
|
||||
print(f"--- {PLAN_PACK}/{rel}")
|
||||
@@ -864,22 +1205,53 @@ def cmd_session(a):
|
||||
return 0
|
||||
if budget + a.settle + 120 > 1700:
|
||||
log("WARNING: plan exceeds ~28 min; mangohud-local log_duration=1800 will truncate the CSV")
|
||||
refuse_pending()
|
||||
env = bench_env()
|
||||
preflight(a.kill_stale)
|
||||
apply_out = apply_variant(a.variant)
|
||||
apply_pins()
|
||||
if a.jvm or a.heap:
|
||||
apply_jvm(a.jvm, a.heap)
|
||||
cfg_hash, cfg_n = tree_hash(GAME / "config", config_skip)
|
||||
opt_hash = sha256_file(GAME / "options.txt")
|
||||
meta = snapshot(env)
|
||||
write_datapack(files)
|
||||
before = crash_files()
|
||||
t_launch = time.time()
|
||||
log(f"session {sid}: {a.variant} {plan} x{a.reps}, timeout {timeout}s")
|
||||
prism = launch(env, WORLD)
|
||||
tail = LogTail(GAME / "logs" / "latest.log")
|
||||
state = {"done": False, "last": None}
|
||||
# SIGTERM (e.g. a matrix runner timeout) unwinds through the restore below
|
||||
signal.signal(signal.SIGTERM, lambda *_: sys.exit(143))
|
||||
journal = Journal.begin(GAME, sid)
|
||||
prism = None
|
||||
state = {"done": False}
|
||||
try:
|
||||
reset_resource_packs(GAME)
|
||||
apply_out = apply_variant(a.variant)
|
||||
apply_pins()
|
||||
if a.jvm or a.heap:
|
||||
apply_jvm(a.jvm, a.heap)
|
||||
apply_shader(GAME, journal, a.shader, a.shader_options)
|
||||
apply_dh(GAME, journal, a.dh == "on")
|
||||
disable_mods(GAME, journal, to_disable)
|
||||
add_mods(GAME, journal, to_add)
|
||||
world_sha = swap_world(GAME, journal, a.world_zip) if a.world_zip else None
|
||||
cfg_hash, cfg_n = tree_hash(GAME / "config", config_skip)
|
||||
opt_hash = sha256_file(GAME / "options.txt")
|
||||
packs = options_values(GAME, dict(RESOURCE_PACK_BASELINE))
|
||||
meta = snapshot(env)
|
||||
journal.record({"op": "tree", "path": plan_dir().relative_to(GAME).as_posix()})
|
||||
write_datapack(files)
|
||||
before = crash_files()
|
||||
t_launch = time.time()
|
||||
log(f"session {sid}: {a.variant} {plan} x{a.reps}, timeout {timeout}s")
|
||||
prism = launch(env, WORLD)
|
||||
tail = LogTail(GAME / "logs" / "latest.log")
|
||||
outcome, detail = _run_session(a, tail, prism, before, timeout, state)
|
||||
finally:
|
||||
if prism is not None:
|
||||
kill_client(prism)
|
||||
failed = restore_pending(GAME)
|
||||
for e, err in failed:
|
||||
log(f"RESTORE FAILED {e}: {err}")
|
||||
if failed:
|
||||
log(f"journal kept at {GAME / PENDING}; fix and run `bench.py restore`")
|
||||
jvm_flags = state.get("jvm_flags")
|
||||
return _record_session(a, sid, plan, tail, t_launch, outcome, detail, jvm_flags,
|
||||
cfg_hash, cfg_n, opt_hash, apply_out, meta, packs, world_sha,
|
||||
to_disable, to_add, failed)
|
||||
|
||||
|
||||
def _run_session(a, tail, prism, before, timeout, state):
|
||||
"""Monitor the running client until BENCH_DONE / crash / exit / timeout."""
|
||||
|
||||
def on_line(line):
|
||||
m = MARK_RE.search(line)
|
||||
@@ -888,19 +1260,21 @@ def cmd_session(a):
|
||||
if DONE_RE.search(line):
|
||||
state["done"] = True
|
||||
|
||||
jvm_flags = None
|
||||
try:
|
||||
def until():
|
||||
nonlocal jvm_flags
|
||||
if jvm_flags is None:
|
||||
jvm_flags = effective_jvm_flags(client_pids())
|
||||
return state["done"]
|
||||
outcome, detail, _ = monitor(tail, prism, before, until, timeout, on_line)
|
||||
time.sleep(2) # let MangoHud flush the tail of the capture
|
||||
tail.poll()
|
||||
finally:
|
||||
kill_client(prism)
|
||||
disarm()
|
||||
state["jvm_flags"] = None
|
||||
|
||||
def until():
|
||||
if state["jvm_flags"] is None:
|
||||
state["jvm_flags"] = effective_jvm_flags(client_pids())
|
||||
return state["done"]
|
||||
outcome, detail, _ = monitor(tail, prism, before, until, timeout, on_line)
|
||||
time.sleep(2) # let MangoHud flush the tail of the capture
|
||||
tail.poll()
|
||||
return outcome, detail
|
||||
|
||||
|
||||
def _record_session(a, sid, plan, tail, t_launch, outcome, detail, jvm_flags, cfg_hash,
|
||||
cfg_n, opt_hash, apply_out, meta, packs, world_sha, to_disable, to_add,
|
||||
restore_failed):
|
||||
mem_after = meminfo_available_mb()
|
||||
lines = tail.lines
|
||||
ev = scan_events(lines)
|
||||
@@ -925,11 +1299,21 @@ def cmd_session(a):
|
||||
"jvm_set": a.jvm, "heap_mb": a.heap, "jvm_flags": jvm_flags,
|
||||
"config_hash": cfg_hash[:16], "config_files": cfg_n, "options_hash": opt_hash[:16],
|
||||
"apply_output": apply_out,
|
||||
"shader": a.shader, "shader_options": a.shader_options,
|
||||
"shader_options_sha256": sha256_file(a.shader_options) if a.shader_options else None,
|
||||
"dh": a.dh, "disabled_mods": to_disable, "added_mods": [p.name for p in to_add],
|
||||
"world_zip": a.world_zip, "world_zip_sha256": world_sha,
|
||||
"resource_packs": packs.get("resourcePacks"),
|
||||
"incompatible_resource_packs": packs.get("incompatibleResourcePacks"),
|
||||
"restore_errors": [err for _, err in restore_failed],
|
||||
"session_outcome": outcome, "session_detail": detail,
|
||||
"startup_s": ev["game_took_s"],
|
||||
"join_s": round(ev["joined_t"] - t_launch, 2) if ev["joined_t"] else None,
|
||||
"atlas": ev["atlas"], "log": log_path, **meta, "mem_available_after_mb": mem_after,
|
||||
}
|
||||
noise = env_flags(meta["gpus"], meta["server_running"])
|
||||
if restore_failed:
|
||||
noise.append("restore_failed")
|
||||
n_ok = 0
|
||||
for scene in plan:
|
||||
warmup = 0 if scene == "s5" else a.warmup
|
||||
@@ -940,7 +1324,7 @@ def cmd_session(a):
|
||||
t_start=ts, t_end=te)
|
||||
if ts is None or te is None:
|
||||
row.update(metrics=None, flags=[{"crash": "crashed", "timeout": "timeout"}
|
||||
.get(outcome, "no_markers")])
|
||||
.get(outcome, "no_markers"), *noise])
|
||||
append_jsonl(RESULTS / "session.jsonl", row)
|
||||
log(f"{scene} rep {rep}: no complete window ({outcome})")
|
||||
continue
|
||||
@@ -955,6 +1339,7 @@ def cmd_session(a):
|
||||
flags = validity_flags(m, te - ts, meta["refresh_rates"], backups, ts, te)
|
||||
if outcome != "ok":
|
||||
flags.append("session_" + outcome) # warn: window itself completed
|
||||
flags += noise
|
||||
row.update(metrics=m, flags=flags)
|
||||
if src:
|
||||
row.update(csv=src[2].name, align=src[1], csv_gpu=src[3].get("gpu", "").strip(),
|
||||
@@ -965,7 +1350,7 @@ def cmd_session(a):
|
||||
log(f"{scene} rep {rep}: n={m['n']} p50={m['p50_ms']} p99={m['p99_ms']} "
|
||||
f"1%low={m['one_low_fps']} avg={m['avg_fps']} {','.join(flags)}")
|
||||
log(f"session {sid} {outcome}: {n_ok} windows recorded -> {RESULTS / 'session.jsonl'}")
|
||||
return 0 if outcome == "ok" else 1
|
||||
return 0 if outcome == "ok" and not restore_failed else 1
|
||||
|
||||
|
||||
def cmd_disarm(a):
|
||||
@@ -973,12 +1358,27 @@ def cmd_disarm(a):
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_restore(a):
|
||||
if client_pids():
|
||||
sys.exit("bench client is running; close it before restoring")
|
||||
if not pending_exists():
|
||||
log("nothing pending" + ("; removed leftover plan pack" if disarm() else ""))
|
||||
return 0
|
||||
failed = restore_pending(GAME)
|
||||
for e, err in failed:
|
||||
log(f"FAILED {e}: {err}")
|
||||
disarm()
|
||||
log(f"restore: {len(failed)} entr{'y' if len(failed) == 1 else 'ies'} failed"
|
||||
if failed else "restored and cleared " + str(GAME / PENDING))
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
def cmd_meta(a):
|
||||
print(json.dumps(snapshot(bench_env()), indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
def build_parser():
|
||||
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0],
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
sub = ap.add_subparsers(dest="cmd", required=True)
|
||||
@@ -1009,14 +1409,35 @@ def main(argv=None):
|
||||
s.add_argument("--any-order", action="store_true")
|
||||
s.add_argument("--kill-stale", action="store_true")
|
||||
s.add_argument("--dry-run", action="store_true", help="print the generated datapack only")
|
||||
s.add_argument("--shader", default="off",
|
||||
help="shaderpack file/dir name in GAME/shaderpacks, or off (Iris disabled)")
|
||||
s.add_argument("--shader-options", help="file copied to shaderpacks/<NAME>.txt for the session")
|
||||
s.add_argument("--dh", choices=["on", "off"], default="on",
|
||||
help="DH rendering via [client.advanced.debugging] rendererMode")
|
||||
s.add_argument("--disable-mods", help="comma list of globs (case-insensitive) of mods/*.jar "
|
||||
"renamed to .jar.disabled for the session")
|
||||
s.add_argument("--add-mods", help="comma list of jar paths copied into mods/ for the session")
|
||||
s.add_argument("--world-zip", help="world zip swapped in for saves/onionbench for the session")
|
||||
for p in (b, s):
|
||||
p.add_argument("--display", choices=["desktop", "gamescope", "gamescope-headless"],
|
||||
default="gamescope-headless")
|
||||
p.add_argument("--res", default="1920x1080", help="WxH (gamescope modes)")
|
||||
s.set_defaults(fn=cmd_session)
|
||||
|
||||
d = sub.add_parser("disarm", help="remove a leftover autoplan datapack from the world")
|
||||
d.set_defaults(fn=cmd_disarm)
|
||||
r = sub.add_parser("restore", help="replay and clear a leftover .harness-pending.json")
|
||||
r.set_defaults(fn=cmd_restore)
|
||||
m = sub.add_parser("meta", help="print the environment/metadata snapshot")
|
||||
m.set_defaults(fn=cmd_meta)
|
||||
return ap
|
||||
|
||||
a = ap.parse_args(argv)
|
||||
|
||||
def main(argv=None):
|
||||
a = build_parser().parse_args(argv)
|
||||
if getattr(a, "display", None):
|
||||
w, h = (int(x) for x in a.res.lower().split("x"))
|
||||
DISPLAY_MODE.update(mode=a.display, res=(w, h))
|
||||
return a.fn(a)
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user