feat(bench): automate B7 shader profiles, wire B12b trials, settle 45s

- gen_shader_profiles.py + shader-profiles/: Iris per-pack option files
  resolved from each pack's profile.* lines (Iris has no profile key)
- bench.py: flag rows shader_not_loaded when Iris falls back to vanilla
  rendering (analyze.py treats it as invalid)
- matrix: shader_options session key, trial jar globs pinned to the
  downloaded Modrinth builds, post-join settle 45 s (B0 v1 showed rep 1
  still warming at 10 s)
This commit is contained in:
logan committed 2026-10-16 23:34:52 -04:00
1 parent a15c912ad0
commit 4fa9a6ab81
13 files changed
+552 -31

No files matched your search

+28 -8
View File
@@ -38,13 +38,13 @@ DISPLAYS = {"desktop", "gamescope", "gamescope-headless"}
PARAM_RE = re.compile(r"\$\{(\w+)\}")
TOP_KEYS = {"defaults", "worlds", "trials", "missing_variants", "block"}
DEFAULT_KEYS = {"reps", "warmup", "capture", "display", "res", "world", "shader", "dh",
DEFAULT_KEYS = {"reps", "warmup", "capture", "settle", "display", "res", "world", "shader", "dh",
"launch_overhead_s"}
BLOCK_KEYS = {"id", "title", "rule", "proxy_note", "manual_checks", "sessions", "params",
"deferred", "manual", "interleave"}
SESSION_KEYS = {"variant", "plan", "reps", "repeat", "tag", "jvm", "heap", "shader",
"shader_profile", "dh", "disable_mods", "add_mods", "world", "warmup",
"capture", "manual", "note", "display", "res"}
"shader_profile", "shader_options", "dh", "disable_mods", "add_mods", "world", "warmup",
"capture", "settle", "manual", "note", "display", "res"}
MISSING_KEYS = {"id", "blocks", "contents"}
# Flags `bench.py session` accepts; bench_argv must emit only these.
BENCH_SESSION_FLAGS = {
@@ -173,7 +173,7 @@ def validate(m, out_dir=OUT_DIR):
def _check_common(s, where, worlds, trials, err):
for k in ("reps", "warmup", "capture"):
for k in ("reps", "warmup", "capture", "settle"):
if k in s and not _pos_int(s[k]):
err.append(f"{where}: {k} must be a positive int")
if "display" in s and s["display"] not in DISPLAYS:
@@ -184,6 +184,8 @@ def _check_common(s, where, worlds, trials, err):
err.append(f"{where}: dh must be on/off")
if "world" in s and s["world"] not in worlds:
err.append(f"{where}: world {s['world']!r} not in [worlds]")
if "shader_options" in s and s.get("shader", "off") == "off":
err.append(f"{where}: shader_options needs shader = <pack>")
def block_map(m):
@@ -247,12 +249,18 @@ def resolve_path(p):
return hits[-1] if hits else None
def shader_options_path(p):
"""shader_options are harness-relative unless absolute / ~."""
p = Path(os.path.expanduser(p))
return p if p.is_absolute() else HERE / p
def bench_argv(m, sess, live_world=False):
"""`bench.py session` argv for one session, plus a list of missing inputs."""
missing = []
a = ["session", "--variant", sess["variant"], "--plan", ",".join(sess["plan"]),
"--reps", str(sess.get("reps", 3)), "--label", sess["label"]]
for k in ("warmup", "capture"):
for k in ("warmup", "capture", "settle"):
if k in sess:
a += [f"--{k}", str(sess[k])]
for k in ("display", "res", "shader", "dh", "jvm"):
@@ -260,6 +268,11 @@ def bench_argv(m, sess, live_world=False):
a += [f"--{k}", str(sess[k])]
if sess.get("heap"):
a += ["--heap", str(sess["heap"])]
if sess.get("shader_options"):
so = shader_options_path(sess["shader_options"])
if not so.is_file():
missing.append(f"shader options {so} (run gen_shader_profiles.py)")
a += ["--shader-options", str(so)]
if sess.get("disable_mods"):
a += ["--disable-mods", ",".join(sess["disable_mods"])]
if sess.get("add_mods"):
@@ -289,11 +302,16 @@ def variant_missing(sess, out_dir=OUT_DIR):
def session_seconds(sess, overhead):
reps, w, c = sess.get("reps", 3), sess.get("warmup", 15), sess.get("capture", 30)
scenes = sum(S5_SECONDS if p == "s5" else w + c for p in sess["plan"]) * reps
return overhead + scenes
return overhead + sess.get("settle", 10) + scenes
def spec_hash(argv, sess):
blob = json.dumps({"argv": argv, "manual": bool(sess.get("manual"))}, sort_keys=True)
spec = {"argv": argv, "manual": bool(sess.get("manual"))}
if sess.get("shader_options"): # a regenerated profile file means a new spec
so = shader_options_path(sess["shader_options"])
spec["shader_options_sha256"] = (hashlib.sha256(so.read_bytes()).hexdigest()
if so.is_file() else None)
blob = json.dumps(spec, sort_keys=True)
return hashlib.sha256(blob.encode()).hexdigest()[:16]
@@ -501,8 +519,10 @@ def cmd_plan(m, a):
argv, miss = bench_argv(m, s, a.live_world)
pre = "# MANUAL: " if s.get("manual") else ""
print(f"{pre}{fmt_cmd(argv)}")
if s.get("shader_profile"):
if s.get("shader_profile") and not s.get("shader_options"):
print(f"# set Iris profile {s['shader_profile']} for {s['shader']} first")
elif s.get("shader_profile"):
print(f"# Iris profile {s['shader_profile']} via shader options file")
if s.get("note"):
print(f"# note: {s['note']}")
if unresolved(s):