logan
a15c912ad0
- bench.py: gamescope-headless display (default, 1920x1080; ~2% cost vs desktop for isolation from the shared Wayland desktop), --shader, --dh, --disable-mods/--add-mods, --world-zip, crash-safe restore journal, spectator mode in the plan (elevated cameras fell to the sea floor and S4 measured underwater darkness) - matrix.toml/matrix.py: declarative, resumable B0-B14 runner keyed on mods hash (warns when B0 must be re-run) - build.py: bench-only resource-pack variants (C1-fa, C1-fa-ext, emf-lod*-fa, C1-bl); every bench variant pins resourcePacks so packs don't leak between sessions; shipped tier output unchanged - analyze.py: tagged labels use their own block's baseline
295 lines
14 KiB
TOML
295 lines
14 KiB
TOML
# Declarative encoding of BENCH-MATRIX.md §4 (blocks B0-B14) for the proxy bench machine
|
||
# (Ryzen 9 9900X / RX 7800 XT). Driven by matrix.py; every session becomes one
|
||
# `python3 bench.py session ...` launch. BENCH-MATRIX.md is authoritative - if this file
|
||
# disagrees with it, fix this file.
|
||
#
|
||
# Session keys:
|
||
# variant proposed-configs/out/<ID> (or an entry in [[missing_variants]] -> blocked)
|
||
# plan scenes in S1->S5 order: s1,s2,s2s,s3,s4,s5 (S6 is not automatable)
|
||
# reps windows per scene inside the session (phase0: 3 runs)
|
||
# repeat separate sessions (= cold launches) of the same config (B0: 3)
|
||
# tag distinguishes sessions that share a variant but differ by flags; the
|
||
# bench label becomes "<block>.<tag>" so analyze.py groups them apart
|
||
# jvm/heap BENCH-MATRIX JVM set / MaxMemAlloc MB
|
||
# shader shaderpack file name or "off"; shader_profile = Iris profile (manual today)
|
||
# dh "on" | "off" (dh-off row of B3)
|
||
# disable_mods globs matched in the instance mods/ (B12a)
|
||
# add_mods jar paths/globs added for the session only (B12b)
|
||
# world key of [worlds]; restored from the zip before the session
|
||
# warmup/capture per-window seconds (bench.py defaults 15 / 30; S5 is fixed 10 + 120)
|
||
# manual true = cannot run unattended; `run` skips it unless --include-manual
|
||
# note free text shown by `plan`
|
||
# ${name} in variant/jvm/plan is a block parameter, supplied with `--param name=value`.
|
||
|
||
[defaults]
|
||
reps = 3
|
||
warmup = 15
|
||
capture = 30
|
||
display = "gamescope-headless"
|
||
res = "1920x1080"
|
||
world = "v1"
|
||
shader = "off" # pin: a shader/DH state must never leak from one session to the next
|
||
dh = "on"
|
||
launch_overhead_s = 60 # launch->join (~28 s measured) + settle 10 s + teardown/restore
|
||
|
||
[worlds]
|
||
# BENCH-MATRIX §2. None of these exist yet: S4 (airships) is unbuilt. Until they do,
|
||
# `run --live-world` uses the instance's saves/onionbench as-is (no restore).
|
||
v1 = "~/onion-bench/worlds/bench-world-v1.zip"
|
||
s4-0 = "~/onion-bench/worlds/bench-world-v1-s4-0.zip"
|
||
s4-1 = "~/onion-bench/worlds/bench-world-v1-s4-1.zip"
|
||
s4-4 = "~/onion-bench/worlds/bench-world-v1.zip" # the main zip carries all 4 ships
|
||
|
||
[trials]
|
||
# B12b trial jars: download the NeoForge 1.21.1 build from Modrinth into this directory.
|
||
gnetum = "~/onion-bench/trials/gnetum-*.jar"
|
||
better_block_entities = "~/onion-bench/trials/betterblockentities-*.jar"
|
||
flerovium = "~/onion-bench/trials/flerovium-*.jar"
|
||
super_resolution = "~/onion-bench/trials/superresolution-*.jar"
|
||
ixeris = "~/onion-bench/trials/ixeris-*.jar"
|
||
iris_flywheel_compat = "~/onion-bench/trials/irisflw-*.jar"
|
||
cull_leaves = "~/onion-bench/trials/cull-leaves-*.jar"
|
||
|
||
# Variants a block needs that build.py does not generate. Sessions using them are
|
||
# reported as blocked until they exist in proposed-configs/out/. (C1-fa, C1-fa-ext,
|
||
# emf-lod20-fa, emf-lod8-fa and C1-bl are now generated by build.py; every bench
|
||
# variant pins options.txt resourcePacks.)
|
||
[[missing_variants]]
|
||
id = "dh-off"
|
||
blocks = []
|
||
contents = """Not needed as a directory: BENCH-MATRIX defines dh-off as C1 with DH rendering \
|
||
toggled off in the GUI. Encoded as variant C1 + `dh = "off"` (bench.py --dh off)."""
|
||
|
||
# ---------------------------------------------------------------------------------------
|
||
|
||
[[block]]
|
||
id = "B0"
|
||
title = "noise floor"
|
||
rule = "Defines each machine's threshold. P2 also checks `MemAvailable` drift (phase0 Step 4)."
|
||
proxy_note = "Single proxy machine. Re-run whenever the mod set changes (matrix.py warns)."
|
||
manual_checks = ["MemAvailable drift: compare mem_available_mb before/after across the 3 sessions (recorded per row)"]
|
||
sessions = [
|
||
{ variant = "C0", plan = "s1", repeat = 3 },
|
||
]
|
||
|
||
[[block]]
|
||
id = "B1"
|
||
title = "baseline"
|
||
rule = "Reference numbers for everything below"
|
||
manual_checks = [
|
||
"S6 (GUI/JEI overlay) needs a key press - run by hand",
|
||
"/sparkc heapsummary after S1 loads",
|
||
"VRAM peak",
|
||
]
|
||
sessions = [
|
||
{ variant = "C0", plan = "s1,s2,s3,s4,s5" },
|
||
]
|
||
|
||
[[block]]
|
||
id = "B2"
|
||
title = "composite"
|
||
rule = "If C1 isn't a win on ≥ 2 machines, stop and bisect with B3–B11 before shipping anything"
|
||
proxy_note = "Proxy runs all three composites; one machine = single-machine evidence, fleet confirmation pending. Baseline is C0 (pooled from B1)."
|
||
manual_checks = ["S6 by hand", "/sparkc heapsummary after S1 loads", "VRAM peak"]
|
||
sessions = [
|
||
{ variant = "C1", plan = "s1,s2,s3,s4,s5" },
|
||
{ variant = "C1-deck", plan = "s1,s2,s3,s4,s5" },
|
||
{ variant = "C1-high", plan = "s1,s2,s3,s4,s5" },
|
||
]
|
||
|
||
[[block]]
|
||
id = "B3"
|
||
title = "Distant Horizons"
|
||
rule = """**Radius per tier** = the largest radius meeting the FPS floor on S1 *and* S5: P0 → deck, P1 → default, P2 → high. \
|
||
| Adopt SSAO-off / LOW-quality for that tier on a win. \
|
||
| Keep 2 threads unless 4 is a win on S5 *and* not a loss on S1"""
|
||
proxy_note = "Absolute floors are fleet values; on the proxy use relative wins only. P0/P1-only rows (dh-ssao, dh-qlow) run here too."
|
||
manual_checks = ["DH memory in F3"]
|
||
sessions = [
|
||
{ variant = "dh-r256", plan = "s1,s5" },
|
||
{ variant = "dh-r192", plan = "s1,s5" },
|
||
{ variant = "C1", plan = "s1,s5" },
|
||
{ variant = "dh-r64", plan = "s1,s5" },
|
||
{ variant = "C1", plan = "s1,s5", dh = "off", tag = "dh-off" },
|
||
{ variant = "dh-ssao", plan = "s1" },
|
||
{ variant = "dh-qlow", plan = "s1" },
|
||
{ variant = "dh-thr4", plan = "s1,s5" },
|
||
]
|
||
|
||
[[block]]
|
||
id = "B4"
|
||
title = "heap & GC: GC"
|
||
rule = "GC = best S5 1 % low. Heap = the smallest size within the spread of the best. Feeds `PLAYER-TUNING.md` §2. P1 isn't run (32 GB, not a constraint)."
|
||
proxy_note = "P2 values (-Xmx6G) on the proxy; the heap ladder (B4h) adds 8 G as the top rung."
|
||
manual_checks = ["/sparkc gc after S5 (pause count / max / total)", "swap delta"]
|
||
sessions = [
|
||
{ variant = "C1", plan = "s2,s5", jvm = "J0", heap = 6144, tag = "J0" },
|
||
{ variant = "C1", plan = "s2,s5", jvm = "J1", heap = 6144, tag = "J1" },
|
||
{ variant = "C1", plan = "s2,s5", jvm = "J2", heap = 6144, tag = "J2" },
|
||
{ variant = "C1", plan = "s2,s5", jvm = "J3", heap = 6144, tag = "J3" },
|
||
]
|
||
|
||
[[block]]
|
||
id = "B4h"
|
||
title = "heap & GC: heap ladder for the GC winner"
|
||
rule = "GC = best S5 1 % low. Heap = the smallest size within the spread of the best. Feeds `PLAYER-TUNING.md` §2. P1 isn't run (32 GB, not a constraint)."
|
||
params = ["gc"]
|
||
proxy_note = "Run after B4 with --param gc=<winning J set>."
|
||
sessions = [
|
||
{ variant = "C1", plan = "s2,s5", jvm = "${gc}", heap = 4096, tag = "heap4" },
|
||
{ variant = "C1", plan = "s2,s5", jvm = "${gc}", heap = 5120, tag = "heap5" },
|
||
{ variant = "C1", plan = "s2,s5", jvm = "${gc}", heap = 6144, tag = "heap6" },
|
||
{ variant = "C1", plan = "s2,s5", jvm = "${gc}", heap = 8192, tag = "heap8" },
|
||
]
|
||
|
||
[[block]]
|
||
id = "B5"
|
||
title = "Sable / SSRD"
|
||
deferred = true
|
||
rule = """SSRD per tier = the largest value meeting the floor with 4 ships. sable-noshade ships only on a win *and* if it looks acceptable. \
|
||
| Decay > spread is a Sable #1011-class finding: report it, no config fix"""
|
||
proxy_note = "Fleet-only (P0/P1) per BENCH-MATRIX; deferred on the proxy. Needs the S4 world zips. The 45-min decay run exceeds mangohud-local log_duration=1800 and needs FPS at 0/15/30/45 min + heapsummary by hand."
|
||
manual_checks = ["sub-level count", "sable-noshade visual acceptability", "/sparkc heapsummary at 0/15/30/45 min"]
|
||
sessions = [
|
||
{ variant = "C1", plan = "s4", world = "s4-0", tag = "ships0" },
|
||
{ variant = "C1", plan = "s4", world = "s4-1", tag = "ships1" },
|
||
{ variant = "C1", plan = "s4", world = "s4-4", tag = "ships4" },
|
||
{ variant = "ssrd-32", plan = "s4", world = "s4-4" },
|
||
{ variant = "ssrd-64", plan = "s4", world = "s4-4" },
|
||
{ variant = "ssrd-128", plan = "s4", world = "s4-4" },
|
||
{ variant = "sable-noshade", plan = "s4", world = "s4-4" },
|
||
{ variant = "C1", plan = "s4", world = "s4-4", jvm = "J0", reps = 1, capture = 2700, repeat = 2, tag = "decay", manual = true, note = "45-min decay; CSV capped at 30 min by mangohud-local" },
|
||
]
|
||
|
||
[[block]]
|
||
id = "B6"
|
||
title = "ModernFix dynamic_resources"
|
||
rule = "Ship `dynamic_resources=true` only if: heap −300 MB or more *or* startup faster beyond spread, **and** zero missing textures, **and** no crash in 3 launches"
|
||
interleave = true
|
||
manual_checks = [
|
||
"S6 by hand",
|
||
"heap after load (/sparkc heapsummary)",
|
||
"Missing-texture check: JEI @chipped/@rechiseled/@handcrafted, 4x4 Rechiseled CTM wall",
|
||
]
|
||
sessions = [
|
||
{ variant = "C1", plan = "s1,s2", repeat = 3 },
|
||
{ variant = "mf-dynres-off", plan = "s1,s2", repeat = 3 },
|
||
]
|
||
|
||
[[block]]
|
||
id = "B7"
|
||
title = "shaders"
|
||
rule = "Data for the owner's shader policy, no auto-decision. Report the FPS cost per profile and whether ship corruption appears with shadow profiles"
|
||
proxy_note = "Iris profiles are set in Iris' GUI; bench.py --shader only selects the pack, so profile rows stay manual until a profile mechanism exists."
|
||
manual_checks = ["Flywheel backend line in F3 (expect off with shaders)", "visual check for Iris #3164 on ships"]
|
||
sessions = [
|
||
{ variant = "C1", plan = "s1,s2,s4", world = "s4-4", shader = "off", tag = "noshader" },
|
||
{ variant = "C1", plan = "s1,s2,s4", world = "s4-4", shader = "MakeUp-UltraFast-9.5f.zip", shader_profile = "shadowless_medium", tag = "makeup-shadowless", manual = true },
|
||
{ variant = "C1", plan = "s1,s2,s4", world = "s4-4", shader = "MakeUp-UltraFast-9.5f.zip", shader_profile = "medium", tag = "makeup-medium", manual = true },
|
||
{ variant = "C1", plan = "s1,s2,s4", world = "s4-4", shader = "ComplementaryReimagined_r5.9.3.zip", shader_profile = "LOW", tag = "compl-low", manual = true },
|
||
{ variant = "C1", plan = "s1,s2,s4", world = "s4-4", shader = "ComplementaryReimagined_r5.9.3.zip", shader_profile = "MEDIUM", tag = "compl-medium", manual = true },
|
||
]
|
||
|
||
[[block]]
|
||
id = "B8"
|
||
title = "entities"
|
||
rule = "EMF: adopt the smallest LOD that's a win with FA on. LDL: per tier, the cheapest mode that's a win. FA guidance → `PLAYER-TUNING.md` §4"
|
||
sessions = [
|
||
{ variant = "C1", plan = "s3" },
|
||
{ variant = "C1-fa", plan = "s3" },
|
||
{ variant = "C1-fa-ext", plan = "s3" },
|
||
{ variant = "emf-lod20-fa", plan = "s3" },
|
||
{ variant = "emf-lod8-fa", plan = "s3" },
|
||
{ variant = "ldl-fancy", plan = "s3" },
|
||
{ variant = "ldl-fastest", plan = "s3" },
|
||
]
|
||
|
||
[[block]]
|
||
id = "B9"
|
||
title = "Flywheel"
|
||
rule = "Per tier: the smoothest value meeting the floor"
|
||
sessions = [
|
||
{ variant = "fw-flat", plan = "s2", shader = "off" },
|
||
{ variant = "C1", plan = "s2", shader = "off" },
|
||
{ variant = "fw-smooth", plan = "s2", shader = "off" },
|
||
]
|
||
|
||
[[block]]
|
||
id = "B10"
|
||
title = "Sound Physics"
|
||
rule = "Adopt each change that's a win. Listen for obvious audio breakage."
|
||
manual_checks = ["listen for audio breakage"]
|
||
sessions = [
|
||
{ variant = "spr-up", plan = "s2s" },
|
||
{ variant = "C1", plan = "s2s" },
|
||
{ variant = "spr-strict", plan = "s2s" },
|
||
{ variant = "spr-nocap", plan = "s2s" },
|
||
]
|
||
|
||
[[block]]
|
||
id = "B11"
|
||
title = "vanilla / Sodium"
|
||
rule = """3 threads only if S5 wins and S1 doesn't lose \
|
||
| Lowest blend radius that's a win, per tier \
|
||
| Mip 4 for mid/high only if VRAM peak ≤ 5.5 GB and no loss"""
|
||
proxy_note = "C1 baseline session added in-block so analyze.py compares within the label."
|
||
manual_checks = ["VRAM peak for opt-mip4"]
|
||
sessions = [
|
||
{ variant = "C1", plan = "s1,s2,s5", tag = "baseline" },
|
||
{ variant = "sod-thr3", plan = "s1,s5" },
|
||
{ variant = "opt-blend2", plan = "s5" },
|
||
{ variant = "opt-blend0", plan = "s5" },
|
||
{ variant = "opt-mip4", plan = "s1,s2" },
|
||
]
|
||
|
||
[[block]]
|
||
id = "B12a"
|
||
title = "added-mod ablation"
|
||
rule = "Keep the mod unless removing it is a win (it costs FPS) or it breaks something visible. Sodium Extra + RSO stay regardless for their per-machine options."
|
||
sessions = [
|
||
{ variant = "C1", plan = "s1,s2s,s3" },
|
||
{ variant = "C1", plan = "s1,s2s,s3", disable_mods = ["BadOptimizations-*.jar"], tag = "no-badopt" },
|
||
{ variant = "C1", plan = "s1,s2s,s3", disable_mods = ["particle_core-*.jar"], tag = "no-particlecore" },
|
||
{ variant = "C1", plan = "s1,s2s,s3", disable_mods = ["sodium-extra-*.jar"], tag = "no-sodiumextra" },
|
||
]
|
||
|
||
[[block]]
|
||
id = "B12b"
|
||
title = "trial mods"
|
||
rule = "Owner pre-approved helpful adds. Add with `packwiz modrinth add` only on a win with no breakage. **Not More Culling**: its NeoForge build declares `minecraft [1.21,1.21.1)`. A new mod set means re-running B0 first."
|
||
proxy_note = "add_mods entries name [trials] keys. Trials are temporary for the session; a permanent add changes the mod set and invalidates B0."
|
||
manual_checks = ["breakage check per trial", "S6 for Gnetum/Flerovium (GUI paths) by hand"]
|
||
sessions = [
|
||
{ variant = "C1", plan = "s1,s2,s3,s5", tag = "baseline" },
|
||
{ variant = "C1", plan = "s1,s2,s3,s5", add_mods = ["gnetum"], tag = "gnetum" },
|
||
{ variant = "C1", plan = "s2,s3", add_mods = ["better_block_entities"], tag = "bbe" },
|
||
{ variant = "C1-fa", plan = "s3", tag = "fa-baseline" },
|
||
{ variant = "C1-fa", plan = "s3", add_mods = ["better_block_entities"], tag = "bbe-fa" },
|
||
{ variant = "C1", plan = "s1,s2,s3,s5", add_mods = ["flerovium"], tag = "flerovium" },
|
||
{ variant = "C1", plan = "s1,s2,s5", add_mods = ["super_resolution"], tag = "superres" },
|
||
{ variant = "C1", plan = "s1,s2,s5", add_mods = ["ixeris"], tag = "ixeris" },
|
||
{ variant = "C1", plan = "s1,s2,s4", world = "s4-4", shader = "MakeUp-UltraFast-9.5f.zip", tag = "shader-baseline" },
|
||
{ variant = "C1", plan = "s1,s2,s4", world = "s4-4", shader = "MakeUp-UltraFast-9.5f.zip", add_mods = ["iris_flywheel_compat"], tag = "irisflw" },
|
||
{ variant = "C1-bl", plan = "s1,s5", tag = "bl-baseline" },
|
||
{ variant = "C1-bl", plan = "s1,s5", add_mods = ["cull_leaves"], tag = "cull-leaves", note = "MidnightLib is already in the pack (1.10.0)" },
|
||
]
|
||
|
||
[[block]]
|
||
id = "B13"
|
||
title = "profile"
|
||
rule = "Flag any mod over 5 % of render-thread time that isn't Sodium, Iris, DH or Flywheel"
|
||
params = ["scene"]
|
||
proxy_note = "Scene = the worst scene from B2 under C1 (--param scene=sN)."
|
||
manual_checks = ["/sparkc profiler start --timeout 60 during the capture; save the report"]
|
||
sessions = [
|
||
{ variant = "C1", plan = "${scene}", reps = 1, capture = 75, manual = true, note = "/sparkc is a client command; start it by hand at BENCH_MARK start" },
|
||
]
|
||
|
||
[[block]]
|
||
id = "B14"
|
||
title = "spawn tuning"
|
||
manual = true
|
||
rule = "Procedure and keep/back-off rule: `SPAWN-TUNING.md` §B14"
|
||
proxy_note = "Singleplayer fresh world `onionspawn`; river, jungle, warm ocean 10 min each; per-species and total entity counts, defaults vs shipped spawn files. Not a frametime session - run by hand."
|
||
sessions = []
|