chore(tools): add FTB Chunks -> Xaero's map reveal migration tooling
The swap does not carry players' explored terrain across. FTB Chunks stores no rendered map (its region zips hold chunks.dat plus five data PNGs, one of which packs height into an image channel), and Xaero's region format is closed and now version 6 with per-pixel NBT blobs, so the community converters no longer work. Rather than convert pixels, prune a copy of the world save down to exactly the chunks a player revealed and let Xaero's own generator map it. Validated against real data: extracting the 167-region / 98,170-chunk mask cross-checks clean against FTB's own pixel data, and Anvil pruning round-trips byte-identically (15.7 MB -> 471 KB on a live region file, slot-to-chunk mapping confirmed against 1024 chunk NBT headers). Deferred for now: needs the world save and one singleplayer pass per player. See analysis/FTB-XAERO-MAP-MIGRATION.md.
This commit is contained in:
logan committed
1 parent
1d02bf1cf6
commit
769af6fa2d
6 files changed
+1066
No files matched your search
Executable
+88
@@ -0,0 +1,88 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Drop Xaero's map regions that a player never actually revealed.
|
||||
|
||||
After a player opens their pruned singleplayer world, Xaero's maps it - but Minecraft
|
||||
also *regenerates* the chunks around wherever that player logged in, so a little
|
||||
terrain they had never seen can sneak onto the map. This compares the harvested map
|
||||
against the reveal mask and removes the strays.
|
||||
|
||||
It works on file names only (`<rx>_<rz>.zip` is one 512x512-block region, exactly the
|
||||
same geometry as an FTB Chunks region file), so it needs no knowledge of Xaero's
|
||||
region format.
|
||||
|
||||
Usage:
|
||||
filter-xaero-regions.py <xaero-map-dir> <mask.json> [--dimension <id>] [--delete]
|
||||
|
||||
Without --delete it only reports. Run it on one dimension folder at a time, e.g.
|
||||
~/.minecraft/xaero/world-map/Multiplayer_advilplug.top/null/
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
REGION_RE = re.compile(r"^(-?\d+)_(-?\d+)\.zip$")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
||||
ap.add_argument("map_dir", help="Xaero's map instance directory holding <rx>_<rz>.zip")
|
||||
ap.add_argument("mask", help="mask JSON from ftbchunks-reveal.py")
|
||||
ap.add_argument("--dimension", default=None,
|
||||
help="dimension to check (default: the mask's only dimension)")
|
||||
ap.add_argument("--delete", action="store_true", help="actually remove the strays")
|
||||
args = ap.parse_args()
|
||||
|
||||
with open(args.mask) as fh:
|
||||
mask = json.load(fh)
|
||||
dims = mask["dimensions"]
|
||||
dim = args.dimension
|
||||
if dim is None:
|
||||
if len(dims) != 1:
|
||||
print(f"mask has {len(dims)} dimensions; pass --dimension", file=sys.stderr)
|
||||
return 2
|
||||
dim = next(iter(dims))
|
||||
if dim not in dims:
|
||||
print(f"no dimension '{dim}' in mask; have {list(dims)}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
revealed = {tuple(int(v) for v in key.split(",")) for key in dims[dim]["regions"]}
|
||||
|
||||
if not os.path.isdir(args.map_dir):
|
||||
print(f"not a directory: {args.map_dir}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
present, strays = [], []
|
||||
for name in sorted(os.listdir(args.map_dir)):
|
||||
m = REGION_RE.match(name)
|
||||
if not m:
|
||||
continue
|
||||
coords = (int(m.group(1)), int(m.group(2)))
|
||||
(present if coords in revealed else strays).append((name, coords))
|
||||
|
||||
print(f"dimension {dim}")
|
||||
print(f" map regions present : {len(present)} (of {len(revealed)} revealed)")
|
||||
print(f" not in the mask : {len(strays)}")
|
||||
for name, _ in strays:
|
||||
print(f" {name}")
|
||||
|
||||
missing = sorted(revealed - {c for _, c in present})
|
||||
if missing:
|
||||
print(f" revealed but absent from the map: {len(missing)} "
|
||||
f"(e.g. {missing[:4]}) - regenerate these, do not delete anything")
|
||||
|
||||
if args.delete and strays:
|
||||
for name, _ in strays:
|
||||
os.remove(os.path.join(args.map_dir, name))
|
||||
print(f"\nremoved {len(strays)} stray region file(s)")
|
||||
elif strays:
|
||||
print("\nre-run with --delete to remove them")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+284
@@ -0,0 +1,284 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract a per-chunk reveal mask from FTB Chunks client map data.
|
||||
|
||||
FTB Chunks stores its explored-terrain map client-side, one zip per 512x512-block
|
||||
region:
|
||||
|
||||
<game>/local/ftbchunks/data/<serverUUID>/<dimension>/<XXXXX-XXXXX>.zip
|
||||
|
||||
Each zip holds `chunks.dat` (the chunks the player has actually seen) plus five
|
||||
PNG data images. Region file names are the region coordinate + 60000, formatted
|
||||
`%05X-%05X`, which is why overworld spawn shows up as `0EA60-0EA60.zip`.
|
||||
|
||||
`chunks.dat` layout (verified against ftb-chunks-neoforge-2101.1.22 bytecode):
|
||||
|
||||
u8 leading byte (0)
|
||||
u8 version (3 on this build)
|
||||
i16 big-endian chunk count
|
||||
per chunk:
|
||||
u8 chunk version (present when version >= 2)
|
||||
u8 x within region (0..31)
|
||||
u8 z within region (0..31)
|
||||
i64 big-endian last-modified millis
|
||||
|
||||
`chunks.dat` was checked against the `data.png` height sentinel (HeightUtils.UNKNOWN
|
||||
= -32767, packed as (height << 16) | (water|light|biome)) across all 167 regions of a
|
||||
real 98,170-chunk dataset: zero disagreements. So chunks.dat alone is a trustworthy
|
||||
reveal mask; --verify re-runs that cross-check anyway.
|
||||
|
||||
Output is a JSON mask consumed by prune-world-to-reveal.py.
|
||||
|
||||
Usage:
|
||||
ftbchunks-reveal.py <input> [--out mask.json] [--dimension <id>] [--verify] [--quiet]
|
||||
|
||||
<input> is either
|
||||
* a client data directory, e.g. .../local/ftbchunks/data/<uuid>/, which is scanned
|
||||
across all dimensions listed in its dimensions.txt, or
|
||||
* a directory or zip containing region zips directly (a single dimension, as when
|
||||
someone has archived just the overworld). Such an input cannot name its own
|
||||
dimension, so pass --dimension to label it.
|
||||
|
||||
Output coordinates are global chunk coordinates, so they are independent of which
|
||||
region file a chunk happened to live in.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import struct
|
||||
import sys
|
||||
import zipfile
|
||||
import zlib
|
||||
|
||||
REGION_RE = re.compile(r"^([0-9A-Fa-f]{4,6})-([0-9A-Fa-f]{4,6})\.zip$")
|
||||
REGION_OFFSET = 60000
|
||||
|
||||
# HeightUtils.UNKNOWN = Short.MIN_VALUE + 1, stored masked to 16 bits.
|
||||
UNKNOWN_RAW = (-32767) & 0xFFFF
|
||||
|
||||
|
||||
def parse_region_name(name: str) -> tuple[int, int] | None:
|
||||
"""`0EA60-0EA5F.zip` -> region coordinate (0, -1)."""
|
||||
m = REGION_RE.match(os.path.basename(name))
|
||||
if not m:
|
||||
return None
|
||||
return int(m.group(1), 16) - REGION_OFFSET, int(m.group(2), 16) - REGION_OFFSET
|
||||
|
||||
|
||||
def parse_chunks_dat(data: bytes) -> set[tuple[int, int]]:
|
||||
"""Return the within-region (x, z) chunk coordinates present in chunks.dat."""
|
||||
if len(data) < 4:
|
||||
return set()
|
||||
buf = io.BytesIO(data)
|
||||
buf.read(1)
|
||||
version = buf.read(1)[0]
|
||||
count = struct.unpack(">h", buf.read(2))[0]
|
||||
chunks: set[tuple[int, int]] = set()
|
||||
for _ in range(count):
|
||||
if version >= 2:
|
||||
buf.read(1)
|
||||
x = buf.read(1)[0]
|
||||
z = buf.read(1)[0]
|
||||
buf.read(8)
|
||||
chunks.add((x, z))
|
||||
return chunks
|
||||
|
||||
|
||||
# --- optional cross-check: decode data.png and find the UNKNOWN sentinel --------
|
||||
|
||||
def _png_decode(data: bytes) -> tuple[int, int, int, bytes]:
|
||||
"""Minimal PNG decoder (8-bit, non-interlaced) -> (w, h, channels, rgba bytes)."""
|
||||
if data[:8] != b"\x89PNG\r\n\x1a\n":
|
||||
raise ValueError("not a PNG")
|
||||
pos, idat = 8, b""
|
||||
width = height = depth = colour = None
|
||||
while pos < len(data):
|
||||
(length,) = struct.unpack(">I", data[pos : pos + 4])
|
||||
kind = data[pos + 4 : pos + 8]
|
||||
body = data[pos + 8 : pos + 8 + length]
|
||||
pos += 12 + length
|
||||
if kind == b"IHDR":
|
||||
width, height, depth, colour, comp, filt, interlace = struct.unpack(
|
||||
">IIBBBBB", body
|
||||
)
|
||||
if depth != 8 or interlace != 0:
|
||||
raise ValueError(f"unsupported PNG depth/interlace {depth}/{interlace}")
|
||||
elif kind == b"IDAT":
|
||||
idat += body
|
||||
elif kind == b"IEND":
|
||||
break
|
||||
channels = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}[colour]
|
||||
stride = width * channels
|
||||
raw = zlib.decompress(idat)
|
||||
out = bytearray()
|
||||
prev = bytearray(stride)
|
||||
p = 0
|
||||
for _ in range(height):
|
||||
ftype = raw[p]
|
||||
p += 1
|
||||
line = bytearray(raw[p : p + stride])
|
||||
p += stride
|
||||
if ftype == 1:
|
||||
for i in range(channels, stride):
|
||||
line[i] = (line[i] + line[i - channels]) & 0xFF
|
||||
elif ftype == 2:
|
||||
for i in range(stride):
|
||||
line[i] = (line[i] + prev[i]) & 0xFF
|
||||
elif ftype == 3:
|
||||
for i in range(stride):
|
||||
a = line[i - channels] if i >= channels else 0
|
||||
line[i] = (line[i] + ((a + prev[i]) >> 1)) & 0xFF
|
||||
elif ftype == 4:
|
||||
for i in range(stride):
|
||||
a = line[i - channels] if i >= channels else 0
|
||||
b = prev[i]
|
||||
c = prev[i - channels] if i >= channels else 0
|
||||
pp = a + b - c
|
||||
pa, pb, pc = abs(pp - a), abs(pp - b), abs(pp - c)
|
||||
pred = a if (pa <= pb and pa <= pc) else (b if pb <= pc else c)
|
||||
line[i] = (line[i] + pred) & 0xFF
|
||||
out += line
|
||||
prev = line
|
||||
return width, height, channels, bytes(out)
|
||||
|
||||
|
||||
def chunks_seen_in_png(png_bytes: bytes) -> set[tuple[int, int]]:
|
||||
"""Chunks (0..31)^2 having at least one non-UNKNOWN pixel."""
|
||||
width, height, channels, px = _png_decode(png_bytes)
|
||||
unknown_r = UNKNOWN_RAW & 0xFF
|
||||
unknown_a = (UNKNOWN_RAW >> 8) & 0xFF
|
||||
seen: set[tuple[int, int]] = set()
|
||||
for cz in range(32):
|
||||
for cx in range(32):
|
||||
hit = False
|
||||
for yy in range(16):
|
||||
base = ((cz * 16 + yy) * width + cx * 16) * channels
|
||||
for xx in range(16):
|
||||
i = base + xx * channels
|
||||
if not (px[i] == unknown_r and px[i + 3] == unknown_a):
|
||||
hit = True
|
||||
break
|
||||
if hit:
|
||||
break
|
||||
if hit:
|
||||
seen.add((cx, cz))
|
||||
return seen
|
||||
|
||||
|
||||
# --- input discovery -----------------------------------------------------------
|
||||
|
||||
def iter_region_zips(path: str):
|
||||
"""Yield (dimension_name, region_filename, bytes) from a dir or an archive."""
|
||||
if zipfile.is_zipfile(path):
|
||||
with zipfile.ZipFile(path) as outer:
|
||||
for info in outer.infolist():
|
||||
if parse_region_name(info.filename) is None:
|
||||
continue
|
||||
yield "(unspecified)", os.path.basename(info.filename), outer.read(info)
|
||||
return
|
||||
if os.path.isdir(path) and any(
|
||||
parse_region_name(n) for n in os.listdir(path)
|
||||
):
|
||||
for name in sorted(os.listdir(path)):
|
||||
if parse_region_name(name):
|
||||
with open(os.path.join(path, name), "rb") as fh:
|
||||
yield "(unspecified)", name, fh.read()
|
||||
return
|
||||
# treat as a client data directory keyed by dimension
|
||||
dims_file = os.path.join(path, "dimensions.txt")
|
||||
dims = []
|
||||
if os.path.exists(dims_file):
|
||||
with open(dims_file) as fh:
|
||||
dims = [ln.strip() for ln in fh if ln.strip()]
|
||||
else:
|
||||
dims = [
|
||||
d
|
||||
for d in sorted(os.listdir(path))
|
||||
if os.path.isdir(os.path.join(path, d))
|
||||
]
|
||||
for dim in dims:
|
||||
dim_dir = os.path.join(path, dim)
|
||||
if not os.path.isdir(dim_dir):
|
||||
continue
|
||||
for name in sorted(os.listdir(dim_dir)):
|
||||
if parse_region_name(name):
|
||||
with open(os.path.join(dim_dir, name), "rb") as fh:
|
||||
yield dim, name, fh.read()
|
||||
|
||||
|
||||
def extract(path: str, verify: bool, quiet: bool, dimension_label: str | None = None) -> dict:
|
||||
dims: dict[str, dict] = {}
|
||||
total = 0
|
||||
mismatches = 0
|
||||
for dim, name, blob in iter_region_zips(path):
|
||||
if dim == "(unspecified)" and dimension_label:
|
||||
dim = dimension_label
|
||||
coords = parse_region_name(name)
|
||||
if coords is None:
|
||||
continue
|
||||
rx, rz = coords
|
||||
with zipfile.ZipFile(io.BytesIO(blob)) as region:
|
||||
listed = parse_chunks_dat(region.read("chunks.dat"))
|
||||
if verify:
|
||||
seen = chunks_seen_in_png(region.read("data.png"))
|
||||
if seen != listed:
|
||||
mismatches += 1
|
||||
print(
|
||||
f" ! {dim}/{name}: chunks.dat={len(listed)} png={len(seen)}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
entry = dims.setdefault(dim, {"regions": {}, "chunks": []})
|
||||
entry["regions"][f"{rx},{rz}"] = len(listed)
|
||||
for cx, cz in sorted(listed):
|
||||
entry["chunks"].append([rx * 32 + cx, rz * 32 + cz])
|
||||
total += len(listed)
|
||||
if not quiet:
|
||||
print(f" {dim:34} {name:>16} {len(listed):>4} chunks", file=sys.stderr)
|
||||
|
||||
for entry in dims.values():
|
||||
entry["chunks"].sort()
|
||||
entry["region_count"] = len(entry["regions"])
|
||||
entry["chunk_count"] = len(entry["chunks"])
|
||||
return {
|
||||
"source": os.path.abspath(path),
|
||||
"verified": verify,
|
||||
"mismatches": mismatches,
|
||||
"total_chunks": total,
|
||||
"dimensions": dims,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
||||
ap.add_argument("input", help="FTB Chunks data dir, or a dir/zip of region zips")
|
||||
ap.add_argument("--out", default="ftb-reveal-mask.json")
|
||||
ap.add_argument("--dimension", default=None,
|
||||
help="dimension id used to label a bare directory/zip of region files, which "
|
||||
"cannot say which dimension it is (e.g. minecraft:overworld)")
|
||||
ap.add_argument("--verify", action="store_true", help="cross-check chunks.dat vs data.png")
|
||||
ap.add_argument("--quiet", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
mask = extract(args.input, args.verify, args.quiet, args.dimension)
|
||||
with open(args.out, "w") as fh:
|
||||
json.dump(mask, fh)
|
||||
|
||||
if args.verify:
|
||||
status = "OK" if mask["mismatches"] == 0 else f"{mask['mismatches']} MISMATCHES"
|
||||
print(f"cross-check chunks.dat vs data.png: {status}", file=sys.stderr)
|
||||
print(
|
||||
f"{mask['total_chunks']:,} revealed chunks in "
|
||||
f"{sum(d['region_count'] for d in mask['dimensions'].values())} regions "
|
||||
f"across {len(mask['dimensions'])} dimension(s) -> {args.out}"
|
||||
)
|
||||
for dim, d in mask["dimensions"].items():
|
||||
print(f" {dim}: {d['chunk_count']:,} chunks / {d['region_count']} regions")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+131
@@ -0,0 +1,131 @@
|
||||
#!/usr/bin/env bash
|
||||
# End-to-end driver for the FTB Chunks -> Xaero's World Map reveal transplant.
|
||||
#
|
||||
# See analysis/FTB-XAERO-MAP-MIGRATION.md for the reasoning. This script does the
|
||||
# parts that can be automated; the singleplayer pass in step 3 has to be done by hand
|
||||
# on a machine that can run the pack.
|
||||
#
|
||||
# Usage:
|
||||
# migrate-player.sh <player> <ftbchunks-data> <world-dir> [--dimension <id>] [--work <dir>]
|
||||
#
|
||||
# <player> label used for output paths, e.g. Stirwick
|
||||
# <ftbchunks-data> that player's local/ftbchunks/data/<uuid>/, or a bare zip/dir
|
||||
# of region files, or an already-extracted mask .json
|
||||
# <world-dir> the world save containing region/
|
||||
#
|
||||
# Example:
|
||||
# tools/migrate-player.sh Stirwick ~/Stirwick-ftbchunks ~/server/world \
|
||||
# --dimension minecraft:overworld --work /var/tmp/xaero-migration
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
if [ $# -lt 3 ]; then
|
||||
sed -n '2,20p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'
|
||||
exit 2
|
||||
fi
|
||||
|
||||
player="$1"; ftb="$2"; world="$3"; shift 3
|
||||
dimension="minecraft:overworld"
|
||||
work="$(pwd)/xaero-migration"
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--dimension) dimension="$2"; shift 2 ;;
|
||||
--work) work="$2"; shift 2 ;;
|
||||
*) echo "unknown option: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
out="$work/$player"
|
||||
mask="$out/mask.json"
|
||||
sp="$out/sp-world"
|
||||
|
||||
echo "player : $player"
|
||||
echo "ftb data : $ftb"
|
||||
echo "world : $world"
|
||||
echo "dimension : $dimension"
|
||||
echo "output : $out"
|
||||
echo
|
||||
|
||||
if [ ! -d "$world/region" ]; then
|
||||
echo "error: $world/region not found" >&2
|
||||
exit 1
|
||||
fi
|
||||
mkdir -p "$out"
|
||||
|
||||
echo "== 1/2 extracting reveal mask =="
|
||||
if [ "${ftb##*.}" = "json" ]; then
|
||||
cp -f "$ftb" "$mask"
|
||||
echo " using prebuilt mask: $ftb"
|
||||
python3 - "$mask" <<'PY'
|
||||
import json, sys
|
||||
m = json.load(open(sys.argv[1]))
|
||||
dims = m["dimensions"]
|
||||
print(f" {m['total_chunks']:,} chunks / "
|
||||
f"{sum(d['region_count'] for d in dims.values())} regions / "
|
||||
f"dimension(s) {list(dims)}")
|
||||
PY
|
||||
else
|
||||
python3 "$here/ftbchunks-reveal.py" "$ftb" --out "$mask" --dimension "$dimension" --verify
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "== 2/2 pruning the world to those chunks (dry run first) =="
|
||||
python3 "$here/prune-world-to-reveal.py" "$world" "$mask" "$sp" \
|
||||
--dimension "$dimension" --minimal --dry-run
|
||||
echo
|
||||
python3 "$here/prune-world-to-reveal.py" "$world" "$mask" "$sp" \
|
||||
--dimension "$dimension" --minimal
|
||||
|
||||
cat <<EOF
|
||||
|
||||
Done. Pruned world: $sp
|
||||
|
||||
Now, by hand.
|
||||
|
||||
3. Copy $sp to a machine that can run the pack and open it in SINGLEPLAYER once,
|
||||
as a world folder (the directory holding level.dat and region/):
|
||||
|
||||
<game dir>/saves/Stirwick-reveal/
|
||||
|
||||
Open it, open the map (M), let it fill, then quit. Xaero's maps what is in the
|
||||
save, and this save holds only the revealed chunks. Do not walk around --
|
||||
Minecraft regenerates chunks near the login position and they will show up.
|
||||
|
||||
Where the result lands: look under <game dir>/xaero/world-map/ for a folder that
|
||||
is not Multiplayer_*, e.g. World/null/<map>/. Sort by modification time if
|
||||
unsure -- those files were just written.
|
||||
|
||||
SANITY CHECK: you should end up with one zip per region in the mask
|
||||
($(python3 -c "import json;print(sum(d['region_count'] for d in json.load(open('$mask'))['dimensions'].values()))") for $player). A handful means it only mapped the spawn area and
|
||||
you should use 'Convert All Regions' in the world map settings, or report back.
|
||||
|
||||
4. Clean the HARVEST before anything else -- this is the step that removes terrain
|
||||
Minecraft regenerated around the login point. Run it on the harvested folder,
|
||||
NOT on the player's own map folder:
|
||||
|
||||
python3 $here/filter-xaero-regions.py \\
|
||||
"<game dir>/xaero/world-map/World/null/<map>/" \\
|
||||
$mask --dimension $dimension
|
||||
|
||||
Check the list, then re-run with --delete.
|
||||
|
||||
5. Merge the surviving <rx>_<rz>.zip files into the player's existing map folder,
|
||||
WITHOUT overwriting anything already there (-n), so any terrain they have
|
||||
explored since the swap is kept:
|
||||
|
||||
cp -n <harvested>/*.zip \\
|
||||
"<their game dir>/xaero/world-map/Multiplayer_advilplug.top/null/<their map instance>/"
|
||||
|
||||
The right instance is the mw\$... folder there. If the player has not joined the
|
||||
server since the swap, have them join once first so Xaero's creates it.
|
||||
|
||||
6. Delete $out -- scratch, usually a few hundred MB.
|
||||
|
||||
Per-dimension: repeat with --dimension <id> and the matching map folder
|
||||
(null = overworld, DIM-1 = nether, DIM1 = end).
|
||||
Other players: each needs their own local/ftbchunks/data/<uuid>/; there is no
|
||||
server-side copy of this data.
|
||||
EOF
|
||||
Executable
+210
@@ -0,0 +1,210 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prune a Minecraft Anvil world save down to a set of revealed chunks.
|
||||
|
||||
This is how we transplant FTB Chunks' reveal footprint onto Xaero's World Map
|
||||
without reverse-engineering Xaero's region format: strip the world save down to
|
||||
exactly the chunks a player had explored, then let Xaero's own generator map that
|
||||
save in singleplayer. Xaero's renders it natively (correct palette, shading,
|
||||
light, biome) and the absent chunks simply never appear on the map.
|
||||
|
||||
Chunk payloads are copied through byte-for-byte -- no NBT re-encoding -- so the
|
||||
output is safe for modded 1.21.1 chunk data. Only the 8 KiB header, the sector
|
||||
offsets and the zero padding are rewritten.
|
||||
|
||||
Usage:
|
||||
prune-world-to-reveal.py <world-save-dir> <mask.json> <out-dir>
|
||||
[--dimension minecraft:overworld] [--dry-run]
|
||||
|
||||
Reads `<world-save-dir>/region/r.<x>.<z>.mca` and writes pruned copies to
|
||||
`<out-dir>/region/`, along with whatever else the world needs. Nested per-dimension
|
||||
`region`/`entities`/`poi` trees are never copied, so the Nether and End are not
|
||||
duplicated at full size; `--minimal` goes further and copies only `level.dat`, which
|
||||
is all Xaero's needs to map the overworld.
|
||||
|
||||
If a full save copy is wanted (level.dat, data/, playerdata/ ...), copy the world
|
||||
first and then run this in place with <out-dir> == <world-save-dir>.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import struct
|
||||
import sys
|
||||
|
||||
SECTOR = 4096
|
||||
HEADER = 2 * SECTOR
|
||||
COMPRESSION_NAMES = {1: "gzip", 2: "zlib", 3: "none", 4: "lz4", 127: "external"}
|
||||
|
||||
|
||||
def read_slot(data: bytes, size: int, slot: int) -> tuple[bytes, int, int] | None:
|
||||
"""Return (payload, timestamp, compression) for an Anvil slot, or None if empty."""
|
||||
off = struct.unpack(">I", b"\x00" + data[slot * 4 : slot * 4 + 3])[0]
|
||||
if off == 0:
|
||||
return None
|
||||
timestamp = struct.unpack(">I", data[SECTOR + slot * 4 : SECTOR + slot * 4 + 4])[0]
|
||||
start = off * SECTOR
|
||||
if start + 5 > len(data):
|
||||
raise ValueError(f"slot {slot}: offset {off} past end of file")
|
||||
(length,) = struct.unpack(">I", data[start : start + 4])
|
||||
compression = data[start + 4]
|
||||
if length < 1 or start + 4 + length > len(data):
|
||||
raise ValueError(f"slot {slot}: bad chunk length {length}")
|
||||
return data[start + 5 : start + 4 + length], timestamp, compression
|
||||
|
||||
|
||||
def prune_region(path: str, keep_slots: set[int], out_path: str, dry: bool) -> dict:
|
||||
with open(path, "rb") as fh:
|
||||
data = fh.read()
|
||||
|
||||
kept: list[tuple[int, bytes, int, int]] = []
|
||||
dropped = 0
|
||||
external = 0
|
||||
for slot in range(1024):
|
||||
entry = read_slot(data, SECTOR, slot)
|
||||
if entry is None:
|
||||
continue
|
||||
payload, timestamp, compression = entry
|
||||
if compression == 127:
|
||||
external += 1
|
||||
if slot in keep_slots:
|
||||
kept.append((slot, payload, timestamp, compression))
|
||||
else:
|
||||
dropped += 1
|
||||
|
||||
stats = {
|
||||
"file": os.path.basename(path),
|
||||
"kept": len(kept),
|
||||
"dropped": dropped,
|
||||
"external_payloads": external,
|
||||
}
|
||||
if dry:
|
||||
return stats
|
||||
|
||||
out = bytearray(HEADER)
|
||||
sector = 2
|
||||
for slot, payload, timestamp, compression in kept:
|
||||
blob = struct.pack(">I", len(payload) + 1) + bytes([compression]) + payload
|
||||
sectors = (len(blob) + SECTOR - 1) // SECTOR
|
||||
if sector + sectors > 0xFFFFFF:
|
||||
raise ValueError(f"{path}: region would exceed the Anvil 24-bit offset limit")
|
||||
out[slot * 4 : slot * 4 + 3] = sector.to_bytes(3, "big")
|
||||
out[slot * 4 + 3] = sectors
|
||||
out[SECTOR + slot * 4 : SECTOR + slot * 4 + 4] = struct.pack(">I", timestamp)
|
||||
out += blob
|
||||
out += b"\x00" * (sectors * SECTOR - len(blob))
|
||||
sector += sectors
|
||||
|
||||
os.makedirs(os.path.dirname(out_path) or ".", exist_ok=True)
|
||||
with open(out_path, "wb") as fh:
|
||||
fh.write(out)
|
||||
return stats
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
||||
ap.add_argument("world", help="world save directory containing region/")
|
||||
ap.add_argument("mask", help="mask JSON from ftbchunks-reveal.py")
|
||||
ap.add_argument("out", help="output world directory (use the same path for in-place)")
|
||||
ap.add_argument("--dimension", default="minecraft:overworld",
|
||||
help="which dimension in the mask to apply (default overworld)")
|
||||
ap.add_argument("--minimal", action="store_true",
|
||||
help="output only level.dat + the pruned region/ tree (smallest usable world)")
|
||||
ap.add_argument("--dry-run", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
with open(args.mask) as fh:
|
||||
mask = json.load(fh)
|
||||
dims = mask["dimensions"]
|
||||
if args.dimension not in dims:
|
||||
print(f"mask has no dimension '{args.dimension}'; available: {list(dims)}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
chunks = {(cx, cz) for cx, cz in dims[args.dimension]["chunks"]}
|
||||
by_region: dict[tuple[int, int], set[int]] = {}
|
||||
for cx, cz in chunks:
|
||||
rx, rz = cx >> 5, cz >> 5
|
||||
by_region.setdefault((rx, rz), set()).add((cx & 31) + (cz & 31) * 32)
|
||||
|
||||
in_region = os.path.join(args.world, "region")
|
||||
if not os.path.isdir(in_region):
|
||||
print(f"no region/ directory under {args.world}", file=sys.stderr)
|
||||
return 2
|
||||
out_region = os.path.join(args.out, "region")
|
||||
os.makedirs(out_region, exist_ok=True)
|
||||
|
||||
# Copy the rest of the world so the pruned save is openable, unless in-place.
|
||||
# Nested `region`/`entities`/`poi` trees (i.e. per-dimension ones like DIM-1/) are never
|
||||
# copied - copying them wholesale would duplicate the Nether and End at full size.
|
||||
if os.path.abspath(args.out) != os.path.abspath(args.world) and not args.dry_run:
|
||||
ignore = shutil.ignore_patterns("region", "entities", "poi")
|
||||
for name in sorted(os.listdir(args.world)):
|
||||
if name in ("region", "entities", "poi"):
|
||||
continue
|
||||
if args.minimal and name not in ("level.dat", "level.dat_old"):
|
||||
continue
|
||||
src, dst = os.path.join(args.world, name), os.path.join(args.out, name)
|
||||
if os.path.exists(dst):
|
||||
continue
|
||||
if os.path.isdir(src):
|
||||
shutil.copytree(src, dst, ignore=ignore)
|
||||
else:
|
||||
shutil.copy2(src, dst)
|
||||
|
||||
total_kept = total_dropped = 0
|
||||
seen_regions = set()
|
||||
failures: list[tuple[str, str]] = []
|
||||
failed_regions: set[tuple[int, int]] = set()
|
||||
for name in sorted(os.listdir(in_region)):
|
||||
if not (name.startswith("r.") and name.endswith(".mca")):
|
||||
continue
|
||||
_, rx, rz, _ = name.split(".")
|
||||
coords = (int(rx), int(rz))
|
||||
keep = by_region.get(coords)
|
||||
if keep is None:
|
||||
continue # region untouched by the reveal set
|
||||
try:
|
||||
stats = prune_region(
|
||||
os.path.join(in_region, name),
|
||||
keep,
|
||||
os.path.join(out_region, name),
|
||||
args.dry_run,
|
||||
)
|
||||
except Exception as exc:
|
||||
# A live server can rewrite a region file mid-read. A torn read is not a reason to
|
||||
# abandon the whole pass - no output is written for a region that fails, so it can
|
||||
# simply be retried.
|
||||
failures.append((name, str(exc)))
|
||||
failed_regions.add(coords)
|
||||
print(f" ! {name}: {exc}", file=sys.stderr)
|
||||
continue
|
||||
seen_regions.add(coords)
|
||||
total_kept += stats["kept"]
|
||||
total_dropped += stats["dropped"]
|
||||
if stats["external_payloads"]:
|
||||
print(f" ! {name}: {stats['external_payloads']} external (.mcc) payload(s) "
|
||||
f"referenced but not copied", file=sys.stderr)
|
||||
print(f" {name}: kept {stats['kept']:>4} / dropped {stats['dropped']:>4}")
|
||||
|
||||
# Any region the player revealed but that has no .mca file.
|
||||
missing = sorted(set(by_region) - seen_regions - failed_regions)
|
||||
if missing:
|
||||
print(f" ! {len(missing)} revealed region(s) have no .mca in the save "
|
||||
f"(e.g. {missing[:3]})", file=sys.stderr)
|
||||
|
||||
verb = "would keep" if args.dry_run else "kept"
|
||||
print(f"\n{verb} {total_kept:,} chunks, dropped {total_dropped:,}, "
|
||||
f"across {len(seen_regions)} region file(s) -> {out_region}")
|
||||
if failures:
|
||||
print(f"\n{len(failures)} region(s) could not be read (not written):", file=sys.stderr)
|
||||
for name, why in failures:
|
||||
print(f" {name}: {why}", file=sys.stderr)
|
||||
print("Re-run to retry them - usually a transient write by a running server.", file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user