logan
769af6fa2d
The swap does not carry players' explored terrain across. FTB Chunks stores no rendered map (its region zips hold chunks.dat plus five data PNGs, one of which packs height into an image channel), and Xaero's region format is closed and now version 6 with per-pixel NBT blobs, so the community converters no longer work. Rather than convert pixels, prune a copy of the world save down to exactly the chunks a player revealed and let Xaero's own generator map it. Validated against real data: extracting the 167-region / 98,170-chunk mask cross-checks clean against FTB's own pixel data, and Anvil pruning round-trips byte-identically (15.7 MB -> 471 KB on a live region file, slot-to-chunk mapping confirmed against 1024 chunk NBT headers). Deferred for now: needs the world save and one singleplayer pass per player. See analysis/FTB-XAERO-MAP-MIGRATION.md.
285 lines
10 KiB
Python
Executable File
285 lines
10 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Extract a per-chunk reveal mask from FTB Chunks client map data.
|
|
|
|
FTB Chunks stores its explored-terrain map client-side, one zip per 512x512-block
|
|
region:
|
|
|
|
<game>/local/ftbchunks/data/<serverUUID>/<dimension>/<XXXXX-XXXXX>.zip
|
|
|
|
Each zip holds `chunks.dat` (the chunks the player has actually seen) plus five
|
|
PNG data images. Region file names are the region coordinate + 60000, formatted
|
|
`%05X-%05X`, which is why overworld spawn shows up as `0EA60-0EA60.zip`.
|
|
|
|
`chunks.dat` layout (verified against ftb-chunks-neoforge-2101.1.22 bytecode):
|
|
|
|
u8 leading byte (0)
|
|
u8 version (3 on this build)
|
|
i16 big-endian chunk count
|
|
per chunk:
|
|
u8 chunk version (present when version >= 2)
|
|
u8 x within region (0..31)
|
|
u8 z within region (0..31)
|
|
i64 big-endian last-modified millis
|
|
|
|
`chunks.dat` was checked against the `data.png` height sentinel (HeightUtils.UNKNOWN
|
|
= -32767, packed as (height << 16) | (water|light|biome)) across all 167 regions of a
|
|
real 98,170-chunk dataset: zero disagreements. So chunks.dat alone is a trustworthy
|
|
reveal mask; --verify re-runs that cross-check anyway.
|
|
|
|
Output is a JSON mask consumed by prune-world-to-reveal.py.
|
|
|
|
Usage:
|
|
ftbchunks-reveal.py <input> [--out mask.json] [--dimension <id>] [--verify] [--quiet]
|
|
|
|
<input> is either
|
|
* a client data directory, e.g. .../local/ftbchunks/data/<uuid>/, which is scanned
|
|
across all dimensions listed in its dimensions.txt, or
|
|
* a directory or zip containing region zips directly (a single dimension, as when
|
|
someone has archived just the overworld). Such an input cannot name its own
|
|
dimension, so pass --dimension to label it.
|
|
|
|
Output coordinates are global chunk coordinates, so they are independent of which
|
|
region file a chunk happened to live in.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import io
|
|
import json
|
|
import os
|
|
import re
|
|
import struct
|
|
import sys
|
|
import zipfile
|
|
import zlib
|
|
|
|
REGION_RE = re.compile(r"^([0-9A-Fa-f]{4,6})-([0-9A-Fa-f]{4,6})\.zip$")
|
|
REGION_OFFSET = 60000
|
|
|
|
# HeightUtils.UNKNOWN = Short.MIN_VALUE + 1, stored masked to 16 bits.
|
|
UNKNOWN_RAW = (-32767) & 0xFFFF
|
|
|
|
|
|
def parse_region_name(name: str) -> tuple[int, int] | None:
|
|
"""`0EA60-0EA5F.zip` -> region coordinate (0, -1)."""
|
|
m = REGION_RE.match(os.path.basename(name))
|
|
if not m:
|
|
return None
|
|
return int(m.group(1), 16) - REGION_OFFSET, int(m.group(2), 16) - REGION_OFFSET
|
|
|
|
|
|
def parse_chunks_dat(data: bytes) -> set[tuple[int, int]]:
|
|
"""Return the within-region (x, z) chunk coordinates present in chunks.dat."""
|
|
if len(data) < 4:
|
|
return set()
|
|
buf = io.BytesIO(data)
|
|
buf.read(1)
|
|
version = buf.read(1)[0]
|
|
count = struct.unpack(">h", buf.read(2))[0]
|
|
chunks: set[tuple[int, int]] = set()
|
|
for _ in range(count):
|
|
if version >= 2:
|
|
buf.read(1)
|
|
x = buf.read(1)[0]
|
|
z = buf.read(1)[0]
|
|
buf.read(8)
|
|
chunks.add((x, z))
|
|
return chunks
|
|
|
|
|
|
# --- optional cross-check: decode data.png and find the UNKNOWN sentinel --------
|
|
|
|
def _png_decode(data: bytes) -> tuple[int, int, int, bytes]:
|
|
"""Minimal PNG decoder (8-bit, non-interlaced) -> (w, h, channels, rgba bytes)."""
|
|
if data[:8] != b"\x89PNG\r\n\x1a\n":
|
|
raise ValueError("not a PNG")
|
|
pos, idat = 8, b""
|
|
width = height = depth = colour = None
|
|
while pos < len(data):
|
|
(length,) = struct.unpack(">I", data[pos : pos + 4])
|
|
kind = data[pos + 4 : pos + 8]
|
|
body = data[pos + 8 : pos + 8 + length]
|
|
pos += 12 + length
|
|
if kind == b"IHDR":
|
|
width, height, depth, colour, comp, filt, interlace = struct.unpack(
|
|
">IIBBBBB", body
|
|
)
|
|
if depth != 8 or interlace != 0:
|
|
raise ValueError(f"unsupported PNG depth/interlace {depth}/{interlace}")
|
|
elif kind == b"IDAT":
|
|
idat += body
|
|
elif kind == b"IEND":
|
|
break
|
|
channels = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}[colour]
|
|
stride = width * channels
|
|
raw = zlib.decompress(idat)
|
|
out = bytearray()
|
|
prev = bytearray(stride)
|
|
p = 0
|
|
for _ in range(height):
|
|
ftype = raw[p]
|
|
p += 1
|
|
line = bytearray(raw[p : p + stride])
|
|
p += stride
|
|
if ftype == 1:
|
|
for i in range(channels, stride):
|
|
line[i] = (line[i] + line[i - channels]) & 0xFF
|
|
elif ftype == 2:
|
|
for i in range(stride):
|
|
line[i] = (line[i] + prev[i]) & 0xFF
|
|
elif ftype == 3:
|
|
for i in range(stride):
|
|
a = line[i - channels] if i >= channels else 0
|
|
line[i] = (line[i] + ((a + prev[i]) >> 1)) & 0xFF
|
|
elif ftype == 4:
|
|
for i in range(stride):
|
|
a = line[i - channels] if i >= channels else 0
|
|
b = prev[i]
|
|
c = prev[i - channels] if i >= channels else 0
|
|
pp = a + b - c
|
|
pa, pb, pc = abs(pp - a), abs(pp - b), abs(pp - c)
|
|
pred = a if (pa <= pb and pa <= pc) else (b if pb <= pc else c)
|
|
line[i] = (line[i] + pred) & 0xFF
|
|
out += line
|
|
prev = line
|
|
return width, height, channels, bytes(out)
|
|
|
|
|
|
def chunks_seen_in_png(png_bytes: bytes) -> set[tuple[int, int]]:
|
|
"""Chunks (0..31)^2 having at least one non-UNKNOWN pixel."""
|
|
width, height, channels, px = _png_decode(png_bytes)
|
|
unknown_r = UNKNOWN_RAW & 0xFF
|
|
unknown_a = (UNKNOWN_RAW >> 8) & 0xFF
|
|
seen: set[tuple[int, int]] = set()
|
|
for cz in range(32):
|
|
for cx in range(32):
|
|
hit = False
|
|
for yy in range(16):
|
|
base = ((cz * 16 + yy) * width + cx * 16) * channels
|
|
for xx in range(16):
|
|
i = base + xx * channels
|
|
if not (px[i] == unknown_r and px[i + 3] == unknown_a):
|
|
hit = True
|
|
break
|
|
if hit:
|
|
break
|
|
if hit:
|
|
seen.add((cx, cz))
|
|
return seen
|
|
|
|
|
|
# --- input discovery -----------------------------------------------------------
|
|
|
|
def iter_region_zips(path: str):
|
|
"""Yield (dimension_name, region_filename, bytes) from a dir or an archive."""
|
|
if zipfile.is_zipfile(path):
|
|
with zipfile.ZipFile(path) as outer:
|
|
for info in outer.infolist():
|
|
if parse_region_name(info.filename) is None:
|
|
continue
|
|
yield "(unspecified)", os.path.basename(info.filename), outer.read(info)
|
|
return
|
|
if os.path.isdir(path) and any(
|
|
parse_region_name(n) for n in os.listdir(path)
|
|
):
|
|
for name in sorted(os.listdir(path)):
|
|
if parse_region_name(name):
|
|
with open(os.path.join(path, name), "rb") as fh:
|
|
yield "(unspecified)", name, fh.read()
|
|
return
|
|
# treat as a client data directory keyed by dimension
|
|
dims_file = os.path.join(path, "dimensions.txt")
|
|
dims = []
|
|
if os.path.exists(dims_file):
|
|
with open(dims_file) as fh:
|
|
dims = [ln.strip() for ln in fh if ln.strip()]
|
|
else:
|
|
dims = [
|
|
d
|
|
for d in sorted(os.listdir(path))
|
|
if os.path.isdir(os.path.join(path, d))
|
|
]
|
|
for dim in dims:
|
|
dim_dir = os.path.join(path, dim)
|
|
if not os.path.isdir(dim_dir):
|
|
continue
|
|
for name in sorted(os.listdir(dim_dir)):
|
|
if parse_region_name(name):
|
|
with open(os.path.join(dim_dir, name), "rb") as fh:
|
|
yield dim, name, fh.read()
|
|
|
|
|
|
def extract(path: str, verify: bool, quiet: bool, dimension_label: str | None = None) -> dict:
|
|
dims: dict[str, dict] = {}
|
|
total = 0
|
|
mismatches = 0
|
|
for dim, name, blob in iter_region_zips(path):
|
|
if dim == "(unspecified)" and dimension_label:
|
|
dim = dimension_label
|
|
coords = parse_region_name(name)
|
|
if coords is None:
|
|
continue
|
|
rx, rz = coords
|
|
with zipfile.ZipFile(io.BytesIO(blob)) as region:
|
|
listed = parse_chunks_dat(region.read("chunks.dat"))
|
|
if verify:
|
|
seen = chunks_seen_in_png(region.read("data.png"))
|
|
if seen != listed:
|
|
mismatches += 1
|
|
print(
|
|
f" ! {dim}/{name}: chunks.dat={len(listed)} png={len(seen)}",
|
|
file=sys.stderr,
|
|
)
|
|
entry = dims.setdefault(dim, {"regions": {}, "chunks": []})
|
|
entry["regions"][f"{rx},{rz}"] = len(listed)
|
|
for cx, cz in sorted(listed):
|
|
entry["chunks"].append([rx * 32 + cx, rz * 32 + cz])
|
|
total += len(listed)
|
|
if not quiet:
|
|
print(f" {dim:34} {name:>16} {len(listed):>4} chunks", file=sys.stderr)
|
|
|
|
for entry in dims.values():
|
|
entry["chunks"].sort()
|
|
entry["region_count"] = len(entry["regions"])
|
|
entry["chunk_count"] = len(entry["chunks"])
|
|
return {
|
|
"source": os.path.abspath(path),
|
|
"verified": verify,
|
|
"mismatches": mismatches,
|
|
"total_chunks": total,
|
|
"dimensions": dims,
|
|
}
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
|
ap.add_argument("input", help="FTB Chunks data dir, or a dir/zip of region zips")
|
|
ap.add_argument("--out", default="ftb-reveal-mask.json")
|
|
ap.add_argument("--dimension", default=None,
|
|
help="dimension id used to label a bare directory/zip of region files, which "
|
|
"cannot say which dimension it is (e.g. minecraft:overworld)")
|
|
ap.add_argument("--verify", action="store_true", help="cross-check chunks.dat vs data.png")
|
|
ap.add_argument("--quiet", action="store_true")
|
|
args = ap.parse_args()
|
|
|
|
mask = extract(args.input, args.verify, args.quiet, args.dimension)
|
|
with open(args.out, "w") as fh:
|
|
json.dump(mask, fh)
|
|
|
|
if args.verify:
|
|
status = "OK" if mask["mismatches"] == 0 else f"{mask['mismatches']} MISMATCHES"
|
|
print(f"cross-check chunks.dat vs data.png: {status}", file=sys.stderr)
|
|
print(
|
|
f"{mask['total_chunks']:,} revealed chunks in "
|
|
f"{sum(d['region_count'] for d in mask['dimensions'].values())} regions "
|
|
f"across {len(mask['dimensions'])} dimension(s) -> {args.out}"
|
|
)
|
|
for dim, d in mask["dimensions"].items():
|
|
print(f" {dim}: {d['chunk_count']:,} chunks / {d['region_count']} regions")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|