#!/usr/bin/env python3 """Extract a per-chunk reveal mask from FTB Chunks client map data. FTB Chunks stores its explored-terrain map client-side, one zip per 512x512-block region: /local/ftbchunks/data///.zip Each zip holds `chunks.dat` (the chunks the player has actually seen) plus five PNG data images. Region file names are the region coordinate + 60000, formatted `%05X-%05X`, which is why overworld spawn shows up as `0EA60-0EA60.zip`. `chunks.dat` layout (verified against ftb-chunks-neoforge-2101.1.22 bytecode): u8 leading byte (0) u8 version (3 on this build) i16 big-endian chunk count per chunk: u8 chunk version (present when version >= 2) u8 x within region (0..31) u8 z within region (0..31) i64 big-endian last-modified millis `chunks.dat` was checked against the `data.png` height sentinel (HeightUtils.UNKNOWN = -32767, packed as (height << 16) | (water|light|biome)) across all 167 regions of a real 98,170-chunk dataset: zero disagreements. So chunks.dat alone is a trustworthy reveal mask; --verify re-runs that cross-check anyway. Output is a JSON mask consumed by prune-world-to-reveal.py. Usage: ftbchunks-reveal.py [--out mask.json] [--dimension ] [--verify] [--quiet] is either * a client data directory, e.g. .../local/ftbchunks/data//, which is scanned across all dimensions listed in its dimensions.txt, or * a directory or zip containing region zips directly (a single dimension, as when someone has archived just the overworld). Such an input cannot name its own dimension, so pass --dimension to label it. Output coordinates are global chunk coordinates, so they are independent of which region file a chunk happened to live in. """ from __future__ import annotations import argparse import io import json import os import re import struct import sys import zipfile import zlib REGION_RE = re.compile(r"^([0-9A-Fa-f]{4,6})-([0-9A-Fa-f]{4,6})\.zip$") REGION_OFFSET = 60000 # HeightUtils.UNKNOWN = Short.MIN_VALUE + 1, stored masked to 16 bits. UNKNOWN_RAW = (-32767) & 0xFFFF def parse_region_name(name: str) -> tuple[int, int] | None: """`0EA60-0EA5F.zip` -> region coordinate (0, -1).""" m = REGION_RE.match(os.path.basename(name)) if not m: return None return int(m.group(1), 16) - REGION_OFFSET, int(m.group(2), 16) - REGION_OFFSET def parse_chunks_dat(data: bytes) -> set[tuple[int, int]]: """Return the within-region (x, z) chunk coordinates present in chunks.dat.""" if len(data) < 4: return set() buf = io.BytesIO(data) buf.read(1) version = buf.read(1)[0] count = struct.unpack(">h", buf.read(2))[0] chunks: set[tuple[int, int]] = set() for _ in range(count): if version >= 2: buf.read(1) x = buf.read(1)[0] z = buf.read(1)[0] buf.read(8) chunks.add((x, z)) return chunks # --- optional cross-check: decode data.png and find the UNKNOWN sentinel -------- def _png_decode(data: bytes) -> tuple[int, int, int, bytes]: """Minimal PNG decoder (8-bit, non-interlaced) -> (w, h, channels, rgba bytes).""" if data[:8] != b"\x89PNG\r\n\x1a\n": raise ValueError("not a PNG") pos, idat = 8, b"" width = height = depth = colour = None while pos < len(data): (length,) = struct.unpack(">I", data[pos : pos + 4]) kind = data[pos + 4 : pos + 8] body = data[pos + 8 : pos + 8 + length] pos += 12 + length if kind == b"IHDR": width, height, depth, colour, comp, filt, interlace = struct.unpack( ">IIBBBBB", body ) if depth != 8 or interlace != 0: raise ValueError(f"unsupported PNG depth/interlace {depth}/{interlace}") elif kind == b"IDAT": idat += body elif kind == b"IEND": break channels = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}[colour] stride = width * channels raw = zlib.decompress(idat) out = bytearray() prev = bytearray(stride) p = 0 for _ in range(height): ftype = raw[p] p += 1 line = bytearray(raw[p : p + stride]) p += stride if ftype == 1: for i in range(channels, stride): line[i] = (line[i] + line[i - channels]) & 0xFF elif ftype == 2: for i in range(stride): line[i] = (line[i] + prev[i]) & 0xFF elif ftype == 3: for i in range(stride): a = line[i - channels] if i >= channels else 0 line[i] = (line[i] + ((a + prev[i]) >> 1)) & 0xFF elif ftype == 4: for i in range(stride): a = line[i - channels] if i >= channels else 0 b = prev[i] c = prev[i - channels] if i >= channels else 0 pp = a + b - c pa, pb, pc = abs(pp - a), abs(pp - b), abs(pp - c) pred = a if (pa <= pb and pa <= pc) else (b if pb <= pc else c) line[i] = (line[i] + pred) & 0xFF out += line prev = line return width, height, channels, bytes(out) def chunks_seen_in_png(png_bytes: bytes) -> set[tuple[int, int]]: """Chunks (0..31)^2 having at least one non-UNKNOWN pixel.""" width, height, channels, px = _png_decode(png_bytes) unknown_r = UNKNOWN_RAW & 0xFF unknown_a = (UNKNOWN_RAW >> 8) & 0xFF seen: set[tuple[int, int]] = set() for cz in range(32): for cx in range(32): hit = False for yy in range(16): base = ((cz * 16 + yy) * width + cx * 16) * channels for xx in range(16): i = base + xx * channels if not (px[i] == unknown_r and px[i + 3] == unknown_a): hit = True break if hit: break if hit: seen.add((cx, cz)) return seen # --- input discovery ----------------------------------------------------------- def iter_region_zips(path: str): """Yield (dimension_name, region_filename, bytes) from a dir or an archive.""" if zipfile.is_zipfile(path): with zipfile.ZipFile(path) as outer: for info in outer.infolist(): if parse_region_name(info.filename) is None: continue yield "(unspecified)", os.path.basename(info.filename), outer.read(info) return if os.path.isdir(path) and any( parse_region_name(n) for n in os.listdir(path) ): for name in sorted(os.listdir(path)): if parse_region_name(name): with open(os.path.join(path, name), "rb") as fh: yield "(unspecified)", name, fh.read() return # treat as a client data directory keyed by dimension dims_file = os.path.join(path, "dimensions.txt") dims = [] if os.path.exists(dims_file): with open(dims_file) as fh: dims = [ln.strip() for ln in fh if ln.strip()] else: dims = [ d for d in sorted(os.listdir(path)) if os.path.isdir(os.path.join(path, d)) ] for dim in dims: dim_dir = os.path.join(path, dim) if not os.path.isdir(dim_dir): continue for name in sorted(os.listdir(dim_dir)): if parse_region_name(name): with open(os.path.join(dim_dir, name), "rb") as fh: yield dim, name, fh.read() def extract(path: str, verify: bool, quiet: bool, dimension_label: str | None = None) -> dict: dims: dict[str, dict] = {} total = 0 mismatches = 0 for dim, name, blob in iter_region_zips(path): if dim == "(unspecified)" and dimension_label: dim = dimension_label coords = parse_region_name(name) if coords is None: continue rx, rz = coords with zipfile.ZipFile(io.BytesIO(blob)) as region: listed = parse_chunks_dat(region.read("chunks.dat")) if verify: seen = chunks_seen_in_png(region.read("data.png")) if seen != listed: mismatches += 1 print( f" ! {dim}/{name}: chunks.dat={len(listed)} png={len(seen)}", file=sys.stderr, ) entry = dims.setdefault(dim, {"regions": {}, "chunks": []}) entry["regions"][f"{rx},{rz}"] = len(listed) for cx, cz in sorted(listed): entry["chunks"].append([rx * 32 + cx, rz * 32 + cz]) total += len(listed) if not quiet: print(f" {dim:34} {name:>16} {len(listed):>4} chunks", file=sys.stderr) for entry in dims.values(): entry["chunks"].sort() entry["region_count"] = len(entry["regions"]) entry["chunk_count"] = len(entry["chunks"]) return { "source": os.path.abspath(path), "verified": verify, "mismatches": mismatches, "total_chunks": total, "dimensions": dims, } def main() -> int: ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) ap.add_argument("input", help="FTB Chunks data dir, or a dir/zip of region zips") ap.add_argument("--out", default="ftb-reveal-mask.json") ap.add_argument("--dimension", default=None, help="dimension id used to label a bare directory/zip of region files, which " "cannot say which dimension it is (e.g. minecraft:overworld)") ap.add_argument("--verify", action="store_true", help="cross-check chunks.dat vs data.png") ap.add_argument("--quiet", action="store_true") args = ap.parse_args() mask = extract(args.input, args.verify, args.quiet, args.dimension) with open(args.out, "w") as fh: json.dump(mask, fh) if args.verify: status = "OK" if mask["mismatches"] == 0 else f"{mask['mismatches']} MISMATCHES" print(f"cross-check chunks.dat vs data.png: {status}", file=sys.stderr) print( f"{mask['total_chunks']:,} revealed chunks in " f"{sum(d['region_count'] for d in mask['dimensions'].values())} regions " f"across {len(mask['dimensions'])} dimension(s) -> {args.out}" ) for dim, d in mask["dimensions"].items(): print(f" {dim}: {d['chunk_count']:,} chunks / {d['region_count']} regions") return 0 if __name__ == "__main__": sys.exit(main())