build(dev): generate a 50k-track library and measure a running app

Plan 007 phase 4 is verified by measurement, not by assertion, and there was no way to produce a number: the fixture library is a few dozen tracks and cannot show any of the findings. - `cmd/gentestdata -bulk N` (`make bulkdata`) writes a ~50 000-track library in 11 s / 466 MB by encoding six clips once and copying them, while still tagging every file through `backend/tagwriter` — a library the app cannot read back measures nothing. - `make sandbox-seed-bulk` seeds from it through the same script and the same discipline as any other seed: by running the app and waiting for the real scan. - `e2e/perf/measure.mjs` (`make perf LABEL=x`, `make perf-compare`) takes fourteen measurements against a running app and writes them to a gitignored `.dev/perf/<label>.json`. It wraps every bound Go method, so "did that refetch the library" is a fact rather than an inference, and records `longtask` entries, which is where a 25 MB JSON parse on the main thread shows up and nowhere else. It is not a spec and does not run in CI.
2026-08-12 01:17:33 -04:00
parent 7de1b4edc1
commit da564f9659
7 changed files with 2592 additions and 17 deletions
@@ -161,7 +161,19 @@ func requireFFmpeg() error {
 // the library's cover-art deduplication is supposed to collapse into a
 // single stored blob.
 func coverJPEG(key string) ([]byte, error) {
-	img := image.NewRGBA(image.Rect(0, 0, coverSizePx, coverSizePx))
+	return coverJPEGSized(key, coverSizePx)
+}
+
+// coverJPEGSized is coverJPEG at an explicit edge length.  The bulk
+// library uses a larger one, because a 64 px cover cannot show the
+// difference between rendering the original artwork and rendering the
+// thumbnail tier that exists for the purpose.
+// coverJPEGSized is coverJPEG at an explicit edge length.  The bulk
+// library uses a larger one, because a 64 px cover cannot show the
+// difference between rendering the original artwork and rendering the
+// thumbnail tier that exists for the purpose.
+func coverJPEGSized(key string, px int) ([]byte, error) {
+	img := image.NewRGBA(image.Rect(0, 0, px, px))

 	// A per-key hue derived from the key's bytes, plus a diagonal
 	// band, so covers are distinguishable by eye in a screenshot.
@@ -177,8 +189,8 @@ func coverJPEG(key string) ([]byte, error) {
 		A: 255,
 	}

-	for y := range coverSizePx {
-		for x := range coverSizePx {
+	for y := range px {
+		for x := range px {
 			c := base
 			if (x+y)%16 < 8 {
 				c.R /= 2
@@ -0,0 +1,462 @@
+package main
+
+// The bulk library exists for one purpose: measuring.  Plan 007
+// phase 4 is verified by numbers rather than assertions, and every
+// number in `.planning/audits/2026-08-11-ui/perf.md` is quoted "at
+// 50 000 tracks" — a size the deterministic fixture library
+// (`libraryFixtures`, a few dozen files) cannot reach and should not
+// try to.  The two are generated by the same command because they are
+// the same problem at two scales, but they share nothing else: the
+// fixture library is a curated set of *cases* selected by name, and
+// this one is a shapeless pile whose only interesting property is how
+// big it is.
+//
+// It is deliberately not committed and deliberately not a dependency
+// of any test.  `make test` must not take four minutes because someone
+// wanted a scroll trace.
+//
+// Generation avoids ffmpeg per file.  Encoding 50 000 files one
+// process at a time is ~40 minutes; encoding a handful of source
+// clips once and copying them is ~20 seconds, and the audio content is
+// not what is being measured.  Tags still go through backend/tagwriter
+// for the same reason the fixture library does — a library the app
+// cannot read back tells you nothing about the app.
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"image"
+	"image/color"
+	"image/jpeg"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"yellowjacket/backend/tagwriter"
+)
+
+// Shape of the generated library.  These ratios are roughly what a
+// real 50 k-track collection looks like, and they matter: several of
+// the findings being measured are O(albums) or O(artists) rather than
+// O(tracks), so a library of 50 000 tracks in one album would measure
+// nothing.
+const (
+	bulkTracksPerAlbum  = 10
+	bulkAlbumsPerArtist = 4
+	bulkSourceVariants  = 6
+	bulkCoverPx         = 300
+	bulkYearBase        = 1968
+	bulkYearSpan        = 56
+)
+
+// bulkGenres is small on purpose — a real library has a long tail of
+// genres but a short head, and the head is what the genres view and
+// the home shelves actually work over.
+//
+//nolint:gochecknoglobals // a package-level constant table.
+var bulkGenres = []string{
+	"Ambient", "Post-Rock", "Dream Pop", "Shoegaze", "Krautrock",
+	"Minimal Techno", "Free Jazz", "Bossa Nova", "Baroque", "Drone",
+	"Noise Rock", "Slowcore", "Trip Hop", "Dub", "Highlife",
+	"Cumbia", "Fado", "Gamelan", "Chamber Pop", "Math Rock",
+	"Field Recording", "Musique Concrète", "Sludge", "Zeuhl",
+}
+
+// Word tables composing artist and album names.  Names are generated
+// rather than numbered so search ranking has something with shared
+// prefixes, shared words and varying lengths to rank — `Artist 04213`
+// would make every query either match everything or nothing.
+//
+//nolint:gochecknoglobals // package-level constant tables.
+var (
+	bulkAdjectives = []string{
+		"Hollow", "Northern", "Quiet", "Endless", "Amber", "Glass",
+		"Distant", "Velvet", "Iron", "Pale", "Golden", "Silent",
+		"Crimson", "Winter", "Coastal", "Electric", "Marble", "Slow",
+	}
+	bulkNouns = []string{
+		"Harbour", "Tideline", "Cartography", "Signal", "Meridian",
+		"Aviary", "Lantern", "Orchard", "Pavilion", "Estuary",
+		"Sequence", "Almanac", "Corridor", "Foundry", "Cascade",
+		"Interval", "Radiance", "Threshold", "Migration", "Beacon",
+	}
+	bulkCollectives = []string{
+		"Ensemble", "Trio", "Quartet", "Society", "Orchestra",
+		"Collective", "Choir", "Union", "Company", "Band",
+	}
+)
+
+// bulkSpec is a fully-resolved request for a bulk library.
+type bulkSpec struct {
+	Out     string
+	Tracks  int
+	CoverPx int
+}
+
+// hash identifies the spec, so regeneration can be skipped the same
+// way the fixture library skips it.
+//
+// The leading version string covers everything about *generation* that
+// the parameters do not — the naming tables, the cover renderer, the
+// source-clip durations.  Bump it when any of those change, or a stale
+// library on disk will be silently accepted as current.
+func (s bulkSpec) hash() string {
+	sum := sha256.Sum256([]byte(fmt.Sprintf(
+		"bulk/v2|%d|%d|%d|%d|%d",
+		s.Tracks, s.CoverPx,
+		bulkTracksPerAlbum, bulkAlbumsPerArtist, bulkSourceVariants,
+	)))
+
+	return hex.EncodeToString(sum[:])
+}
+
+// bulkTrack is one generated file, resolved from its index alone.
+// Everything is a pure function of the index, so generation
+// parallelises without coordination and two runs agree exactly.
+type bulkTrack struct {
+	Rel     string
+	Variant int
+	Album   int
+	Tags    tags
+}
+
+// bulkTrackAt derives track i's identity.
+func bulkTrackAt(i int) bulkTrack {
+	album := i / bulkTracksPerAlbum
+	artist := album / bulkAlbumsPerArtist
+	trackNo := i%bulkTracksPerAlbum + 1
+
+	artistName := bulkArtistName(artist)
+	albumName := bulkAlbumName(album)
+	title := bulkTitle(i)
+
+	return bulkTrack{
+		Rel: filepath.Join(
+			sanitizePathSegment(artistName),
+			sanitizePathSegment(albumName),
+			fmt.Sprintf("%02d %s.mp3", trackNo, sanitizePathSegment(title)),
+		),
+		Variant: i % bulkSourceVariants,
+		Album:   album,
+		Tags: tags{
+			Title:       title,
+			Artist:      artistName,
+			Album:       albumName,
+			AlbumArtist: artistName,
+			Genre:       bulkGenres[album%len(bulkGenres)],
+			Year:        bulkYearBase + album%bulkYearSpan,
+			TrackNumber: trackNo,
+			DiscNumber:  1,
+		},
+	}
+}
+
+func bulkArtistName(n int) string {
+	switch n % 3 {
+	case 0:
+		return bulkAdjectives[n%len(bulkAdjectives)] + " " +
+			bulkNouns[(n/len(bulkAdjectives))%len(bulkNouns)]
+	case 1:
+		return "The " + bulkNouns[n%len(bulkNouns)] + " " +
+			bulkCollectives[(n/len(bulkNouns))%len(bulkCollectives)]
+	default:
+		return bulkNouns[n%len(bulkNouns)] + " & the " +
+			bulkAdjectives[(n/len(bulkNouns))%len(bulkAdjectives)] + "s"
+	}
+}
+
+func bulkAlbumName(n int) string {
+	return bulkAdjectives[(n*7)%len(bulkAdjectives)] + " " +
+		bulkNouns[(n*13)%len(bulkNouns)] +
+		" " + strconv.Itoa(n%97)
+}
+
+func bulkTitle(n int) string {
+	return bulkNouns[(n*3)%len(bulkNouns)] + " " +
+		bulkAdjectives[(n*11)%len(bulkAdjectives)] +
+		" " + strconv.Itoa(n%211)
+}
+
+// sanitizePathSegment keeps generated names usable as path components
+// on every platform the app builds for.
+func sanitizePathSegment(s string) string {
+	const bad = `/\:*?"<>|`
+
+	out := []rune(s)
+	for i, r := range out {
+		for _, b := range bad {
+			if r == b {
+				out[i] = '_'
+			}
+		}
+	}
+
+	return string(out)
+}
+
+// generateBulk builds (or skips) a bulk library.
+func generateBulk(spec bulkSpec, force bool) error {
+	manifestPath := spec.Out + ".manifest.json"
+
+	if !force && readManifestHash(manifestPath) == spec.hash() {
+		if _, err := os.Stat(spec.Out); err == nil {
+			fmt.Printf(
+				"bulk: up to date (%d tracks, hash %s)\n",
+				spec.Tracks, spec.hash()[:12],
+			)
+
+			return nil
+		}
+	}
+
+	if err := requireFFmpeg(); err != nil {
+		return err
+	}
+
+	if err := os.RemoveAll(spec.Out); err != nil {
+		return fmt.Errorf("clean %s: %w", spec.Out, err)
+	}
+
+	started := time.Now()
+
+	sources, cleanup, err := bulkSources(spec.Out)
+	if err != nil {
+		return err
+	}
+	defer cleanup()
+
+	covers, err := bulkCovers(spec, sources)
+	if err != nil {
+		return err
+	}
+
+	if err := bulkWrite(spec, sources, covers); err != nil {
+		return err
+	}
+
+	if err := writeManifest(manifestPath, &manifest{
+		Version:     manifestVersion,
+		Generator:   "gentestdata -bulk",
+		Hash:        spec.hash(),
+		LibraryRoot: filepath.ToSlash(spec.Out),
+		TrackCount:  spec.Tracks,
+	}); err != nil {
+		return err
+	}
+
+	fmt.Printf(
+		"bulk: generated %d tracks in %s (%s, hash %s)\n",
+		spec.Tracks, spec.Out,
+		time.Since(started).Round(time.Second),
+		spec.hash()[:12],
+	)
+
+	return nil
+}
+
+// bulkSources encodes the handful of distinct clips every generated
+// file is copied from.  Durations vary so "time between tracks" is not
+// measuring one number repeatedly, and stay short so a measurement run
+// that plays through a dozen tracks takes seconds.
+func bulkSources(out string) ([][]byte, func(), error) {
+	tmp, err := os.MkdirTemp("", "yj-bulk-src-")
+	if err != nil {
+		return nil, nil, fmt.Errorf("temp dir: %w", err)
+	}
+
+	cleanup := func() { _ = os.RemoveAll(tmp) }
+
+	if err := os.MkdirAll(out, dirPerm); err != nil {
+		cleanup()
+
+		return nil, nil, fmt.Errorf("mkdir %s: %w", out, err)
+	}
+
+	sources := make([][]byte, bulkSourceVariants)
+
+	for i := range bulkSourceVariants {
+		wav := filepath.Join(tmp, strconv.Itoa(i)+".wav")
+		mp3 := filepath.Join(tmp, strconv.Itoa(i)+".mp3")
+		dur := time.Duration(2+i%3) * time.Second
+
+		if err := synthesizeWAV(wav, dur, 220+float64(i)*55); err != nil {
+			cleanup()
+
+			return nil, nil, err
+		}
+
+		if err := transcode(wav, mp3, tagwriter.FormatMP3); err != nil {
+			cleanup()
+
+			return nil, nil, err
+		}
+
+		raw, err := os.ReadFile(mp3) //nolint:gosec // generated path.
+		if err != nil {
+			cleanup()
+
+			return nil, nil, fmt.Errorf("read source clip: %w", err)
+		}
+
+		sources[i] = raw
+	}
+
+	return sources, cleanup, nil
+}
+
+// bulkCovers renders one cover per album up front.  Rendering it per
+// track would be the dominant cost of generation and would defeat the
+// library's cover-art deduplication, which is part of what a scan at
+// this size is being measured on.
+func bulkCovers(spec bulkSpec, _ [][]byte) ([][]byte, error) {
+	albums := (spec.Tracks + bulkTracksPerAlbum - 1) / bulkTracksPerAlbum
+	covers := make([][]byte, albums)
+
+	for i := range albums {
+		img, err := bulkCoverJPEG(bulkAlbumName(i), spec.CoverPx)
+		if err != nil {
+			return nil, err
+		}
+
+		covers[i] = img
+	}
+
+	return covers, nil
+}
+
+// bulkCoverJPEG renders a bulk album cover.
+//
+// Deliberately not coverJPEGSized: the fixture cover's diagonal band
+// is 37 hard edges at 300 px, which is the worst case for a DCT and
+// costs ~35 kB per album — 2 GB across a 50 k-track library, most of
+// it JPEG artefacts around a pattern nobody looks at.  A smooth
+// two-axis gradient is ~6 kB, still distinct per album by eye, and
+// still deterministic per key.
+func bulkCoverJPEG(key string, px int) ([]byte, error) {
+	var seed uint32
+	for _, b := range []byte(key) {
+		seed = seed*31 + uint32(b)
+	}
+
+	img := image.NewRGBA(image.Rect(0, 0, px, px))
+	base := color.RGBA{
+		R: uint8(seed >> 16),
+		G: uint8(seed >> 8),
+		B: uint8(seed),
+		A: 255,
+	}
+
+	for y := range px {
+		shade := float64(y) / float64(px)
+
+		for x := range px {
+			tint := (shade + float64(x)/float64(px)) / 2
+
+			img.Set(x, y, color.RGBA{
+				R: uint8(float64(base.R) * (0.4 + 0.6*tint)),
+				G: uint8(float64(base.G) * (0.4 + 0.6*tint)),
+				B: uint8(float64(base.B) * (0.4 + 0.6*tint)),
+				A: 255,
+			})
+		}
+	}
+
+	var buf bytes.Buffer
+
+	if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: jpegQuality}); err != nil {
+		return nil, fmt.Errorf("encode bulk cover %q: %w", key, err)
+	}
+
+	return buf.Bytes(), nil
+}
+
+// bulkWrite copies and tags every file, in parallel.  tagwriter works
+// on one path at a time with no shared state, so the only coordination
+// needed is the error and the progress counter.
+func bulkWrite(spec bulkSpec, sources, covers [][]byte) error {
+	logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
+		Level: slog.LevelError,
+	}))
+
+	jobs := make(chan int, runtime.NumCPU()*2)
+
+	var (
+		wg      sync.WaitGroup
+		done    atomic.Int64
+		failed  atomic.Pointer[error]
+		workers = runtime.NumCPU()
+	)
+
+	for range workers {
+		wg.Add(1)
+
+		go func() {
+			defer wg.Done()
+
+			for i := range jobs {
+				if failed.Load() != nil {
+					return
+				}
+
+				if err := bulkWriteOne(
+					logger, spec, sources, covers, i,
+				); err != nil {
+					failed.CompareAndSwap(nil, &err)
+
+					return
+				}
+
+				if n := done.Add(1); n%5000 == 0 {
+					fmt.Printf("bulk: %d/%d\n", n, spec.Tracks)
+				}
+			}
+		}()
+	}
+
+	for i := range spec.Tracks {
+		jobs <- i
+	}
+
+	close(jobs)
+	wg.Wait()
+
+	if err := failed.Load(); err != nil {
+		return *err
+	}
+
+	return nil
+}
+
+func bulkWriteOne(
+	logger *slog.Logger,
+	spec bulkSpec,
+	sources, covers [][]byte,
+	i int,
+) error {
+	t := bulkTrackAt(i)
+	dst := filepath.Join(spec.Out, t.Rel)
+
+	if err := ensureDir(dst); err != nil {
+		return err
+	}
+
+	if err := os.WriteFile(dst, sources[t.Variant], filePerm); err != nil {
+		return fmt.Errorf("write %s: %w", dst, err)
+	}
+
+	changes := t.Tags.changes()
+	changes[tagwriter.FieldCoverArt] = covers[t.Album]
+
+	if err := tagwriter.WriteFileTags(logger, dst, changes); err != nil {
+		return fmt.Errorf("tag %s: %w", t.Rel, err)
+	}
+
+	return stampMTime(dst)
+}
@@ -13,6 +13,11 @@
 //
 //	go run ./cmd/gentestdata            # generate if out of date
 //	go run ./cmd/gentestdata -force     # regenerate unconditionally
+//	go run ./cmd/gentestdata -bulk 50000  # the measurement library
+//
+// The -bulk library is a separate thing with a separate purpose; see
+// bulk.go.  It is not committed, not a test dependency, and generating
+// it does not regenerate the fixture library.
 package main

 import (
@@ -44,6 +49,9 @@ func main() {
 		brokenDir   string
 		manifestOut string
 		force       bool
+		bulkTracks  int
+		bulkOut     string
+		bulkCover   int
 	)

 	flag.StringVar(
@@ -62,9 +70,33 @@ func main() {
 		&force, "force", false,
 		"regenerate even when the manifest is already up to date",
 	)
+	flag.IntVar(
+		&bulkTracks, "bulk", 0,
+		"generate a bulk measurement library of N tracks instead",
+	)
+	flag.StringVar(
+		&bulkOut, "bulk-out", ".dev/music_library_bulk",
+		"library root for -bulk (gitignored; not a test fixture)",
+	)
+	flag.IntVar(
+		&bulkCover, "bulk-cover-px", bulkCoverPx,
+		"edge length of the embedded cover art for -bulk",
+	)
 	flag.Parse()

-	if err := run(outDir, brokenDir, manifestOut, force); err != nil {
+	var err error
+
+	if bulkTracks > 0 {
+		err = generateBulk(bulkSpec{
+			Out:     bulkOut,
+			Tracks:  bulkTracks,
+			CoverPx: bulkCover,
+		}, force)
+	} else {
+		err = run(outDir, brokenDir, manifestOut, force)
+	}
+
+	if err != nil {
 		fmt.Fprintln(os.Stderr, "gentestdata:", err)
 		os.Exit(1)
 	}
@@ -34,15 +34,19 @@ type manifestTrack struct {
 // stamps its own encoder strings, so byte hashes differ between ffmpeg
 // builds while the fixtures they describe are identical.
 type manifest struct {
-	Version     int                 `json:"version"`
-	Generator   string              `json:"generator"`
-	Hash        string              `json:"hash"`
-	LibraryRoot string              `json:"libraryRoot"`
-	BrokenRoot  string              `json:"brokenRoot"`
-	Cases       map[string][]string `json:"cases"`
-	Tracks      []manifestTrack     `json:"tracks"`
-	Extras      []string            `json:"extras"`
-	Broken      []string            `json:"broken"`
+	Version     int    `json:"version"`
+	Generator   string `json:"generator"`
+	Hash        string `json:"hash"`
+	LibraryRoot string `json:"libraryRoot"`
+	BrokenRoot  string `json:"brokenRoot"`
+	// TrackCount is written by the bulk library, which has tens of
+	// thousands of tracks and no reason to describe each one: nothing
+	// selects a bulk track by name, only the total matters.
+	TrackCount int                 `json:"trackCount,omitempty"`
+	Cases      map[string][]string `json:"cases"`
+	Tracks     []manifestTrack     `json:"tracks"`
+	Extras     []string            `json:"extras"`
+	Broken     []string            `json:"broken"`
 }

 // buildManifest derives the manifest from the spec alone.  It runs