yellowjacket/cmd/gentestdata/bulk.go

package main

// The bulk library exists for one purpose: measuring.  Plan 007
// phase 4 is verified by numbers rather than assertions, and every
// number in `.planning/audits/2026-08-11-ui/perf.md` is quoted "at
// 50 000 tracks" — a size the deterministic fixture library
// (`libraryFixtures`, a few dozen files) cannot reach and should not
// try to.  The two are generated by the same command because they are
// the same problem at two scales, but they share nothing else: the
// fixture library is a curated set of *cases* selected by name, and
// this one is a shapeless pile whose only interesting property is how
// big it is.
//
// It is deliberately not committed and deliberately not a dependency
// of any test.  `make test` must not take four minutes because someone
// wanted a scroll trace.
//
// Generation avoids ffmpeg per file.  Encoding 50 000 files one
// process at a time is ~40 minutes; encoding a handful of source
// clips once and copying them is ~20 seconds, and the audio content is
// not what is being measured.  Tags still go through backend/tagwriter
// for the same reason the fixture library does — a library the app
// cannot read back tells you nothing about the app.

import (
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"image"
	"image/color"
	"image/jpeg"
	"log/slog"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"sync"
	"sync/atomic"
	"time"

	"yellowjacket/backend/tagwriter"
)

// Shape of the generated library.  These ratios are roughly what a
// real 50 k-track collection looks like, and they matter: several of
// the findings being measured are O(albums) or O(artists) rather than
// O(tracks), so a library of 50 000 tracks in one album would measure
// nothing.
const (
	bulkTracksPerAlbum  = 10
	bulkAlbumsPerArtist = 4
	bulkSourceVariants  = 6
	bulkCoverPx         = 300
	bulkYearBase        = 1968
	bulkYearSpan        = 56
)

// bulkGenres is small on purpose — a real library has a long tail of
// genres but a short head, and the head is what the genres view and
// the home shelves actually work over.
//
//nolint:gochecknoglobals // a package-level constant table.
var bulkGenres = []string{
	"Ambient", "Post-Rock", "Dream Pop", "Shoegaze", "Krautrock",
	"Minimal Techno", "Free Jazz", "Bossa Nova", "Baroque", "Drone",
	"Noise Rock", "Slowcore", "Trip Hop", "Dub", "Highlife",
	"Cumbia", "Fado", "Gamelan", "Chamber Pop", "Math Rock",
	"Field Recording", "Musique Concrète", "Sludge", "Zeuhl",
}

// Word tables composing artist and album names.  Names are generated
// rather than numbered so search ranking has something with shared
// prefixes, shared words and varying lengths to rank — `Artist 04213`
// would make every query either match everything or nothing.
//
//nolint:gochecknoglobals // package-level constant tables.
var (
	bulkAdjectives = []string{
		"Hollow", "Northern", "Quiet", "Endless", "Amber", "Glass",
		"Distant", "Velvet", "Iron", "Pale", "Golden", "Silent",
		"Crimson", "Winter", "Coastal", "Electric", "Marble", "Slow",
	}
	bulkNouns = []string{
		"Harbour", "Tideline", "Cartography", "Signal", "Meridian",
		"Aviary", "Lantern", "Orchard", "Pavilion", "Estuary",
		"Sequence", "Almanac", "Corridor", "Foundry", "Cascade",
		"Interval", "Radiance", "Threshold", "Migration", "Beacon",
	}
	bulkCollectives = []string{
		"Ensemble", "Trio", "Quartet", "Society", "Orchestra",
		"Collective", "Choir", "Union", "Company", "Band",
	}
)

// bulkSpec is a fully-resolved request for a bulk library.
type bulkSpec struct {
	Out     string
	Tracks  int
	CoverPx int
}

// hash identifies the spec, so regeneration can be skipped the same
// way the fixture library skips it.
//
// The leading version string covers everything about *generation* that
// the parameters do not — the naming tables, the cover renderer, the
// source-clip durations.  Bump it when any of those change, or a stale
// library on disk will be silently accepted as current.
func (s bulkSpec) hash() string {
	sum := sha256.Sum256([]byte(fmt.Sprintf(
		"bulk/v2|%d|%d|%d|%d|%d",
		s.Tracks, s.CoverPx,
		bulkTracksPerAlbum, bulkAlbumsPerArtist, bulkSourceVariants,
	)))

	return hex.EncodeToString(sum[:])
}

// bulkTrack is one generated file, resolved from its index alone.
// Everything is a pure function of the index, so generation
// parallelises without coordination and two runs agree exactly.
type bulkTrack struct {
	Rel     string
	Variant int
	Album   int
	Tags    tags
}

// bulkTrackAt derives track i's identity.
func bulkTrackAt(i int) bulkTrack {
	album := i / bulkTracksPerAlbum
	artist := album / bulkAlbumsPerArtist
	trackNo := i%bulkTracksPerAlbum + 1

	artistName := bulkArtistName(artist)
	albumName := bulkAlbumName(album)
	title := bulkTitle(i)

	return bulkTrack{
		Rel: filepath.Join(
			sanitizePathSegment(artistName),
			sanitizePathSegment(albumName),
			fmt.Sprintf("%02d %s.mp3", trackNo, sanitizePathSegment(title)),
		),
		Variant: i % bulkSourceVariants,
		Album:   album,
		Tags: tags{
			Title:       title,
			Artist:      artistName,
			Album:       albumName,
			AlbumArtist: artistName,
			Genre:       bulkGenres[album%len(bulkGenres)],
			Year:        bulkYearBase + album%bulkYearSpan,
			TrackNumber: trackNo,
			DiscNumber:  1,
		},
	}
}

func bulkArtistName(n int) string {
	switch n % 3 {
	case 0:
		return bulkAdjectives[n%len(bulkAdjectives)] + " " +
			bulkNouns[(n/len(bulkAdjectives))%len(bulkNouns)]
	case 1:
		return "The " + bulkNouns[n%len(bulkNouns)] + " " +
			bulkCollectives[(n/len(bulkNouns))%len(bulkCollectives)]
	default:
		return bulkNouns[n%len(bulkNouns)] + " & the " +
			bulkAdjectives[(n/len(bulkNouns))%len(bulkAdjectives)] + "s"
	}
}

func bulkAlbumName(n int) string {
	return bulkAdjectives[(n*7)%len(bulkAdjectives)] + " " +
		bulkNouns[(n*13)%len(bulkNouns)] +
		" " + strconv.Itoa(n%97)
}

func bulkTitle(n int) string {
	return bulkNouns[(n*3)%len(bulkNouns)] + " " +
		bulkAdjectives[(n*11)%len(bulkAdjectives)] +
		" " + strconv.Itoa(n%211)
}

// sanitizePathSegment keeps generated names usable as path components
// on every platform the app builds for.
func sanitizePathSegment(s string) string {
	const bad = `/\:*?"<>|`

	out := []rune(s)
	for i, r := range out {
		for _, b := range bad {
			if r == b {
				out[i] = '_'
			}
		}
	}

	return string(out)
}

// generateBulk builds (or skips) a bulk library.
func generateBulk(spec bulkSpec, force bool) error {
	manifestPath := spec.Out + ".manifest.json"

	if !force && readManifestHash(manifestPath) == spec.hash() {
		if _, err := os.Stat(spec.Out); err == nil {
			fmt.Printf(
				"bulk: up to date (%d tracks, hash %s)\n",
				spec.Tracks, spec.hash()[:12],
			)

			return nil
		}
	}

	if err := requireFFmpeg(); err != nil {
		return err
	}

	if err := os.RemoveAll(spec.Out); err != nil {
		return fmt.Errorf("clean %s: %w", spec.Out, err)
	}

	started := time.Now()

	sources, cleanup, err := bulkSources(spec.Out)
	if err != nil {
		return err
	}
	defer cleanup()

	covers, err := bulkCovers(spec, sources)
	if err != nil {
		return err
	}

	if err := bulkWrite(spec, sources, covers); err != nil {
		return err
	}

	if err := writeManifest(manifestPath, &manifest{
		Version:     manifestVersion,
		Generator:   "gentestdata -bulk",
		Hash:        spec.hash(),
		LibraryRoot: filepath.ToSlash(spec.Out),
		TrackCount:  spec.Tracks,
	}); err != nil {
		return err
	}

	fmt.Printf(
		"bulk: generated %d tracks in %s (%s, hash %s)\n",
		spec.Tracks, spec.Out,
		time.Since(started).Round(time.Second),
		spec.hash()[:12],
	)

	return nil
}

// bulkSources encodes the handful of distinct clips every generated
// file is copied from.  Durations vary so "time between tracks" is not
// measuring one number repeatedly, and stay short so a measurement run
// that plays through a dozen tracks takes seconds.
func bulkSources(out string) ([][]byte, func(), error) {
	tmp, err := os.MkdirTemp("", "yj-bulk-src-")
	if err != nil {
		return nil, nil, fmt.Errorf("temp dir: %w", err)
	}

	cleanup := func() { _ = os.RemoveAll(tmp) }

	if err := os.MkdirAll(out, dirPerm); err != nil {
		cleanup()

		return nil, nil, fmt.Errorf("mkdir %s: %w", out, err)
	}

	sources := make([][]byte, bulkSourceVariants)

	for i := range bulkSourceVariants {
		wav := filepath.Join(tmp, strconv.Itoa(i)+".wav")
		mp3 := filepath.Join(tmp, strconv.Itoa(i)+".mp3")
		dur := time.Duration(2+i%3) * time.Second

		if err := synthesizeWAV(wav, dur, 220+float64(i)*55); err != nil {
			cleanup()

			return nil, nil, err
		}

		if err := transcode(wav, mp3, tagwriter.FormatMP3); err != nil {
			cleanup()

			return nil, nil, err
		}

		raw, err := os.ReadFile(mp3) //nolint:gosec // generated path.
		if err != nil {
			cleanup()

			return nil, nil, fmt.Errorf("read source clip: %w", err)
		}

		sources[i] = raw
	}

	return sources, cleanup, nil
}

// bulkCovers renders one cover per album up front.  Rendering it per
// track would be the dominant cost of generation and would defeat the
// library's cover-art deduplication, which is part of what a scan at
// this size is being measured on.
func bulkCovers(spec bulkSpec, _ [][]byte) ([][]byte, error) {
	albums := (spec.Tracks + bulkTracksPerAlbum - 1) / bulkTracksPerAlbum
	covers := make([][]byte, albums)

	for i := range albums {
		img, err := bulkCoverJPEG(bulkAlbumName(i), spec.CoverPx)
		if err != nil {
			return nil, err
		}

		covers[i] = img
	}

	return covers, nil
}

// bulkCoverJPEG renders a bulk album cover.
//
// Deliberately not coverJPEGSized: the fixture cover's diagonal band
// is 37 hard edges at 300 px, which is the worst case for a DCT and
// costs ~35 kB per album — 2 GB across a 50 k-track library, most of
// it JPEG artefacts around a pattern nobody looks at.  A smooth
// two-axis gradient is ~6 kB, still distinct per album by eye, and
// still deterministic per key.
func bulkCoverJPEG(key string, px int) ([]byte, error) {
	var seed uint32
	for _, b := range []byte(key) {
		seed = seed*31 + uint32(b)
	}

	img := image.NewRGBA(image.Rect(0, 0, px, px))
	base := color.RGBA{
		R: uint8(seed >> 16),
		G: uint8(seed >> 8),
		B: uint8(seed),
		A: 255,
	}

	for y := range px {
		shade := float64(y) / float64(px)

		for x := range px {
			tint := (shade + float64(x)/float64(px)) / 2

			img.Set(x, y, color.RGBA{
				R: uint8(float64(base.R) * (0.4 + 0.6*tint)),
				G: uint8(float64(base.G) * (0.4 + 0.6*tint)),
				B: uint8(float64(base.B) * (0.4 + 0.6*tint)),
				A: 255,
			})
		}
	}

	var buf bytes.Buffer

	if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: jpegQuality}); err != nil {
		return nil, fmt.Errorf("encode bulk cover %q: %w", key, err)
	}

	return buf.Bytes(), nil
}

// bulkWrite copies and tags every file, in parallel.  tagwriter works
// on one path at a time with no shared state, so the only coordination
// needed is the error and the progress counter.
func bulkWrite(spec bulkSpec, sources, covers [][]byte) error {
	logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
		Level: slog.LevelError,
	}))

	jobs := make(chan int, runtime.NumCPU()*2)

	var (
		wg      sync.WaitGroup
		done    atomic.Int64
		failed  atomic.Pointer[error]
		workers = runtime.NumCPU()
	)

	for range workers {
		wg.Add(1)

		go func() {
			defer wg.Done()

			for i := range jobs {
				if failed.Load() != nil {
					return
				}

				if err := bulkWriteOne(
					logger, spec, sources, covers, i,
				); err != nil {
					failed.CompareAndSwap(nil, &err)

					return
				}

				if n := done.Add(1); n%5000 == 0 {
					fmt.Printf("bulk: %d/%d\n", n, spec.Tracks)
				}
			}
		}()
	}

	for i := range spec.Tracks {
		jobs <- i
	}

	close(jobs)
	wg.Wait()

	if err := failed.Load(); err != nil {
		return *err
	}

	return nil
}

func bulkWriteOne(
	logger *slog.Logger,
	spec bulkSpec,
	sources, covers [][]byte,
	i int,
) error {
	t := bulkTrackAt(i)
	dst := filepath.Join(spec.Out, t.Rel)

	if err := ensureDir(dst); err != nil {
		return err
	}

	if err := os.WriteFile(dst, sources[t.Variant], filePerm); err != nil {
		return fmt.Errorf("write %s: %w", dst, err)
	}

	changes := t.Tags.changes()
	changes[tagwriter.FieldCoverArt] = covers[t.Album]

	if err := tagwriter.WriteFileTags(logger, dst, changes); err != nil {
		return fmt.Errorf("tag %s: %w", t.Rel, err)
	}

	return stampMTime(dst)
}