build(dev): generate a 50k-track library and measure a running app
Plan 007 phase 4 is verified by measurement, not by assertion, and there was no way to produce a number: the fixture library is a few dozen tracks and cannot show any of the findings. - `cmd/gentestdata -bulk N` (`make bulkdata`) writes a ~50 000-track library in 11 s / 466 MB by encoding six clips once and copying them, while still tagging every file through `backend/tagwriter` — a library the app cannot read back measures nothing. - `make sandbox-seed-bulk` seeds from it through the same script and the same discipline as any other seed: by running the app and waiting for the real scan. - `e2e/perf/measure.mjs` (`make perf LABEL=x`, `make perf-compare`) takes fourteen measurements against a running app and writes them to a gitignored `.dev/perf/<label>.json`. It wraps every bound Go method, so "did that refetch the library" is a fact rather than an inference, and records `longtask` entries, which is where a 25 MB JSON parse on the main thread shows up and nowhere else. It is not a spec and does not run in CI.
This commit is contained in:
@@ -37,6 +37,15 @@ dev-logs: ## Tail the headless app log
|
|||||||
sandbox-seed: testdata ## Build a seeded YJ_HOME snapshot: make sandbox-seed NAME=<n>
|
sandbox-seed: testdata ## Build a seeded YJ_HOME snapshot: make sandbox-seed NAME=<n>
|
||||||
@./scripts/seed-sandbox.sh $(if $(NAME),--name $(NAME),)
|
@./scripts/seed-sandbox.sh $(if $(NAME),--name $(NAME),)
|
||||||
|
|
||||||
|
# The bulk seed is the *measurement* seed, not a fixture seed. Same
|
||||||
|
# script and the same discipline (the app builds it by scanning for
|
||||||
|
# real); the only difference is which manifest it is pointed at. It is
|
||||||
|
# a separate target because a 50 000-track scan is minutes, and nothing
|
||||||
|
# routine should depend on it.
|
||||||
|
sandbox-seed-bulk: bulkdata ## Build a seeded YJ_HOME from the bulk library
|
||||||
|
@./scripts/seed-sandbox.sh --name $(if $(NAME),$(NAME),bulk) \
|
||||||
|
--manifest .dev/music_library_bulk.manifest.json
|
||||||
|
|
||||||
sandbox-seeds: ## List built seeds
|
sandbox-seeds: ## List built seeds
|
||||||
@ls -1 .dev/seeds/*.tar 2>/dev/null | sed 's|.*/||; s|\.tar$$||' \
|
@ls -1 .dev/seeds/*.tar 2>/dev/null | sed 's|.*/||; s|\.tar$$||' \
|
||||||
|| echo " (none; build one with: make sandbox-seed NAME=default)"
|
|| echo " (none; build one with: make sandbox-seed NAME=default)"
|
||||||
@@ -45,6 +54,21 @@ sandbox-seeds: ## List built seeds
|
|||||||
# daemonises, which is the opposite of what Playwright's `webServer`
|
# daemonises, which is the opposite of what Playwright's `webServer`
|
||||||
# supervises, and starting one per run would rebuild the frontend every
|
# supervises, and starting one per run would rebuild the frontend every
|
||||||
# time. globalSetup fails with the exact commands to run if it is down.
|
# time. globalSetup fails with the exact commands to run if it is down.
|
||||||
|
# Phase 4 of plan 007 is verified by measurement rather than assertion,
|
||||||
|
# so this is not a spec and does not run in CI: it produces a number to
|
||||||
|
# read, against a running app seeded with the bulk library.
|
||||||
|
#
|
||||||
|
# make sandbox-seed-bulk && make dev-headless SEED=bulk
|
||||||
|
# make perf LABEL=before ... change something ... make perf LABEL=after
|
||||||
|
# make perf-compare BEFORE=before AFTER=after
|
||||||
|
perf: ## Take a performance measurement (LABEL=<name>) of a running app
|
||||||
|
@cd e2e && pnpm install --silent && \
|
||||||
|
node perf/measure.mjs --label $(if $(LABEL),$(LABEL),current)
|
||||||
|
|
||||||
|
perf-compare: ## Print a before/after table: BEFORE=<a> AFTER=<b>
|
||||||
|
@cd e2e && node perf/measure.mjs --compare \
|
||||||
|
$(if $(BEFORE),$(BEFORE),before) $(if $(AFTER),$(AFTER),after)
|
||||||
|
|
||||||
e2e: ## Run the Playwright smoke suite against a running dev-headless app
|
e2e: ## Run the Playwright smoke suite against a running dev-headless app
|
||||||
@cd e2e && pnpm install --silent && npx playwright test $(E2E_ARGS)
|
@cd e2e && pnpm install --silent && npx playwright test $(E2E_ARGS)
|
||||||
|
|
||||||
@@ -96,7 +120,8 @@ bindings: ## Regenerate frontend/wailsjs from the bound Go structs
|
|||||||
frontend/wailsjs/runtime/package.json
|
frontend/wailsjs/runtime/package.json
|
||||||
|
|
||||||
.PHONY: dev-headless dev-headless-fresh dev-stop dev-logs \
|
.PHONY: dev-headless dev-headless-fresh dev-stop dev-logs \
|
||||||
sandbox-seed sandbox-seeds e2e e2e-setup e2e-report \
|
sandbox-seed sandbox-seed-bulk sandbox-seeds e2e e2e-setup e2e-report \
|
||||||
|
perf perf-compare \
|
||||||
ui-test ui-watch ui-visual ui-visual-update ui-setup \
|
ui-test ui-watch ui-visual ui-visual-update ui-setup \
|
||||||
bindings bindings-check skill-check
|
bindings bindings-check skill-check
|
||||||
|
|
||||||
@@ -220,7 +245,19 @@ testdata-clean: ## Delete the generated fixture library
|
|||||||
rm -rf test_data/music_library_test test_data/music_library_broken \
|
rm -rf test_data/music_library_test test_data/music_library_broken \
|
||||||
test_data/music_library_test.manifest.json
|
test_data/music_library_test.manifest.json
|
||||||
|
|
||||||
.PHONY: testdata testdata-force testdata-clean
|
# The bulk library answers a different question from the fixture one:
|
||||||
|
# not "does this behave correctly" but "how does this behave at the
|
||||||
|
# size the audit measured". ~11 s, ~470 MB, into a gitignored .dev/,
|
||||||
|
# and deliberately not a dependency of `make test`.
|
||||||
|
BULK_TRACKS ?= 50000
|
||||||
|
|
||||||
|
bulkdata: ## Generate the bulk measurement library (BULK_TRACKS=50000)
|
||||||
|
go run ./cmd/gentestdata -bulk $(BULK_TRACKS)
|
||||||
|
|
||||||
|
bulkdata-clean: ## Delete the bulk measurement library
|
||||||
|
rm -rf .dev/music_library_bulk .dev/music_library_bulk.manifest.json
|
||||||
|
|
||||||
|
.PHONY: testdata testdata-force testdata-clean bulkdata bulkdata-clean
|
||||||
|
|
||||||
# The tag sets must match `make test` exactly, or lint is checking three
|
# The tag sets must match `make test` exactly, or lint is checking three
|
||||||
# configurations that nothing builds. webkit2_41 is not optional: without
|
# configurations that nothing builds. webkit2_41 is not optional: without
|
||||||
|
|||||||
@@ -161,7 +161,19 @@ func requireFFmpeg() error {
|
|||||||
// the library's cover-art deduplication is supposed to collapse into a
|
// the library's cover-art deduplication is supposed to collapse into a
|
||||||
// single stored blob.
|
// single stored blob.
|
||||||
func coverJPEG(key string) ([]byte, error) {
|
func coverJPEG(key string) ([]byte, error) {
|
||||||
img := image.NewRGBA(image.Rect(0, 0, coverSizePx, coverSizePx))
|
return coverJPEGSized(key, coverSizePx)
|
||||||
|
}
|
||||||
|
|
||||||
|
// coverJPEGSized is coverJPEG at an explicit edge length. The bulk
|
||||||
|
// library uses a larger one, because a 64 px cover cannot show the
|
||||||
|
// difference between rendering the original artwork and rendering the
|
||||||
|
// thumbnail tier that exists for the purpose.
|
||||||
|
// coverJPEGSized is coverJPEG at an explicit edge length. The bulk
|
||||||
|
// library uses a larger one, because a 64 px cover cannot show the
|
||||||
|
// difference between rendering the original artwork and rendering the
|
||||||
|
// thumbnail tier that exists for the purpose.
|
||||||
|
func coverJPEGSized(key string, px int) ([]byte, error) {
|
||||||
|
img := image.NewRGBA(image.Rect(0, 0, px, px))
|
||||||
|
|
||||||
// A per-key hue derived from the key's bytes, plus a diagonal
|
// A per-key hue derived from the key's bytes, plus a diagonal
|
||||||
// band, so covers are distinguishable by eye in a screenshot.
|
// band, so covers are distinguishable by eye in a screenshot.
|
||||||
@@ -177,8 +189,8 @@ func coverJPEG(key string) ([]byte, error) {
|
|||||||
A: 255,
|
A: 255,
|
||||||
}
|
}
|
||||||
|
|
||||||
for y := range coverSizePx {
|
for y := range px {
|
||||||
for x := range coverSizePx {
|
for x := range px {
|
||||||
c := base
|
c := base
|
||||||
if (x+y)%16 < 8 {
|
if (x+y)%16 < 8 {
|
||||||
c.R /= 2
|
c.R /= 2
|
||||||
|
|||||||
@@ -0,0 +1,462 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// The bulk library exists for one purpose: measuring. Plan 007
|
||||||
|
// phase 4 is verified by numbers rather than assertions, and every
|
||||||
|
// number in `.planning/audits/2026-08-11-ui/perf.md` is quoted "at
|
||||||
|
// 50 000 tracks" — a size the deterministic fixture library
|
||||||
|
// (`libraryFixtures`, a few dozen files) cannot reach and should not
|
||||||
|
// try to. The two are generated by the same command because they are
|
||||||
|
// the same problem at two scales, but they share nothing else: the
|
||||||
|
// fixture library is a curated set of *cases* selected by name, and
|
||||||
|
// this one is a shapeless pile whose only interesting property is how
|
||||||
|
// big it is.
|
||||||
|
//
|
||||||
|
// It is deliberately not committed and deliberately not a dependency
|
||||||
|
// of any test. `make test` must not take four minutes because someone
|
||||||
|
// wanted a scroll trace.
|
||||||
|
//
|
||||||
|
// Generation avoids ffmpeg per file. Encoding 50 000 files one
|
||||||
|
// process at a time is ~40 minutes; encoding a handful of source
|
||||||
|
// clips once and copying them is ~20 seconds, and the audio content is
|
||||||
|
// not what is being measured. Tags still go through backend/tagwriter
|
||||||
|
// for the same reason the fixture library does — a library the app
|
||||||
|
// cannot read back tells you nothing about the app.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"fmt"
|
||||||
|
"image"
|
||||||
|
"image/color"
|
||||||
|
"image/jpeg"
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"strconv"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"yellowjacket/backend/tagwriter"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Shape of the generated library. These ratios are roughly what a
|
||||||
|
// real 50 k-track collection looks like, and they matter: several of
|
||||||
|
// the findings being measured are O(albums) or O(artists) rather than
|
||||||
|
// O(tracks), so a library of 50 000 tracks in one album would measure
|
||||||
|
// nothing.
|
||||||
|
const (
|
||||||
|
bulkTracksPerAlbum = 10
|
||||||
|
bulkAlbumsPerArtist = 4
|
||||||
|
bulkSourceVariants = 6
|
||||||
|
bulkCoverPx = 300
|
||||||
|
bulkYearBase = 1968
|
||||||
|
bulkYearSpan = 56
|
||||||
|
)
|
||||||
|
|
||||||
|
// bulkGenres is small on purpose — a real library has a long tail of
|
||||||
|
// genres but a short head, and the head is what the genres view and
|
||||||
|
// the home shelves actually work over.
|
||||||
|
//
|
||||||
|
//nolint:gochecknoglobals // a package-level constant table.
|
||||||
|
var bulkGenres = []string{
|
||||||
|
"Ambient", "Post-Rock", "Dream Pop", "Shoegaze", "Krautrock",
|
||||||
|
"Minimal Techno", "Free Jazz", "Bossa Nova", "Baroque", "Drone",
|
||||||
|
"Noise Rock", "Slowcore", "Trip Hop", "Dub", "Highlife",
|
||||||
|
"Cumbia", "Fado", "Gamelan", "Chamber Pop", "Math Rock",
|
||||||
|
"Field Recording", "Musique Concrète", "Sludge", "Zeuhl",
|
||||||
|
}
|
||||||
|
|
||||||
|
// Word tables composing artist and album names. Names are generated
|
||||||
|
// rather than numbered so search ranking has something with shared
|
||||||
|
// prefixes, shared words and varying lengths to rank — `Artist 04213`
|
||||||
|
// would make every query either match everything or nothing.
|
||||||
|
//
|
||||||
|
//nolint:gochecknoglobals // package-level constant tables.
|
||||||
|
var (
|
||||||
|
bulkAdjectives = []string{
|
||||||
|
"Hollow", "Northern", "Quiet", "Endless", "Amber", "Glass",
|
||||||
|
"Distant", "Velvet", "Iron", "Pale", "Golden", "Silent",
|
||||||
|
"Crimson", "Winter", "Coastal", "Electric", "Marble", "Slow",
|
||||||
|
}
|
||||||
|
bulkNouns = []string{
|
||||||
|
"Harbour", "Tideline", "Cartography", "Signal", "Meridian",
|
||||||
|
"Aviary", "Lantern", "Orchard", "Pavilion", "Estuary",
|
||||||
|
"Sequence", "Almanac", "Corridor", "Foundry", "Cascade",
|
||||||
|
"Interval", "Radiance", "Threshold", "Migration", "Beacon",
|
||||||
|
}
|
||||||
|
bulkCollectives = []string{
|
||||||
|
"Ensemble", "Trio", "Quartet", "Society", "Orchestra",
|
||||||
|
"Collective", "Choir", "Union", "Company", "Band",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
// bulkSpec is a fully-resolved request for a bulk library.
|
||||||
|
type bulkSpec struct {
|
||||||
|
Out string
|
||||||
|
Tracks int
|
||||||
|
CoverPx int
|
||||||
|
}
|
||||||
|
|
||||||
|
// hash identifies the spec, so regeneration can be skipped the same
|
||||||
|
// way the fixture library skips it.
|
||||||
|
//
|
||||||
|
// The leading version string covers everything about *generation* that
|
||||||
|
// the parameters do not — the naming tables, the cover renderer, the
|
||||||
|
// source-clip durations. Bump it when any of those change, or a stale
|
||||||
|
// library on disk will be silently accepted as current.
|
||||||
|
func (s bulkSpec) hash() string {
|
||||||
|
sum := sha256.Sum256([]byte(fmt.Sprintf(
|
||||||
|
"bulk/v2|%d|%d|%d|%d|%d",
|
||||||
|
s.Tracks, s.CoverPx,
|
||||||
|
bulkTracksPerAlbum, bulkAlbumsPerArtist, bulkSourceVariants,
|
||||||
|
)))
|
||||||
|
|
||||||
|
return hex.EncodeToString(sum[:])
|
||||||
|
}
|
||||||
|
|
||||||
|
// bulkTrack is one generated file, resolved from its index alone.
|
||||||
|
// Everything is a pure function of the index, so generation
|
||||||
|
// parallelises without coordination and two runs agree exactly.
|
||||||
|
type bulkTrack struct {
|
||||||
|
Rel string
|
||||||
|
Variant int
|
||||||
|
Album int
|
||||||
|
Tags tags
|
||||||
|
}
|
||||||
|
|
||||||
|
// bulkTrackAt derives track i's identity.
|
||||||
|
func bulkTrackAt(i int) bulkTrack {
|
||||||
|
album := i / bulkTracksPerAlbum
|
||||||
|
artist := album / bulkAlbumsPerArtist
|
||||||
|
trackNo := i%bulkTracksPerAlbum + 1
|
||||||
|
|
||||||
|
artistName := bulkArtistName(artist)
|
||||||
|
albumName := bulkAlbumName(album)
|
||||||
|
title := bulkTitle(i)
|
||||||
|
|
||||||
|
return bulkTrack{
|
||||||
|
Rel: filepath.Join(
|
||||||
|
sanitizePathSegment(artistName),
|
||||||
|
sanitizePathSegment(albumName),
|
||||||
|
fmt.Sprintf("%02d %s.mp3", trackNo, sanitizePathSegment(title)),
|
||||||
|
),
|
||||||
|
Variant: i % bulkSourceVariants,
|
||||||
|
Album: album,
|
||||||
|
Tags: tags{
|
||||||
|
Title: title,
|
||||||
|
Artist: artistName,
|
||||||
|
Album: albumName,
|
||||||
|
AlbumArtist: artistName,
|
||||||
|
Genre: bulkGenres[album%len(bulkGenres)],
|
||||||
|
Year: bulkYearBase + album%bulkYearSpan,
|
||||||
|
TrackNumber: trackNo,
|
||||||
|
DiscNumber: 1,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func bulkArtistName(n int) string {
|
||||||
|
switch n % 3 {
|
||||||
|
case 0:
|
||||||
|
return bulkAdjectives[n%len(bulkAdjectives)] + " " +
|
||||||
|
bulkNouns[(n/len(bulkAdjectives))%len(bulkNouns)]
|
||||||
|
case 1:
|
||||||
|
return "The " + bulkNouns[n%len(bulkNouns)] + " " +
|
||||||
|
bulkCollectives[(n/len(bulkNouns))%len(bulkCollectives)]
|
||||||
|
default:
|
||||||
|
return bulkNouns[n%len(bulkNouns)] + " & the " +
|
||||||
|
bulkAdjectives[(n/len(bulkNouns))%len(bulkAdjectives)] + "s"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func bulkAlbumName(n int) string {
|
||||||
|
return bulkAdjectives[(n*7)%len(bulkAdjectives)] + " " +
|
||||||
|
bulkNouns[(n*13)%len(bulkNouns)] +
|
||||||
|
" " + strconv.Itoa(n%97)
|
||||||
|
}
|
||||||
|
|
||||||
|
func bulkTitle(n int) string {
|
||||||
|
return bulkNouns[(n*3)%len(bulkNouns)] + " " +
|
||||||
|
bulkAdjectives[(n*11)%len(bulkAdjectives)] +
|
||||||
|
" " + strconv.Itoa(n%211)
|
||||||
|
}
|
||||||
|
|
||||||
|
// sanitizePathSegment keeps generated names usable as path components
|
||||||
|
// on every platform the app builds for.
|
||||||
|
func sanitizePathSegment(s string) string {
|
||||||
|
const bad = `/\:*?"<>|`
|
||||||
|
|
||||||
|
out := []rune(s)
|
||||||
|
for i, r := range out {
|
||||||
|
for _, b := range bad {
|
||||||
|
if r == b {
|
||||||
|
out[i] = '_'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return string(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
// generateBulk builds (or skips) a bulk library.
|
||||||
|
func generateBulk(spec bulkSpec, force bool) error {
|
||||||
|
manifestPath := spec.Out + ".manifest.json"
|
||||||
|
|
||||||
|
if !force && readManifestHash(manifestPath) == spec.hash() {
|
||||||
|
if _, err := os.Stat(spec.Out); err == nil {
|
||||||
|
fmt.Printf(
|
||||||
|
"bulk: up to date (%d tracks, hash %s)\n",
|
||||||
|
spec.Tracks, spec.hash()[:12],
|
||||||
|
)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := requireFFmpeg(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.RemoveAll(spec.Out); err != nil {
|
||||||
|
return fmt.Errorf("clean %s: %w", spec.Out, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
started := time.Now()
|
||||||
|
|
||||||
|
sources, cleanup, err := bulkSources(spec.Out)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
|
covers, err := bulkCovers(spec, sources)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := bulkWrite(spec, sources, covers); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := writeManifest(manifestPath, &manifest{
|
||||||
|
Version: manifestVersion,
|
||||||
|
Generator: "gentestdata -bulk",
|
||||||
|
Hash: spec.hash(),
|
||||||
|
LibraryRoot: filepath.ToSlash(spec.Out),
|
||||||
|
TrackCount: spec.Tracks,
|
||||||
|
}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf(
|
||||||
|
"bulk: generated %d tracks in %s (%s, hash %s)\n",
|
||||||
|
spec.Tracks, spec.Out,
|
||||||
|
time.Since(started).Round(time.Second),
|
||||||
|
spec.hash()[:12],
|
||||||
|
)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// bulkSources encodes the handful of distinct clips every generated
|
||||||
|
// file is copied from. Durations vary so "time between tracks" is not
|
||||||
|
// measuring one number repeatedly, and stay short so a measurement run
|
||||||
|
// that plays through a dozen tracks takes seconds.
|
||||||
|
func bulkSources(out string) ([][]byte, func(), error) {
|
||||||
|
tmp, err := os.MkdirTemp("", "yj-bulk-src-")
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("temp dir: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanup := func() { _ = os.RemoveAll(tmp) }
|
||||||
|
|
||||||
|
if err := os.MkdirAll(out, dirPerm); err != nil {
|
||||||
|
cleanup()
|
||||||
|
|
||||||
|
return nil, nil, fmt.Errorf("mkdir %s: %w", out, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
sources := make([][]byte, bulkSourceVariants)
|
||||||
|
|
||||||
|
for i := range bulkSourceVariants {
|
||||||
|
wav := filepath.Join(tmp, strconv.Itoa(i)+".wav")
|
||||||
|
mp3 := filepath.Join(tmp, strconv.Itoa(i)+".mp3")
|
||||||
|
dur := time.Duration(2+i%3) * time.Second
|
||||||
|
|
||||||
|
if err := synthesizeWAV(wav, dur, 220+float64(i)*55); err != nil {
|
||||||
|
cleanup()
|
||||||
|
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := transcode(wav, mp3, tagwriter.FormatMP3); err != nil {
|
||||||
|
cleanup()
|
||||||
|
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
raw, err := os.ReadFile(mp3) //nolint:gosec // generated path.
|
||||||
|
if err != nil {
|
||||||
|
cleanup()
|
||||||
|
|
||||||
|
return nil, nil, fmt.Errorf("read source clip: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
sources[i] = raw
|
||||||
|
}
|
||||||
|
|
||||||
|
return sources, cleanup, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// bulkCovers renders one cover per album up front. Rendering it per
|
||||||
|
// track would be the dominant cost of generation and would defeat the
|
||||||
|
// library's cover-art deduplication, which is part of what a scan at
|
||||||
|
// this size is being measured on.
|
||||||
|
func bulkCovers(spec bulkSpec, _ [][]byte) ([][]byte, error) {
|
||||||
|
albums := (spec.Tracks + bulkTracksPerAlbum - 1) / bulkTracksPerAlbum
|
||||||
|
covers := make([][]byte, albums)
|
||||||
|
|
||||||
|
for i := range albums {
|
||||||
|
img, err := bulkCoverJPEG(bulkAlbumName(i), spec.CoverPx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
covers[i] = img
|
||||||
|
}
|
||||||
|
|
||||||
|
return covers, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// bulkCoverJPEG renders a bulk album cover.
|
||||||
|
//
|
||||||
|
// Deliberately not coverJPEGSized: the fixture cover's diagonal band
|
||||||
|
// is 37 hard edges at 300 px, which is the worst case for a DCT and
|
||||||
|
// costs ~35 kB per album — 2 GB across a 50 k-track library, most of
|
||||||
|
// it JPEG artefacts around a pattern nobody looks at. A smooth
|
||||||
|
// two-axis gradient is ~6 kB, still distinct per album by eye, and
|
||||||
|
// still deterministic per key.
|
||||||
|
func bulkCoverJPEG(key string, px int) ([]byte, error) {
|
||||||
|
var seed uint32
|
||||||
|
for _, b := range []byte(key) {
|
||||||
|
seed = seed*31 + uint32(b)
|
||||||
|
}
|
||||||
|
|
||||||
|
img := image.NewRGBA(image.Rect(0, 0, px, px))
|
||||||
|
base := color.RGBA{
|
||||||
|
R: uint8(seed >> 16),
|
||||||
|
G: uint8(seed >> 8),
|
||||||
|
B: uint8(seed),
|
||||||
|
A: 255,
|
||||||
|
}
|
||||||
|
|
||||||
|
for y := range px {
|
||||||
|
shade := float64(y) / float64(px)
|
||||||
|
|
||||||
|
for x := range px {
|
||||||
|
tint := (shade + float64(x)/float64(px)) / 2
|
||||||
|
|
||||||
|
img.Set(x, y, color.RGBA{
|
||||||
|
R: uint8(float64(base.R) * (0.4 + 0.6*tint)),
|
||||||
|
G: uint8(float64(base.G) * (0.4 + 0.6*tint)),
|
||||||
|
B: uint8(float64(base.B) * (0.4 + 0.6*tint)),
|
||||||
|
A: 255,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var buf bytes.Buffer
|
||||||
|
|
||||||
|
if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: jpegQuality}); err != nil {
|
||||||
|
return nil, fmt.Errorf("encode bulk cover %q: %w", key, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return buf.Bytes(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// bulkWrite copies and tags every file, in parallel. tagwriter works
|
||||||
|
// on one path at a time with no shared state, so the only coordination
|
||||||
|
// needed is the error and the progress counter.
|
||||||
|
func bulkWrite(spec bulkSpec, sources, covers [][]byte) error {
|
||||||
|
logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
|
||||||
|
Level: slog.LevelError,
|
||||||
|
}))
|
||||||
|
|
||||||
|
jobs := make(chan int, runtime.NumCPU()*2)
|
||||||
|
|
||||||
|
var (
|
||||||
|
wg sync.WaitGroup
|
||||||
|
done atomic.Int64
|
||||||
|
failed atomic.Pointer[error]
|
||||||
|
workers = runtime.NumCPU()
|
||||||
|
)
|
||||||
|
|
||||||
|
for range workers {
|
||||||
|
wg.Add(1)
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
|
||||||
|
for i := range jobs {
|
||||||
|
if failed.Load() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := bulkWriteOne(
|
||||||
|
logger, spec, sources, covers, i,
|
||||||
|
); err != nil {
|
||||||
|
failed.CompareAndSwap(nil, &err)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if n := done.Add(1); n%5000 == 0 {
|
||||||
|
fmt.Printf("bulk: %d/%d\n", n, spec.Tracks)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
for i := range spec.Tracks {
|
||||||
|
jobs <- i
|
||||||
|
}
|
||||||
|
|
||||||
|
close(jobs)
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
if err := failed.Load(); err != nil {
|
||||||
|
return *err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func bulkWriteOne(
|
||||||
|
logger *slog.Logger,
|
||||||
|
spec bulkSpec,
|
||||||
|
sources, covers [][]byte,
|
||||||
|
i int,
|
||||||
|
) error {
|
||||||
|
t := bulkTrackAt(i)
|
||||||
|
dst := filepath.Join(spec.Out, t.Rel)
|
||||||
|
|
||||||
|
if err := ensureDir(dst); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.WriteFile(dst, sources[t.Variant], filePerm); err != nil {
|
||||||
|
return fmt.Errorf("write %s: %w", dst, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
changes := t.Tags.changes()
|
||||||
|
changes[tagwriter.FieldCoverArt] = covers[t.Album]
|
||||||
|
|
||||||
|
if err := tagwriter.WriteFileTags(logger, dst, changes); err != nil {
|
||||||
|
return fmt.Errorf("tag %s: %w", t.Rel, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return stampMTime(dst)
|
||||||
|
}
|
||||||
+33
-1
@@ -13,6 +13,11 @@
|
|||||||
//
|
//
|
||||||
// go run ./cmd/gentestdata # generate if out of date
|
// go run ./cmd/gentestdata # generate if out of date
|
||||||
// go run ./cmd/gentestdata -force # regenerate unconditionally
|
// go run ./cmd/gentestdata -force # regenerate unconditionally
|
||||||
|
// go run ./cmd/gentestdata -bulk 50000 # the measurement library
|
||||||
|
//
|
||||||
|
// The -bulk library is a separate thing with a separate purpose; see
|
||||||
|
// bulk.go. It is not committed, not a test dependency, and generating
|
||||||
|
// it does not regenerate the fixture library.
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -44,6 +49,9 @@ func main() {
|
|||||||
brokenDir string
|
brokenDir string
|
||||||
manifestOut string
|
manifestOut string
|
||||||
force bool
|
force bool
|
||||||
|
bulkTracks int
|
||||||
|
bulkOut string
|
||||||
|
bulkCover int
|
||||||
)
|
)
|
||||||
|
|
||||||
flag.StringVar(
|
flag.StringVar(
|
||||||
@@ -62,9 +70,33 @@ func main() {
|
|||||||
&force, "force", false,
|
&force, "force", false,
|
||||||
"regenerate even when the manifest is already up to date",
|
"regenerate even when the manifest is already up to date",
|
||||||
)
|
)
|
||||||
|
flag.IntVar(
|
||||||
|
&bulkTracks, "bulk", 0,
|
||||||
|
"generate a bulk measurement library of N tracks instead",
|
||||||
|
)
|
||||||
|
flag.StringVar(
|
||||||
|
&bulkOut, "bulk-out", ".dev/music_library_bulk",
|
||||||
|
"library root for -bulk (gitignored; not a test fixture)",
|
||||||
|
)
|
||||||
|
flag.IntVar(
|
||||||
|
&bulkCover, "bulk-cover-px", bulkCoverPx,
|
||||||
|
"edge length of the embedded cover art for -bulk",
|
||||||
|
)
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
|
||||||
if err := run(outDir, brokenDir, manifestOut, force); err != nil {
|
var err error
|
||||||
|
|
||||||
|
if bulkTracks > 0 {
|
||||||
|
err = generateBulk(bulkSpec{
|
||||||
|
Out: bulkOut,
|
||||||
|
Tracks: bulkTracks,
|
||||||
|
CoverPx: bulkCover,
|
||||||
|
}, force)
|
||||||
|
} else {
|
||||||
|
err = run(outDir, brokenDir, manifestOut, force)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
fmt.Fprintln(os.Stderr, "gentestdata:", err)
|
fmt.Fprintln(os.Stderr, "gentestdata:", err)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -34,15 +34,19 @@ type manifestTrack struct {
|
|||||||
// stamps its own encoder strings, so byte hashes differ between ffmpeg
|
// stamps its own encoder strings, so byte hashes differ between ffmpeg
|
||||||
// builds while the fixtures they describe are identical.
|
// builds while the fixtures they describe are identical.
|
||||||
type manifest struct {
|
type manifest struct {
|
||||||
Version int `json:"version"`
|
Version int `json:"version"`
|
||||||
Generator string `json:"generator"`
|
Generator string `json:"generator"`
|
||||||
Hash string `json:"hash"`
|
Hash string `json:"hash"`
|
||||||
LibraryRoot string `json:"libraryRoot"`
|
LibraryRoot string `json:"libraryRoot"`
|
||||||
BrokenRoot string `json:"brokenRoot"`
|
BrokenRoot string `json:"brokenRoot"`
|
||||||
Cases map[string][]string `json:"cases"`
|
// TrackCount is written by the bulk library, which has tens of
|
||||||
Tracks []manifestTrack `json:"tracks"`
|
// thousands of tracks and no reason to describe each one: nothing
|
||||||
Extras []string `json:"extras"`
|
// selects a bulk track by name, only the total matters.
|
||||||
Broken []string `json:"broken"`
|
TrackCount int `json:"trackCount,omitempty"`
|
||||||
|
Cases map[string][]string `json:"cases"`
|
||||||
|
Tracks []manifestTrack `json:"tracks"`
|
||||||
|
Extras []string `json:"extras"`
|
||||||
|
Broken []string `json:"broken"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// buildManifest derives the manifest from the spec alone. It runs
|
// buildManifest derives the manifest from the spec alone. It runs
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
+24
-2
@@ -16,6 +16,13 @@
|
|||||||
#
|
#
|
||||||
# Usage:
|
# Usage:
|
||||||
# scripts/seed-sandbox.sh [--name NAME] [--port N] [--no-build]
|
# scripts/seed-sandbox.sh [--name NAME] [--port N] [--no-build]
|
||||||
|
# [--manifest PATH]
|
||||||
|
#
|
||||||
|
# --manifest points at a different generated library's manifest, which
|
||||||
|
# is how the bulk measurement library (`make bulkdata`) gets seeded:
|
||||||
|
# same script, same discipline, different pile of files. A manifest
|
||||||
|
# either lists its tracks or states a trackCount; both are accepted,
|
||||||
|
# because describing 50 000 tracks individually would serve nobody.
|
||||||
#
|
#
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
@@ -31,6 +38,9 @@ NAME="default"
|
|||||||
PORT=34115
|
PORT=34115
|
||||||
SESSION="yj-seed"
|
SESSION="yj-seed"
|
||||||
BUILD_ARGS=()
|
BUILD_ARGS=()
|
||||||
|
# Replaced below once the manifest says how many tracks are coming: a
|
||||||
|
# 50 000-track scan is minutes, and a fixed 180 s deadline would abort
|
||||||
|
# a healthy run rather than an unhealthy one.
|
||||||
SCAN_TIMEOUT=180
|
SCAN_TIMEOUT=180
|
||||||
|
|
||||||
while [ $# -gt 0 ]; do
|
while [ $# -gt 0 ]; do
|
||||||
@@ -47,6 +57,10 @@ while [ $# -gt 0 ]; do
|
|||||||
BUILD_ARGS+=(--no-build)
|
BUILD_ARGS+=(--no-build)
|
||||||
shift
|
shift
|
||||||
;;
|
;;
|
||||||
|
--manifest)
|
||||||
|
MANIFEST="${2:?--manifest needs a path}"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
*)
|
*)
|
||||||
echo "seed-sandbox: unknown argument: $1" >&2
|
echo "seed-sandbox: unknown argument: $1" >&2
|
||||||
exit 2
|
exit 2
|
||||||
@@ -64,12 +78,14 @@ need playwright-cli
|
|||||||
need jq
|
need jq
|
||||||
|
|
||||||
if [ ! -f "$MANIFEST" ]; then
|
if [ ! -f "$MANIFEST" ]; then
|
||||||
echo "seed-sandbox: fixtures missing; run 'make testdata'" >&2
|
echo "seed-sandbox: no manifest at $MANIFEST" >&2
|
||||||
|
echo " run 'make testdata' (fixtures) or 'make bulkdata' (measurement)" >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
LIBRARY_DIR="$REPO_ROOT/$(jq -r .libraryRoot "$MANIFEST")"
|
LIBRARY_DIR="$REPO_ROOT/$(jq -r .libraryRoot "$MANIFEST")"
|
||||||
WANT_TRACKS="$(jq '.tracks | length' "$MANIFEST")"
|
WANT_TRACKS="$(jq '.trackCount // (.tracks | length)' "$MANIFEST")"
|
||||||
|
SCAN_TIMEOUT=$((180 + WANT_TRACKS / 50))
|
||||||
FIXTURE_HASH="$(jq -r .hash "$MANIFEST")"
|
FIXTURE_HASH="$(jq -r .hash "$MANIFEST")"
|
||||||
|
|
||||||
cleanup() {
|
cleanup() {
|
||||||
@@ -138,6 +154,12 @@ while [ "$SECONDS" -lt "$deadline" ]; do
|
|||||||
|
|
||||||
[ "$got" = "$WANT_TRACKS" ] && break
|
[ "$got" = "$WANT_TRACKS" ] && break
|
||||||
|
|
||||||
|
# A large scan is minutes of silence otherwise, which is
|
||||||
|
# indistinguishable from a hang.
|
||||||
|
if [ $((SECONDS % 15)) -eq 0 ]; then
|
||||||
|
echo " ... $got/$WANT_TRACKS"
|
||||||
|
fi
|
||||||
|
|
||||||
sleep 1
|
sleep 1
|
||||||
done
|
done
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user