build(dev): generate a 50k-track library and measure a running app
Plan 007 phase 4 is verified by measurement, not by assertion, and there was no way to produce a number: the fixture library is a few dozen tracks and cannot show any of the findings. - `cmd/gentestdata -bulk N` (`make bulkdata`) writes a ~50 000-track library in 11 s / 466 MB by encoding six clips once and copying them, while still tagging every file through `backend/tagwriter` — a library the app cannot read back measures nothing. - `make sandbox-seed-bulk` seeds from it through the same script and the same discipline as any other seed: by running the app and waiting for the real scan. - `e2e/perf/measure.mjs` (`make perf LABEL=x`, `make perf-compare`) takes fourteen measurements against a running app and writes them to a gitignored `.dev/perf/<label>.json`. It wraps every bound Go method, so "did that refetch the library" is a fact rather than an inference, and records `longtask` entries, which is where a 25 MB JSON parse on the main thread shows up and nowhere else. It is not a spec and does not run in CI.
This commit is contained in:
@@ -37,6 +37,15 @@ dev-logs: ## Tail the headless app log
|
||||
sandbox-seed: testdata ## Build a seeded YJ_HOME snapshot: make sandbox-seed NAME=<n>
|
||||
@./scripts/seed-sandbox.sh $(if $(NAME),--name $(NAME),)
|
||||
|
||||
# The bulk seed is the *measurement* seed, not a fixture seed. Same
|
||||
# script and the same discipline (the app builds it by scanning for
|
||||
# real); the only difference is which manifest it is pointed at. It is
|
||||
# a separate target because a 50 000-track scan is minutes, and nothing
|
||||
# routine should depend on it.
|
||||
sandbox-seed-bulk: bulkdata ## Build a seeded YJ_HOME from the bulk library
|
||||
@./scripts/seed-sandbox.sh --name $(if $(NAME),$(NAME),bulk) \
|
||||
--manifest .dev/music_library_bulk.manifest.json
|
||||
|
||||
sandbox-seeds: ## List built seeds
|
||||
@ls -1 .dev/seeds/*.tar 2>/dev/null | sed 's|.*/||; s|\.tar$$||' \
|
||||
|| echo " (none; build one with: make sandbox-seed NAME=default)"
|
||||
@@ -45,6 +54,21 @@ sandbox-seeds: ## List built seeds
|
||||
# daemonises, which is the opposite of what Playwright's `webServer`
|
||||
# supervises, and starting one per run would rebuild the frontend every
|
||||
# time. globalSetup fails with the exact commands to run if it is down.
|
||||
# Phase 4 of plan 007 is verified by measurement rather than assertion,
|
||||
# so this is not a spec and does not run in CI: it produces a number to
|
||||
# read, against a running app seeded with the bulk library.
|
||||
#
|
||||
# make sandbox-seed-bulk && make dev-headless SEED=bulk
|
||||
# make perf LABEL=before ... change something ... make perf LABEL=after
|
||||
# make perf-compare BEFORE=before AFTER=after
|
||||
perf: ## Take a performance measurement (LABEL=<name>) of a running app
|
||||
@cd e2e && pnpm install --silent && \
|
||||
node perf/measure.mjs --label $(if $(LABEL),$(LABEL),current)
|
||||
|
||||
perf-compare: ## Print a before/after table: BEFORE=<a> AFTER=<b>
|
||||
@cd e2e && node perf/measure.mjs --compare \
|
||||
$(if $(BEFORE),$(BEFORE),before) $(if $(AFTER),$(AFTER),after)
|
||||
|
||||
e2e: ## Run the Playwright smoke suite against a running dev-headless app
|
||||
@cd e2e && pnpm install --silent && npx playwright test $(E2E_ARGS)
|
||||
|
||||
@@ -96,7 +120,8 @@ bindings: ## Regenerate frontend/wailsjs from the bound Go structs
|
||||
frontend/wailsjs/runtime/package.json
|
||||
|
||||
.PHONY: dev-headless dev-headless-fresh dev-stop dev-logs \
|
||||
sandbox-seed sandbox-seeds e2e e2e-setup e2e-report \
|
||||
sandbox-seed sandbox-seed-bulk sandbox-seeds e2e e2e-setup e2e-report \
|
||||
perf perf-compare \
|
||||
ui-test ui-watch ui-visual ui-visual-update ui-setup \
|
||||
bindings bindings-check skill-check
|
||||
|
||||
@@ -220,7 +245,19 @@ testdata-clean: ## Delete the generated fixture library
|
||||
rm -rf test_data/music_library_test test_data/music_library_broken \
|
||||
test_data/music_library_test.manifest.json
|
||||
|
||||
.PHONY: testdata testdata-force testdata-clean
|
||||
# The bulk library answers a different question from the fixture one:
|
||||
# not "does this behave correctly" but "how does this behave at the
|
||||
# size the audit measured". ~11 s, ~470 MB, into a gitignored .dev/,
|
||||
# and deliberately not a dependency of `make test`.
|
||||
BULK_TRACKS ?= 50000
|
||||
|
||||
bulkdata: ## Generate the bulk measurement library (BULK_TRACKS=50000)
|
||||
go run ./cmd/gentestdata -bulk $(BULK_TRACKS)
|
||||
|
||||
bulkdata-clean: ## Delete the bulk measurement library
|
||||
rm -rf .dev/music_library_bulk .dev/music_library_bulk.manifest.json
|
||||
|
||||
.PHONY: testdata testdata-force testdata-clean bulkdata bulkdata-clean
|
||||
|
||||
# The tag sets must match `make test` exactly, or lint is checking three
|
||||
# configurations that nothing builds. webkit2_41 is not optional: without
|
||||
|
||||
@@ -161,7 +161,19 @@ func requireFFmpeg() error {
|
||||
// the library's cover-art deduplication is supposed to collapse into a
|
||||
// single stored blob.
|
||||
func coverJPEG(key string) ([]byte, error) {
|
||||
img := image.NewRGBA(image.Rect(0, 0, coverSizePx, coverSizePx))
|
||||
return coverJPEGSized(key, coverSizePx)
|
||||
}
|
||||
|
||||
// coverJPEGSized is coverJPEG at an explicit edge length. The bulk
|
||||
// library uses a larger one, because a 64 px cover cannot show the
|
||||
// difference between rendering the original artwork and rendering the
|
||||
// thumbnail tier that exists for the purpose.
|
||||
// coverJPEGSized is coverJPEG at an explicit edge length. The bulk
|
||||
// library uses a larger one, because a 64 px cover cannot show the
|
||||
// difference between rendering the original artwork and rendering the
|
||||
// thumbnail tier that exists for the purpose.
|
||||
func coverJPEGSized(key string, px int) ([]byte, error) {
|
||||
img := image.NewRGBA(image.Rect(0, 0, px, px))
|
||||
|
||||
// A per-key hue derived from the key's bytes, plus a diagonal
|
||||
// band, so covers are distinguishable by eye in a screenshot.
|
||||
@@ -177,8 +189,8 @@ func coverJPEG(key string) ([]byte, error) {
|
||||
A: 255,
|
||||
}
|
||||
|
||||
for y := range coverSizePx {
|
||||
for x := range coverSizePx {
|
||||
for y := range px {
|
||||
for x := range px {
|
||||
c := base
|
||||
if (x+y)%16 < 8 {
|
||||
c.R /= 2
|
||||
|
||||
@@ -0,0 +1,462 @@
|
||||
package main
|
||||
|
||||
// The bulk library exists for one purpose: measuring. Plan 007
|
||||
// phase 4 is verified by numbers rather than assertions, and every
|
||||
// number in `.planning/audits/2026-08-11-ui/perf.md` is quoted "at
|
||||
// 50 000 tracks" — a size the deterministic fixture library
|
||||
// (`libraryFixtures`, a few dozen files) cannot reach and should not
|
||||
// try to. The two are generated by the same command because they are
|
||||
// the same problem at two scales, but they share nothing else: the
|
||||
// fixture library is a curated set of *cases* selected by name, and
|
||||
// this one is a shapeless pile whose only interesting property is how
|
||||
// big it is.
|
||||
//
|
||||
// It is deliberately not committed and deliberately not a dependency
|
||||
// of any test. `make test` must not take four minutes because someone
|
||||
// wanted a scroll trace.
|
||||
//
|
||||
// Generation avoids ffmpeg per file. Encoding 50 000 files one
|
||||
// process at a time is ~40 minutes; encoding a handful of source
|
||||
// clips once and copying them is ~20 seconds, and the audio content is
|
||||
// not what is being measured. Tags still go through backend/tagwriter
|
||||
// for the same reason the fixture library does — a library the app
|
||||
// cannot read back tells you nothing about the app.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/jpeg"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"yellowjacket/backend/tagwriter"
|
||||
)
|
||||
|
||||
// Shape of the generated library. These ratios are roughly what a
|
||||
// real 50 k-track collection looks like, and they matter: several of
|
||||
// the findings being measured are O(albums) or O(artists) rather than
|
||||
// O(tracks), so a library of 50 000 tracks in one album would measure
|
||||
// nothing.
|
||||
const (
|
||||
bulkTracksPerAlbum = 10
|
||||
bulkAlbumsPerArtist = 4
|
||||
bulkSourceVariants = 6
|
||||
bulkCoverPx = 300
|
||||
bulkYearBase = 1968
|
||||
bulkYearSpan = 56
|
||||
)
|
||||
|
||||
// bulkGenres is small on purpose — a real library has a long tail of
|
||||
// genres but a short head, and the head is what the genres view and
|
||||
// the home shelves actually work over.
|
||||
//
|
||||
//nolint:gochecknoglobals // a package-level constant table.
|
||||
var bulkGenres = []string{
|
||||
"Ambient", "Post-Rock", "Dream Pop", "Shoegaze", "Krautrock",
|
||||
"Minimal Techno", "Free Jazz", "Bossa Nova", "Baroque", "Drone",
|
||||
"Noise Rock", "Slowcore", "Trip Hop", "Dub", "Highlife",
|
||||
"Cumbia", "Fado", "Gamelan", "Chamber Pop", "Math Rock",
|
||||
"Field Recording", "Musique Concrète", "Sludge", "Zeuhl",
|
||||
}
|
||||
|
||||
// Word tables composing artist and album names. Names are generated
|
||||
// rather than numbered so search ranking has something with shared
|
||||
// prefixes, shared words and varying lengths to rank — `Artist 04213`
|
||||
// would make every query either match everything or nothing.
|
||||
//
|
||||
//nolint:gochecknoglobals // package-level constant tables.
|
||||
var (
|
||||
bulkAdjectives = []string{
|
||||
"Hollow", "Northern", "Quiet", "Endless", "Amber", "Glass",
|
||||
"Distant", "Velvet", "Iron", "Pale", "Golden", "Silent",
|
||||
"Crimson", "Winter", "Coastal", "Electric", "Marble", "Slow",
|
||||
}
|
||||
bulkNouns = []string{
|
||||
"Harbour", "Tideline", "Cartography", "Signal", "Meridian",
|
||||
"Aviary", "Lantern", "Orchard", "Pavilion", "Estuary",
|
||||
"Sequence", "Almanac", "Corridor", "Foundry", "Cascade",
|
||||
"Interval", "Radiance", "Threshold", "Migration", "Beacon",
|
||||
}
|
||||
bulkCollectives = []string{
|
||||
"Ensemble", "Trio", "Quartet", "Society", "Orchestra",
|
||||
"Collective", "Choir", "Union", "Company", "Band",
|
||||
}
|
||||
)
|
||||
|
||||
// bulkSpec is a fully-resolved request for a bulk library.
|
||||
type bulkSpec struct {
|
||||
Out string
|
||||
Tracks int
|
||||
CoverPx int
|
||||
}
|
||||
|
||||
// hash identifies the spec, so regeneration can be skipped the same
|
||||
// way the fixture library skips it.
|
||||
//
|
||||
// The leading version string covers everything about *generation* that
|
||||
// the parameters do not — the naming tables, the cover renderer, the
|
||||
// source-clip durations. Bump it when any of those change, or a stale
|
||||
// library on disk will be silently accepted as current.
|
||||
func (s bulkSpec) hash() string {
|
||||
sum := sha256.Sum256([]byte(fmt.Sprintf(
|
||||
"bulk/v2|%d|%d|%d|%d|%d",
|
||||
s.Tracks, s.CoverPx,
|
||||
bulkTracksPerAlbum, bulkAlbumsPerArtist, bulkSourceVariants,
|
||||
)))
|
||||
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// bulkTrack is one generated file, resolved from its index alone.
|
||||
// Everything is a pure function of the index, so generation
|
||||
// parallelises without coordination and two runs agree exactly.
|
||||
type bulkTrack struct {
|
||||
Rel string
|
||||
Variant int
|
||||
Album int
|
||||
Tags tags
|
||||
}
|
||||
|
||||
// bulkTrackAt derives track i's identity.
|
||||
func bulkTrackAt(i int) bulkTrack {
|
||||
album := i / bulkTracksPerAlbum
|
||||
artist := album / bulkAlbumsPerArtist
|
||||
trackNo := i%bulkTracksPerAlbum + 1
|
||||
|
||||
artistName := bulkArtistName(artist)
|
||||
albumName := bulkAlbumName(album)
|
||||
title := bulkTitle(i)
|
||||
|
||||
return bulkTrack{
|
||||
Rel: filepath.Join(
|
||||
sanitizePathSegment(artistName),
|
||||
sanitizePathSegment(albumName),
|
||||
fmt.Sprintf("%02d %s.mp3", trackNo, sanitizePathSegment(title)),
|
||||
),
|
||||
Variant: i % bulkSourceVariants,
|
||||
Album: album,
|
||||
Tags: tags{
|
||||
Title: title,
|
||||
Artist: artistName,
|
||||
Album: albumName,
|
||||
AlbumArtist: artistName,
|
||||
Genre: bulkGenres[album%len(bulkGenres)],
|
||||
Year: bulkYearBase + album%bulkYearSpan,
|
||||
TrackNumber: trackNo,
|
||||
DiscNumber: 1,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func bulkArtistName(n int) string {
|
||||
switch n % 3 {
|
||||
case 0:
|
||||
return bulkAdjectives[n%len(bulkAdjectives)] + " " +
|
||||
bulkNouns[(n/len(bulkAdjectives))%len(bulkNouns)]
|
||||
case 1:
|
||||
return "The " + bulkNouns[n%len(bulkNouns)] + " " +
|
||||
bulkCollectives[(n/len(bulkNouns))%len(bulkCollectives)]
|
||||
default:
|
||||
return bulkNouns[n%len(bulkNouns)] + " & the " +
|
||||
bulkAdjectives[(n/len(bulkNouns))%len(bulkAdjectives)] + "s"
|
||||
}
|
||||
}
|
||||
|
||||
func bulkAlbumName(n int) string {
|
||||
return bulkAdjectives[(n*7)%len(bulkAdjectives)] + " " +
|
||||
bulkNouns[(n*13)%len(bulkNouns)] +
|
||||
" " + strconv.Itoa(n%97)
|
||||
}
|
||||
|
||||
func bulkTitle(n int) string {
|
||||
return bulkNouns[(n*3)%len(bulkNouns)] + " " +
|
||||
bulkAdjectives[(n*11)%len(bulkAdjectives)] +
|
||||
" " + strconv.Itoa(n%211)
|
||||
}
|
||||
|
||||
// sanitizePathSegment keeps generated names usable as path components
|
||||
// on every platform the app builds for.
|
||||
func sanitizePathSegment(s string) string {
|
||||
const bad = `/\:*?"<>|`
|
||||
|
||||
out := []rune(s)
|
||||
for i, r := range out {
|
||||
for _, b := range bad {
|
||||
if r == b {
|
||||
out[i] = '_'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return string(out)
|
||||
}
|
||||
|
||||
// generateBulk builds (or skips) a bulk library.
|
||||
func generateBulk(spec bulkSpec, force bool) error {
|
||||
manifestPath := spec.Out + ".manifest.json"
|
||||
|
||||
if !force && readManifestHash(manifestPath) == spec.hash() {
|
||||
if _, err := os.Stat(spec.Out); err == nil {
|
||||
fmt.Printf(
|
||||
"bulk: up to date (%d tracks, hash %s)\n",
|
||||
spec.Tracks, spec.hash()[:12],
|
||||
)
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
if err := requireFFmpeg(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := os.RemoveAll(spec.Out); err != nil {
|
||||
return fmt.Errorf("clean %s: %w", spec.Out, err)
|
||||
}
|
||||
|
||||
started := time.Now()
|
||||
|
||||
sources, cleanup, err := bulkSources(spec.Out)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cleanup()
|
||||
|
||||
covers, err := bulkCovers(spec, sources)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := bulkWrite(spec, sources, covers); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := writeManifest(manifestPath, &manifest{
|
||||
Version: manifestVersion,
|
||||
Generator: "gentestdata -bulk",
|
||||
Hash: spec.hash(),
|
||||
LibraryRoot: filepath.ToSlash(spec.Out),
|
||||
TrackCount: spec.Tracks,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Printf(
|
||||
"bulk: generated %d tracks in %s (%s, hash %s)\n",
|
||||
spec.Tracks, spec.Out,
|
||||
time.Since(started).Round(time.Second),
|
||||
spec.hash()[:12],
|
||||
)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// bulkSources encodes the handful of distinct clips every generated
|
||||
// file is copied from. Durations vary so "time between tracks" is not
|
||||
// measuring one number repeatedly, and stay short so a measurement run
|
||||
// that plays through a dozen tracks takes seconds.
|
||||
func bulkSources(out string) ([][]byte, func(), error) {
|
||||
tmp, err := os.MkdirTemp("", "yj-bulk-src-")
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("temp dir: %w", err)
|
||||
}
|
||||
|
||||
cleanup := func() { _ = os.RemoveAll(tmp) }
|
||||
|
||||
if err := os.MkdirAll(out, dirPerm); err != nil {
|
||||
cleanup()
|
||||
|
||||
return nil, nil, fmt.Errorf("mkdir %s: %w", out, err)
|
||||
}
|
||||
|
||||
sources := make([][]byte, bulkSourceVariants)
|
||||
|
||||
for i := range bulkSourceVariants {
|
||||
wav := filepath.Join(tmp, strconv.Itoa(i)+".wav")
|
||||
mp3 := filepath.Join(tmp, strconv.Itoa(i)+".mp3")
|
||||
dur := time.Duration(2+i%3) * time.Second
|
||||
|
||||
if err := synthesizeWAV(wav, dur, 220+float64(i)*55); err != nil {
|
||||
cleanup()
|
||||
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
if err := transcode(wav, mp3, tagwriter.FormatMP3); err != nil {
|
||||
cleanup()
|
||||
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
raw, err := os.ReadFile(mp3) //nolint:gosec // generated path.
|
||||
if err != nil {
|
||||
cleanup()
|
||||
|
||||
return nil, nil, fmt.Errorf("read source clip: %w", err)
|
||||
}
|
||||
|
||||
sources[i] = raw
|
||||
}
|
||||
|
||||
return sources, cleanup, nil
|
||||
}
|
||||
|
||||
// bulkCovers renders one cover per album up front. Rendering it per
|
||||
// track would be the dominant cost of generation and would defeat the
|
||||
// library's cover-art deduplication, which is part of what a scan at
|
||||
// this size is being measured on.
|
||||
func bulkCovers(spec bulkSpec, _ [][]byte) ([][]byte, error) {
|
||||
albums := (spec.Tracks + bulkTracksPerAlbum - 1) / bulkTracksPerAlbum
|
||||
covers := make([][]byte, albums)
|
||||
|
||||
for i := range albums {
|
||||
img, err := bulkCoverJPEG(bulkAlbumName(i), spec.CoverPx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
covers[i] = img
|
||||
}
|
||||
|
||||
return covers, nil
|
||||
}
|
||||
|
||||
// bulkCoverJPEG renders a bulk album cover.
|
||||
//
|
||||
// Deliberately not coverJPEGSized: the fixture cover's diagonal band
|
||||
// is 37 hard edges at 300 px, which is the worst case for a DCT and
|
||||
// costs ~35 kB per album — 2 GB across a 50 k-track library, most of
|
||||
// it JPEG artefacts around a pattern nobody looks at. A smooth
|
||||
// two-axis gradient is ~6 kB, still distinct per album by eye, and
|
||||
// still deterministic per key.
|
||||
func bulkCoverJPEG(key string, px int) ([]byte, error) {
|
||||
var seed uint32
|
||||
for _, b := range []byte(key) {
|
||||
seed = seed*31 + uint32(b)
|
||||
}
|
||||
|
||||
img := image.NewRGBA(image.Rect(0, 0, px, px))
|
||||
base := color.RGBA{
|
||||
R: uint8(seed >> 16),
|
||||
G: uint8(seed >> 8),
|
||||
B: uint8(seed),
|
||||
A: 255,
|
||||
}
|
||||
|
||||
for y := range px {
|
||||
shade := float64(y) / float64(px)
|
||||
|
||||
for x := range px {
|
||||
tint := (shade + float64(x)/float64(px)) / 2
|
||||
|
||||
img.Set(x, y, color.RGBA{
|
||||
R: uint8(float64(base.R) * (0.4 + 0.6*tint)),
|
||||
G: uint8(float64(base.G) * (0.4 + 0.6*tint)),
|
||||
B: uint8(float64(base.B) * (0.4 + 0.6*tint)),
|
||||
A: 255,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: jpegQuality}); err != nil {
|
||||
return nil, fmt.Errorf("encode bulk cover %q: %w", key, err)
|
||||
}
|
||||
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
// bulkWrite copies and tags every file, in parallel. tagwriter works
|
||||
// on one path at a time with no shared state, so the only coordination
|
||||
// needed is the error and the progress counter.
|
||||
func bulkWrite(spec bulkSpec, sources, covers [][]byte) error {
|
||||
logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
|
||||
Level: slog.LevelError,
|
||||
}))
|
||||
|
||||
jobs := make(chan int, runtime.NumCPU()*2)
|
||||
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
done atomic.Int64
|
||||
failed atomic.Pointer[error]
|
||||
workers = runtime.NumCPU()
|
||||
)
|
||||
|
||||
for range workers {
|
||||
wg.Add(1)
|
||||
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
for i := range jobs {
|
||||
if failed.Load() != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if err := bulkWriteOne(
|
||||
logger, spec, sources, covers, i,
|
||||
); err != nil {
|
||||
failed.CompareAndSwap(nil, &err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if n := done.Add(1); n%5000 == 0 {
|
||||
fmt.Printf("bulk: %d/%d\n", n, spec.Tracks)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
for i := range spec.Tracks {
|
||||
jobs <- i
|
||||
}
|
||||
|
||||
close(jobs)
|
||||
wg.Wait()
|
||||
|
||||
if err := failed.Load(); err != nil {
|
||||
return *err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func bulkWriteOne(
|
||||
logger *slog.Logger,
|
||||
spec bulkSpec,
|
||||
sources, covers [][]byte,
|
||||
i int,
|
||||
) error {
|
||||
t := bulkTrackAt(i)
|
||||
dst := filepath.Join(spec.Out, t.Rel)
|
||||
|
||||
if err := ensureDir(dst); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := os.WriteFile(dst, sources[t.Variant], filePerm); err != nil {
|
||||
return fmt.Errorf("write %s: %w", dst, err)
|
||||
}
|
||||
|
||||
changes := t.Tags.changes()
|
||||
changes[tagwriter.FieldCoverArt] = covers[t.Album]
|
||||
|
||||
if err := tagwriter.WriteFileTags(logger, dst, changes); err != nil {
|
||||
return fmt.Errorf("tag %s: %w", t.Rel, err)
|
||||
}
|
||||
|
||||
return stampMTime(dst)
|
||||
}
|
||||
+33
-1
@@ -13,6 +13,11 @@
|
||||
//
|
||||
// go run ./cmd/gentestdata # generate if out of date
|
||||
// go run ./cmd/gentestdata -force # regenerate unconditionally
|
||||
// go run ./cmd/gentestdata -bulk 50000 # the measurement library
|
||||
//
|
||||
// The -bulk library is a separate thing with a separate purpose; see
|
||||
// bulk.go. It is not committed, not a test dependency, and generating
|
||||
// it does not regenerate the fixture library.
|
||||
package main
|
||||
|
||||
import (
|
||||
@@ -44,6 +49,9 @@ func main() {
|
||||
brokenDir string
|
||||
manifestOut string
|
||||
force bool
|
||||
bulkTracks int
|
||||
bulkOut string
|
||||
bulkCover int
|
||||
)
|
||||
|
||||
flag.StringVar(
|
||||
@@ -62,9 +70,33 @@ func main() {
|
||||
&force, "force", false,
|
||||
"regenerate even when the manifest is already up to date",
|
||||
)
|
||||
flag.IntVar(
|
||||
&bulkTracks, "bulk", 0,
|
||||
"generate a bulk measurement library of N tracks instead",
|
||||
)
|
||||
flag.StringVar(
|
||||
&bulkOut, "bulk-out", ".dev/music_library_bulk",
|
||||
"library root for -bulk (gitignored; not a test fixture)",
|
||||
)
|
||||
flag.IntVar(
|
||||
&bulkCover, "bulk-cover-px", bulkCoverPx,
|
||||
"edge length of the embedded cover art for -bulk",
|
||||
)
|
||||
flag.Parse()
|
||||
|
||||
if err := run(outDir, brokenDir, manifestOut, force); err != nil {
|
||||
var err error
|
||||
|
||||
if bulkTracks > 0 {
|
||||
err = generateBulk(bulkSpec{
|
||||
Out: bulkOut,
|
||||
Tracks: bulkTracks,
|
||||
CoverPx: bulkCover,
|
||||
}, force)
|
||||
} else {
|
||||
err = run(outDir, brokenDir, manifestOut, force)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "gentestdata:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -34,15 +34,19 @@ type manifestTrack struct {
|
||||
// stamps its own encoder strings, so byte hashes differ between ffmpeg
|
||||
// builds while the fixtures they describe are identical.
|
||||
type manifest struct {
|
||||
Version int `json:"version"`
|
||||
Generator string `json:"generator"`
|
||||
Hash string `json:"hash"`
|
||||
LibraryRoot string `json:"libraryRoot"`
|
||||
BrokenRoot string `json:"brokenRoot"`
|
||||
Cases map[string][]string `json:"cases"`
|
||||
Tracks []manifestTrack `json:"tracks"`
|
||||
Extras []string `json:"extras"`
|
||||
Broken []string `json:"broken"`
|
||||
Version int `json:"version"`
|
||||
Generator string `json:"generator"`
|
||||
Hash string `json:"hash"`
|
||||
LibraryRoot string `json:"libraryRoot"`
|
||||
BrokenRoot string `json:"brokenRoot"`
|
||||
// TrackCount is written by the bulk library, which has tens of
|
||||
// thousands of tracks and no reason to describe each one: nothing
|
||||
// selects a bulk track by name, only the total matters.
|
||||
TrackCount int `json:"trackCount,omitempty"`
|
||||
Cases map[string][]string `json:"cases"`
|
||||
Tracks []manifestTrack `json:"tracks"`
|
||||
Extras []string `json:"extras"`
|
||||
Broken []string `json:"broken"`
|
||||
}
|
||||
|
||||
// buildManifest derives the manifest from the spec alone. It runs
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+24
-2
@@ -16,6 +16,13 @@
|
||||
#
|
||||
# Usage:
|
||||
# scripts/seed-sandbox.sh [--name NAME] [--port N] [--no-build]
|
||||
# [--manifest PATH]
|
||||
#
|
||||
# --manifest points at a different generated library's manifest, which
|
||||
# is how the bulk measurement library (`make bulkdata`) gets seeded:
|
||||
# same script, same discipline, different pile of files. A manifest
|
||||
# either lists its tracks or states a trackCount; both are accepted,
|
||||
# because describing 50 000 tracks individually would serve nobody.
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
@@ -31,6 +38,9 @@ NAME="default"
|
||||
PORT=34115
|
||||
SESSION="yj-seed"
|
||||
BUILD_ARGS=()
|
||||
# Replaced below once the manifest says how many tracks are coming: a
|
||||
# 50 000-track scan is minutes, and a fixed 180 s deadline would abort
|
||||
# a healthy run rather than an unhealthy one.
|
||||
SCAN_TIMEOUT=180
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
@@ -47,6 +57,10 @@ while [ $# -gt 0 ]; do
|
||||
BUILD_ARGS+=(--no-build)
|
||||
shift
|
||||
;;
|
||||
--manifest)
|
||||
MANIFEST="${2:?--manifest needs a path}"
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
echo "seed-sandbox: unknown argument: $1" >&2
|
||||
exit 2
|
||||
@@ -64,12 +78,14 @@ need playwright-cli
|
||||
need jq
|
||||
|
||||
if [ ! -f "$MANIFEST" ]; then
|
||||
echo "seed-sandbox: fixtures missing; run 'make testdata'" >&2
|
||||
echo "seed-sandbox: no manifest at $MANIFEST" >&2
|
||||
echo " run 'make testdata' (fixtures) or 'make bulkdata' (measurement)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
LIBRARY_DIR="$REPO_ROOT/$(jq -r .libraryRoot "$MANIFEST")"
|
||||
WANT_TRACKS="$(jq '.tracks | length' "$MANIFEST")"
|
||||
WANT_TRACKS="$(jq '.trackCount // (.tracks | length)' "$MANIFEST")"
|
||||
SCAN_TIMEOUT=$((180 + WANT_TRACKS / 50))
|
||||
FIXTURE_HASH="$(jq -r .hash "$MANIFEST")"
|
||||
|
||||
cleanup() {
|
||||
@@ -138,6 +154,12 @@ while [ "$SECONDS" -lt "$deadline" ]; do
|
||||
|
||||
[ "$got" = "$WANT_TRACKS" ] && break
|
||||
|
||||
# A large scan is minutes of silence otherwise, which is
|
||||
# indistinguishable from a hang.
|
||||
if [ $((SECONDS % 15)) -eq 0 ]; then
|
||||
echo " ... $got/$WANT_TRACKS"
|
||||
fi
|
||||
|
||||
sleep 1
|
||||
done
|
||||
|
||||
|
||||
Reference in New Issue
Block a user