perf(explore): make the owned-artist backfill yield, mark, and stop

The post-scan backfills share MusicBrainz's rate limiters with every page the user can open, and both were FIFO — so a thousand-artist enrichment put an album page behind an hour of queued work. WithBackgroundLane/WithBackgroundPriority add a slower second lane: a marked wait takes no token while any interactive wait is outstanding. It is a context marker rather than a parameter because a backfill calls the same client methods a detail page does. A long backfill also has to be visible and stoppable, so jobs.KindCatalogEnrich registers both with progress and cancel — after the work is counted, since these passes are a no-op on every launch once the library is covered. What it does not fetch is the point. It ran for hours against a 900-artist library and marked nothing, because three of the four things it did per artist were work nobody asked for: similar artists, which the artist page already resolves on view, and a full GetArtistImage (fanart.tv, TheAudioDB, Wikidata, Wikipedia, ten portraits) reached only to warm the MB artist lookup EnsureArtistRels does alone. It was also serial across artists while every limiter is per-host and idle. The marks are a table rather than more explore_index columns, because artifactimport merges by column list and a flag added there is a second place to remember. BrowseReleaseGroupsAll pages to exhaustion, where the old call silently cut a prolific artist at 100 release groups. One portrait is downloaded now; the rest are remembered as URLs. resolveAllSources downloaded every candidate, up to ten, full size, while nothing reads anything but primary.jpg — 5.3 GB measured on a real cache, 4.1 GB of it unreachable. OrphanedArtistImagesJob is why that survived: it joined the bare MBID onto the images directory, but artist directories are sharded under a two-character prefix, so it named a path that never existed and deleted the rows that were the only record of the files it left behind. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UDCbcCZQepnpSQYJ6SxxZm
2026-08-14 13:33:54 -04:00
parent 878cf4b561
commit 20fbf28f2a
22 changed files with 2058 additions and 236 deletions
@@ -11,6 +11,7 @@ import (
 	"time"

 	"yellowjacket/backend/database"
+	"yellowjacket/backend/explore"
 )

 // errTestJobFailed stands in for a job returning an error.
@@ -287,6 +288,12 @@ func TestOrphanedCoverFilesJob_EmptyLiveSetIsNoOp(t *testing.T) {

 // Artwork for an artist in the library is kept regardless of age;
 // artwork for a browsed artist ages out.
+//
+// The directories are laid out by explore.ArtistImageDir rather than by
+// this test, which is the point: the job used to join the bare MBID,
+// name a path that has never existed, delete the rows and leave every
+// file on disk.  A test that invents its own flat layout agrees with
+// the bug.
 func TestOrphanedArtistImagesJob(t *testing.T) {
 	t.Parallel()

@@ -300,7 +307,7 @@ func TestOrphanedArtistImagesJob(t *testing.T) {
 	)

 	for _, mbid := range []string{ownedMBID, browsedMBID, recentMBID} {
-		artistDir := filepath.Join(dir, mbid)
+		artistDir := explore.ArtistImageDir(dir, mbid)
 		if err := os.MkdirAll(artistDir, 0o755); err != nil {
 			t.Fatalf("mkdir %s: %v", mbid, err)
 		}
@@ -334,27 +341,27 @@ func TestOrphanedArtistImagesJob(t *testing.T) {
 			   (artist_mbid, source, source_url, file_path, created_at)
 			 VALUES (?, 'test', 'http://x', ?, ?)`,
 			tc.mbid,
-			filepath.Join(dir, tc.mbid, "primary.jpg"),
+			filepath.Join(explore.ArtistImageDir(dir, tc.mbid), "primary.jpg"),
 			tc.created,
 		); err != nil {
 			t.Fatalf("seed artist_images for %s: %v", tc.mbid, err)
 		}
 	}

-	if _, err := OrphanedArtistImagesJob(db, dir).
+	if _, err := OrphanedArtistImagesJob(db, dir, explore.ArtistImageDir).
 		Run(context.Background()); err != nil {
 		t.Fatalf("run job: %v", err)
 	}

-	if _, err := os.Stat(filepath.Join(dir, ownedMBID)); err != nil {
+	if _, err := os.Stat(explore.ArtistImageDir(dir, ownedMBID)); err != nil {
 		t.Error("artwork for a library artist was evicted")
 	}

-	if _, err := os.Stat(filepath.Join(dir, recentMBID)); err != nil {
+	if _, err := os.Stat(explore.ArtistImageDir(dir, recentMBID)); err != nil {
 		t.Error("recently fetched artwork was evicted")
 	}

-	if _, err := os.Stat(filepath.Join(dir, browsedMBID)); !os.IsNotExist(err) {
+	if _, err := os.Stat(explore.ArtistImageDir(dir, browsedMBID)); !os.IsNotExist(err) {
 		t.Error("stale browsed artwork survived the sweep")
 	}

@@ -432,3 +439,85 @@ func TestSweepMissingDirectory(t *testing.T) {
 		t.Errorf("FilesDeleted = %d, want 0", result.FilesDeleted)
 	}
 }
+
+// A directory holding an artist's portrait plus the candidates an older
+// version downloaded keeps the portrait and loses the candidates.
+func TestStrayArtistImageFilesJob(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+
+	const mbid = "44444444-4444-4444-4444-444444444444"
+
+	artistDir := explore.ArtistImageDir(dir, mbid)
+	if err := os.MkdirAll(artistDir, 0o755); err != nil {
+		t.Fatalf("mkdir: %v", err)
+	}
+
+	keep := []string{
+		"primary.jpg", "primary_sm.jpg", "primary_md.jpg",
+		"primary_lg.jpg", ".miss",
+	}
+	strays := []string{"audiodb_0.jpg", "fanart_1.jpg", "wikidata_3.jpg"}
+
+	for _, name := range append(append([]string{}, keep...), strays...) {
+		if err := os.WriteFile(
+			filepath.Join(artistDir, name), []byte("xx"), 0o600,
+		); err != nil {
+			t.Fatalf("write %s: %v", name, err)
+		}
+	}
+
+	result, err := StrayArtistImageFilesJob(dir, explore.ArtistImageKeepNames()).
+		Run(context.Background())
+	if err != nil {
+		t.Fatalf("run job: %v", err)
+	}
+
+	if result.FilesDeleted != int64(len(strays)) {
+		t.Errorf("FilesDeleted = %d, want %d", result.FilesDeleted, len(strays))
+	}
+
+	for _, name := range keep {
+		if _, err := os.Stat(filepath.Join(artistDir, name)); err != nil {
+			t.Errorf("%s was swept and should not have been", name)
+		}
+	}
+
+	for _, name := range strays {
+		if _, err := os.Stat(
+			filepath.Join(artistDir, name),
+		); !os.IsNotExist(err) {
+			t.Errorf("%s survived the sweep", name)
+		}
+	}
+}
+
+// An empty keep set would condemn every file, which is never what a
+// caller means — it is a failed lookup, not an empty live set.
+func TestStrayArtistImageFilesJobRefusesEmptyKeepSet(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+
+	const mbid = "55555555-5555-5555-5555-555555555555"
+
+	artistDir := explore.ArtistImageDir(dir, mbid)
+	if err := os.MkdirAll(artistDir, 0o755); err != nil {
+		t.Fatalf("mkdir: %v", err)
+	}
+
+	primary := filepath.Join(artistDir, "primary.jpg")
+	if err := os.WriteFile(primary, []byte("xx"), 0o600); err != nil {
+		t.Fatalf("write: %v", err)
+	}
+
+	if _, err := StrayArtistImageFilesJob(dir, nil).
+		Run(context.Background()); err != nil {
+		t.Fatalf("run job: %v", err)
+	}
+
+	if _, err := os.Stat(primary); err != nil {
+		t.Error("an empty keep set emptied the directory")
+	}
+}
@@ -140,7 +140,20 @@ func liveCoverFiles(
 // an artist the user owns is kept indefinitely, and everything else ages
 // out. Rows whose file has vanished are dropped so the table matches
 // what is actually on disk.
-func OrphanedArtistImagesJob(db *database.DB, artistImagesDir string) Job {
+//
+// dirFor maps an artist MBID to its directory.  It is a parameter, as
+// OrphanedCoverFilesJob's expandVariants is, because that layout is the
+// image provider's business and this job had been guessing it: artist
+// directories are sharded under a two-character prefix, so joining the
+// bare MBID named a path that has never existed.  RemoveAll succeeds on
+// a missing path, so the job reported success, freed nothing, and
+// deleted the rows that were the only record of the files it left
+// behind.
+func OrphanedArtistImagesJob(
+	db *database.DB,
+	artistImagesDir string,
+	dirFor func(baseDir, mbid string) string,
+) Job {
 	return Job{
 		Name:        "artist-images-sweep",
 		MinInterval: dailyInterval,
@@ -162,7 +175,7 @@ func OrphanedArtistImagesJob(db *database.DB, artistImagesDir string) Job {
 					return result, nil
 				}

-				dir := filepath.Join(artistImagesDir, mbid)
+				dir := dirFor(artistImagesDir, mbid)

 				freed, files := dirSize(dir)

@@ -188,6 +201,82 @@ func OrphanedArtistImagesJob(db *database.DB, artistImagesDir string) Job {
 	}
 }

+// StrayArtistImageFilesJob removes downloaded image candidates that
+// were never the artist's portrait.
+//
+// The image provider used to download every candidate an upstream
+// offered — up to ten, full size — and keep them all, while nothing in
+// the app has ever read anything but primary.jpg and its three size
+// tiers.  It now downloads candidates in priority order until one
+// succeeds and records the rest as URLs, so nothing new lands here;
+// this reclaims what earlier versions left, which on a real cache was
+// about four fifths of it.
+//
+// keep is the set of names an artist directory is allowed to hold.  It
+// is a parameter for the same reason dirFor above is: the provider owns
+// the naming, and a sweep that guesses it deletes the wrong files.
+func StrayArtistImageFilesJob(artistImagesDir string, keep map[string]bool) Job {
+	return Job{
+		Name:        "artist-images-strays",
+		MinInterval: dailyInterval,
+		Run: func(ctx context.Context) (Result, error) {
+			// An empty keep set would mean "every file is garbage",
+			// which is never the intent — refuse, as the covers sweep
+			// refuses an empty live set.
+			if len(keep) == 0 {
+				return Result{}, nil
+			}
+
+			var result Result
+
+			shards, err := os.ReadDir(artistImagesDir)
+			if err != nil {
+				if os.IsNotExist(err) {
+					return Result{}, nil
+				}
+
+				return Result{}, fmt.Errorf("read %s: %w", artistImagesDir, err)
+			}
+
+			for _, shard := range shards {
+				if !shard.IsDir() {
+					continue
+				}
+
+				shardDir := filepath.Join(artistImagesDir, shard.Name())
+
+				artists, err := os.ReadDir(shardDir)
+				if err != nil {
+					continue
+				}
+
+				for _, artist := range artists {
+					if ctx.Err() != nil {
+						return result, nil
+					}
+
+					if !artist.IsDir() {
+						continue
+					}
+
+					swept, err := sweepDir(ctx,
+						filepath.Join(shardDir, artist.Name()),
+						func(name string) bool { return !keep[name] },
+					)
+					if err != nil {
+						continue
+					}
+
+					result.FilesDeleted += swept.FilesDeleted
+					result.BytesFreed += swept.BytesFreed
+				}
+			}
+
+			return result, nil
+		},
+	}
+}
+
 // staleArtistMBIDs returns artist MBIDs whose cached artwork may be
 // evicted: fetched before the cutoff and not an artist in the library.
 func staleArtistMBIDs(