perf(explore): make the owned-artist backfill yield, mark, and stop

The post-scan backfills share MusicBrainz's rate limiters with every
page the user can open, and both were FIFO — so a thousand-artist
enrichment put an album page behind an hour of queued work.
WithBackgroundLane/WithBackgroundPriority add a slower second lane: a
marked wait takes no token while any interactive wait is outstanding.
It is a context marker rather than a parameter because a backfill calls
the same client methods a detail page does. A long backfill also has to
be visible and stoppable, so jobs.KindCatalogEnrich registers both with
progress and cancel — after the work is counted, since these passes are
a no-op on every launch once the library is covered.

What it does not fetch is the point. It ran for hours against a
900-artist library and marked nothing, because three of the four things
it did per artist were work nobody asked for: similar artists, which
the artist page already resolves on view, and a full GetArtistImage
(fanart.tv, TheAudioDB, Wikidata, Wikipedia, ten portraits) reached
only to warm the MB artist lookup EnsureArtistRels does alone. It was
also serial across artists while every limiter is per-host and idle.

The marks are a table rather than more explore_index columns, because
artifactimport merges by column list and a flag added there is a second
place to remember. BrowseReleaseGroupsAll pages to exhaustion, where
the old call silently cut a prolific artist at 100 release groups.

One portrait is downloaded now; the rest are remembered as URLs.
resolveAllSources downloaded every candidate, up to ten, full size,
while nothing reads anything but primary.jpg — 5.3 GB measured on a
real cache, 4.1 GB of it unreachable. OrphanedArtistImagesJob is why
that survived: it joined the bare MBID onto the images directory, but
artist directories are sharded under a two-character prefix, so it
named a path that never existed and deleted the rows that were the only
record of the files it left behind.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UDCbcCZQepnpSQYJ6SxxZm

backend/database/database.go

@@ -193,7 +193,26 @@ var exploreIndexFTSTriggers = []string{
 		INSERT INTO explore_index_fts(explore_index_fts, rowid, title, artist_name, aliases)
 		VALUES ('delete', old.id, old.title, old.artist_name, old.aliases);
 	END`,
-	`CREATE TRIGGER explore_index_au AFTER UPDATE ON explore_index BEGIN
+	// Narrowed to the three columns the FTS table actually indexes, and
+	// guarded on them having changed.  An UPDATE that leaves all three
+	// alone has nothing to re-index, and re-indexing it is not free: an
+	// FTS5 delete has to find the old row's postings in a multi-million
+	// row index, which is the ~31 rows/s figure below.
+	//
+	// This is not a micro-optimisation.  Every writer here upserts, and
+	// the merge rules keep existing values (`CASE WHEN excluded.title
+	// != '' ...`), so the common write is a row arriving unchanged: the
+	// discography backfill re-browsing a known artist, the incremental
+	// dump refreshing popularity.  Each of those used to pay a full
+	// delete + insert against the FTS index while holding the single
+	// writer connection — measured at 91% of the app's CPU, with the
+	// play path queued behind it.
+	`CREATE TRIGGER explore_index_au AFTER UPDATE OF title, artist_name, aliases
+	ON explore_index
+	WHEN old.title IS NOT new.title
+		OR old.artist_name IS NOT new.artist_name
+		OR old.aliases IS NOT new.aliases
+	BEGIN
 		INSERT INTO explore_index_fts(explore_index_fts, rowid, title, artist_name, aliases)
 		VALUES ('delete', old.id, old.title, old.artist_name, old.aliases);
 		INSERT INTO explore_index_fts(rowid, title, artist_name, aliases)
@@ -203,7 +222,17 @@ var exploreIndexFTSTriggers = []string{
 
 // createExploreIndexFTSTriggers installs the sync triggers.  Safe to
 // call on a database that already has them.
+//
+// It drops first rather than tolerating "already exists", because a
+// trigger is a definition and not a row: an install that already has
+// the old one would otherwise keep it forever, and these definitions
+// are exactly where this table's write cost is decided.  Three DDL
+// statements against a table with no rows to rewrite, on open.
 func createExploreIndexFTSTriggers(ctx context.Context, db *sql.DB) error {
+	if err := dropExploreIndexFTSTriggers(ctx, db); err != nil {
+		return err
+	}
+
 	for _, stmt := range exploreIndexFTSTriggers {
 		if _, err := db.ExecContext(ctx, stmt); err != nil &&
 			!strings.Contains(err.Error(), "already exists") {
@@ -214,6 +243,23 @@ func createExploreIndexFTSTriggers(ctx context.Context, db *sql.DB) error {
 	return nil
 }
 
+// exploreIndexFTSTriggerNames is what both the drop paths remove.
+var exploreIndexFTSTriggerNames = []string{
+	"explore_index_ai", "explore_index_ad", "explore_index_au",
+}
+
+func dropExploreIndexFTSTriggers(ctx context.Context, db *sql.DB) error {
+	for _, name := range exploreIndexFTSTriggerNames {
+		if _, err := db.ExecContext(
+			ctx, "DROP TRIGGER IF EXISTS "+name,
+		); err != nil {
+			return fmt.Errorf("drop explore FTS trigger %s: %w", name, err)
+		}
+	}
+
+	return nil
+}
+
 // SuspendExploreIndexFTS drops the FTS sync triggers so a bulk load can
 // write explore_index without paying per-row FTS maintenance.
 //
@@ -227,12 +273,8 @@ func createExploreIndexFTSTriggers(ctx context.Context, db *sql.DB) error {
 // Callers MUST pair this with ResumeExploreIndexFTS — while suspended,
 // explore_index_fts stops tracking the table and search goes stale.
 func (d *DB) SuspendExploreIndexFTS() error {
-	for _, name := range []string{
-		"explore_index_ai", "explore_index_ad", "explore_index_au",
-	} {
-		if _, err := d.db.ExecContext(d.Ctx, "DROP TRIGGER IF EXISTS "+name); err != nil {
-			return fmt.Errorf("suspend explore FTS: drop %s: %w", name, err)
-		}
+	if err := dropExploreIndexFTSTriggers(d.Ctx, d.db); err != nil {
+		return fmt.Errorf("suspend explore FTS: %w", err)
 	}
 
 	return nil

backend/database/explorefts_test.go

@@ -1,6 +1,7 @@
 package database
 
 import (
+	"strings"
 	"testing"
 )
 
@@ -157,3 +158,160 @@ func TestExploreFTSSuspendIsIdempotent(t *testing.T) {
 		t.Fatalf("resume: %v", err)
 	}
 }
+
+// ftsSegmentCount reports how much the FTS index itself has been
+// written to.  Every delete + insert the update trigger performs
+// appends to the shadow content table, so this is the observable that
+// tells "the trigger re-indexed the row" from "the trigger declined
+// to".  Search results cannot: a no-op re-index leaves the same
+// matches behind.
+func ftsSegmentCount(t *testing.T, db *DB) int {
+	t.Helper()
+
+	rows, err := db.QueryContext("SELECT COUNT(*) FROM explore_index_fts_data")
+	if err != nil {
+		t.Fatalf("fts data count: %v", err)
+	}
+
+	defer func() { _ = rows.Close() }()
+
+	n := 0
+
+	if rows.Next() {
+		if err := rows.Scan(&n); err != nil {
+			t.Fatalf("scan fts data count: %v", err)
+		}
+	}
+
+	return n
+}
+
+// The common write in this schema is an upsert whose merge rules keep
+// every existing value — the discography backfill re-browsing a known
+// artist, the incremental dump refreshing popularity.  Re-indexing
+// those cost an FTS5 delete against a multi-million row index while
+// holding the single writer connection, which is what starved the
+// playback path.  An update that leaves title, artist_name and aliases
+// alone must not touch the FTS index at all.
+func TestExploreFTSUpdateSkipsUnchangedText(t *testing.T) {
+	db := NewTestDB(t)
+
+	seedExploreRow(t, db, "mbid-1", "Unchanged Title", "Steady Artist")
+
+	before := ftsSegmentCount(t, db)
+
+	// A popularity refresh: an FTS column is not named at all.
+	if _, err := db.ExecContext(
+		"UPDATE explore_index SET popularity = 42 WHERE mbid = 'mbid-1'",
+	); err != nil {
+		t.Fatalf("popularity update: %v", err)
+	}
+
+	// An upsert-shaped write that re-states the text identically, which
+	// is what the merge rules produce for a row that has not changed.
+	if _, err := db.ExecContext(`
+		UPDATE explore_index
+		SET title = 'Unchanged Title', artist_name = 'Steady Artist', popularity = 43
+		WHERE mbid = 'mbid-1'
+	`); err != nil {
+		t.Fatalf("no-op text update: %v", err)
+	}
+
+	if got := ftsSegmentCount(t, db); got != before {
+		t.Errorf(
+			"FTS index written by an update that changed no text: %d rows, want %d",
+			got, before,
+		)
+	}
+
+	if got := ftsMatches(t, db, "Unchanged"); got != 1 {
+		t.Errorf("matches after unchanged updates = %d, want 1", got)
+	}
+}
+
+// The other half of the same guard: a real rename still re-indexes,
+// old term gone and new term found.
+func TestExploreFTSUpdateReindexesChangedText(t *testing.T) {
+	db := NewTestDB(t)
+
+	seedExploreRow(t, db, "mbid-2", "Original Title", "Some Artist")
+
+	if _, err := db.ExecContext(
+		"UPDATE explore_index SET title = 'Corrected Title' WHERE mbid = 'mbid-2'",
+	); err != nil {
+		t.Fatalf("rename: %v", err)
+	}
+
+	if got := ftsMatches(t, db, "Original"); got != 0 {
+		t.Errorf("matches for the old title = %d, want 0", got)
+	}
+
+	if got := ftsMatches(t, db, "Corrected"); got != 1 {
+		t.Errorf("matches for the new title = %d, want 1", got)
+	}
+
+	// The same for the other two indexed columns.
+	if _, err := db.ExecContext(
+		"UPDATE explore_index SET artist_name = 'Renamed Artist', aliases = 'AKA Thing' WHERE mbid = 'mbid-2'",
+	); err != nil {
+		t.Fatalf("artist rename: %v", err)
+	}
+
+	if got := ftsMatches(t, db, "Renamed"); got != 1 {
+		t.Errorf("matches for the new artist = %d, want 1", got)
+	}
+
+	if got := ftsMatches(t, db, "AKA"); got != 1 {
+		t.Errorf("matches for the new alias = %d, want 1", got)
+	}
+}
+
+// An existing install already carries the previous, unguarded trigger,
+// and a create that tolerated "already exists" would leave it there
+// forever — so the definition has to be replaced on open, not merely
+// offered.
+func TestExploreFTSTriggersAreReplacedOnOpen(t *testing.T) {
+	db := NewTestDB(t)
+
+	if err := db.SuspendExploreIndexFTS(); err != nil {
+		t.Fatalf("suspend: %v", err)
+	}
+
+	// The shape that shipped before: fires on every UPDATE.
+	if _, err := db.ExecContext(`
+		CREATE TRIGGER explore_index_au AFTER UPDATE ON explore_index BEGIN
+			INSERT INTO explore_index_fts(explore_index_fts, rowid, title, artist_name, aliases)
+			VALUES ('delete', old.id, old.title, old.artist_name, old.aliases);
+			INSERT INTO explore_index_fts(rowid, title, artist_name, aliases)
+			VALUES (new.id, new.title, new.artist_name, new.aliases);
+		END
+	`); err != nil {
+		t.Fatalf("install old trigger: %v", err)
+	}
+
+	if err := createExploreIndexFTSTriggers(db.Ctx, db.db); err != nil {
+		t.Fatalf("recreate triggers: %v", err)
+	}
+
+	rows, err := db.QueryContext(
+		"SELECT sql FROM sqlite_master WHERE type = 'trigger' AND name = 'explore_index_au'",
+	)
+	if err != nil {
+		t.Fatalf("read trigger sql: %v", err)
+	}
+
+	defer func() { _ = rows.Close() }()
+
+	definition := ""
+
+	if rows.Next() {
+		if err := rows.Scan(&definition); err != nil {
+			t.Fatalf("scan trigger sql: %v", err)
+		}
+	}
+
+	if !strings.Contains(definition, "UPDATE OF") ||
+		!strings.Contains(definition, "WHEN") {
+		t.Errorf("explore_index_au was not replaced; definition is:\n%s", definition)
+	}
+}

backend/database/sql/schemas/artist_enrichment.sql

@@ -0,0 +1,31 @@
+-- Per-artist record of which catalog enrichment passes have completed,
+-- so the owned-artist backfill knows what is left to do.
+--
+-- It is a table rather than more flag columns on `explore_index` for one
+-- reason: the downloaded catalog artifact is merged into that table by
+-- column list (see artifactimport.go), so a flag added there is a second
+-- place to remember, and forgetting it silently wipes every mark on the
+-- next catalog update.  These marks are about *this install's* fetching,
+-- which the artifact knows nothing about.
+--
+-- Each column is a separate fetch with its own failure mode, which is
+-- why they are not one boolean: an MB browse failing must not claim the
+-- similar-artists fetch, or vice versa.  NULL means "not done" — the
+-- timestamp is for debugging and for any future re-fetch policy, not
+-- for expiry.  Nothing expires these today.
+--
+-- `explore_index.discog_fetched` is deliberately NOT duplicated here: it
+-- means "this artist's top release groups and recordings are present",
+-- which the artifact legitimately answers for artists it covers.
+
+CREATE TABLE IF NOT EXISTS artist_enrichment (
+  artist_mbid  TEXT PRIMARY KEY,
+
+  -- The full MusicBrainz browse-by-artist landed: every release group,
+  -- with primary and secondary types.  ListenBrainz's top-release-groups
+  -- endpoint gives neither the tail nor the types.
+  browsed_at   DATETIME,
+
+  -- similar_artist_map has been filled for this artist.
+  similar_at   DATETIME
+);

backend/database/sql/sqlcgen/models.go

@@ -26,6 +26,12 @@ type ArtistCreditArtist struct {
 	CreditID int64
 }
 
+type ArtistEnrichment struct {
+	ArtistMbid string
+	BrowsedAt  sql.NullTime
+	SimilarAt  sql.NullTime
+}
+
 type ArtistImage struct {
 	ID         int64
 	ArtistMbid string