perf(explore): make the owned-artist backfill yield, mark, and stop

The post-scan backfills share MusicBrainz's rate limiters with every
page the user can open, and both were FIFO — so a thousand-artist
enrichment put an album page behind an hour of queued work.
WithBackgroundLane/WithBackgroundPriority add a slower second lane: a
marked wait takes no token while any interactive wait is outstanding.
It is a context marker rather than a parameter because a backfill calls
the same client methods a detail page does. A long backfill also has to
be visible and stoppable, so jobs.KindCatalogEnrich registers both with
progress and cancel — after the work is counted, since these passes are
a no-op on every launch once the library is covered.

What it does not fetch is the point. It ran for hours against a
900-artist library and marked nothing, because three of the four things
it did per artist were work nobody asked for: similar artists, which
the artist page already resolves on view, and a full GetArtistImage
(fanart.tv, TheAudioDB, Wikidata, Wikipedia, ten portraits) reached
only to warm the MB artist lookup EnsureArtistRels does alone. It was
also serial across artists while every limiter is per-host and idle.

The marks are a table rather than more explore_index columns, because
artifactimport merges by column list and a flag added there is a second
place to remember. BrowseReleaseGroupsAll pages to exhaustion, where
the old call silently cut a prolific artist at 100 release groups.

One portrait is downloaded now; the rest are remembered as URLs.
resolveAllSources downloaded every candidate, up to ten, full size,
while nothing reads anything but primary.jpg — 5.3 GB measured on a
real cache, 4.1 GB of it unreachable. OrphanedArtistImagesJob is why
that survived: it joined the bare MBID onto the images directory, but
artist directories are sharded under a two-character prefix, so it
named a path that never existed and deleted the rows that were the only
record of the files it left behind.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UDCbcCZQepnpSQYJ6SxxZm
This commit is contained in:
2026-08-14 13:33:54 -04:00
co-authored by Claude Opus 5
parent 878cf4b561
commit 20fbf28f2a
22 changed files with 2058 additions and 236 deletions
+49 -7
View File
@@ -193,7 +193,26 @@ var exploreIndexFTSTriggers = []string{
INSERT INTO explore_index_fts(explore_index_fts, rowid, title, artist_name, aliases)
VALUES ('delete', old.id, old.title, old.artist_name, old.aliases);
END`,
`CREATE TRIGGER explore_index_au AFTER UPDATE ON explore_index BEGIN
// Narrowed to the three columns the FTS table actually indexes, and
// guarded on them having changed. An UPDATE that leaves all three
// alone has nothing to re-index, and re-indexing it is not free: an
// FTS5 delete has to find the old row's postings in a multi-million
// row index, which is the ~31 rows/s figure below.
//
// This is not a micro-optimisation. Every writer here upserts, and
// the merge rules keep existing values (`CASE WHEN excluded.title
// != '' ...`), so the common write is a row arriving unchanged: the
// discography backfill re-browsing a known artist, the incremental
// dump refreshing popularity. Each of those used to pay a full
// delete + insert against the FTS index while holding the single
// writer connection — measured at 91% of the app's CPU, with the
// play path queued behind it.
`CREATE TRIGGER explore_index_au AFTER UPDATE OF title, artist_name, aliases
ON explore_index
WHEN old.title IS NOT new.title
OR old.artist_name IS NOT new.artist_name
OR old.aliases IS NOT new.aliases
BEGIN
INSERT INTO explore_index_fts(explore_index_fts, rowid, title, artist_name, aliases)
VALUES ('delete', old.id, old.title, old.artist_name, old.aliases);
INSERT INTO explore_index_fts(rowid, title, artist_name, aliases)
@@ -203,7 +222,17 @@ var exploreIndexFTSTriggers = []string{
// createExploreIndexFTSTriggers installs the sync triggers. Safe to
// call on a database that already has them.
//
// It drops first rather than tolerating "already exists", because a
// trigger is a definition and not a row: an install that already has
// the old one would otherwise keep it forever, and these definitions
// are exactly where this table's write cost is decided. Three DDL
// statements against a table with no rows to rewrite, on open.
func createExploreIndexFTSTriggers(ctx context.Context, db *sql.DB) error {
if err := dropExploreIndexFTSTriggers(ctx, db); err != nil {
return err
}
for _, stmt := range exploreIndexFTSTriggers {
if _, err := db.ExecContext(ctx, stmt); err != nil &&
!strings.Contains(err.Error(), "already exists") {
@@ -214,6 +243,23 @@ func createExploreIndexFTSTriggers(ctx context.Context, db *sql.DB) error {
return nil
}
// exploreIndexFTSTriggerNames is what both the drop paths remove.
var exploreIndexFTSTriggerNames = []string{
"explore_index_ai", "explore_index_ad", "explore_index_au",
}
func dropExploreIndexFTSTriggers(ctx context.Context, db *sql.DB) error {
for _, name := range exploreIndexFTSTriggerNames {
if _, err := db.ExecContext(
ctx, "DROP TRIGGER IF EXISTS "+name,
); err != nil {
return fmt.Errorf("drop explore FTS trigger %s: %w", name, err)
}
}
return nil
}
// SuspendExploreIndexFTS drops the FTS sync triggers so a bulk load can
// write explore_index without paying per-row FTS maintenance.
//
@@ -227,12 +273,8 @@ func createExploreIndexFTSTriggers(ctx context.Context, db *sql.DB) error {
// Callers MUST pair this with ResumeExploreIndexFTS — while suspended,
// explore_index_fts stops tracking the table and search goes stale.
func (d *DB) SuspendExploreIndexFTS() error {
for _, name := range []string{
"explore_index_ai", "explore_index_ad", "explore_index_au",
} {
if _, err := d.db.ExecContext(d.Ctx, "DROP TRIGGER IF EXISTS "+name); err != nil {
return fmt.Errorf("suspend explore FTS: drop %s: %w", name, err)
}
if err := dropExploreIndexFTSTriggers(d.Ctx, d.db); err != nil {
return fmt.Errorf("suspend explore FTS: %w", err)
}
return nil