perf(explore): make the owned-artist backfill yield, mark, and stop
The post-scan backfills share MusicBrainz's rate limiters with every page the user can open, and both were FIFO — so a thousand-artist enrichment put an album page behind an hour of queued work. WithBackgroundLane/WithBackgroundPriority add a slower second lane: a marked wait takes no token while any interactive wait is outstanding. It is a context marker rather than a parameter because a backfill calls the same client methods a detail page does. A long backfill also has to be visible and stoppable, so jobs.KindCatalogEnrich registers both with progress and cancel — after the work is counted, since these passes are a no-op on every launch once the library is covered. What it does not fetch is the point. It ran for hours against a 900-artist library and marked nothing, because three of the four things it did per artist were work nobody asked for: similar artists, which the artist page already resolves on view, and a full GetArtistImage (fanart.tv, TheAudioDB, Wikidata, Wikipedia, ten portraits) reached only to warm the MB artist lookup EnsureArtistRels does alone. It was also serial across artists while every limiter is per-host and idle. The marks are a table rather than more explore_index columns, because artifactimport merges by column list and a flag added there is a second place to remember. BrowseReleaseGroupsAll pages to exhaustion, where the old call silently cut a prolific artist at 100 release groups. One portrait is downloaded now; the rest are remembered as URLs. resolveAllSources downloaded every candidate, up to ten, full size, while nothing reads anything but primary.jpg — 5.3 GB measured on a real cache, 4.1 GB of it unreachable. OrphanedArtistImagesJob is why that survived: it joined the bare MBID onto the images directory, but artist directories are sharded under a two-character prefix, so it named a path that never existed and deleted the rows that were the only record of the files it left behind. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UDCbcCZQepnpSQYJ6SxxZm
This commit is contained in:
@@ -193,7 +193,26 @@ var exploreIndexFTSTriggers = []string{
|
||||
INSERT INTO explore_index_fts(explore_index_fts, rowid, title, artist_name, aliases)
|
||||
VALUES ('delete', old.id, old.title, old.artist_name, old.aliases);
|
||||
END`,
|
||||
`CREATE TRIGGER explore_index_au AFTER UPDATE ON explore_index BEGIN
|
||||
// Narrowed to the three columns the FTS table actually indexes, and
|
||||
// guarded on them having changed. An UPDATE that leaves all three
|
||||
// alone has nothing to re-index, and re-indexing it is not free: an
|
||||
// FTS5 delete has to find the old row's postings in a multi-million
|
||||
// row index, which is the ~31 rows/s figure below.
|
||||
//
|
||||
// This is not a micro-optimisation. Every writer here upserts, and
|
||||
// the merge rules keep existing values (`CASE WHEN excluded.title
|
||||
// != '' ...`), so the common write is a row arriving unchanged: the
|
||||
// discography backfill re-browsing a known artist, the incremental
|
||||
// dump refreshing popularity. Each of those used to pay a full
|
||||
// delete + insert against the FTS index while holding the single
|
||||
// writer connection — measured at 91% of the app's CPU, with the
|
||||
// play path queued behind it.
|
||||
`CREATE TRIGGER explore_index_au AFTER UPDATE OF title, artist_name, aliases
|
||||
ON explore_index
|
||||
WHEN old.title IS NOT new.title
|
||||
OR old.artist_name IS NOT new.artist_name
|
||||
OR old.aliases IS NOT new.aliases
|
||||
BEGIN
|
||||
INSERT INTO explore_index_fts(explore_index_fts, rowid, title, artist_name, aliases)
|
||||
VALUES ('delete', old.id, old.title, old.artist_name, old.aliases);
|
||||
INSERT INTO explore_index_fts(rowid, title, artist_name, aliases)
|
||||
@@ -203,7 +222,17 @@ var exploreIndexFTSTriggers = []string{
|
||||
|
||||
// createExploreIndexFTSTriggers installs the sync triggers. Safe to
|
||||
// call on a database that already has them.
|
||||
//
|
||||
// It drops first rather than tolerating "already exists", because a
|
||||
// trigger is a definition and not a row: an install that already has
|
||||
// the old one would otherwise keep it forever, and these definitions
|
||||
// are exactly where this table's write cost is decided. Three DDL
|
||||
// statements against a table with no rows to rewrite, on open.
|
||||
func createExploreIndexFTSTriggers(ctx context.Context, db *sql.DB) error {
|
||||
if err := dropExploreIndexFTSTriggers(ctx, db); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, stmt := range exploreIndexFTSTriggers {
|
||||
if _, err := db.ExecContext(ctx, stmt); err != nil &&
|
||||
!strings.Contains(err.Error(), "already exists") {
|
||||
@@ -214,6 +243,23 @@ func createExploreIndexFTSTriggers(ctx context.Context, db *sql.DB) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// exploreIndexFTSTriggerNames is what both the drop paths remove.
|
||||
var exploreIndexFTSTriggerNames = []string{
|
||||
"explore_index_ai", "explore_index_ad", "explore_index_au",
|
||||
}
|
||||
|
||||
func dropExploreIndexFTSTriggers(ctx context.Context, db *sql.DB) error {
|
||||
for _, name := range exploreIndexFTSTriggerNames {
|
||||
if _, err := db.ExecContext(
|
||||
ctx, "DROP TRIGGER IF EXISTS "+name,
|
||||
); err != nil {
|
||||
return fmt.Errorf("drop explore FTS trigger %s: %w", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SuspendExploreIndexFTS drops the FTS sync triggers so a bulk load can
|
||||
// write explore_index without paying per-row FTS maintenance.
|
||||
//
|
||||
@@ -227,12 +273,8 @@ func createExploreIndexFTSTriggers(ctx context.Context, db *sql.DB) error {
|
||||
// Callers MUST pair this with ResumeExploreIndexFTS — while suspended,
|
||||
// explore_index_fts stops tracking the table and search goes stale.
|
||||
func (d *DB) SuspendExploreIndexFTS() error {
|
||||
for _, name := range []string{
|
||||
"explore_index_ai", "explore_index_ad", "explore_index_au",
|
||||
} {
|
||||
if _, err := d.db.ExecContext(d.Ctx, "DROP TRIGGER IF EXISTS "+name); err != nil {
|
||||
return fmt.Errorf("suspend explore FTS: drop %s: %w", name, err)
|
||||
}
|
||||
if err := dropExploreIndexFTSTriggers(d.Ctx, d.db); err != nil {
|
||||
return fmt.Errorf("suspend explore FTS: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
Reference in New Issue
Block a user