From 37f75d50e5869b30a43c1ef4486f7894840d6fc1 Mon Sep 17 00:00:00 2001 From: Caleb Allen Date: Fri, 24 Jul 2026 13:38:26 -0400 Subject: [PATCH] feat(explore): backfill owned artists' discographies offline post-scan Enrich owned artists whose discography hasn't been fetched yet in a bounded, resumable background pass so their wider catalogue is searchable offline right after a scan, instead of only on first artist-page view. Keyed off the persistent discog_fetched flag via LEFT JOIN, so already- enriched artists never reappear and the run is a cheap no-op once every owned artist is covered. Capped at discogBackfillMaxPerRun per run and routed through discogSF to avoid double-fetching an artist a concurrent interactive EnsureArtistDiscography is handling. Invoked on both scan completion (OnStartup) and OnDomReady to resume a capped/interrupted run. Co-Authored-By: Claude Opus 4.8 --- backend/app.go | 11 ++++ backend/explore/explore.go | 9 ++++ backend/explore/searchindex.go | 99 ++++++++++++++++++++++++++++++++++ 3 files changed, 119 insertions(+) diff --git a/backend/app.go b/backend/app.go index da78050..aa740ee 100644 --- a/backend/app.go +++ b/backend/app.go @@ -253,6 +253,12 @@ func (yj *YellowJacketApp) OnStartup(ctx context.Context) { // with no API calls. Deep discographies stay lazy. yj.explore.PopulateLocalCrossReferences() + // Enrich any owned artists whose discography hasn't been + // fetched yet so their wider catalogue is searchable offline + // right after the scan. Background, bounded, resumable, and a + // no-op once every owned artist is covered. + yj.explore.BackfillLibraryDiscographies() + // Start (or resume) the dump-based index build. Skips // itself once the one-time import has completed, so this // is cheap on every startup. @@ -438,6 +444,11 @@ func (yj *YellowJacketApp) OnDomReady(ctx context.Context) { // to fill any remaining gaps. yj.explore.RebuildLyricsIndexIfNeeded() yj.explore.BackfillLibraryLyrics() + + // Continue enriching any owned artists still missing their + // discography (e.g. a prior run was capped or interrupted). + // Cheap no-op once every owned artist is covered. + yj.explore.BackfillLibraryDiscographies() } // Kick off the autotag prefetch worker so any unscored diff --git a/backend/explore/explore.go b/backend/explore/explore.go index 419361e..e368f4c 100644 --- a/backend/explore/explore.go +++ b/backend/explore/explore.go @@ -177,6 +177,15 @@ func (e *Service) PopulateLocalCrossReferencesIfNeeded() { e.index.PopulateLocalCrossReferences() } +// BackfillLibraryDiscographies enriches owned artists that have not had +// their discography fetched yet, in the background. Idempotent and +// bounded — the query only returns unenriched artists and each is marked +// discog_fetched on success, so this is cheap (an empty query) once every +// owned artist is covered and safe to call on every scan and launch. +func (e *Service) BackfillLibraryDiscographies() { + go e.index.BackfillLibraryDiscographies(e.ctx) +} + // InvalidateLibrarySync clears the "ready" markers guarding the gated // library-sync steps so they re-run on the next launch. Call after a // mutation that changes owned content outside a scan (e.g. removing a diff --git a/backend/explore/searchindex.go b/backend/explore/searchindex.go index 86f9988..354f417 100644 --- a/backend/explore/searchindex.go +++ b/backend/explore/searchindex.go @@ -86,6 +86,13 @@ const ( // indexer's dedicated rate limiter (LB allows 30/10s). indexerRate = 3 + // discogBackfillMaxPerRun bounds how many owned artists a single + // post-scan discography backfill run enriches before yielding. The + // remainder stay unenriched (discog_fetched = 0) and are picked up by + // the next run, so a large first-scan library spreads its enrichment + // across launches instead of running for the better part of an hour. + discogBackfillMaxPerRun = 2000 + // indexSimilarPerArtist is how many similar artists to store // per library artist in similar_artist_map. indexSimilarPerArtist = 20 @@ -345,6 +352,98 @@ func (si *SearchIndex) artistDiscogFetched(mbid string) bool { return rows.Next() } +// unenrichedLibraryArtistMBIDs returns MBIDs for owned artists whose +// discography has not yet been fetched — either they have no index row +// or their row is still discog_fetched = 0. The LEFT JOIN keys off the +// persistent flag, so an artist enriched on a prior run (interactively or +// by an earlier backfill) never reappears, giving "new artists only" for +// free. Ordered by owned-track count so the artists the user has most of +// are enriched first. The limit bounds a single run (see +// discogBackfillMaxPerRun). +func (si *SearchIndex) unenrichedLibraryArtistMBIDs(limit int) []string { + rows, err := si.db.QueryContext(` + SELECT a.mbid + FROM artists a + LEFT JOIN explore_index ei + ON ei.entity_type = 'artist' AND ei.mbid = a.mbid + WHERE a.mbid IS NOT NULL AND a.mbid != '' + AND (ei.id IS NULL OR ei.discog_fetched = 0) + GROUP BY a.mbid + ORDER BY COUNT(*) DESC + LIMIT ? + `, limit) + if err != nil { + si.logger.Warn("discography backfill: query failed", "error", err) + + return nil + } + + defer func() { _ = rows.Close() }() + + var mbids []string + + for rows.Next() { + var mbid string + if err := rows.Scan(&mbid); err == nil { + mbids = append(mbids, mbid) + } + } + + return mbids +} + +// BackfillLibraryDiscographies fetches top release groups and recordings +// for every owned artist that has not been enriched yet, so an artist's +// wider catalogue is searchable offline right after a scan instead of +// only on first artist-page view. It is bounded (discogBackfillMaxPerRun) +// and resumable — each artist is marked discog_fetched on success, so a +// cancelled or capped run simply continues on the next call. Runs through +// discogSF so it never double-fetches an artist a concurrent interactive +// EnsureArtistDiscography is already handling. +func (si *SearchIndex) BackfillLibraryDiscographies(ctx context.Context) { + if si.lb == nil { + return + } + + mbids := si.unenrichedLibraryArtistMBIDs(discogBackfillMaxPerRun) + if len(mbids) == 0 { + return + } + + // One shared rate limiter paces the whole run, unlike the per-call + // client EnsureArtistDiscography builds for interactive fetches. + indexLB := NewListenBrainzClient( + NewRateLimiterN(indexerRate), si.lb.cache, si.logger.WithGroup("indexer"), + ) + + done := 0 + + for _, mbid := range mbids { + if ctx.Err() != nil { + return + } + + _, _, _ = si.discogSF.Do(mbid, func() (any, error) { + // Re-check under the singleflight: an interactive fetch may + // have enriched this artist since the query above. + if si.artistDiscogFetched(mbid) { + return nil, nil + } + + si.indexOneArtist(ctx, indexLB, lbSitewideArtist{ + ArtistMBID: mbid, + ArtistName: si.artistDisplayName(mbid), + }) + + return nil, nil + }) + + done++ + } + + si.logger.Info("discography backfill complete", "artists", done) +} + // artistDisplayName resolves a human-readable name for an artist MBID, // preferring the index title, then the local library, then the MBID // itself. Used to seed the discography fetch's artist entry.