diff --git a/CLAUDE.md b/CLAUDE.md index 65b5f99..a23a31f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -86,4 +86,4 @@ Tests use `database.NewTestDB(t)` for in-memory SQLite with full schema. Test au ## Git Workflow -Direct push to `main` is blocked by lefthook — use feature branches and PRs. Pre-commit runs vet, lint, codegen check, and frontend typecheck in parallel. Pre-push runs the full test suite. +Feature branches and PRs are the norm, but direct pushes to `main` are allowed. Pre-commit runs vet, lint, codegen check, and frontend typecheck in parallel. Pre-push runs the full test suite. diff --git a/backend/app.go b/backend/app.go index da78050..aa740ee 100644 --- a/backend/app.go +++ b/backend/app.go @@ -253,6 +253,12 @@ func (yj *YellowJacketApp) OnStartup(ctx context.Context) { // with no API calls. Deep discographies stay lazy. yj.explore.PopulateLocalCrossReferences() + // Enrich any owned artists whose discography hasn't been + // fetched yet so their wider catalogue is searchable offline + // right after the scan. Background, bounded, resumable, and a + // no-op once every owned artist is covered. + yj.explore.BackfillLibraryDiscographies() + // Start (or resume) the dump-based index build. Skips // itself once the one-time import has completed, so this // is cheap on every startup. @@ -438,6 +444,11 @@ func (yj *YellowJacketApp) OnDomReady(ctx context.Context) { // to fill any remaining gaps. yj.explore.RebuildLyricsIndexIfNeeded() yj.explore.BackfillLibraryLyrics() + + // Continue enriching any owned artists still missing their + // discography (e.g. a prior run was capped or interrupted). + // Cheap no-op once every owned artist is covered. + yj.explore.BackfillLibraryDiscographies() } // Kick off the autotag prefetch worker so any unscored diff --git a/backend/explore/explore.go b/backend/explore/explore.go index 419361e..e368f4c 100644 --- a/backend/explore/explore.go +++ b/backend/explore/explore.go @@ -177,6 +177,15 @@ func (e *Service) PopulateLocalCrossReferencesIfNeeded() { e.index.PopulateLocalCrossReferences() } +// BackfillLibraryDiscographies enriches owned artists that have not had +// their discography fetched yet, in the background. Idempotent and +// bounded — the query only returns unenriched artists and each is marked +// discog_fetched on success, so this is cheap (an empty query) once every +// owned artist is covered and safe to call on every scan and launch. +func (e *Service) BackfillLibraryDiscographies() { + go e.index.BackfillLibraryDiscographies(e.ctx) +} + // InvalidateLibrarySync clears the "ready" markers guarding the gated // library-sync steps so they re-run on the next launch. Call after a // mutation that changes owned content outside a scan (e.g. removing a diff --git a/backend/explore/searchindex.go b/backend/explore/searchindex.go index 86f9988..354f417 100644 --- a/backend/explore/searchindex.go +++ b/backend/explore/searchindex.go @@ -86,6 +86,13 @@ const ( // indexer's dedicated rate limiter (LB allows 30/10s). indexerRate = 3 + // discogBackfillMaxPerRun bounds how many owned artists a single + // post-scan discography backfill run enriches before yielding. The + // remainder stay unenriched (discog_fetched = 0) and are picked up by + // the next run, so a large first-scan library spreads its enrichment + // across launches instead of running for the better part of an hour. + discogBackfillMaxPerRun = 2000 + // indexSimilarPerArtist is how many similar artists to store // per library artist in similar_artist_map. indexSimilarPerArtist = 20 @@ -345,6 +352,98 @@ func (si *SearchIndex) artistDiscogFetched(mbid string) bool { return rows.Next() } +// unenrichedLibraryArtistMBIDs returns MBIDs for owned artists whose +// discography has not yet been fetched — either they have no index row +// or their row is still discog_fetched = 0. The LEFT JOIN keys off the +// persistent flag, so an artist enriched on a prior run (interactively or +// by an earlier backfill) never reappears, giving "new artists only" for +// free. Ordered by owned-track count so the artists the user has most of +// are enriched first. The limit bounds a single run (see +// discogBackfillMaxPerRun). +func (si *SearchIndex) unenrichedLibraryArtistMBIDs(limit int) []string { + rows, err := si.db.QueryContext(` + SELECT a.mbid + FROM artists a + LEFT JOIN explore_index ei + ON ei.entity_type = 'artist' AND ei.mbid = a.mbid + WHERE a.mbid IS NOT NULL AND a.mbid != '' + AND (ei.id IS NULL OR ei.discog_fetched = 0) + GROUP BY a.mbid + ORDER BY COUNT(*) DESC + LIMIT ? + `, limit) + if err != nil { + si.logger.Warn("discography backfill: query failed", "error", err) + + return nil + } + + defer func() { _ = rows.Close() }() + + var mbids []string + + for rows.Next() { + var mbid string + if err := rows.Scan(&mbid); err == nil { + mbids = append(mbids, mbid) + } + } + + return mbids +} + +// BackfillLibraryDiscographies fetches top release groups and recordings +// for every owned artist that has not been enriched yet, so an artist's +// wider catalogue is searchable offline right after a scan instead of +// only on first artist-page view. It is bounded (discogBackfillMaxPerRun) +// and resumable — each artist is marked discog_fetched on success, so a +// cancelled or capped run simply continues on the next call. Runs through +// discogSF so it never double-fetches an artist a concurrent interactive +// EnsureArtistDiscography is already handling. +func (si *SearchIndex) BackfillLibraryDiscographies(ctx context.Context) { + if si.lb == nil { + return + } + + mbids := si.unenrichedLibraryArtistMBIDs(discogBackfillMaxPerRun) + if len(mbids) == 0 { + return + } + + // One shared rate limiter paces the whole run, unlike the per-call + // client EnsureArtistDiscography builds for interactive fetches. + indexLB := NewListenBrainzClient( + NewRateLimiterN(indexerRate), si.lb.cache, si.logger.WithGroup("indexer"), + ) + + done := 0 + + for _, mbid := range mbids { + if ctx.Err() != nil { + return + } + + _, _, _ = si.discogSF.Do(mbid, func() (any, error) { + // Re-check under the singleflight: an interactive fetch may + // have enriched this artist since the query above. + if si.artistDiscogFetched(mbid) { + return nil, nil + } + + si.indexOneArtist(ctx, indexLB, lbSitewideArtist{ + ArtistMBID: mbid, + ArtistName: si.artistDisplayName(mbid), + }) + + return nil, nil + }) + + done++ + } + + si.logger.Info("discography backfill complete", "artists", done) +} + // artistDisplayName resolves a human-readable name for an artist MBID, // preferring the index title, then the local library, then the MBID // itself. Used to seed the discography fetch's artist entry. diff --git a/lefthook.yml b/lefthook.yml index fb2f309..b05cce6 100644 --- a/lefthook.yml +++ b/lefthook.yml @@ -31,17 +31,6 @@ pre-commit: pre-push: parallel: true commands: - protect-main: - run: | - branch=$(git rev-parse --abbrev-ref HEAD) - if [ "$branch" = "main" ]; then - echo "Direct push to main is not allowed. Use a pull request instead." - exit 1 - fi - skip: - - merge - - rebase - go-test: glob: "*.go" run: go test -tags webkit2_41 -race -count=1 -timeout 120s ./...