perf(explore): make the owned-artist backfill yield, mark, and stop
The post-scan backfills share MusicBrainz's rate limiters with every page the user can open, and both were FIFO — so a thousand-artist enrichment put an album page behind an hour of queued work. WithBackgroundLane/WithBackgroundPriority add a slower second lane: a marked wait takes no token while any interactive wait is outstanding. It is a context marker rather than a parameter because a backfill calls the same client methods a detail page does. A long backfill also has to be visible and stoppable, so jobs.KindCatalogEnrich registers both with progress and cancel — after the work is counted, since these passes are a no-op on every launch once the library is covered. What it does not fetch is the point. It ran for hours against a 900-artist library and marked nothing, because three of the four things it did per artist were work nobody asked for: similar artists, which the artist page already resolves on view, and a full GetArtistImage (fanart.tv, TheAudioDB, Wikidata, Wikipedia, ten portraits) reached only to warm the MB artist lookup EnsureArtistRels does alone. It was also serial across artists while every limiter is per-host and idle. The marks are a table rather than more explore_index columns, because artifactimport merges by column list and a flag added there is a second place to remember. BrowseReleaseGroupsAll pages to exhaustion, where the old call silently cut a prolific artist at 100 release groups. One portrait is downloaded now; the rest are remembered as URLs. resolveAllSources downloaded every candidate, up to ten, full size, while nothing reads anything but primary.jpg — 5.3 GB measured on a real cache, 4.1 GB of it unreachable. OrphanedArtistImagesJob is why that survived: it joined the bare MBID onto the images directory, but artist directories are sharded under a two-character prefix, so it named a path that never existed and deleted the rows that were the only record of the files it left behind. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UDCbcCZQepnpSQYJ6SxxZm
This commit is contained in:
@@ -0,0 +1,181 @@
|
||||
package explore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// artistEnrichment records which catalog passes have completed for one
|
||||
// artist. See sql/schemas/artist_enrichment.sql for why these live
|
||||
// beside explore_index rather than in it.
|
||||
type artistEnrichment struct {
|
||||
// Browsed is true once the full MusicBrainz browse has landed.
|
||||
Browsed bool
|
||||
// Similar is true once similar_artist_map has been filled.
|
||||
Similar bool
|
||||
}
|
||||
|
||||
// enrichmentFor reads an artist's marks. A missing row is the zero
|
||||
// value — nothing done — so a read error degrades to re-fetching rather
|
||||
// than to skipping, which is the safe direction for a resumable pass.
|
||||
func (si *SearchIndex) enrichmentFor(mbid string) artistEnrichment {
|
||||
var mark artistEnrichment
|
||||
|
||||
rows, err := si.db.QueryContext(
|
||||
"SELECT browsed_at IS NOT NULL, similar_at IS NOT NULL "+
|
||||
"FROM artist_enrichment WHERE artist_mbid = ?",
|
||||
mbid,
|
||||
)
|
||||
if err != nil {
|
||||
return mark
|
||||
}
|
||||
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
if rows.Next() {
|
||||
_ = rows.Scan(&mark.Browsed, &mark.Similar)
|
||||
}
|
||||
|
||||
return mark
|
||||
}
|
||||
|
||||
// artistBrowsed reports whether the full MB browse has run for an
|
||||
// artist.
|
||||
func (si *SearchIndex) artistBrowsed(mbid string) bool {
|
||||
return si.enrichmentFor(mbid).Browsed
|
||||
}
|
||||
|
||||
// markArtistBrowsed records that the full MB browse has landed.
|
||||
func (si *SearchIndex) markArtistBrowsed(mbid string) {
|
||||
si.markArtistEnrichment(mbid, "browsed_at")
|
||||
}
|
||||
|
||||
// markArtistSimilar records that similar artists have been persisted.
|
||||
func (si *SearchIndex) markArtistSimilar(mbid string) {
|
||||
si.markArtistEnrichment(mbid, "similar_at")
|
||||
}
|
||||
|
||||
// markArtistEnrichment stamps one column, leaving the other alone. The
|
||||
// column name is never user input — the two callers above are the only
|
||||
// ones, and each passes a literal.
|
||||
func (si *SearchIndex) markArtistEnrichment(mbid, column string) {
|
||||
if mbid == "" {
|
||||
return
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
|
||||
_, err := si.db.ExecContext(
|
||||
"INSERT INTO artist_enrichment (artist_mbid, "+column+") VALUES (?, ?) "+
|
||||
"ON CONFLICT(artist_mbid) DO UPDATE SET "+column+" = excluded."+column,
|
||||
mbid, now,
|
||||
)
|
||||
if err != nil {
|
||||
si.logger.Warn("artist enrichment: mark failed",
|
||||
"mbid", mbid, "column", column, "error", err,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// SetMusicBrainz wires the shared MB client so the owned-artist backfill
|
||||
// can complete a discography the ListenBrainz endpoints can only sketch.
|
||||
// Without it the backfill still runs; it just skips the browse.
|
||||
func (si *SearchIndex) SetMusicBrainz(mb *MusicBrainzClient) {
|
||||
si.mu.Lock()
|
||||
si.mb = mb
|
||||
si.mu.Unlock()
|
||||
}
|
||||
|
||||
// musicBrainz returns the wired MB client, or nil.
|
||||
func (si *SearchIndex) musicBrainz() *MusicBrainzClient {
|
||||
si.mu.RLock()
|
||||
defer si.mu.RUnlock()
|
||||
|
||||
return si.mb
|
||||
}
|
||||
|
||||
// browseFullDiscography fetches every release group MusicBrainz has for
|
||||
// an artist and merges it into the index, then marks the artist browsed.
|
||||
//
|
||||
// This is what makes an owned artist's discography *complete* and
|
||||
// *typed*: `fetchTopReleaseGroups` takes ListenBrainz's top 50 by listen
|
||||
// count, above a popularity floor, and LB returns no secondary types at
|
||||
// all — so without this an artist's page shows their popular albums as
|
||||
// one undifferentiated list, with the tail missing entirely.
|
||||
//
|
||||
// The MBID-keyed mark, rather than the old "does any row have secondary
|
||||
// types" heuristic, is what makes it run once: an artist whose every
|
||||
// release is a plain album has no secondary types to find, so the
|
||||
// heuristic was permanently unsatisfied and re-browsed forever.
|
||||
func (si *SearchIndex) browseFullDiscography(ctx context.Context, mbid string) bool {
|
||||
mb := si.musicBrainz()
|
||||
if mb == nil || mbid == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
rgs, err := mb.BrowseReleaseGroupsAll(ctx, mbid)
|
||||
if err != nil {
|
||||
si.logger.Debug("discography backfill: browse failed",
|
||||
"mbid", mbid, "error", err,
|
||||
)
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// An artist with genuinely no release groups is still browsed —
|
||||
// marking it stops the pass asking again every run. Only an error
|
||||
// above leaves it unmarked.
|
||||
if len(rgs) > 0 {
|
||||
si.AddFromCache(si.browsedArtistName(mbid, rgs), mbid, rgs)
|
||||
}
|
||||
|
||||
si.markArtistBrowsed(mbid)
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// browsedArtistName picks the name AddFromCache will stamp onto every
|
||||
// release group a browse returned.
|
||||
//
|
||||
// MB's browse-by-artist does not echo the artist credit on each item
|
||||
// (the artist is the query parameter), so the name has to come from
|
||||
// somewhere. The canonical local name is preferred — the index title,
|
||||
// then the library row, both via artistDisplayName — and a release
|
||||
// group's own credit is the fallback for an artist neither knows.
|
||||
//
|
||||
// The featuring guard is why that fallback is not simply "the first
|
||||
// credit": a release group credited "X feat. Y" names a collaboration,
|
||||
// not the artist whose page this is, and stamping it on every row is
|
||||
// how one artist's discography came to be filed under a collaboration's
|
||||
// name. Only true featuring markers count — "&", "x" and "," appear
|
||||
// inside real artist names.
|
||||
func (si *SearchIndex) browsedArtistName(mbid string, rgs []MBReleaseGroup) string {
|
||||
if name := si.artistDisplayName(mbid); name != "" && name != mbid {
|
||||
return name
|
||||
}
|
||||
|
||||
for _, rg := range rgs {
|
||||
if rg.ArtistCredit != "" && !looksLikeFeaturingCredit(rg.ArtistCredit) {
|
||||
return rg.ArtistCredit
|
||||
}
|
||||
}
|
||||
|
||||
// Empty rather than the MBID: AddFromCache's "non-empty wins" rule
|
||||
// then leaves whatever real name arrives later untouched.
|
||||
return ""
|
||||
}
|
||||
|
||||
// looksLikeFeaturingCredit reports whether a credit string carries a
|
||||
// "featuring" clause — i.e. it names a collaboration rather than a
|
||||
// single artist.
|
||||
func looksLikeFeaturingCredit(credit string) bool {
|
||||
lower := strings.ToLower(credit)
|
||||
for _, sep := range []string{" feat. ", " feat ", " featuring ", " ft. ", " ft "} {
|
||||
if strings.Contains(lower, sep) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
Reference in New Issue
Block a user