Files
yellowjacket/backend/explore/artistenrichment.go
T
yonluandClaude Opus 5 20fbf28f2a perf(explore): make the owned-artist backfill yield, mark, and stop
The post-scan backfills share MusicBrainz's rate limiters with every
page the user can open, and both were FIFO — so a thousand-artist
enrichment put an album page behind an hour of queued work.
WithBackgroundLane/WithBackgroundPriority add a slower second lane: a
marked wait takes no token while any interactive wait is outstanding.
It is a context marker rather than a parameter because a backfill calls
the same client methods a detail page does. A long backfill also has to
be visible and stoppable, so jobs.KindCatalogEnrich registers both with
progress and cancel — after the work is counted, since these passes are
a no-op on every launch once the library is covered.

What it does not fetch is the point. It ran for hours against a
900-artist library and marked nothing, because three of the four things
it did per artist were work nobody asked for: similar artists, which
the artist page already resolves on view, and a full GetArtistImage
(fanart.tv, TheAudioDB, Wikidata, Wikipedia, ten portraits) reached
only to warm the MB artist lookup EnsureArtistRels does alone. It was
also serial across artists while every limiter is per-host and idle.

The marks are a table rather than more explore_index columns, because
artifactimport merges by column list and a flag added there is a second
place to remember. BrowseReleaseGroupsAll pages to exhaustion, where
the old call silently cut a prolific artist at 100 release groups.

One portrait is downloaded now; the rest are remembered as URLs.
resolveAllSources downloaded every candidate, up to ten, full size,
while nothing reads anything but primary.jpg — 5.3 GB measured on a
real cache, 4.1 GB of it unreachable. OrphanedArtistImagesJob is why
that survived: it joined the bare MBID onto the images directory, but
artist directories are sharded under a two-character prefix, so it
named a path that never existed and deleted the rows that were the only
record of the files it left behind.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UDCbcCZQepnpSQYJ6SxxZm
2026-08-14 13:33:54 -04:00

182 lines
5.7 KiB
Go

package explore
import (
"context"
"strings"
"time"
)
// artistEnrichment records which catalog passes have completed for one
// artist. See sql/schemas/artist_enrichment.sql for why these live
// beside explore_index rather than in it.
type artistEnrichment struct {
// Browsed is true once the full MusicBrainz browse has landed.
Browsed bool
// Similar is true once similar_artist_map has been filled.
Similar bool
}
// enrichmentFor reads an artist's marks. A missing row is the zero
// value — nothing done — so a read error degrades to re-fetching rather
// than to skipping, which is the safe direction for a resumable pass.
func (si *SearchIndex) enrichmentFor(mbid string) artistEnrichment {
var mark artistEnrichment
rows, err := si.db.QueryContext(
"SELECT browsed_at IS NOT NULL, similar_at IS NOT NULL "+
"FROM artist_enrichment WHERE artist_mbid = ?",
mbid,
)
if err != nil {
return mark
}
defer func() { _ = rows.Close() }()
if rows.Next() {
_ = rows.Scan(&mark.Browsed, &mark.Similar)
}
return mark
}
// artistBrowsed reports whether the full MB browse has run for an
// artist.
func (si *SearchIndex) artistBrowsed(mbid string) bool {
return si.enrichmentFor(mbid).Browsed
}
// markArtistBrowsed records that the full MB browse has landed.
func (si *SearchIndex) markArtistBrowsed(mbid string) {
si.markArtistEnrichment(mbid, "browsed_at")
}
// markArtistSimilar records that similar artists have been persisted.
func (si *SearchIndex) markArtistSimilar(mbid string) {
si.markArtistEnrichment(mbid, "similar_at")
}
// markArtistEnrichment stamps one column, leaving the other alone. The
// column name is never user input — the two callers above are the only
// ones, and each passes a literal.
func (si *SearchIndex) markArtistEnrichment(mbid, column string) {
if mbid == "" {
return
}
now := time.Now().UTC()
_, err := si.db.ExecContext(
"INSERT INTO artist_enrichment (artist_mbid, "+column+") VALUES (?, ?) "+
"ON CONFLICT(artist_mbid) DO UPDATE SET "+column+" = excluded."+column,
mbid, now,
)
if err != nil {
si.logger.Warn("artist enrichment: mark failed",
"mbid", mbid, "column", column, "error", err,
)
}
}
// SetMusicBrainz wires the shared MB client so the owned-artist backfill
// can complete a discography the ListenBrainz endpoints can only sketch.
// Without it the backfill still runs; it just skips the browse.
func (si *SearchIndex) SetMusicBrainz(mb *MusicBrainzClient) {
si.mu.Lock()
si.mb = mb
si.mu.Unlock()
}
// musicBrainz returns the wired MB client, or nil.
func (si *SearchIndex) musicBrainz() *MusicBrainzClient {
si.mu.RLock()
defer si.mu.RUnlock()
return si.mb
}
// browseFullDiscography fetches every release group MusicBrainz has for
// an artist and merges it into the index, then marks the artist browsed.
//
// This is what makes an owned artist's discography *complete* and
// *typed*: `fetchTopReleaseGroups` takes ListenBrainz's top 50 by listen
// count, above a popularity floor, and LB returns no secondary types at
// all — so without this an artist's page shows their popular albums as
// one undifferentiated list, with the tail missing entirely.
//
// The MBID-keyed mark, rather than the old "does any row have secondary
// types" heuristic, is what makes it run once: an artist whose every
// release is a plain album has no secondary types to find, so the
// heuristic was permanently unsatisfied and re-browsed forever.
func (si *SearchIndex) browseFullDiscography(ctx context.Context, mbid string) bool {
mb := si.musicBrainz()
if mb == nil || mbid == "" {
return false
}
rgs, err := mb.BrowseReleaseGroupsAll(ctx, mbid)
if err != nil {
si.logger.Debug("discography backfill: browse failed",
"mbid", mbid, "error", err,
)
return false
}
// An artist with genuinely no release groups is still browsed —
// marking it stops the pass asking again every run. Only an error
// above leaves it unmarked.
if len(rgs) > 0 {
si.AddFromCache(si.browsedArtistName(mbid, rgs), mbid, rgs)
}
si.markArtistBrowsed(mbid)
return true
}
// browsedArtistName picks the name AddFromCache will stamp onto every
// release group a browse returned.
//
// MB's browse-by-artist does not echo the artist credit on each item
// (the artist is the query parameter), so the name has to come from
// somewhere. The canonical local name is preferred — the index title,
// then the library row, both via artistDisplayName — and a release
// group's own credit is the fallback for an artist neither knows.
//
// The featuring guard is why that fallback is not simply "the first
// credit": a release group credited "X feat. Y" names a collaboration,
// not the artist whose page this is, and stamping it on every row is
// how one artist's discography came to be filed under a collaboration's
// name. Only true featuring markers count — "&", "x" and "," appear
// inside real artist names.
func (si *SearchIndex) browsedArtistName(mbid string, rgs []MBReleaseGroup) string {
if name := si.artistDisplayName(mbid); name != "" && name != mbid {
return name
}
for _, rg := range rgs {
if rg.ArtistCredit != "" && !looksLikeFeaturingCredit(rg.ArtistCredit) {
return rg.ArtistCredit
}
}
// Empty rather than the MBID: AddFromCache's "non-empty wins" rule
// then leaves whatever real name arrives later untouched.
return ""
}
// looksLikeFeaturingCredit reports whether a credit string carries a
// "featuring" clause — i.e. it names a collaboration rather than a
// single artist.
func looksLikeFeaturingCredit(credit string) bool {
lower := strings.ToLower(credit)
for _, sep := range []string{" feat. ", " feat ", " featuring ", " ft. ", " ft "} {
if strings.Contains(lower, sep) {
return true
}
}
return false
}