perf: unified per-artist indexing — discography + image in parallel

Restructure indexOneArtist to run LB discography fetches and MB
artist image resolution concurrently. They use different rate
limiters (LB: 3 req/s, MB: 1 req/s) so they overlap without
contention.

Per artist, the indexer now runs two parallel pipelines:
  LB pipeline: top-release-groups + top-recordings
  MB pipeline: url-rels → Wikidata P18 → Wikimedia image fetch

All artist images are pre-cached during the index build instead
of being resolved on-demand during search. Total build time
drops from ~105 min (sequential) to ~63 min (parallel, MB-bound).

SearchIndex now takes ArtistImageProvider as a dependency. The
Service constructor creates artistImg before the index so both
can share it.
This commit is contained in:
2026-03-26 08:54:43 -04:00
parent 963269b753
commit c9b3c86f37
2 changed files with 44 additions and 10 deletions
+1 -1
View File
@@ -36,11 +36,11 @@ func NewExploreService(logger *slog.Logger, db *database.DB) *Service {
limiter := NewRateLimiter() limiter := NewRateLimiter()
mb := NewMusicBrainzClient(cache, logger.WithGroup("musicbrainz")) mb := NewMusicBrainzClient(cache, logger.WithGroup("musicbrainz"))
lb := NewListenBrainzClient(limiter, cache, logger.WithGroup("listenbrainz")) lb := NewListenBrainzClient(limiter, cache, logger.WithGroup("listenbrainz"))
index := NewSearchIndex(db, lb, logger.WithGroup("search-index"))
artProxy := NewCoverArtProxy(db, limiter) artProxy := NewCoverArtProxy(db, limiter)
artistImg := NewArtistImageProvider( artistImg := NewArtistImageProvider(
db, cache, NewRateLimiter(), logger.WithGroup("artist-image"), db, cache, NewRateLimiter(), logger.WithGroup("artist-image"),
) )
index := NewSearchIndex(db, lb, artistImg, logger.WithGroup("search-index"))
logger.Info("explore service created") logger.Info("explore service created")
+43 -9
View File
@@ -102,9 +102,10 @@ type lbSitewideArtist struct {
// - Tier 4: similar artists to library artists (background, ~24min) // - Tier 4: similar artists to library artists (background, ~24min)
// - Tier 5: organic growth from user browsing (ongoing, free) // - Tier 5: organic growth from user browsing (ongoing, free)
type SearchIndex struct { type SearchIndex struct {
db *database.DB db *database.DB
lb *ListenBrainzClient lb *ListenBrainzClient
logger *slog.Logger artistImg *ArtistImageProvider
logger *slog.Logger
cancel context.CancelFunc cancel context.CancelFunc
done chan struct{} done chan struct{}
@@ -119,13 +120,15 @@ type SearchIndex struct {
func NewSearchIndex( func NewSearchIndex(
db *database.DB, db *database.DB,
lb *ListenBrainzClient, lb *ListenBrainzClient,
artistImg *ArtistImageProvider,
logger *slog.Logger, logger *slog.Logger,
) *SearchIndex { ) *SearchIndex {
return &SearchIndex{ return &SearchIndex{
db: db, db: db,
lb: lb, lb: lb,
logger: logger, artistImg: artistImg,
done: make(chan struct{}), logger: logger,
done: make(chan struct{}),
} }
} }
@@ -932,9 +935,40 @@ func (si *SearchIndex) indexOneArtist(
} }
rgLimit, recLimit := si.scaledLimits(artist.ListenCount) rgLimit, recLimit := si.scaledLimits(artist.ListenCount)
rgs := si.fetchTopReleaseGroups(ctx, lb, artist, rgLimit)
recs := si.fetchTopRecordings(ctx, lb, artist, recLimit)
// Run LB discography fetches and MB artist image resolution
// concurrently — they use different rate limiters so they
// don't block each other.
var (
rgs []SearchIndexResult
recs []SearchIndexResult
wg sync.WaitGroup
)
// LB pipeline: top release groups + top recordings.
wg.Add(1)
go func() {
defer wg.Done()
rgs = si.fetchTopReleaseGroups(ctx, lb, artist, rgLimit)
recs = si.fetchTopRecordings(ctx, lb, artist, recLimit)
}()
// MB pipeline: resolve + cache artist image (uses MB rate limiter).
wg.Add(1)
go func() {
defer wg.Done()
if si.artistImg != nil {
si.artistImg.GetArtistImage(artist.ArtistMBID)
}
}()
wg.Wait()
// Batch write discography results.
all := make([]SearchIndexResult, 0, len(rgs)+len(recs)) all := make([]SearchIndexResult, 0, len(rgs)+len(recs))
all = append(all, rgs...) all = append(all, rgs...)
all = append(all, recs...) all = append(all, recs...)