From c9b3c86f378402614f2113aacbb87bc54a427127 Mon Sep 17 00:00:00 2001
From: Caleb Allen <caleballen1276@gmail.com>
Date: Thu, 26 Mar 2026 08:54:43 -0400
Subject: [PATCH] =?UTF-8?q?perf:=20unified=20per-artist=20indexing=20?=
 =?UTF-8?q?=E2=80=94=20discography=20+=20image=20in=20parallel?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Restructure indexOneArtist to run LB discography fetches and MB
artist image resolution concurrently. They use different rate
limiters (LB: 3 req/s, MB: 1 req/s) so they overlap without
contention.

Per artist, the indexer now runs two parallel pipelines:
  LB pipeline: top-release-groups + top-recordings
  MB pipeline: url-rels → Wikidata P18 → Wikimedia image fetch

All artist images are pre-cached during the index build instead
of being resolved on-demand during search. Total build time
drops from ~105 min (sequential) to ~63 min (parallel, MB-bound).

SearchIndex now takes ArtistImageProvider as a dependency. The
Service constructor creates artistImg before the index so both
can share it.
---
 backend/explore/explore.go     |  2 +-
 backend/explore/searchindex.go | 52 ++++++++++++++++++++++++++++------
 2 files changed, 44 insertions(+), 10 deletions(-)

diff --git a/backend/explore/explore.go b/backend/explore/explore.go
index 5008a29..42785d3 100644
--- a/backend/explore/explore.go
+++ b/backend/explore/explore.go
@@ -36,11 +36,11 @@ func NewExploreService(logger *slog.Logger, db *database.DB) *Service {
 	limiter := NewRateLimiter()
 	mb := NewMusicBrainzClient(cache, logger.WithGroup("musicbrainz"))
 	lb := NewListenBrainzClient(limiter, cache, logger.WithGroup("listenbrainz"))
-	index := NewSearchIndex(db, lb, logger.WithGroup("search-index"))
 	artProxy := NewCoverArtProxy(db, limiter)
 	artistImg := NewArtistImageProvider(
 		db, cache, NewRateLimiter(), logger.WithGroup("artist-image"),
 	)
+	index := NewSearchIndex(db, lb, artistImg, logger.WithGroup("search-index"))
 
 	logger.Info("explore service created")
 
diff --git a/backend/explore/searchindex.go b/backend/explore/searchindex.go
index d4369be..be20e01 100644
--- a/backend/explore/searchindex.go
+++ b/backend/explore/searchindex.go
@@ -102,9 +102,10 @@ type lbSitewideArtist struct {
 //   - Tier 4: similar artists to library artists (background, ~24min)
 //   - Tier 5: organic growth from user browsing (ongoing, free)
 type SearchIndex struct {
-	db     *database.DB
-	lb     *ListenBrainzClient
-	logger *slog.Logger
+	db        *database.DB
+	lb        *ListenBrainzClient
+	artistImg *ArtistImageProvider
+	logger    *slog.Logger
 
 	cancel context.CancelFunc
 	done   chan struct{}
@@ -119,13 +120,15 @@ type SearchIndex struct {
 func NewSearchIndex(
 	db *database.DB,
 	lb *ListenBrainzClient,
+	artistImg *ArtistImageProvider,
 	logger *slog.Logger,
 ) *SearchIndex {
 	return &SearchIndex{
-		db:     db,
-		lb:     lb,
-		logger: logger,
-		done:   make(chan struct{}),
+		db:        db,
+		lb:        lb,
+		artistImg: artistImg,
+		logger:    logger,
+		done:      make(chan struct{}),
 	}
 }
 
@@ -932,9 +935,40 @@ func (si *SearchIndex) indexOneArtist(
 	}
 
 	rgLimit, recLimit := si.scaledLimits(artist.ListenCount)
-	rgs := si.fetchTopReleaseGroups(ctx, lb, artist, rgLimit)
-	recs := si.fetchTopRecordings(ctx, lb, artist, recLimit)
 
+	// Run LB discography fetches and MB artist image resolution
+	// concurrently — they use different rate limiters so they
+	// don't block each other.
+	var (
+		rgs  []SearchIndexResult
+		recs []SearchIndexResult
+		wg   sync.WaitGroup
+	)
+
+	// LB pipeline: top release groups + top recordings.
+	wg.Add(1)
+
+	go func() {
+		defer wg.Done()
+
+		rgs = si.fetchTopReleaseGroups(ctx, lb, artist, rgLimit)
+		recs = si.fetchTopRecordings(ctx, lb, artist, recLimit)
+	}()
+
+	// MB pipeline: resolve + cache artist image (uses MB rate limiter).
+	wg.Add(1)
+
+	go func() {
+		defer wg.Done()
+
+		if si.artistImg != nil {
+			si.artistImg.GetArtistImage(artist.ArtistMBID)
+		}
+	}()
+
+	wg.Wait()
+
+	// Batch write discography results.
 	all := make([]SearchIndexResult, 0, len(rgs)+len(recs))
 	all = append(all, rgs...)
 	all = append(all, recs...)