perf: unified per-artist indexing — discography + image in parallel
Restructure indexOneArtist to run LB discography fetches and MB artist image resolution concurrently. They use different rate limiters (LB: 3 req/s, MB: 1 req/s) so they overlap without contention. Per artist, the indexer now runs two parallel pipelines: LB pipeline: top-release-groups + top-recordings MB pipeline: url-rels → Wikidata P18 → Wikimedia image fetch All artist images are pre-cached during the index build instead of being resolved on-demand during search. Total build time drops from ~105 min (sequential) to ~63 min (parallel, MB-bound). SearchIndex now takes ArtistImageProvider as a dependency. The Service constructor creates artistImg before the index so both can share it.
This commit is contained in:
@@ -36,11 +36,11 @@ func NewExploreService(logger *slog.Logger, db *database.DB) *Service {
|
|||||||
limiter := NewRateLimiter()
|
limiter := NewRateLimiter()
|
||||||
mb := NewMusicBrainzClient(cache, logger.WithGroup("musicbrainz"))
|
mb := NewMusicBrainzClient(cache, logger.WithGroup("musicbrainz"))
|
||||||
lb := NewListenBrainzClient(limiter, cache, logger.WithGroup("listenbrainz"))
|
lb := NewListenBrainzClient(limiter, cache, logger.WithGroup("listenbrainz"))
|
||||||
index := NewSearchIndex(db, lb, logger.WithGroup("search-index"))
|
|
||||||
artProxy := NewCoverArtProxy(db, limiter)
|
artProxy := NewCoverArtProxy(db, limiter)
|
||||||
artistImg := NewArtistImageProvider(
|
artistImg := NewArtistImageProvider(
|
||||||
db, cache, NewRateLimiter(), logger.WithGroup("artist-image"),
|
db, cache, NewRateLimiter(), logger.WithGroup("artist-image"),
|
||||||
)
|
)
|
||||||
|
index := NewSearchIndex(db, lb, artistImg, logger.WithGroup("search-index"))
|
||||||
|
|
||||||
logger.Info("explore service created")
|
logger.Info("explore service created")
|
||||||
|
|
||||||
|
|||||||
@@ -102,9 +102,10 @@ type lbSitewideArtist struct {
|
|||||||
// - Tier 4: similar artists to library artists (background, ~24min)
|
// - Tier 4: similar artists to library artists (background, ~24min)
|
||||||
// - Tier 5: organic growth from user browsing (ongoing, free)
|
// - Tier 5: organic growth from user browsing (ongoing, free)
|
||||||
type SearchIndex struct {
|
type SearchIndex struct {
|
||||||
db *database.DB
|
db *database.DB
|
||||||
lb *ListenBrainzClient
|
lb *ListenBrainzClient
|
||||||
logger *slog.Logger
|
artistImg *ArtistImageProvider
|
||||||
|
logger *slog.Logger
|
||||||
|
|
||||||
cancel context.CancelFunc
|
cancel context.CancelFunc
|
||||||
done chan struct{}
|
done chan struct{}
|
||||||
@@ -119,13 +120,15 @@ type SearchIndex struct {
|
|||||||
func NewSearchIndex(
|
func NewSearchIndex(
|
||||||
db *database.DB,
|
db *database.DB,
|
||||||
lb *ListenBrainzClient,
|
lb *ListenBrainzClient,
|
||||||
|
artistImg *ArtistImageProvider,
|
||||||
logger *slog.Logger,
|
logger *slog.Logger,
|
||||||
) *SearchIndex {
|
) *SearchIndex {
|
||||||
return &SearchIndex{
|
return &SearchIndex{
|
||||||
db: db,
|
db: db,
|
||||||
lb: lb,
|
lb: lb,
|
||||||
logger: logger,
|
artistImg: artistImg,
|
||||||
done: make(chan struct{}),
|
logger: logger,
|
||||||
|
done: make(chan struct{}),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -932,9 +935,40 @@ func (si *SearchIndex) indexOneArtist(
|
|||||||
}
|
}
|
||||||
|
|
||||||
rgLimit, recLimit := si.scaledLimits(artist.ListenCount)
|
rgLimit, recLimit := si.scaledLimits(artist.ListenCount)
|
||||||
rgs := si.fetchTopReleaseGroups(ctx, lb, artist, rgLimit)
|
|
||||||
recs := si.fetchTopRecordings(ctx, lb, artist, recLimit)
|
|
||||||
|
|
||||||
|
// Run LB discography fetches and MB artist image resolution
|
||||||
|
// concurrently — they use different rate limiters so they
|
||||||
|
// don't block each other.
|
||||||
|
var (
|
||||||
|
rgs []SearchIndexResult
|
||||||
|
recs []SearchIndexResult
|
||||||
|
wg sync.WaitGroup
|
||||||
|
)
|
||||||
|
|
||||||
|
// LB pipeline: top release groups + top recordings.
|
||||||
|
wg.Add(1)
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
|
||||||
|
rgs = si.fetchTopReleaseGroups(ctx, lb, artist, rgLimit)
|
||||||
|
recs = si.fetchTopRecordings(ctx, lb, artist, recLimit)
|
||||||
|
}()
|
||||||
|
|
||||||
|
// MB pipeline: resolve + cache artist image (uses MB rate limiter).
|
||||||
|
wg.Add(1)
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
|
||||||
|
if si.artistImg != nil {
|
||||||
|
si.artistImg.GetArtistImage(artist.ArtistMBID)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
// Batch write discography results.
|
||||||
all := make([]SearchIndexResult, 0, len(rgs)+len(recs))
|
all := make([]SearchIndexResult, 0, len(rgs)+len(recs))
|
||||||
all = append(all, rgs...)
|
all = append(all, rgs...)
|
||||||
all = append(all, recs...)
|
all = append(all, recs...)
|
||||||
|
|||||||
Reference in New Issue
Block a user