From 59d5b7272ef6a21d1252fb29e3ba8996db3239a5 Mon Sep 17 00:00:00 2001 From: Caleb Allen Date: Wed, 25 Mar 2026 13:11:15 -0400 Subject: [PATCH] feat: popularity-scaled per-artist index budgets Instead of fixed 20 RGs + 100 recordings for every artist, scale the budget by popularity using a power curve (exponent 0.3): Radiohead (2.5M listens): 20 RGs, 100 recordings Hans Zimmer (715K): 15 RGs, 71 recordings Clutch (178K): 11 RGs, 50 recordings Similar (~10K): 7 RGs, 27 recordings Organic (unknown): 5 RGs, 10 recordings Saves ~53% index size (~29 MB vs ~62 MB) with identical API calls. The savings come from T4 similar artists (long tail) where full discographies were wasteful. Top artists still get full coverage. --- backend/explore/searchindex.go | 74 ++++++++++++++++++++++++++++------ 1 file changed, 61 insertions(+), 13 deletions(-) diff --git a/backend/explore/searchindex.go b/backend/explore/searchindex.go index d50eadd..554aeaf 100644 --- a/backend/explore/searchindex.go +++ b/backend/explore/searchindex.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "log/slog" + "math" "net/http" "strings" "sync" @@ -23,13 +24,17 @@ const ( // from the LB sitewide endpoint. indexTopArtists = 1000 - // indexRGsPerArtist is the number of top release groups to - // store per artist. - indexRGsPerArtist = 20 + // indexMaxRGs is the ceiling for release groups per artist. + indexMaxRGs = 20 - // indexRecsPerArtist is the number of top recordings to store - // per artist. - indexRecsPerArtist = 100 + // indexMinRGs is the floor for release groups per artist. + indexMinRGs = 5 + + // indexMaxRecs is the ceiling for recordings per artist. + indexMaxRecs = 100 + + // indexMinRecs is the floor for recordings per artist. + indexMinRecs = 10 // indexMinPopularity is the minimum listen count for an entry // to be indexed. Cuts noise from long-tail entries. @@ -49,6 +54,12 @@ const ( // per library artist for Tier 4 expansion. indexSimilarPerArtist = 50 + // indexPopularityExponent controls how steeply the per-artist + // budget scales with popularity. Lower = steeper curve. + // 0.3 means an artist with 1/10th the listens of the max gets + // ~50% of the budget, not 10%. + indexPopularityExponent = 0.3 + // labsBaseURL is the base URL for the ListenBrainz labs API. labsBaseURL = "https://labs.api.listenbrainz.org" @@ -93,8 +104,9 @@ type SearchIndex struct { cancel context.CancelFunc done chan struct{} - mu sync.RWMutex - ready bool + mu sync.RWMutex + ready bool + maxListens int // highest artist listen count seen, for scaling } // NewSearchIndex creates a search index backed by the given @@ -411,13 +423,23 @@ func (si *SearchIndex) buildTier1Sitewide( si.upsertSearchResults(rgs) } - // Insert all artists. + // Insert all artists and track max popularity. artists := make([]lbSitewideArtist, 0, len(artistMap)) + maxL := 0 + for _, a := range artistMap { artists = append(artists, a) + + if a.ListenCount > maxL { + maxL = a.ListenCount + } } + si.mu.Lock() + si.maxListens = maxL + si.mu.Unlock() + si.upsertArtists(artists) si.logger.Info("search index: Tier 1 indexed", @@ -891,8 +913,9 @@ func (si *SearchIndex) indexOneArtist( return } - rgs := si.fetchTopReleaseGroups(ctx, lb, artist) - recs := si.fetchTopRecordings(ctx, lb, artist) + rgLimit, recLimit := si.scaledLimits(artist.ListenCount) + rgs := si.fetchTopReleaseGroups(ctx, lb, artist, rgLimit) + recs := si.fetchTopRecordings(ctx, lb, artist, recLimit) all := make([]SearchIndexResult, 0, len(rgs)+len(recs)) all = append(all, rgs...) @@ -912,6 +935,7 @@ func (si *SearchIndex) fetchTopReleaseGroups( ctx context.Context, lb *ListenBrainzClient, artist lbSitewideArtist, + maxCount int, ) []SearchIndexResult { url := fmt.Sprintf( "%s/1/popularity/top-release-groups-for-artist/%s", @@ -947,7 +971,7 @@ func (si *SearchIndex) fetchTopReleaseGroups( return nil } - limit := indexRGsPerArtist + limit := maxCount if limit > len(raw) { limit = len(raw) } @@ -987,6 +1011,7 @@ func (si *SearchIndex) fetchTopRecordings( ctx context.Context, lb *ListenBrainzClient, artist lbSitewideArtist, + maxCount int, ) []SearchIndexResult { url := fmt.Sprintf( "%s/1/popularity/top-recordings-for-artist/%s", @@ -1003,7 +1028,7 @@ func (si *SearchIndex) fetchTopRecordings( return nil } - limit := indexRecsPerArtist + limit := maxCount if limit > len(raw) { limit = len(raw) } @@ -1180,6 +1205,29 @@ func (si *SearchIndex) markReadyIfPopulated() { } } +// scaledLimits returns the number of release groups and recordings +// to index for an artist with the given listen count, scaled by +// popularity relative to the most popular artist in the index. +func (si *SearchIndex) scaledLimits(listenCount int) (rgs, recs int) { + si.mu.RLock() + maxL := si.maxListens + si.mu.RUnlock() + + if maxL <= 0 || listenCount <= 0 { + return indexMinRGs, indexMinRecs + } + + ratio := math.Pow(float64(listenCount)/float64(maxL), indexPopularityExponent) + + rgs = int(float64(indexMinRGs) + ratio*float64(indexMaxRGs-indexMinRGs)) + recs = int(float64(indexMinRecs) + ratio*float64(indexMaxRecs-indexMinRecs)) + + rgs = max(indexMinRGs, min(indexMaxRGs, rgs)) + recs = max(indexMinRecs, min(indexMaxRecs, recs)) + + return rgs, recs +} + // newLBRequest creates an HTTP GET request with the LB User-Agent. func newLBRequest(ctx context.Context, url string) (*http.Request, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)