From 176ac26f91bcb90067ff4f728d0f930f5de213ef Mon Sep 17 00:00:00 2001 From: Caleb Allen Date: Tue, 24 Mar 2026 22:07:16 -0400 Subject: [PATCH] feat: popularity-boosted search reranking via ListenBrainz MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After MB search returns text-relevance-scored results, fetch bulk popularity data from ListenBrainz (POST /1/popularity/{artist, recording,release-group}) for all result MBIDs. Blend scores: final = 0.6 * mb_relevance + 0.4 * log10_popularity Log-scale normalization ensures massive artists don't drown out everything, but popular results rise above obscure exact matches. Release groups (no MB score) sort by raw popularity. Three LB POST calls run concurrently — each hits a different endpoint. All are rate-limited and cached (24h TTL). Example: searching 'tatsuro' now ranks Tatsuro Yamashita (2.5M LB listens, score 97) above 'tatsuro' vocaloid producer (4 listens, score 64) despite the latter being an exact name match on MB. --- backend/explore/explore.go | 199 +++++++++++++++++++++++++++++++- backend/explore/listenbrainz.go | 191 +++++++++++++++++++++++++++++- 2 files changed, 382 insertions(+), 8 deletions(-) diff --git a/backend/explore/explore.go b/backend/explore/explore.go index b3fcca1..bd7fbe6 100644 --- a/backend/explore/explore.go +++ b/backend/explore/explore.go @@ -3,6 +3,8 @@ package explore import ( "context" "log/slog" + "math" + "sort" "sync" "yellowjacket/backend/database" @@ -127,12 +129,17 @@ func (e *Service) CoverArtGroupURL(releaseGroupMBID string) string { } // Search concurrently queries MusicBrainz for artists, release -// groups, and recordings matching the query, returning aggregated -// results in a single round-trip. If any sub-search fails the -// error is logged and the remaining results are still returned. +// groups, and recordings matching the query, then boosts results +// using ListenBrainz popularity data. The final score blends +// text relevance (60%) with log-scaled listen counts (40%). +// +// If any sub-search or popularity lookup fails the error is logged +// and the remaining results are still returned — popularity +// failures degrade to MB-only ordering. func (e *Service) Search(query string) (*MBSearchResult, error) { e.logger.Info("search started", "query", query) + // Phase 1: concurrent MB search (3 goroutines, library-limited). var ( result MBSearchResult mu sync.Mutex @@ -216,6 +223,18 @@ func (e *Service) Search(query string) (*MBSearchResult, error) { wg.Wait() + e.logger.Info("search MB complete", + "query", query, + "artists", len(result.Artists), + "releaseGroups", len(result.ReleaseGroups), + "recordings", len(result.Recordings), + ) + + // Phase 2: concurrent LB popularity lookups (3 goroutines, + // rate-limited). Each hits a different endpoint so they can + // overlap on different rate-limiter tokens. + e.boostWithPopularity(&result) + e.logger.Info("search completed", "query", query, "artists", len(result.Artists), @@ -225,3 +244,177 @@ func (e *Service) Search(query string) (*MBSearchResult, error) { return &result, nil } + +// --------------------------------------------------------------------------- +// Popularity-boosted reranking +// --------------------------------------------------------------------------- + +const ( + // Blending weights for final score. + relevanceWeight = 0.6 + popularityWeight = 0.4 +) + +// boostWithPopularity fetches ListenBrainz listen counts for all +// entities in result and re-sorts each slice using a blended score +// of MB text relevance + log-scaled popularity. Modifies result +// in place. Failures are logged and degrade to MB-only ordering. +func (e *Service) boostWithPopularity(result *MBSearchResult) { + // Collect MBIDs per entity type. + artistMBIDs := make([]string, len(result.Artists)) + for i, a := range result.Artists { + artistMBIDs[i] = a.MBID + } + + recordingMBIDs := make([]string, len(result.Recordings)) + for i, r := range result.Recordings { + recordingMBIDs[i] = r.MBID + } + + rgMBIDs := make([]string, len(result.ReleaseGroups)) + for i, rg := range result.ReleaseGroups { + rgMBIDs[i] = rg.MBID + } + + // Fetch popularity concurrently. + var ( + artistPop map[string]int + recordingPop map[string]int + rgPop map[string]int + wg sync.WaitGroup + ) + + wg.Add(3) //nolint:mnd + + go func() { + defer wg.Done() + + pop, err := e.lb.ArtistPopularity(e.ctx, artistMBIDs) + if err != nil { + e.logger.Warn("popularity lookup failed", "entity", "artist", "error", err) + + return + } + + artistPop = pop + }() + + go func() { + defer wg.Done() + + pop, err := e.lb.RecordingPopularity(e.ctx, recordingMBIDs) + if err != nil { + e.logger.Warn("popularity lookup failed", "entity", "recording", "error", err) + + return + } + + recordingPop = pop + }() + + go func() { + defer wg.Done() + + pop, err := e.lb.ReleaseGroupPopularity(e.ctx, rgMBIDs) + if err != nil { + e.logger.Warn("popularity lookup failed", "entity", "releaseGroup", "error", err) + + return + } + + rgPop = pop + }() + + wg.Wait() + + // Rerank each entity type. + rerankArtists(result.Artists, artistPop) + rerankRecordings(result.Recordings, recordingPop) + rerankReleaseGroups(result.ReleaseGroups, rgPop) +} + +// rerankArtists sorts artists by blended score and updates their +// Score field to the new value (0–100 scale). +func rerankArtists(artists []MBArtist, pop map[string]int) { + if len(artists) == 0 { + return + } + + maxPop := maxListenCount(pop) + + sort.SliceStable(artists, func(i, j int) bool { + si := blendedScore(float64(artists[i].Score)/100.0, pop[artists[i].MBID], maxPop) + sj := blendedScore(float64(artists[j].Score)/100.0, pop[artists[j].MBID], maxPop) + + return si > sj + }) + + // Update Score field so the frontend's top-results section can + // use it directly. + maxPop2 := maxListenCount(pop) + + for i := range artists { + s := blendedScore(float64(artists[i].Score)/100.0, pop[artists[i].MBID], maxPop2) + artists[i].Score = int(s * 100) + } +} + +// rerankRecordings sorts recordings by blended score and updates +// their Score field. +func rerankRecordings(recordings []MBRecording, pop map[string]int) { + if len(recordings) == 0 { + return + } + + maxPop := maxListenCount(pop) + + sort.SliceStable(recordings, func(i, j int) bool { + si := blendedScore(float64(recordings[i].Score)/100.0, pop[recordings[i].MBID], maxPop) + sj := blendedScore(float64(recordings[j].Score)/100.0, pop[recordings[j].MBID], maxPop) + + return si > sj + }) + + for i := range recordings { + s := blendedScore(float64(recordings[i].Score)/100.0, pop[recordings[i].MBID], maxPop) + recordings[i].Score = int(s * 100) + } +} + +// rerankReleaseGroups sorts release groups by popularity only +// (they have no MB score field). +func rerankReleaseGroups(rgs []MBReleaseGroup, pop map[string]int) { + if len(rgs) == 0 || len(pop) == 0 { + return + } + + sort.SliceStable(rgs, func(i, j int) bool { + return pop[rgs[i].MBID] > pop[rgs[j].MBID] + }) +} + +// blendedScore computes relevanceWeight*relevance + popularityWeight*logPop. +// relevance is 0–1. listenCount is raw; maxListenCount is the +// maximum in the result set (for normalization). +func blendedScore(relevance float64, listenCount, maxListenCount int) float64 { + if maxListenCount <= 0 { + return relevance + } + + logPop := math.Log10(float64(listenCount)+1) / math.Log10(float64(maxListenCount)+1) + + return relevanceWeight*relevance + popularityWeight*logPop +} + +// maxListenCount returns the highest listen count in the map. +func maxListenCount(pop map[string]int) int { + maxVal := 0 + + for _, v := range pop { + if v > maxVal { + maxVal = v + } + } + + return maxVal +} diff --git a/backend/explore/listenbrainz.go b/backend/explore/listenbrainz.go index 763967b..e31b6a2 100644 --- a/backend/explore/listenbrainz.go +++ b/backend/explore/listenbrainz.go @@ -1,13 +1,18 @@ package explore import ( + "bytes" "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "errors" "fmt" "io" "log/slog" "net/http" + "slices" + "strings" "time" ) @@ -133,6 +138,153 @@ func (c *ListenBrainzClient) SimilarArtists( return out, nil } +// --------------------------------------------------------------------------- +// Bulk popularity lookups (POST endpoints) +// --------------------------------------------------------------------------- + +// lbPopularityResult is the response shape for all three bulk +// popularity endpoints. The JSON field names are snake_case from +// the ListenBrainz API. +type lbPopularityResult struct { + MBID string `json:"artist_mbid"` + RecordingMBID string `json:"recording_mbid"` + ReleaseGroupMBID string `json:"release_group_mbid"` + TotalListenCount *int `json:"total_listen_count"` + TotalUserCount *int `json:"total_user_count"` +} + +// ArtistPopularity fetches total listen counts for a batch of +// artist MBIDs. Returns a map[mbid]→listenCount. Artists with +// null counts (unknown to LB) are omitted from the map. +func (c *ListenBrainzClient) ArtistPopularity( + ctx context.Context, mbids []string, +) (map[string]int, error) { + if len(mbids) == 0 { + return nil, nil //nolint:nilnil + } + + url := listenBrainzBaseURL + "/1/popularity/artist" + cacheKey := "lb:pop:artist:" + hashMBIDs(mbids) + + if data, ok := c.cache.Get(cacheKey); ok { + var out map[string]int + if err := json.Unmarshal(data, &out); err == nil { + return out, nil + } + } + + body, err := c.doPost(ctx, url, map[string][]string{ + "artist_mbids": mbids, + }) + if err != nil { + return nil, fmt.Errorf("artist popularity: %w", err) + } + + return c.parsePopularity(cacheKey, body, func(r lbPopularityResult) string { + return r.MBID + }) +} + +// RecordingPopularity fetches total listen counts for a batch of +// recording MBIDs. Returns a map[mbid]→listenCount. +func (c *ListenBrainzClient) RecordingPopularity( + ctx context.Context, mbids []string, +) (map[string]int, error) { + if len(mbids) == 0 { + return nil, nil //nolint:nilnil + } + + url := listenBrainzBaseURL + "/1/popularity/recording" + cacheKey := "lb:pop:recording:" + hashMBIDs(mbids) + + if data, ok := c.cache.Get(cacheKey); ok { + var out map[string]int + if err := json.Unmarshal(data, &out); err == nil { + return out, nil + } + } + + body, err := c.doPost(ctx, url, map[string][]string{ + "recording_mbids": mbids, + }) + if err != nil { + return nil, fmt.Errorf("recording popularity: %w", err) + } + + return c.parsePopularity(cacheKey, body, func(r lbPopularityResult) string { + return r.RecordingMBID + }) +} + +// ReleaseGroupPopularity fetches total listen counts for a batch of +// release group MBIDs. Returns a map[mbid]→listenCount. +func (c *ListenBrainzClient) ReleaseGroupPopularity( + ctx context.Context, mbids []string, +) (map[string]int, error) { + if len(mbids) == 0 { + return nil, nil //nolint:nilnil + } + + url := listenBrainzBaseURL + "/1/popularity/release-group" + cacheKey := "lb:pop:release-group:" + hashMBIDs(mbids) + + if data, ok := c.cache.Get(cacheKey); ok { + var out map[string]int + if err := json.Unmarshal(data, &out); err == nil { + return out, nil + } + } + + body, err := c.doPost(ctx, url, map[string][]string{ + "release_group_mbids": mbids, + }) + if err != nil { + return nil, fmt.Errorf("release group popularity: %w", err) + } + + return c.parsePopularity(cacheKey, body, func(r lbPopularityResult) string { + return r.ReleaseGroupMBID + }) +} + +// parsePopularity unmarshals a bulk popularity response, extracts +// the MBID→listenCount mapping, caches it, and returns it. +func (c *ListenBrainzClient) parsePopularity( + cacheKey string, + body []byte, + extractMBID func(lbPopularityResult) string, +) (map[string]int, error) { + var raw []lbPopularityResult + if err := json.Unmarshal(body, &raw); err != nil { + return nil, fmt.Errorf("popularity unmarshal: %w", err) + } + + out := make(map[string]int, len(raw)) + + for _, r := range raw { + mbid := extractMBID(r) + if mbid != "" && r.TotalListenCount != nil { + out[mbid] = *r.TotalListenCount + } + } + + c.cacheJSON(cacheKey, out, cacheTTLSearch, "", "") + + return out, nil +} + +// hashMBIDs produces a short deterministic key from a slice of +// MBIDs by sorting and hashing. Used for cache keys. +func hashMBIDs(mbids []string) string { + sorted := make([]string, len(mbids)) + copy(sorted, mbids) + slices.Sort(sorted) + + h := sha256.Sum256([]byte(strings.Join(sorted, "|"))) + + return hex.EncodeToString(h[:8]) +} + // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- @@ -141,6 +293,26 @@ func (c *ListenBrainzClient) SimilarArtists( // body. Non-2xx status codes are returned as errors. func (c *ListenBrainzClient) doGet( ctx context.Context, url string, +) ([]byte, error) { + return c.doRequest(ctx, http.MethodGet, url, nil) +} + +// doPost performs a rate-limited POST request with a JSON body and +// returns the response body. +func (c *ListenBrainzClient) doPost( + ctx context.Context, url string, body any, +) ([]byte, error) { + payload, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("marshal POST body: %w", err) + } + + return c.doRequest(ctx, http.MethodPost, url, payload) +} + +// doRequest is the shared HTTP helper for GET and POST. +func (c *ListenBrainzClient) doRequest( + ctx context.Context, method string, url string, body []byte, ) ([]byte, error) { c.logger.Debug("listenbrainz rate limiter wait", "url", url) @@ -148,15 +320,24 @@ func (c *ListenBrainzClient) doGet( return nil, fmt.Errorf("rate limiter: %w", err) } - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + var bodyReader io.Reader + if body != nil { + bodyReader = bytes.NewReader(body) + } + + req, err := http.NewRequestWithContext(ctx, method, url, bodyReader) if err != nil { return nil, err } req.Header.Set("User-Agent", lbUserAgent) + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + c.logger.Info("listenbrainz request", - "method", http.MethodGet, + "method", method, "url", url, ) @@ -167,7 +348,7 @@ func (c *ListenBrainzClient) doGet( defer func() { _ = resp.Body.Close() }() - body, err := io.ReadAll(resp.Body) + respBody, err := io.ReadAll(resp.Body) if err != nil { return nil, fmt.Errorf("read body: %w", err) } @@ -179,11 +360,11 @@ func (c *ListenBrainzClient) doGet( if resp.StatusCode < 200 || resp.StatusCode >= 300 { return nil, fmt.Errorf( - "%w: %d %s", ErrListenBrainzHTTP, resp.StatusCode, truncateBody(body), + "%w: %d %s", ErrListenBrainzHTTP, resp.StatusCode, truncateBody(respBody), ) } - return body, nil + return respBody, nil } // cacheJSON marshals v to JSON and stores it in the cache.