From a93a83c4d2357295011a548fe407bf94f115f012 Mon Sep 17 00:00:00 2001 From: Caleb Allen Date: Sun, 29 Mar 2026 16:37:51 -0400 Subject: [PATCH] fix: shared MB rate limiter prevents search/indexer 429 collisions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, the MusicBrainzClient had no proactive rate limiter — it relied on the musicbrainzws2 library's retry-on-429 backoff. When the background indexer was resolving artist images (hitting MB at 1.5 req/sec) and a user search fired 3+ concurrent MB calls, the combined burst triggered 429s with cascading retries up to 60s. Now a single shared RateLimiter (1 req/sec) gates all MB API calls: - MusicBrainzClient search/lookup/browse methods - ArtistImageProvider fetchMBRels (was on a separate 1.5 req/sec limiter) The limiter serializes access proactively, preventing 429s entirely. The musicbrainzws2 retry logic remains as a safety net. Also split the old shared limiter into separate lbLimiter (for ListenBrainz + CoverArt) and mbLimiter (for MusicBrainz) so the two APIs don't block each other. --- backend/explore/explore.go | 11 ++--- backend/explore/musicbrainz.go | 55 +++++++++++++++++++----- frontend/wailsjs/go/explore/Service.d.ts | 2 + frontend/wailsjs/go/explore/Service.js | 4 ++ 4 files changed, 56 insertions(+), 16 deletions(-) diff --git a/backend/explore/explore.go b/backend/explore/explore.go index 6f3b107..f085e67 100644 --- a/backend/explore/explore.go +++ b/backend/explore/explore.go @@ -35,12 +35,13 @@ type Service struct { // client, and ListenBrainz client internally. func NewExploreService(logger *slog.Logger, db *database.DB) *Service { cache := NewCache(db, logger.WithGroup("cache")) - limiter := NewRateLimiter() - mb := NewMusicBrainzClient(cache, logger.WithGroup("musicbrainz")) - lb := NewListenBrainzClient(limiter, cache, logger.WithGroup("listenbrainz")) - artProxy := NewCoverArtProxy(db, limiter) + lbLimiter := NewRateLimiter() + mbLimiter := NewRateLimiter() // 1 req/sec, shared across all MB consumers + mb := NewMusicBrainzClient(cache, mbLimiter, logger.WithGroup("musicbrainz")) + lb := NewListenBrainzClient(lbLimiter, cache, logger.WithGroup("listenbrainz")) + artProxy := NewCoverArtProxy(db, lbLimiter) artistImg := NewArtistImageProvider( - db, cache, NewRateLimiterF(1.5), logger.WithGroup("artist-image"), + db, cache, mbLimiter, logger.WithGroup("artist-image"), ) index := NewSearchIndex(db, lb, artistImg, logger.WithGroup("search-index")) libMBID := NewLibraryMBIDIndex(db) diff --git a/backend/explore/musicbrainz.go b/backend/explore/musicbrainz.go index dcf4831..01c20a3 100644 --- a/backend/explore/musicbrainz.go +++ b/backend/explore/musicbrainz.go @@ -23,18 +23,22 @@ const ( // response cache. Every API call checks the cache first and stores // successful responses for future hits. // -// The underlying musicbrainzws2.Client handles MusicBrainz-specific -// rate limiting via retries on HTTP 429, so we do not use the -// RateLimiter from this package (that is reserved for ListenBrainz). +// A proactive rate limiter gates all outgoing requests at 1 req/sec +// to avoid triggering MusicBrainz 429 responses. The underlying +// musicbrainzws2.Client still retries on 429 as a safety net, but +// the limiter should prevent most rate-limit hits. type MusicBrainzClient struct { - mb *musicbrainzws2.Client - cache *Cache - logger *slog.Logger + mb *musicbrainzws2.Client + cache *Cache + limiter *RateLimiter + logger *slog.Logger } // NewMusicBrainzClient creates a MusicBrainz API client that caches -// responses in the given Cache. -func NewMusicBrainzClient(cache *Cache, logger *slog.Logger) *MusicBrainzClient { +// responses in the given Cache. The provided rate limiter is shared +// with all other MB consumers (e.g. artist image resolution) to +// prevent concurrent bursts from triggering 429s. +func NewMusicBrainzClient(cache *Cache, limiter *RateLimiter, logger *slog.Logger) *MusicBrainzClient { mb := musicbrainzws2.NewClient(musicbrainzws2.AppInfo{ Name: "YellowJacket", Version: "dev", @@ -42,9 +46,10 @@ func NewMusicBrainzClient(cache *Cache, logger *slog.Logger) *MusicBrainzClient }) return &MusicBrainzClient{ - mb: mb, - cache: cache, - logger: logger, + mb: mb, + cache: cache, + limiter: limiter, + logger: logger, } } @@ -71,6 +76,10 @@ func (c *MusicBrainzClient) SearchArtists( } } + if err := c.limiter.Wait(ctx); err != nil { + return nil, err + } + c.logger.Info("musicbrainz search artists", "query", query, "limit", limit, @@ -105,6 +114,10 @@ func (c *MusicBrainzClient) SearchReleaseGroups( } } + if err := c.limiter.Wait(ctx); err != nil { + return nil, err + } + c.logger.Info("musicbrainz search release groups", "query", query, "limit", limit, @@ -139,6 +152,10 @@ func (c *MusicBrainzClient) SearchRecordings( } } + if err := c.limiter.Wait(ctx); err != nil { + return nil, err + } + c.logger.Info("musicbrainz search recordings", "query", query, "limit", limit, @@ -176,6 +193,10 @@ func (c *MusicBrainzClient) LookupArtist( } } + if err := c.limiter.Wait(ctx); err != nil { + return nil, err + } + c.logger.Info("musicbrainz lookup artist", "mbid", mbid) a, err := c.mb.LookupArtist(ctx, @@ -206,6 +227,10 @@ func (c *MusicBrainzClient) LookupReleaseGroup( } } + if err := c.limiter.Wait(ctx); err != nil { + return nil, err + } + c.logger.Info("musicbrainz lookup release group", "mbid", mbid) rg, err := c.mb.LookupReleaseGroup(ctx, @@ -241,6 +266,10 @@ func (c *MusicBrainzClient) BrowseReleaseGroups( } } + if err := c.limiter.Wait(ctx); err != nil { + return nil, err + } + c.logger.Info("musicbrainz browse release groups", "artistMBID", artistMBID, ) @@ -276,6 +305,10 @@ func (c *MusicBrainzClient) BrowseReleases( } } + if err := c.limiter.Wait(ctx); err != nil { + return nil, err + } + c.logger.Info("musicbrainz browse releases", "releaseGroupMBID", releaseGroupMBID, ) diff --git a/frontend/wailsjs/go/explore/Service.d.ts b/frontend/wailsjs/go/explore/Service.d.ts index bdf37d1..1ba3e4e 100755 --- a/frontend/wailsjs/go/explore/Service.d.ts +++ b/frontend/wailsjs/go/explore/Service.d.ts @@ -23,6 +23,8 @@ export function GetThumbnail(arg1:string,arg2:string,arg3:string):Promise):Promise>; +export function IndexNewArtists():Promise; + export function InvalidateIndexDiscographies():Promise; export function LookupArtist(arg1:string):Promise; diff --git a/frontend/wailsjs/go/explore/Service.js b/frontend/wailsjs/go/explore/Service.js index d8ca373..217a451 100755 --- a/frontend/wailsjs/go/explore/Service.js +++ b/frontend/wailsjs/go/explore/Service.js @@ -42,6 +42,10 @@ export function GetThumbnails(arg1) { return window['go']['explore']['Service']['GetThumbnails'](arg1); } +export function IndexNewArtists() { + return window['go']['explore']['Service']['IndexNewArtists'](); +} + export function InvalidateIndexDiscographies() { return window['go']['explore']['Service']['InvalidateIndexDiscographies'](); }