fix: shared MB rate limiter prevents search/indexer 429 collisions

Previously, the MusicBrainzClient had no proactive rate limiter —
it relied on the musicbrainzws2 library's retry-on-429 backoff.
When the background indexer was resolving artist images (hitting MB
at 1.5 req/sec) and a user search fired 3+ concurrent MB calls,
the combined burst triggered 429s with cascading retries up to 60s.

Now a single shared RateLimiter (1 req/sec) gates all MB API calls:
- MusicBrainzClient search/lookup/browse methods
- ArtistImageProvider fetchMBRels (was on a separate 1.5 req/sec limiter)

The limiter serializes access proactively, preventing 429s entirely.
The musicbrainzws2 retry logic remains as a safety net.

Also split the old shared limiter into separate lbLimiter (for
ListenBrainz + CoverArt) and mbLimiter (for MusicBrainz) so the
two APIs don't block each other.
This commit is contained in:
2026-03-29 16:37:51 -04:00
parent 0f2a82256c
commit a93a83c4d2
4 changed files with 56 additions and 16 deletions
+6 -5
View File
@@ -35,12 +35,13 @@ type Service struct {
// client, and ListenBrainz client internally.
func NewExploreService(logger *slog.Logger, db *database.DB) *Service {
cache := NewCache(db, logger.WithGroup("cache"))
limiter := NewRateLimiter()
mb := NewMusicBrainzClient(cache, logger.WithGroup("musicbrainz"))
lb := NewListenBrainzClient(limiter, cache, logger.WithGroup("listenbrainz"))
artProxy := NewCoverArtProxy(db, limiter)
lbLimiter := NewRateLimiter()
mbLimiter := NewRateLimiter() // 1 req/sec, shared across all MB consumers
mb := NewMusicBrainzClient(cache, mbLimiter, logger.WithGroup("musicbrainz"))
lb := NewListenBrainzClient(lbLimiter, cache, logger.WithGroup("listenbrainz"))
artProxy := NewCoverArtProxy(db, lbLimiter)
artistImg := NewArtistImageProvider(
db, cache, NewRateLimiterF(1.5), logger.WithGroup("artist-image"),
db, cache, mbLimiter, logger.WithGroup("artist-image"),
)
index := NewSearchIndex(db, lb, artistImg, logger.WithGroup("search-index"))
libMBID := NewLibraryMBIDIndex(db)
+44 -11
View File
@@ -23,18 +23,22 @@ const (
// response cache. Every API call checks the cache first and stores
// successful responses for future hits.
//
// The underlying musicbrainzws2.Client handles MusicBrainz-specific
// rate limiting via retries on HTTP 429, so we do not use the
// RateLimiter from this package (that is reserved for ListenBrainz).
// A proactive rate limiter gates all outgoing requests at 1 req/sec
// to avoid triggering MusicBrainz 429 responses. The underlying
// musicbrainzws2.Client still retries on 429 as a safety net, but
// the limiter should prevent most rate-limit hits.
type MusicBrainzClient struct {
mb *musicbrainzws2.Client
cache *Cache
logger *slog.Logger
mb *musicbrainzws2.Client
cache *Cache
limiter *RateLimiter
logger *slog.Logger
}
// NewMusicBrainzClient creates a MusicBrainz API client that caches
// responses in the given Cache.
func NewMusicBrainzClient(cache *Cache, logger *slog.Logger) *MusicBrainzClient {
// responses in the given Cache. The provided rate limiter is shared
// with all other MB consumers (e.g. artist image resolution) to
// prevent concurrent bursts from triggering 429s.
func NewMusicBrainzClient(cache *Cache, limiter *RateLimiter, logger *slog.Logger) *MusicBrainzClient {
mb := musicbrainzws2.NewClient(musicbrainzws2.AppInfo{
Name: "YellowJacket",
Version: "dev",
@@ -42,9 +46,10 @@ func NewMusicBrainzClient(cache *Cache, logger *slog.Logger) *MusicBrainzClient
})
return &MusicBrainzClient{
mb: mb,
cache: cache,
logger: logger,
mb: mb,
cache: cache,
limiter: limiter,
logger: logger,
}
}
@@ -71,6 +76,10 @@ func (c *MusicBrainzClient) SearchArtists(
}
}
if err := c.limiter.Wait(ctx); err != nil {
return nil, err
}
c.logger.Info("musicbrainz search artists",
"query", query,
"limit", limit,
@@ -105,6 +114,10 @@ func (c *MusicBrainzClient) SearchReleaseGroups(
}
}
if err := c.limiter.Wait(ctx); err != nil {
return nil, err
}
c.logger.Info("musicbrainz search release groups",
"query", query,
"limit", limit,
@@ -139,6 +152,10 @@ func (c *MusicBrainzClient) SearchRecordings(
}
}
if err := c.limiter.Wait(ctx); err != nil {
return nil, err
}
c.logger.Info("musicbrainz search recordings",
"query", query,
"limit", limit,
@@ -176,6 +193,10 @@ func (c *MusicBrainzClient) LookupArtist(
}
}
if err := c.limiter.Wait(ctx); err != nil {
return nil, err
}
c.logger.Info("musicbrainz lookup artist", "mbid", mbid)
a, err := c.mb.LookupArtist(ctx,
@@ -206,6 +227,10 @@ func (c *MusicBrainzClient) LookupReleaseGroup(
}
}
if err := c.limiter.Wait(ctx); err != nil {
return nil, err
}
c.logger.Info("musicbrainz lookup release group", "mbid", mbid)
rg, err := c.mb.LookupReleaseGroup(ctx,
@@ -241,6 +266,10 @@ func (c *MusicBrainzClient) BrowseReleaseGroups(
}
}
if err := c.limiter.Wait(ctx); err != nil {
return nil, err
}
c.logger.Info("musicbrainz browse release groups",
"artistMBID", artistMBID,
)
@@ -276,6 +305,10 @@ func (c *MusicBrainzClient) BrowseReleases(
}
}
if err := c.limiter.Wait(ctx); err != nil {
return nil, err
}
c.logger.Info("musicbrainz browse releases",
"releaseGroupMBID", releaseGroupMBID,
)
+2
View File
@@ -23,6 +23,8 @@ export function GetThumbnail(arg1:string,arg2:string,arg3:string):Promise<string
export function GetThumbnails(arg1:Array<explore.ThumbnailRequest>):Promise<Record<string, string>>;
export function IndexNewArtists():Promise<void>;
export function InvalidateIndexDiscographies():Promise<void>;
export function LookupArtist(arg1:string):Promise<explore.MBArtist>;
+4
View File
@@ -42,6 +42,10 @@ export function GetThumbnails(arg1) {
return window['go']['explore']['Service']['GetThumbnails'](arg1);
}
export function IndexNewArtists() {
return window['go']['explore']['Service']['IndexNewArtists']();
}
export function InvalidateIndexDiscographies() {
return window['go']['explore']['Service']['InvalidateIndexDiscographies']();
}