From 5bb1c89fba9571e41427362b61b12153f6507766 Mon Sep 17 00:00:00 2001 From: Caleb Allen Date: Sun, 29 Mar 2026 12:00:36 -0400 Subject: [PATCH] =?UTF-8?q?perf:=20optimize=20index=20build=20=E2=80=94=20?= =?UTF-8?q?permanent=20caches=20+=20faster=20MB=20rate=20limit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three changes to reduce subsequent index build times: 1. Positive image/rels cache TTL: 30 days → 365 days Artist url-rels and resolved image URLs rarely change. Already-indexed artists make zero API calls on rebuild. 2. Negative cache (misses) stays at 30 days so new images are discovered within a month of being added upstream. 3. MB rate limiter for background indexing: 1.0 → 1.5 req/s url-rels lookups are lightweight; 1.5/s is well within what MB handles (Picard and Kodi both use similar rates). Cuts the MB-bound portion of index build by ~33%. Also adds NewRateLimiterF for fractional rates and caches MB rels fetch failures (30-day miss TTL) to avoid retrying unreachable artists every build. --- backend/explore/artistimage.go | 36 +++++++++++++++++++--------------- backend/explore/explore.go | 2 +- backend/explore/ratelimiter.go | 8 ++++++++ 3 files changed, 29 insertions(+), 17 deletions(-) diff --git a/backend/explore/artistimage.go b/backend/explore/artistimage.go index cf94e7f..e2eb501 100644 --- a/backend/explore/artistimage.go +++ b/backend/explore/artistimage.go @@ -28,17 +28,18 @@ import ( var ErrArtistImage = errors.New("artist image fetch failed") const ( - wikimediaThumbBase = "https://upload.wikimedia.org/wikipedia/commons/thumb" - wikidataAPIBase = "https://www.wikidata.org/w/api.php" - wikipediaAPIBase = "https://en.wikipedia.org/w/api.php" - fanartTVAPIBase = "https://webservice.fanart.tv/v3/music" - audioDBAPIBase = "https://www.theaudiodb.com/api/v1/json/2" - artistImageTimeout = 10 * time.Second - artistImageCacheTTL = 30 * 24 * time.Hour - artistImageBaseDir = "artist-images" - artistImageMaxBytes = 2 * 1024 * 1024 - artistImageMaxSize = 500 // max dimension for stored full-res images - maxImagesPerArtist = 10 + wikimediaThumbBase = "https://upload.wikimedia.org/wikipedia/commons/thumb" + wikidataAPIBase = "https://www.wikidata.org/w/api.php" + wikipediaAPIBase = "https://en.wikipedia.org/w/api.php" + fanartTVAPIBase = "https://webservice.fanart.tv/v3/music" + audioDBAPIBase = "https://www.theaudiodb.com/api/v1/json/2" + artistImageTimeout = 10 * time.Second + artistImageCacheTTL = 365 * 24 * time.Hour // positive results: ~permanent + artistImageMissCacheTTL = 30 * 24 * time.Hour // negative results: retry monthly + artistImageBaseDir = "artist-images" + artistImageMaxBytes = 2 * 1024 * 1024 + artistImageMaxSize = 500 // max dimension for stored full-res images + maxImagesPerArtist = 10 ) // fanartTVProjectKey is the project API key for fanart.tv. @@ -460,7 +461,7 @@ func (p *ArtistImageProvider) fetchFanartTV(artistMBID string) []string { body, err := p.fetchURL(url) if err != nil { // Cache empty result to avoid re-fetching. - p.cache.Set(cacheKey, []byte("[]"), artistImageCacheTTL, artistMBID, "artist") + p.cache.Set(cacheKey, []byte("[]"), artistImageMissCacheTTL, artistMBID, "artist") return nil } @@ -473,7 +474,7 @@ func (p *ArtistImageProvider) fetchFanartTV(artistMBID string) []string { } if err := json.Unmarshal(body, &response); err != nil || len(response.ArtistThumb) == 0 { - p.cache.Set(cacheKey, []byte("[]"), artistImageCacheTTL, artistMBID, "artist") + p.cache.Set(cacheKey, []byte("[]"), artistImageMissCacheTTL, artistMBID, "artist") return nil } @@ -516,7 +517,7 @@ func (p *ArtistImageProvider) fetchAudioDB(artistMBID string) []string { body, err := p.fetchURL(url) if err != nil { - p.cache.Set(cacheKey, []byte("[]"), artistImageCacheTTL, artistMBID, "artist") + p.cache.Set(cacheKey, []byte("[]"), artistImageMissCacheTTL, artistMBID, "artist") return nil } @@ -531,7 +532,7 @@ func (p *ArtistImageProvider) fetchAudioDB(artistMBID string) []string { } if err := json.Unmarshal(body, &response); err != nil || len(response.Artists) == 0 { - p.cache.Set(cacheKey, []byte("[]"), artistImageCacheTTL, artistMBID, "artist") + p.cache.Set(cacheKey, []byte("[]"), artistImageMissCacheTTL, artistMBID, "artist") return nil } @@ -581,6 +582,9 @@ func (p *ArtistImageProvider) fetchMBRels(artistMBID string) []mbRelation { body, err := p.fetchURL(url) if err != nil { + // Cache the miss so we don't re-request on every build. + p.cache.Set(cacheKey, []byte("{}"), artistImageMissCacheTTL, artistMBID, "artist") + return nil } @@ -687,7 +691,7 @@ func (p *ArtistImageProvider) fetchWikipediaLeadImage(qid string) string { enwiki, ok := entity.Sitelinks["enwiki"] if !ok || enwiki.Title == "" { - p.cache.Set(cacheKey, []byte(""), artistImageCacheTTL, "", "") + p.cache.Set(cacheKey, []byte(""), artistImageMissCacheTTL, "", "") return "" } diff --git a/backend/explore/explore.go b/backend/explore/explore.go index d2defb2..958d69e 100644 --- a/backend/explore/explore.go +++ b/backend/explore/explore.go @@ -39,7 +39,7 @@ func NewExploreService(logger *slog.Logger, db *database.DB) *Service { lb := NewListenBrainzClient(limiter, cache, logger.WithGroup("listenbrainz")) artProxy := NewCoverArtProxy(db, limiter) artistImg := NewArtistImageProvider( - db, cache, NewRateLimiter(), logger.WithGroup("artist-image"), + db, cache, NewRateLimiterF(1.5), logger.WithGroup("artist-image"), ) index := NewSearchIndex(db, lb, artistImg, logger.WithGroup("search-index")) libMBID := NewLibraryMBIDIndex(db) diff --git a/backend/explore/ratelimiter.go b/backend/explore/ratelimiter.go index cbb0444..854f116 100644 --- a/backend/explore/ratelimiter.go +++ b/backend/explore/ratelimiter.go @@ -38,6 +38,14 @@ func NewRateLimiterN(n int) *RateLimiter { } } +// NewRateLimiterF returns a rate limiter that allows f requests +// per second with a burst of 1. +func NewRateLimiterF(f float64) *RateLimiter { + return &RateLimiter{ + limiter: rate.NewLimiter(rate.Limit(f), 1), + } +} + // Wait blocks until the rate limiter allows the caller to proceed // or the context is cancelled. Returns ctx.Err() if the context // expires before a token becomes available.