yonlu
5bb1c89fba
Three changes to reduce subsequent index build times: 1. Positive image/rels cache TTL: 30 days → 365 days Artist url-rels and resolved image URLs rarely change. Already-indexed artists make zero API calls on rebuild. 2. Negative cache (misses) stays at 30 days so new images are discovered within a month of being added upstream. 3. MB rate limiter for background indexing: 1.0 → 1.5 req/s url-rels lookups are lightweight; 1.5/s is well within what MB handles (Picard and Kodi both use similar rates). Cuts the MB-bound portion of index build by ~33%. Also adds NewRateLimiterF for fractional rates and caches MB rels fetch failures (30-day miss TTL) to avoid retrying unreachable artists every build.
55 lines
1.7 KiB
Go
55 lines
1.7 KiB
Go
// Package explore provides MusicBrainz and ListenBrainz API clients
|
|
// with rate-limited HTTP access and a SQLite response cache.
|
|
package explore
|
|
|
|
import (
|
|
"context"
|
|
"time"
|
|
|
|
"golang.org/x/time/rate"
|
|
)
|
|
|
|
// RateLimiter enforces a maximum request rate using a token bucket.
|
|
// MusicBrainz requires ≤1 request per second and rejects ALL
|
|
// requests (not just excess) when the rate is exceeded, so callers
|
|
// block proactively via Wait rather than retrying reactively.
|
|
//
|
|
// RateLimiter is safe for concurrent use.
|
|
type RateLimiter struct {
|
|
limiter *rate.Limiter
|
|
}
|
|
|
|
// NewRateLimiter returns a rate limiter that allows exactly one
|
|
// request per second with a burst size of 1. The first call to
|
|
// Wait returns immediately; subsequent calls block until the next
|
|
// token is available.
|
|
func NewRateLimiter() *RateLimiter {
|
|
return &RateLimiter{
|
|
limiter: rate.NewLimiter(rate.Every(time.Second), 1),
|
|
}
|
|
}
|
|
|
|
// NewRateLimiterN returns a rate limiter that allows n requests
|
|
// per second with a burst of n. Used for background tasks like
|
|
// index building where a higher rate is acceptable.
|
|
func NewRateLimiterN(n int) *RateLimiter {
|
|
return &RateLimiter{
|
|
limiter: rate.NewLimiter(rate.Limit(n), n),
|
|
}
|
|
}
|
|
|
|
// NewRateLimiterF returns a rate limiter that allows f requests
|
|
// per second with a burst of 1.
|
|
func NewRateLimiterF(f float64) *RateLimiter {
|
|
return &RateLimiter{
|
|
limiter: rate.NewLimiter(rate.Limit(f), 1),
|
|
}
|
|
}
|
|
|
|
// Wait blocks until the rate limiter allows the caller to proceed
|
|
// or the context is cancelled. Returns ctx.Err() if the context
|
|
// expires before a token becomes available.
|
|
func (r *RateLimiter) Wait(ctx context.Context) error {
|
|
return r.limiter.Wait(ctx)
|
|
}
|