perf(S01/T01): Add token-bucket rate limiter (1 req/sec), SQLite respon…

- backend/explore/ratelimiter.go
- backend/explore/cache.go
- backend/database/sql/schemas/explore_cache.sql
- backend/database/database.go
This commit is contained in:
2026-03-23 07:55:25 -04:00
parent 398fd5aaae
commit 8fc075c24a
9 changed files with 652 additions and 1 deletions
+73 -1
View File
@@ -349,6 +349,16 @@ func runMigrations(
}
}
// Migration 11: explore_cache table for MusicBrainz/ListenBrainz
// API response caching with TTL expiry and MBID lookups.
if version < 11 {
if err := migration11ExploreCache(
ctx, db, logger,
); err != nil {
return err
}
}
return nil
}
@@ -1222,7 +1232,7 @@ func migration9SmartPlaylists(
// migration10PlayHistory adds play history tracking:
// - play_history table for timestamped play log
// - play_count and last_played columns on audio_files
// - Recreates track_metadata VIEW to expose the new columns
// - Recreates track_metadata VIEW to expose the new columns.
func migration10PlayHistory(
ctx context.Context,
db *sql.DB,
@@ -1352,6 +1362,68 @@ func migration10PlayHistory(
return nil
}
// migration11ExploreCache creates the explore_cache table for
// MusicBrainz and ListenBrainz API response caching. The table
// stores raw JSON keyed by URL with TTL-based expiry and optional
// MBID columns for future autotagging lookups.
func migration11ExploreCache(
ctx context.Context,
db *sql.DB,
logger *slog.Logger,
) error {
logger.Info(
"applying migration 11: explore_cache table",
)
if _, err := db.ExecContext(ctx, `
CREATE TABLE IF NOT EXISTS explore_cache (
url_key TEXT PRIMARY KEY,
response TEXT NOT NULL,
mbid TEXT,
entity_type TEXT,
expires_at DATETIME NOT NULL,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
)
`); err != nil {
return fmt.Errorf(
"migration 11: could not create explore_cache table: %w",
err,
)
}
if _, err := db.ExecContext(ctx, `
CREATE INDEX IF NOT EXISTS idx_explore_cache_expires
ON explore_cache(expires_at)
`); err != nil {
return fmt.Errorf(
"migration 11: could not create expires index: %w",
err,
)
}
if _, err := db.ExecContext(ctx, `
CREATE INDEX IF NOT EXISTS idx_explore_cache_mbid
ON explore_cache(mbid)
`); err != nil {
return fmt.Errorf(
"migration 11: could not create mbid index: %w",
err,
)
}
if _, err := db.ExecContext(
ctx, "PRAGMA user_version = 11",
); err != nil {
return fmt.Errorf(
"could not set user_version to 11: %w", err,
)
}
logger.Info("migration 11 complete")
return nil
}
// readLibraryDirFromTOML reads the TOML config file and returns
// the Library.DirectoryPath value, or "" if not configured.
func readLibraryDirFromTOML(logger *slog.Logger) string {
+191
View File
@@ -1041,3 +1041,194 @@ func TestMigration10PlayHistory(t *testing.T) {
t.Errorf("track_metadata play_count = %d, want 1", viewPlayCount)
}
}
// ---------------------------------------------------------------------------
// Migration 11 — explore_cache table
// ---------------------------------------------------------------------------
func TestMigration11ExploreCache(t *testing.T) {
t.Parallel()
db := NewTestDB(t)
// Verify user_version >= 11.
var version int
verRows, err := db.QueryContext("PRAGMA user_version")
if err != nil {
t.Fatalf("PRAGMA user_version: %v", err)
}
if !verRows.Next() {
_ = verRows.Close()
t.Fatal("PRAGMA user_version: no row returned")
}
if err := verRows.Scan(&version); err != nil {
_ = verRows.Close()
t.Fatalf("scan user_version: %v", err)
}
_ = verRows.Close()
if version < 11 {
t.Errorf("user_version = %d, want >= 11", version)
}
// Verify explore_cache table exists.
var tableCount int64
tblRows, err := db.QueryContext(
"SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='explore_cache'",
)
if err != nil {
t.Fatalf("query sqlite_master: %v", err)
}
if !tblRows.Next() {
_ = tblRows.Close()
t.Fatal("no row from sqlite_master query")
}
if err := tblRows.Scan(&tableCount); err != nil {
_ = tblRows.Close()
t.Fatalf("scan table count: %v", err)
}
_ = tblRows.Close()
if tableCount != 1 {
t.Errorf("explore_cache table count = %d, want 1", tableCount)
}
// Verify all expected columns exist.
expectedCols := map[string]bool{
"url_key": false,
"response": false,
"mbid": false,
"entity_type": false,
"expires_at": false,
"created_at": false,
}
colRows, err := db.QueryContext(
"PRAGMA table_info(explore_cache)",
)
if err != nil {
t.Fatalf("PRAGMA table_info(explore_cache): %v", err)
}
for colRows.Next() {
var (
cid int64
name string
colType string
notNull int64
dfltValue sql.NullString
pk int64
)
if err := colRows.Scan(
&cid, &name, &colType, &notNull, &dfltValue, &pk,
); err != nil {
_ = colRows.Close()
t.Fatalf("scan table_info row: %v", err)
}
if _, ok := expectedCols[name]; ok {
expectedCols[name] = true
}
}
_ = colRows.Close()
for col, found := range expectedCols {
if !found {
t.Errorf("explore_cache missing column: %s", col)
}
}
// Verify indexes exist.
idxRows, err := db.QueryContext(
"SELECT name FROM sqlite_master WHERE type='index' AND tbl_name='explore_cache'",
)
if err != nil {
t.Fatalf("query indexes: %v", err)
}
indexes := map[string]bool{}
for idxRows.Next() {
var name string
if err := idxRows.Scan(&name); err != nil {
_ = idxRows.Close()
t.Fatalf("scan index name: %v", err)
}
indexes[name] = true
}
_ = idxRows.Close()
if !indexes["idx_explore_cache_expires"] {
t.Error("missing index: idx_explore_cache_expires")
}
if !indexes["idx_explore_cache_mbid"] {
t.Error("missing index: idx_explore_cache_mbid")
}
// Round-trip: insert and read back.
_, err = db.ExecContext(
`INSERT INTO explore_cache (url_key, response, mbid, entity_type, expires_at)
VALUES ('test-key', '{"data":"value"}', 'abc-123', 'artist', datetime('now', '+1 hour'))`,
)
if err != nil {
t.Fatalf("insert explore_cache: %v", err)
}
rows, err := db.QueryContext(
"SELECT url_key, response, mbid, entity_type FROM explore_cache WHERE url_key = 'test-key'",
)
if err != nil {
t.Fatalf("query explore_cache: %v", err)
}
if !rows.Next() {
_ = rows.Close()
t.Fatal("explore_cache row not found")
}
var (
urlKey string
response string
mbid sql.NullString
entityType sql.NullString
)
if err := rows.Scan(&urlKey, &response, &mbid, &entityType); err != nil {
_ = rows.Close()
t.Fatalf("scan explore_cache row: %v", err)
}
_ = rows.Close()
if urlKey != "test-key" {
t.Errorf("url_key = %q, want %q", urlKey, "test-key")
}
if response != `{"data":"value"}` {
t.Errorf("response = %q, want %q", response, `{"data":"value"}`)
}
if !mbid.Valid || mbid.String != "abc-123" {
t.Errorf("mbid = %v, want abc-123", mbid)
}
if !entityType.Valid || entityType.String != "artist" {
t.Errorf("entity_type = %v, want artist", entityType)
}
}
@@ -0,0 +1,10 @@
CREATE TABLE IF NOT EXISTS explore_cache (
url_key TEXT PRIMARY KEY,
response TEXT NOT NULL,
mbid TEXT,
entity_type TEXT,
expires_at DATETIME NOT NULL,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_explore_cache_expires ON explore_cache(expires_at);
CREATE INDEX IF NOT EXISTS idx_explore_cache_mbid ON explore_cache(mbid);
+123
View File
@@ -0,0 +1,123 @@
package explore
import (
"fmt"
"log/slog"
"time"
"yellowjacket/backend/database"
)
// Cache provides a SQLite-backed response cache with TTL expiry.
// It stores raw JSON API responses keyed by URL and supports
// optional MBID columns for future autotagging lookups.
//
// All operations use the shared database.DB connection and its
// single-writer constraint (SetMaxOpenConns(1)).
type Cache struct {
db *database.DB
logger *slog.Logger
}
// NewCache returns a cache backed by the given database connection.
func NewCache(db *database.DB, logger *slog.Logger) *Cache {
return &Cache{db: db, logger: logger}
}
// Get returns the cached response for the given URL key if it
// exists and has not expired. Returns (data, true) on a cache hit
// and (nil, false) on a miss or expired entry.
func (c *Cache) Get(key string) ([]byte, bool) {
rows, err := c.db.QueryContext(
"SELECT response FROM explore_cache WHERE url_key = ? AND expires_at > datetime('now')",
key,
)
if err != nil {
c.logger.Warn("explore cache get error",
"key", key,
"err", err,
)
return nil, false
}
defer func() { _ = rows.Close() }()
if !rows.Next() {
c.logger.Debug("explore cache miss", "key", key)
return nil, false
}
var response string
if err := rows.Scan(&response); err != nil {
c.logger.Warn("explore cache scan error",
"key", key,
"err", err,
)
return nil, false
}
c.logger.Debug("explore cache hit", "key", key)
return []byte(response), true
}
// Set stores a response in the cache with the given TTL. If mbid
// and entityType are non-empty they are stored for future
// autotagging lookups; otherwise they are stored as NULL.
func (c *Cache) Set(
key string,
data []byte,
ttl time.Duration,
mbid string,
entityType string,
) {
seconds := int(ttl.Seconds())
if seconds < 1 {
seconds = 1
}
expr := fmt.Sprintf("datetime('now', '+%d seconds')", seconds)
query := fmt.Sprintf(
`INSERT OR REPLACE INTO explore_cache
(url_key, response, mbid, entity_type, expires_at)
VALUES (?, ?, NULLIF(?, ''), NULLIF(?, ''), %s)`,
expr,
)
if _, err := c.db.ExecContext(query, key, string(data), mbid, entityType); err != nil {
c.logger.Warn("explore cache set error",
"key", key,
"err", err,
)
} else {
c.logger.Debug("explore cache set",
"key", key,
"ttl", ttl,
"mbid", mbid,
"entityType", entityType,
)
}
}
// Evict removes all expired entries from the cache.
func (c *Cache) Evict() {
result, err := c.db.ExecContext(
"DELETE FROM explore_cache WHERE expires_at < datetime('now')",
)
if err != nil {
c.logger.Warn("explore cache evict error", "err", err)
return
}
if n, _ := result.RowsAffected(); n > 0 {
c.logger.Info("explore cache evicted expired entries",
"count", n,
)
}
}
+153
View File
@@ -0,0 +1,153 @@
package explore
import (
"database/sql"
"log/slog"
"testing"
"time"
"yellowjacket/backend/database"
)
func newTestCache(t *testing.T) *Cache {
t.Helper()
db := database.NewTestDB(t)
return NewCache(db, slog.Default())
}
func TestCacheSetGet(t *testing.T) {
t.Parallel()
c := newTestCache(t)
data := []byte(`{"artist":"Radiohead"}`)
c.Set("https://musicbrainz.org/ws/2/artist?query=radiohead", data, 5*time.Minute, "", "")
got, ok := c.Get("https://musicbrainz.org/ws/2/artist?query=radiohead")
if !ok {
t.Fatal("expected cache hit, got miss")
}
if string(got) != string(data) {
t.Errorf("got %q, want %q", string(got), string(data))
}
}
func TestCacheMiss(t *testing.T) {
t.Parallel()
c := newTestCache(t)
_, ok := c.Get("https://nonexistent.example.com/api")
if ok {
t.Error("expected cache miss, got hit")
}
}
func TestCacheTTLExpiry(t *testing.T) {
c := newTestCache(t)
data := []byte(`{"ephemeral":true}`)
c.Set("ttl-test-key", data, 1*time.Second, "", "")
// Verify it's there immediately.
if _, ok := c.Get("ttl-test-key"); !ok {
t.Fatal("expected cache hit immediately after set")
}
// Wait for expiry.
time.Sleep(2 * time.Second)
if _, ok := c.Get("ttl-test-key"); ok {
t.Error("expected cache miss after TTL expiry, got hit")
}
}
func TestCacheMBID(t *testing.T) {
t.Parallel()
c := newTestCache(t)
data := []byte(`{"name":"OK Computer"}`)
c.Set(
"mbid-test-key",
data,
10*time.Minute,
"b3b40b1b-3c03-4b8a-8291-8e1f2d09e211",
"release_group",
)
// Query the MBID column directly to verify it was stored.
db := c.db
rows, err := db.QueryContext(
"SELECT mbid, entity_type FROM explore_cache WHERE url_key = ?",
"mbid-test-key",
)
if err != nil {
t.Fatalf("query explore_cache: %v", err)
}
defer func() { _ = rows.Close() }()
if !rows.Next() {
t.Fatal("explore_cache row not found")
}
var (
mbid sql.NullString
entityType sql.NullString
)
if err := rows.Scan(&mbid, &entityType); err != nil {
t.Fatalf("scan: %v", err)
}
if !mbid.Valid || mbid.String != "b3b40b1b-3c03-4b8a-8291-8e1f2d09e211" {
t.Errorf("mbid = %v, want b3b40b1b-3c03-4b8a-8291-8e1f2d09e211", mbid)
}
if !entityType.Valid || entityType.String != "release_group" {
t.Errorf("entity_type = %v, want release_group", entityType)
}
}
func TestCacheEvict(t *testing.T) {
c := newTestCache(t)
// Insert an entry that expires in 1 second.
c.Set("evict-key", []byte(`{}`), 1*time.Second, "", "")
time.Sleep(2 * time.Second)
// Evict expired entries.
c.Evict()
// Verify the row is gone entirely (not just expired-but-present).
db := c.db
rows, err := db.QueryContext(
"SELECT COUNT(*) FROM explore_cache WHERE url_key = ?",
"evict-key",
)
if err != nil {
t.Fatalf("query: %v", err)
}
defer func() { _ = rows.Close() }()
if !rows.Next() {
t.Fatal("no row returned")
}
var count int64
if err := rows.Scan(&count); err != nil {
t.Fatalf("scan: %v", err)
}
if count != 0 {
t.Errorf("expected 0 rows after evict, got %d", count)
}
}
+37
View File
@@ -0,0 +1,37 @@
// Package explore provides MusicBrainz and ListenBrainz API clients
// with rate-limited HTTP access and a SQLite response cache.
package explore
import (
"context"
"time"
"golang.org/x/time/rate"
)
// RateLimiter enforces a maximum request rate using a token bucket.
// MusicBrainz requires ≤1 request per second and rejects ALL
// requests (not just excess) when the rate is exceeded, so callers
// block proactively via Wait rather than retrying reactively.
//
// RateLimiter is safe for concurrent use.
type RateLimiter struct {
limiter *rate.Limiter
}
// NewRateLimiter returns a rate limiter that allows exactly one
// request per second with a burst size of 1. The first call to
// Wait returns immediately; subsequent calls block until the next
// token is available.
func NewRateLimiter() *RateLimiter {
return &RateLimiter{
limiter: rate.NewLimiter(rate.Every(time.Second), 1),
}
}
// Wait blocks until the rate limiter allows the caller to proceed
// or the context is cancelled. Returns ctx.Err() if the context
// expires before a token becomes available.
func (r *RateLimiter) Wait(ctx context.Context) error {
return r.limiter.Wait(ctx)
}
+62
View File
@@ -0,0 +1,62 @@
package explore
import (
"context"
"errors"
"testing"
"time"
)
func TestRateLimiterBurst(t *testing.T) {
rl := NewRateLimiter()
ctx := context.Background()
const n = 5
start := time.Now()
for i := range n {
if err := rl.Wait(ctx); err != nil {
t.Fatalf("Wait %d: %v", i, err)
}
}
elapsed := time.Since(start)
// First request is immediate; 4 more at 1/sec = ≥4s total.
if elapsed < 4*time.Second {
t.Errorf(
"elapsed %v, want ≥ 4s (rate limiter too fast)", elapsed,
)
}
// Generous upper bound to avoid CI flakes.
if elapsed > 7*time.Second {
t.Errorf(
"elapsed %v, want ≤ 7s (rate limiter too slow)", elapsed,
)
}
}
func TestRateLimiterContextCancel(t *testing.T) {
t.Parallel()
rl := NewRateLimiter()
// Drain the initial token so the next Wait must block.
if err := rl.Wait(context.Background()); err != nil {
t.Fatalf("drain token: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel() // cancel immediately
err := rl.Wait(ctx)
if err == nil {
t.Fatal("expected error from cancelled context, got nil")
}
if !errors.Is(err, context.Canceled) {
t.Errorf("error = %v, want context.Canceled", err)
}
}
+1
View File
@@ -354,6 +354,7 @@ require (
golang.org/x/sys v0.41.0 // indirect
golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4 // indirect
golang.org/x/term v0.40.0 // indirect
golang.org/x/time v0.15.0 // indirect
golang.org/x/tools v0.42.0 // indirect
golang.org/x/vuln v1.1.4 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250707201910-8d1bb00bc6a7 // indirect
+2
View File
@@ -1224,6 +1224,8 @@ golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=