diff --git a/backend/database/database.go b/backend/database/database.go index e625499..dc0e9f7 100644 --- a/backend/database/database.go +++ b/backend/database/database.go @@ -349,6 +349,16 @@ func runMigrations( } } + // Migration 11: explore_cache table for MusicBrainz/ListenBrainz + // API response caching with TTL expiry and MBID lookups. + if version < 11 { + if err := migration11ExploreCache( + ctx, db, logger, + ); err != nil { + return err + } + } + return nil } @@ -1222,7 +1232,7 @@ func migration9SmartPlaylists( // migration10PlayHistory adds play history tracking: // - play_history table for timestamped play log // - play_count and last_played columns on audio_files -// - Recreates track_metadata VIEW to expose the new columns +// - Recreates track_metadata VIEW to expose the new columns. func migration10PlayHistory( ctx context.Context, db *sql.DB, @@ -1352,6 +1362,68 @@ func migration10PlayHistory( return nil } +// migration11ExploreCache creates the explore_cache table for +// MusicBrainz and ListenBrainz API response caching. The table +// stores raw JSON keyed by URL with TTL-based expiry and optional +// MBID columns for future autotagging lookups. +func migration11ExploreCache( + ctx context.Context, + db *sql.DB, + logger *slog.Logger, +) error { + logger.Info( + "applying migration 11: explore_cache table", + ) + + if _, err := db.ExecContext(ctx, ` + CREATE TABLE IF NOT EXISTS explore_cache ( + url_key TEXT PRIMARY KEY, + response TEXT NOT NULL, + mbid TEXT, + entity_type TEXT, + expires_at DATETIME NOT NULL, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP + ) + `); err != nil { + return fmt.Errorf( + "migration 11: could not create explore_cache table: %w", + err, + ) + } + + if _, err := db.ExecContext(ctx, ` + CREATE INDEX IF NOT EXISTS idx_explore_cache_expires + ON explore_cache(expires_at) + `); err != nil { + return fmt.Errorf( + "migration 11: could not create expires index: %w", + err, + ) + } + + if _, err := db.ExecContext(ctx, ` + CREATE INDEX IF NOT EXISTS idx_explore_cache_mbid + ON explore_cache(mbid) + `); err != nil { + return fmt.Errorf( + "migration 11: could not create mbid index: %w", + err, + ) + } + + if _, err := db.ExecContext( + ctx, "PRAGMA user_version = 11", + ); err != nil { + return fmt.Errorf( + "could not set user_version to 11: %w", err, + ) + } + + logger.Info("migration 11 complete") + + return nil +} + // readLibraryDirFromTOML reads the TOML config file and returns // the Library.DirectoryPath value, or "" if not configured. func readLibraryDirFromTOML(logger *slog.Logger) string { diff --git a/backend/database/database_test.go b/backend/database/database_test.go index d632835..a07b12a 100644 --- a/backend/database/database_test.go +++ b/backend/database/database_test.go @@ -1041,3 +1041,194 @@ func TestMigration10PlayHistory(t *testing.T) { t.Errorf("track_metadata play_count = %d, want 1", viewPlayCount) } } + +// --------------------------------------------------------------------------- +// Migration 11 — explore_cache table +// --------------------------------------------------------------------------- + +func TestMigration11ExploreCache(t *testing.T) { + t.Parallel() + + db := NewTestDB(t) + + // Verify user_version >= 11. + var version int + + verRows, err := db.QueryContext("PRAGMA user_version") + if err != nil { + t.Fatalf("PRAGMA user_version: %v", err) + } + + if !verRows.Next() { + _ = verRows.Close() + t.Fatal("PRAGMA user_version: no row returned") + } + + if err := verRows.Scan(&version); err != nil { + _ = verRows.Close() + t.Fatalf("scan user_version: %v", err) + } + + _ = verRows.Close() + + if version < 11 { + t.Errorf("user_version = %d, want >= 11", version) + } + + // Verify explore_cache table exists. + var tableCount int64 + + tblRows, err := db.QueryContext( + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='explore_cache'", + ) + if err != nil { + t.Fatalf("query sqlite_master: %v", err) + } + + if !tblRows.Next() { + _ = tblRows.Close() + + t.Fatal("no row from sqlite_master query") + } + + if err := tblRows.Scan(&tableCount); err != nil { + _ = tblRows.Close() + t.Fatalf("scan table count: %v", err) + } + + _ = tblRows.Close() + + if tableCount != 1 { + t.Errorf("explore_cache table count = %d, want 1", tableCount) + } + + // Verify all expected columns exist. + expectedCols := map[string]bool{ + "url_key": false, + "response": false, + "mbid": false, + "entity_type": false, + "expires_at": false, + "created_at": false, + } + + colRows, err := db.QueryContext( + "PRAGMA table_info(explore_cache)", + ) + if err != nil { + t.Fatalf("PRAGMA table_info(explore_cache): %v", err) + } + + for colRows.Next() { + var ( + cid int64 + name string + colType string + notNull int64 + dfltValue sql.NullString + pk int64 + ) + + if err := colRows.Scan( + &cid, &name, &colType, ¬Null, &dfltValue, &pk, + ); err != nil { + _ = colRows.Close() + t.Fatalf("scan table_info row: %v", err) + } + + if _, ok := expectedCols[name]; ok { + expectedCols[name] = true + } + } + + _ = colRows.Close() + + for col, found := range expectedCols { + if !found { + t.Errorf("explore_cache missing column: %s", col) + } + } + + // Verify indexes exist. + idxRows, err := db.QueryContext( + "SELECT name FROM sqlite_master WHERE type='index' AND tbl_name='explore_cache'", + ) + if err != nil { + t.Fatalf("query indexes: %v", err) + } + + indexes := map[string]bool{} + + for idxRows.Next() { + var name string + + if err := idxRows.Scan(&name); err != nil { + _ = idxRows.Close() + t.Fatalf("scan index name: %v", err) + } + + indexes[name] = true + } + + _ = idxRows.Close() + + if !indexes["idx_explore_cache_expires"] { + t.Error("missing index: idx_explore_cache_expires") + } + + if !indexes["idx_explore_cache_mbid"] { + t.Error("missing index: idx_explore_cache_mbid") + } + + // Round-trip: insert and read back. + _, err = db.ExecContext( + `INSERT INTO explore_cache (url_key, response, mbid, entity_type, expires_at) + VALUES ('test-key', '{"data":"value"}', 'abc-123', 'artist', datetime('now', '+1 hour'))`, + ) + if err != nil { + t.Fatalf("insert explore_cache: %v", err) + } + + rows, err := db.QueryContext( + "SELECT url_key, response, mbid, entity_type FROM explore_cache WHERE url_key = 'test-key'", + ) + if err != nil { + t.Fatalf("query explore_cache: %v", err) + } + + if !rows.Next() { + _ = rows.Close() + t.Fatal("explore_cache row not found") + } + + var ( + urlKey string + response string + mbid sql.NullString + entityType sql.NullString + ) + + if err := rows.Scan(&urlKey, &response, &mbid, &entityType); err != nil { + _ = rows.Close() + + t.Fatalf("scan explore_cache row: %v", err) + } + + _ = rows.Close() + + if urlKey != "test-key" { + t.Errorf("url_key = %q, want %q", urlKey, "test-key") + } + + if response != `{"data":"value"}` { + t.Errorf("response = %q, want %q", response, `{"data":"value"}`) + } + + if !mbid.Valid || mbid.String != "abc-123" { + t.Errorf("mbid = %v, want abc-123", mbid) + } + + if !entityType.Valid || entityType.String != "artist" { + t.Errorf("entity_type = %v, want artist", entityType) + } +} diff --git a/backend/database/sql/schemas/explore_cache.sql b/backend/database/sql/schemas/explore_cache.sql new file mode 100644 index 0000000..f28643d --- /dev/null +++ b/backend/database/sql/schemas/explore_cache.sql @@ -0,0 +1,10 @@ +CREATE TABLE IF NOT EXISTS explore_cache ( + url_key TEXT PRIMARY KEY, + response TEXT NOT NULL, + mbid TEXT, + entity_type TEXT, + expires_at DATETIME NOT NULL, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP +); +CREATE INDEX IF NOT EXISTS idx_explore_cache_expires ON explore_cache(expires_at); +CREATE INDEX IF NOT EXISTS idx_explore_cache_mbid ON explore_cache(mbid); diff --git a/backend/explore/cache.go b/backend/explore/cache.go new file mode 100644 index 0000000..13ed5d7 --- /dev/null +++ b/backend/explore/cache.go @@ -0,0 +1,123 @@ +package explore + +import ( + "fmt" + "log/slog" + "time" + + "yellowjacket/backend/database" +) + +// Cache provides a SQLite-backed response cache with TTL expiry. +// It stores raw JSON API responses keyed by URL and supports +// optional MBID columns for future autotagging lookups. +// +// All operations use the shared database.DB connection and its +// single-writer constraint (SetMaxOpenConns(1)). +type Cache struct { + db *database.DB + logger *slog.Logger +} + +// NewCache returns a cache backed by the given database connection. +func NewCache(db *database.DB, logger *slog.Logger) *Cache { + return &Cache{db: db, logger: logger} +} + +// Get returns the cached response for the given URL key if it +// exists and has not expired. Returns (data, true) on a cache hit +// and (nil, false) on a miss or expired entry. +func (c *Cache) Get(key string) ([]byte, bool) { + rows, err := c.db.QueryContext( + "SELECT response FROM explore_cache WHERE url_key = ? AND expires_at > datetime('now')", + key, + ) + if err != nil { + c.logger.Warn("explore cache get error", + "key", key, + "err", err, + ) + + return nil, false + } + + defer func() { _ = rows.Close() }() + + if !rows.Next() { + c.logger.Debug("explore cache miss", "key", key) + + return nil, false + } + + var response string + + if err := rows.Scan(&response); err != nil { + c.logger.Warn("explore cache scan error", + "key", key, + "err", err, + ) + + return nil, false + } + + c.logger.Debug("explore cache hit", "key", key) + + return []byte(response), true +} + +// Set stores a response in the cache with the given TTL. If mbid +// and entityType are non-empty they are stored for future +// autotagging lookups; otherwise they are stored as NULL. +func (c *Cache) Set( + key string, + data []byte, + ttl time.Duration, + mbid string, + entityType string, +) { + seconds := int(ttl.Seconds()) + if seconds < 1 { + seconds = 1 + } + + expr := fmt.Sprintf("datetime('now', '+%d seconds')", seconds) + + query := fmt.Sprintf( + `INSERT OR REPLACE INTO explore_cache + (url_key, response, mbid, entity_type, expires_at) + VALUES (?, ?, NULLIF(?, ''), NULLIF(?, ''), %s)`, + expr, + ) + + if _, err := c.db.ExecContext(query, key, string(data), mbid, entityType); err != nil { + c.logger.Warn("explore cache set error", + "key", key, + "err", err, + ) + } else { + c.logger.Debug("explore cache set", + "key", key, + "ttl", ttl, + "mbid", mbid, + "entityType", entityType, + ) + } +} + +// Evict removes all expired entries from the cache. +func (c *Cache) Evict() { + result, err := c.db.ExecContext( + "DELETE FROM explore_cache WHERE expires_at < datetime('now')", + ) + if err != nil { + c.logger.Warn("explore cache evict error", "err", err) + + return + } + + if n, _ := result.RowsAffected(); n > 0 { + c.logger.Info("explore cache evicted expired entries", + "count", n, + ) + } +} diff --git a/backend/explore/cache_test.go b/backend/explore/cache_test.go new file mode 100644 index 0000000..02e946b --- /dev/null +++ b/backend/explore/cache_test.go @@ -0,0 +1,153 @@ +package explore + +import ( + "database/sql" + "log/slog" + "testing" + "time" + + "yellowjacket/backend/database" +) + +func newTestCache(t *testing.T) *Cache { + t.Helper() + + db := database.NewTestDB(t) + + return NewCache(db, slog.Default()) +} + +func TestCacheSetGet(t *testing.T) { + t.Parallel() + + c := newTestCache(t) + + data := []byte(`{"artist":"Radiohead"}`) + c.Set("https://musicbrainz.org/ws/2/artist?query=radiohead", data, 5*time.Minute, "", "") + + got, ok := c.Get("https://musicbrainz.org/ws/2/artist?query=radiohead") + if !ok { + t.Fatal("expected cache hit, got miss") + } + + if string(got) != string(data) { + t.Errorf("got %q, want %q", string(got), string(data)) + } +} + +func TestCacheMiss(t *testing.T) { + t.Parallel() + + c := newTestCache(t) + + _, ok := c.Get("https://nonexistent.example.com/api") + if ok { + t.Error("expected cache miss, got hit") + } +} + +func TestCacheTTLExpiry(t *testing.T) { + c := newTestCache(t) + + data := []byte(`{"ephemeral":true}`) + c.Set("ttl-test-key", data, 1*time.Second, "", "") + + // Verify it's there immediately. + if _, ok := c.Get("ttl-test-key"); !ok { + t.Fatal("expected cache hit immediately after set") + } + + // Wait for expiry. + time.Sleep(2 * time.Second) + + if _, ok := c.Get("ttl-test-key"); ok { + t.Error("expected cache miss after TTL expiry, got hit") + } +} + +func TestCacheMBID(t *testing.T) { + t.Parallel() + + c := newTestCache(t) + + data := []byte(`{"name":"OK Computer"}`) + c.Set( + "mbid-test-key", + data, + 10*time.Minute, + "b3b40b1b-3c03-4b8a-8291-8e1f2d09e211", + "release_group", + ) + + // Query the MBID column directly to verify it was stored. + db := c.db + + rows, err := db.QueryContext( + "SELECT mbid, entity_type FROM explore_cache WHERE url_key = ?", + "mbid-test-key", + ) + if err != nil { + t.Fatalf("query explore_cache: %v", err) + } + + defer func() { _ = rows.Close() }() + + if !rows.Next() { + t.Fatal("explore_cache row not found") + } + + var ( + mbid sql.NullString + entityType sql.NullString + ) + + if err := rows.Scan(&mbid, &entityType); err != nil { + t.Fatalf("scan: %v", err) + } + + if !mbid.Valid || mbid.String != "b3b40b1b-3c03-4b8a-8291-8e1f2d09e211" { + t.Errorf("mbid = %v, want b3b40b1b-3c03-4b8a-8291-8e1f2d09e211", mbid) + } + + if !entityType.Valid || entityType.String != "release_group" { + t.Errorf("entity_type = %v, want release_group", entityType) + } +} + +func TestCacheEvict(t *testing.T) { + c := newTestCache(t) + + // Insert an entry that expires in 1 second. + c.Set("evict-key", []byte(`{}`), 1*time.Second, "", "") + + time.Sleep(2 * time.Second) + + // Evict expired entries. + c.Evict() + + // Verify the row is gone entirely (not just expired-but-present). + db := c.db + + rows, err := db.QueryContext( + "SELECT COUNT(*) FROM explore_cache WHERE url_key = ?", + "evict-key", + ) + if err != nil { + t.Fatalf("query: %v", err) + } + + defer func() { _ = rows.Close() }() + + if !rows.Next() { + t.Fatal("no row returned") + } + + var count int64 + if err := rows.Scan(&count); err != nil { + t.Fatalf("scan: %v", err) + } + + if count != 0 { + t.Errorf("expected 0 rows after evict, got %d", count) + } +} diff --git a/backend/explore/ratelimiter.go b/backend/explore/ratelimiter.go new file mode 100644 index 0000000..34161f4 --- /dev/null +++ b/backend/explore/ratelimiter.go @@ -0,0 +1,37 @@ +// Package explore provides MusicBrainz and ListenBrainz API clients +// with rate-limited HTTP access and a SQLite response cache. +package explore + +import ( + "context" + "time" + + "golang.org/x/time/rate" +) + +// RateLimiter enforces a maximum request rate using a token bucket. +// MusicBrainz requires ≤1 request per second and rejects ALL +// requests (not just excess) when the rate is exceeded, so callers +// block proactively via Wait rather than retrying reactively. +// +// RateLimiter is safe for concurrent use. +type RateLimiter struct { + limiter *rate.Limiter +} + +// NewRateLimiter returns a rate limiter that allows exactly one +// request per second with a burst size of 1. The first call to +// Wait returns immediately; subsequent calls block until the next +// token is available. +func NewRateLimiter() *RateLimiter { + return &RateLimiter{ + limiter: rate.NewLimiter(rate.Every(time.Second), 1), + } +} + +// Wait blocks until the rate limiter allows the caller to proceed +// or the context is cancelled. Returns ctx.Err() if the context +// expires before a token becomes available. +func (r *RateLimiter) Wait(ctx context.Context) error { + return r.limiter.Wait(ctx) +} diff --git a/backend/explore/ratelimiter_test.go b/backend/explore/ratelimiter_test.go new file mode 100644 index 0000000..6c80544 --- /dev/null +++ b/backend/explore/ratelimiter_test.go @@ -0,0 +1,62 @@ +package explore + +import ( + "context" + "errors" + "testing" + "time" +) + +func TestRateLimiterBurst(t *testing.T) { + rl := NewRateLimiter() + ctx := context.Background() + + const n = 5 + + start := time.Now() + + for i := range n { + if err := rl.Wait(ctx); err != nil { + t.Fatalf("Wait %d: %v", i, err) + } + } + + elapsed := time.Since(start) + + // First request is immediate; 4 more at 1/sec = ≥4s total. + if elapsed < 4*time.Second { + t.Errorf( + "elapsed %v, want ≥ 4s (rate limiter too fast)", elapsed, + ) + } + + // Generous upper bound to avoid CI flakes. + if elapsed > 7*time.Second { + t.Errorf( + "elapsed %v, want ≤ 7s (rate limiter too slow)", elapsed, + ) + } +} + +func TestRateLimiterContextCancel(t *testing.T) { + t.Parallel() + + rl := NewRateLimiter() + + // Drain the initial token so the next Wait must block. + if err := rl.Wait(context.Background()); err != nil { + t.Fatalf("drain token: %v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() // cancel immediately + + err := rl.Wait(ctx) + if err == nil { + t.Fatal("expected error from cancelled context, got nil") + } + + if !errors.Is(err, context.Canceled) { + t.Errorf("error = %v, want context.Canceled", err) + } +} diff --git a/go.mod b/go.mod index 507e560..9398771 100644 --- a/go.mod +++ b/go.mod @@ -354,6 +354,7 @@ require ( golang.org/x/sys v0.41.0 // indirect golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4 // indirect golang.org/x/term v0.40.0 // indirect + golang.org/x/time v0.15.0 // indirect golang.org/x/tools v0.42.0 // indirect golang.org/x/vuln v1.1.4 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250707201910-8d1bb00bc6a7 // indirect diff --git a/go.sum b/go.sum index ff58046..74a7143 100644 --- a/go.sum +++ b/go.sum @@ -1224,6 +1224,8 @@ golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=