feat: per-tier refresh intervals with incremental discography builds

Replace the single 7-day full rebuild with per-tier scheduling:

- Tier 1 (sitewide top lists): weekly refresh, 12 API calls
- Tiers 2-4 (discographies): monthly refresh, incremental —
  only fetches discographies for artists not already indexed

On subsequent runs:
- If T1 is fresh, load cached artists from the index (~0 calls)
- If discographies are fresh, skip Tiers 2-4 entirely (~0 calls)
- If discographies are stale, diff against indexed set and only
  fetch new artists that appeared in the sitewide lists

Add helpers: isMetaFresh (per-key freshness check),
loadCachedSitewideArtists (read artists from existing index),
filterUnindexed (diff artist list against indexed set).

After first build: typical startup is <5s (T1 cache load).
Monthly incremental: ~50-100 calls for newly appeared artists.

backend/explore/searchindex.go

@@ -17,8 +17,13 @@ import (
 
 // Index build parameters.
 const (
-	// indexRebuildInterval is the minimum time between full rebuilds.
+	// indexTier1Interval is the minimum time between Tier 1
-	indexRebuildInterval = 7 * 24 * time.Hour
+	// (sitewide top lists) refreshes.  Cheap — 12 API calls.
+	indexTier1Interval = 7 * 24 * time.Hour
+
+	// indexTier2Interval is the minimum time between Tier 2/4
+	// (discography) refreshes.  Incremental — only new artists.
+	indexTier2Interval = 30 * 24 * time.Hour
 
 	// indexTopArtists is the number of artists to fetch per range
 	// from the LB sitewide endpoint.
@@ -318,16 +323,6 @@ func isWordChar(r rune) bool {
 func (si *SearchIndex) build(ctx context.Context) {
 	start := time.Now()
 
-	if si.isFresh() {
-		si.logger.Info("search index is fresh, skipping rebuild")
-
-		si.mu.Lock()
-		si.ready = true
-		si.mu.Unlock()
-
-		return
-	}
-
 	si.logger.Info("search index build starting")
 
 	// Mark ready from existing rows so search works during the build.
@@ -336,11 +331,23 @@ func (si *SearchIndex) build(ctx context.Context) {
 	indexLimiter := NewRateLimiterN(indexerRate)
 	indexLB := NewListenBrainzClient(indexLimiter, si.lb.cache, si.logger.WithGroup("indexer"))
 
-	// Tier 1: sitewide instant — top lists across all time ranges.
+	// Tier 1: sitewide instant — refresh weekly (12 calls, <5s).
-	sitewideArtists := si.buildTier1Sitewide(ctx, indexLB)
+	tier1Fresh := si.isMetaFresh("tier1_built", indexTier1Interval)
 
-	if ctx.Err() != nil {
+	var sitewideArtists []lbSitewideArtist
-		return
+
+	if tier1Fresh {
+		si.logger.Info("search index: Tier 1 fresh, loading cached artists")
+
+		sitewideArtists = si.loadCachedSitewideArtists()
+	} else {
+		sitewideArtists = si.buildTier1Sitewide(ctx, indexLB)
+
+		if ctx.Err() != nil {
+			return
+		}
+
+		si.setMeta("tier1_built", time.Now().UTC().Format(time.RFC3339))
 	}
 
 	si.mu.Lock()
@@ -349,35 +356,56 @@ func (si *SearchIndex) build(ctx context.Context) {
 
 	si.logger.Info("search index: Tier 1 complete (sitewide instant)")
 
-	// Tier 2: full discographies of sitewide artists.
+	// Tiers 2-4: discographies — refresh monthly, incremental.
-	si.buildTier2Discographies(ctx, indexLB, sitewideArtists)
+	// Only fetch discographies for artists not already indexed.
+	discogFresh := si.isMetaFresh("discog_built", indexTier2Interval)
 
-	if ctx.Err() != nil {
+	if discogFresh {
-		return
+		si.logger.Info("search index: discographies fresh, skipping Tiers 2-4")
+	} else {
+		indexed := si.indexedArtistMBIDs()
+
+		// Tier 2: sitewide artists' discographies (incremental).
+		newSitewide := filterUnindexed(sitewideArtists, indexed)
+
+		si.logger.Info("search index: Tier 2 starting",
+			"total", len(sitewideArtists),
+			"alreadyIndexed", len(sitewideArtists)-len(newSitewide),
+			"new", len(newSitewide),
+		)
+
+		si.indexArtistDiscographies(ctx, indexLB, newSitewide, "Tier 2")
+
+		if ctx.Err() != nil {
+			return
+		}
+
+		si.logger.Info("search index: Tier 2 complete (sitewide discographies)")
+
+		// Tier 3: library artists' discographies (incremental).
+		// Re-read indexed set since Tier 2 added entries.
+		indexed = si.indexedArtistMBIDs()
+		libraryMBIDs := si.buildTier3Library(ctx, indexLB, sitewideArtists, indexed)
+
+		if ctx.Err() != nil {
+			return
+		}
+
+		si.logger.Info("search index: Tier 3 complete (library discographies)")
+
+		// Tier 4: similar artists (incremental).
+		indexed = si.indexedArtistMBIDs()
+		si.buildTier4Similar(ctx, indexLB, libraryMBIDs, indexed)
+
+		if ctx.Err() != nil {
+			return
+		}
+
+		si.logger.Info("search index: Tier 4 complete (similar artists)")
+
+		si.setMeta("discog_built", time.Now().UTC().Format(time.RFC3339))
 	}
 
-	si.logger.Info("search index: Tier 2 complete (sitewide discographies)")
-
-	// Tier 3: library artists' full discographies.
-	libraryMBIDs := si.buildTier3Library(ctx, indexLB, sitewideArtists)
-
-	if ctx.Err() != nil {
-		return
-	}
-
-	si.logger.Info("search index: Tier 3 complete (library discographies)")
-
-	// Tier 4: similar artists to library artists.
-	si.buildTier4Similar(ctx, indexLB, libraryMBIDs)
-
-	if ctx.Err() != nil {
-		return
-	}
-
-	si.logger.Info("search index: Tier 4 complete (similar artists)")
-
-	si.setMeta("last_built", time.Now().UTC().Format(time.RFC3339))
-
 	si.logger.Info("search index build complete", "elapsed", time.Since(start).Round(time.Second))
 }
 
@@ -610,14 +638,6 @@ func (si *SearchIndex) fetchSitewideReleaseGroups(
 // Tier 2: sitewide artists' full discographies
 // ---------------------------------------------------------------------------
 
-func (si *SearchIndex) buildTier2Discographies(
-	ctx context.Context,
-	lb *ListenBrainzClient,
-	artists []lbSitewideArtist,
-) {
-	si.indexArtistDiscographies(ctx, lb, artists, "Tier 2")
-}
-
 // ---------------------------------------------------------------------------
 // Tier 3: library artists' full discographies
 // ---------------------------------------------------------------------------
@@ -629,6 +649,7 @@ func (si *SearchIndex) buildTier3Library(
 	ctx context.Context,
 	lb *ListenBrainzClient,
 	sitewideArtists []lbSitewideArtist,
+	indexed map[string]bool,
 ) []string {
 	// Build a name→artist map from sitewide (lowercased).
 	nameMap := make(map[string]lbSitewideArtist, len(sitewideArtists))
@@ -670,8 +691,6 @@ func (si *SearchIndex) buildTier3Library(
 	defer func() { _ = libRows.Close() }()
 
 	// Collect MBIDs for matched library artists.
-	indexedMBIDs := si.indexedArtistMBIDs()
-
 	var matched []lbSitewideArtist
 
 	var resolvedMBIDs []string
@@ -698,7 +717,7 @@ func (si *SearchIndex) buildTier3Library(
 			resolvedMBIDs = append(resolvedMBIDs, a.ArtistMBID)
 
 			// Only index if not already in the index from Tier 2.
-			if !indexedMBIDs[a.ArtistMBID] {
+			if !indexed[a.ArtistMBID] {
 				matched = append(matched, a)
 			}
 		}
@@ -724,13 +743,12 @@ func (si *SearchIndex) buildTier4Similar(
 	ctx context.Context,
 	lb *ListenBrainzClient,
 	libraryMBIDs []string,
+	indexed map[string]bool,
 ) {
 	if len(libraryMBIDs) == 0 {
 		return
 	}
 
-	indexedMBIDs := si.indexedArtistMBIDs()
-
 	// Fetch similar artists for each library artist.
 	newArtistMap := make(map[string]lbSitewideArtist)
 
@@ -762,7 +780,7 @@ func (si *SearchIndex) buildTier4Similar(
 			mu.Lock()
 
 			for _, s := range similar {
-				if !indexedMBIDs[s.ArtistMBID] {
+				if !indexed[s.ArtistMBID] {
 					if _, exists := newArtistMap[s.ArtistMBID]; !exists {
 						newArtistMap[s.ArtistMBID] = lbSitewideArtist{
 							ArtistMBID: s.ArtistMBID,
@@ -1149,9 +1167,9 @@ func (si *SearchIndex) indexedArtistMBIDs() map[string]bool {
 	return result
 }
 
-func (si *SearchIndex) isFresh() bool {
+func (si *SearchIndex) isMetaFresh(key string, maxAge time.Duration) bool {
 	rows, err := si.db.QueryContext(
-		"SELECT value FROM explore_index_meta WHERE key = 'last_built'",
+		"SELECT value FROM explore_index_meta WHERE key = ?", key,
 	)
 	if err != nil {
 		return false
@@ -1173,7 +1191,58 @@ func (si *SearchIndex) isFresh() bool {
 		return false
 	}
 
-	return time.Since(t) < indexRebuildInterval
+	return time.Since(t) < maxAge
+}
+
+// loadCachedSitewideArtists reads artist entries from the existing
+// index when Tier 1 is fresh and doesn't need re-fetching.
+func (si *SearchIndex) loadCachedSitewideArtists() []lbSitewideArtist {
+	rows, err := si.db.QueryContext(`
+		SELECT mbid, title, popularity
+		FROM explore_index
+		WHERE entity_type = 'artist'
+		ORDER BY popularity DESC
+	`)
+	if err != nil {
+		return nil
+	}
+
+	defer func() { _ = rows.Close() }()
+
+	var artists []lbSitewideArtist
+
+	maxL := 0
+
+	for rows.Next() {
+		var a lbSitewideArtist
+		if err := rows.Scan(&a.ArtistMBID, &a.ArtistName, &a.ListenCount); err == nil {
+			artists = append(artists, a)
+
+			if a.ListenCount > maxL {
+				maxL = a.ListenCount
+			}
+		}
+	}
+
+	si.mu.Lock()
+	si.maxListens = maxL
+	si.mu.Unlock()
+
+	return artists
+}
+
+// filterUnindexed returns artists whose MBIDs are not in the
+// indexed set.
+func filterUnindexed(artists []lbSitewideArtist, indexed map[string]bool) []lbSitewideArtist {
+	var out []lbSitewideArtist
+
+	for _, a := range artists {
+		if !indexed[a.ArtistMBID] {
+			out = append(out, a)
+		}
+	}
+
+	return out
 }
 
 func (si *SearchIndex) setMeta(key, value string) {