feat: data lifecycle rewrite, download clients, wanted list, and central catalog index

Ships the fresh-start schema cleanup: rebuilt explore catalog index
pipeline (dump import, artifact fetch/build, incremental listen-count
refresh), a new download subsystem (Lidarr/Prowlarr/qBittorrent/SABnzbd/
slskd/yt-dlp providers, staging, reconciliation, wanted list), and the
supporting schema/query/store changes across backend and frontend.

Also includes two smaller follow-ups: bump the central index's
rebuild-after cadence from 90 to 180 days, and remove the Explore
"library only" online/offline toggle entirely (frontend-only, no
backend counterpart) rather than carry unused UI/state.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Y2Agd9af5hE7qzti2ackiS

backend/explore/indexpatch.go

@@ -0,0 +1,213 @@
+//go:build indexbuild
+
+package explore
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"regexp"
+	"sort"
+	"sync"
+
+	"yellowjacket/backend/jobs"
+)
+
+// Helpers used only while building the catalog from the MetaBrainz
+// dumps: the similar-artist patch pass and the per-stage error
+// reporting that goes with it.  They live behind the `indexbuild` tag
+// with the importer they serve, so the app binary carries neither.
+
+const (
+	// indexSimilarPerArtist is how many similar artists to store per
+	// library artist in similar_artist_map.
+	indexSimilarPerArtist = 20
+
+	// similarArtistsBatchSize is the number of seed MBIDs processed in
+	// one logging "batch".  The labs multi-seed POST form is broken, so
+	// one GET per seed is issued (concurrency bounded by indexerRate);
+	// batching here just keeps progress log output bounded.
+	similarArtistsBatchSize = 50
+)
+
+// setTierError marks a build stage as errored.
+func (si *SearchIndex) setTierError(name, errMsg string) {
+	si.mu.Lock()
+
+	for i := range si.buildStatus.Tiers {
+		if si.buildStatus.Tiers[i].Name == name {
+			si.buildStatus.Tiers[i].State = "error"
+			si.buildStatus.Tiers[i].Error = errMsg
+			si.mu.Unlock()
+
+			si.logIndexJob(jobs.LevelError, name+": "+errMsg)
+			si.emitStatus()
+
+			return
+		}
+	}
+
+	si.mu.Unlock()
+}
+
+// fetchSimilarArtistsBatch queries the labs similar-artists endpoint
+// for multiple seed MBIDs.  Despite the name, this actually fans
+// out one request per seed: the labs API's multi-seed mode is
+// broken (results for different seeds get mis-labeled, and some
+// seeds return zero), so batching with multiple artist_mbids is
+// not viable.  Concurrency is bounded by indexerRate to respect
+// the labs rate limit; each call goes through the provided LB
+// client's rate limiter and cache.
+func (si *SearchIndex) fetchSimilarArtistsBatch(
+	ctx context.Context, lb *ListenBrainzClient, seedMBIDs []string,
+) map[string][]lbSimilarArtistWire {
+	if len(seedMBIDs) == 0 {
+		return nil
+	}
+
+	var (
+		mu      sync.Mutex
+		grouped = make(map[string][]lbSimilarArtistWire, len(seedMBIDs))
+		wg      sync.WaitGroup
+	)
+
+	sem := make(chan struct{}, indexerRate)
+
+	for _, seedMBID := range seedMBIDs {
+		if ctx.Err() != nil {
+			break
+		}
+
+		sem <- struct{}{}
+
+		wg.Add(1)
+
+		go func(seed string) {
+			defer func() {
+				<-sem
+				wg.Done()
+			}()
+
+			// Use the LB client's per-seed GET form — goes through
+			// the shared rate limiter and cache.  The multi-seed
+			// POST form is not viable (see function comment).
+			similar, err := lb.SimilarArtists(ctx, seed)
+			if err != nil || len(similar) == 0 {
+				return
+			}
+
+			// Convert to the internal wire type used by the caller
+			// and trim to indexSimilarPerArtist.
+			if len(similar) > indexSimilarPerArtist {
+				similar = similar[:indexSimilarPerArtist]
+			}
+
+			results := make([]lbSimilarArtistWire, len(similar))
+			for i, s := range similar {
+				results[i] = lbSimilarArtistWire{
+					ArtistMBID:    s.ArtistMBID,
+					Name:          s.Name,
+					Score:         int(s.Score),
+					ReferenceMBID: seed,
+				}
+			}
+
+			mu.Lock()
+			grouped[seed] = results
+			mu.Unlock()
+		}(seedMBID)
+	}
+
+	wg.Wait()
+
+	return grouped
+}
+
+// chunkStrings splits a slice into chunks of at most size n.
+func chunkStrings(s []string, n int) [][]string {
+	var chunks [][]string
+
+	for i := 0; i < len(s); i += n {
+		end := i + n
+		if end > len(s) {
+			end = len(s)
+		}
+
+		chunks = append(chunks, s[i:end])
+	}
+
+	return chunks
+}
+
+// getLibraryArtistMBIDs returns MBIDs for all library artists that have one.
+// Used when Tier 3 was skipped but Tier 4 needs the library MBID list.
+func (si *SearchIndex) getLibraryArtistMBIDs() []string {
+	rows, err := si.db.QueryContext(
+		"SELECT DISTINCT mbid FROM artists WHERE mbid IS NOT NULL AND mbid != ''",
+	)
+	if err != nil {
+		return nil
+	}
+
+	defer func() { _ = rows.Close() }()
+
+	var mbids []string
+
+	for rows.Next() {
+		var mbid string
+		if err := rows.Scan(&mbid); err == nil {
+			mbids = append(mbids, mbid)
+		}
+	}
+
+	return mbids
+}
+
+// discoverDumpFile walks a dump base directory, finds subdirectories
+// matching dirRe (newest first, lexicographically — MetaBrainz dump
+// directory names embed sortable timestamps), and returns the full URL
+// of the first file inside matching fileRe.  Directories that don't
+// contain a matching file (e.g. partial uploads) are skipped.
+func discoverDumpFile(
+	ctx context.Context,
+	client *http.Client,
+	baseURL string,
+	dirRe, fileRe *regexp.Regexp,
+) (string, error) {
+	hrefs, err := listHrefs(ctx, client, baseURL)
+	if err != nil {
+		return "", err
+	}
+
+	var dirs []string
+
+	for _, h := range hrefs {
+		trimmed := trimTrailingSlash(h)
+		if dirRe.MatchString(trimmed) {
+			dirs = append(dirs, trimmed)
+		}
+	}
+
+	if len(dirs) == 0 {
+		return "", fmt.Errorf("%w: no dump directories under %s", ErrDumpDiscovery, baseURL)
+	}
+
+	sort.Sort(sort.Reverse(sort.StringSlice(dirs)))
+
+	for _, dir := range dirs {
+		dirURL := baseURL + dir + "/"
+
+		files, err := listHrefs(ctx, client, dirURL)
+		if err != nil {
+			continue
+		}
+
+		for _, f := range files {
+			if fileRe.MatchString(f) {
+				return dirURL + f, nil
+			}
+		}
+	}
+
+	return "", fmt.Errorf("%w: no matching dump file under %s", ErrDumpDiscovery, baseURL)
+}