feat: data lifecycle rewrite, download clients, wanted list, and central catalog index

Ships the fresh-start schema cleanup: rebuilt explore catalog index
pipeline (dump import, artifact fetch/build, incremental listen-count
refresh), a new download subsystem (Lidarr/Prowlarr/qBittorrent/SABnzbd/
slskd/yt-dlp providers, staging, reconciliation, wanted list), and the
supporting schema/query/store changes across backend and frontend.

Also includes two smaller follow-ups: bump the central index's
rebuild-after cadence from 90 to 180 days, and remove the Explore
"library only" online/offline toggle entirely (frontend-only, no
backend counterpart) rather than carry unused UI/state.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Y2Agd9af5hE7qzti2ackiS

backend/datamap/datamap.go

@@ -0,0 +1,410 @@
+// Package datamap is the catalog of everything YellowJacket persists.
+//
+// It exists because deletion logic used to be written per call site and
+// lived far from the thing being deleted.  Nobody adding a table could
+// know the full set of places that needed updating, so tables leaked
+// (rows nothing ever removed), files leaked (thumbnails whose database
+// row was gone), and in one case a new table's foreign key silently made
+// libraries unremovable.
+//
+// Every table, view, and on-disk asset directory is classified along two
+// axes that between them determine every policy worth having:
+//
+//   - Kind — what it costs to lose the data.
+//   - Lifetime — how rows or files are removed.
+//
+// The catalog is plain data with no dependency on any service, so tests
+// can assert it against a live schema.  The rule that gives it teeth:
+// every table in the database must be claimed by exactly one entry here,
+// enforced by TestCatalogCoversSchema.  A new table fails the build until
+// somebody states what it is and how it dies.
+package datamap
+
+import "strings"
+
+// Kind classifies persisted data by what losing it costs.
+type Kind string
+
+const (
+	// Owned data is a projection of the user's audio files.  The files
+	// on disk are the source of truth; a rescan rebuilds all of it.
+	Owned Kind = "owned"
+
+	// Authored data was created by the user and exists nowhere else.
+	// Losing it is unrecoverable data loss — it must never be deleted
+	// as a side effect of anything.
+	Authored Kind = "authored"
+
+	// Derived data is computed from owned data and is cheap to rebuild.
+	// It may be deleted freely and must never block deletion of the
+	// data it was derived from.
+	Derived Kind = "derived"
+
+	// Cache data came from the network or a MusicBrainz dump.  It is
+	// rebuildable but expensive — rate limits, or a multi-hour index
+	// build — so it is evicted on its own schedule rather than being
+	// tied to the lifetime of anything else.
+	Cache Kind = "cache"
+)
+
+// Lifetime says how rows leave a table.  It is declared here and checked
+// against the live schema by TestLifetimesMatchSchema, so a foreign key
+// that disagrees with the stated policy fails the tests.
+type Lifetime string
+
+const (
+	// Cascade means a foreign key with ON DELETE CASCADE removes rows
+	// when the parent goes.  No application code required.
+	Cascade Lifetime = "cascade"
+
+	// SetNull means a foreign key with ON DELETE SET NULL orphans the
+	// row deliberately, keeping it alive without its parent.
+	SetNull Lifetime = "set-null"
+
+	// Swept means application code must delete these rows explicitly —
+	// either in a removal path or in the maintenance janitor.  Any table
+	// with a NO ACTION foreign key must declare this, because such a key
+	// blocks its parent's deletion until someone clears the child rows.
+	Swept Lifetime = "swept"
+
+	// Retained means rows are never removed automatically.  Only an
+	// explicit user action deletes them.
+	Retained Lifetime = "retained"
+)
+
+// Table is one catalog entry.
+type Table struct {
+	// Name is the SQL table or view name.
+	Name string
+	// Kind is what the data costs to lose.
+	Kind Kind
+	// Lifetime is how rows are removed.
+	Lifetime Lifetime
+	// FTS marks an FTS5 virtual table, which SQLite backs with four
+	// shadow tables (_config, _data, _docsize, _idx).  Those are
+	// implementation detail and are resolved to this entry.
+	FTS bool
+	// Note records why the classification is what it is, particularly
+	// where a table is not purely one kind.
+	Note string
+}
+
+// ftsShadowSuffixes are the tables SQLite creates behind an FTS5 virtual
+// table.  They belong to their parent and are never catalogued directly.
+var ftsShadowSuffixes = []string{
+	"_config", "_data", "_docsize", "_idx",
+}
+
+// internalTables are SQLite's own bookkeeping, outside our control.
+var internalTables = map[string]bool{
+	"sqlite_sequence": true,
+	"sqlite_stat1":    true,
+	"sqlite_stat4":    true,
+}
+
+// tables is the catalog.  Keep it alphabetical.
+var tables = []Table{
+	{
+		Name: "artist_credit", Kind: Owned, Lifetime: Swept,
+		Note: "Credit strings parsed from file tags. Orphan-swept when " +
+			"no recording references them.",
+	},
+	{
+		Name: "artist_credit_artist", Kind: Owned, Lifetime: Swept,
+		Note: "Join table between credits and artists.",
+	},
+	{
+		Name: "artist_images", Kind: Cache, Lifetime: Swept,
+		Note: "Artist photos from fanart.tv/MusicBrainz. Rows point at " +
+			"files under the artist-images directory; the janitor sweeps " +
+			"both together.",
+	},
+	{
+		Name: "artist_metadata", Kind: Cache, Lifetime: Swept,
+		Note: "Fetched artist bios and metadata. No TTL — swept when the " +
+			"artist is no longer referenced.",
+	},
+	{
+		Name: "artists", Kind: Owned, Lifetime: Swept,
+		Note: "Artists parsed from file tags. Orphan-swept.",
+	},
+	{
+		Name: "audio_files", Kind: Owned, Lifetime: Swept,
+		Note: "MIXED KIND. Mostly an owned projection of files on disk, " +
+			"but play_count, last_played and tag_status are authored and " +
+			"exist nowhere else. Deleting a row to rebuild it destroys " +
+			"that authored state — which is why a file rename currently " +
+			"loses play counts. See the data architecture plan.",
+	},
+	{
+		Name: "cover_art", Kind: Owned, Lifetime: Swept,
+		Note: "Extracted embedded artwork. file_path names the original " +
+			"only; the sized variants beside it are derived filenames and " +
+			"must be expanded when deleting (see library.coverArtFileSet).",
+	},
+	{
+		Name: "download_items", Kind: Authored, Lifetime: Cascade,
+		Note: "One grab attempt per row, with the ranked candidate stored " +
+			"as JSON. Cascades from download_requests. The candidate blob " +
+			"is kept rather than re-derived because a provider's result " +
+			"set is ephemeral — the peer that had the files may be gone, " +
+			"and the row still has to explain why it was chosen.",
+	},
+	{
+		Name: "download_providers", Kind: Authored, Lifetime: Retained,
+		Note: "Download clients the user connected. Removed only by the " +
+			"user. Holds no secrets: API keys live in a 0600 file keyed " +
+			"by this row's id, so the table can be dumped into a bug " +
+			"report without redaction.",
+	},
+	{
+		Name: "download_requests", Kind: Authored, Lifetime: Cascade,
+		Note: "One row per 'go find me this'. Cascades from libraries, " +
+			"and cascades onward to download_items. Terminal rows are " +
+			"history the user clears explicitly.",
+	},
+	{
+		Name: "download_wants", Kind: Authored, Lifetime: Cascade,
+		Note: "The wanted list: one MBID per row, plus retry bookkeeping. " +
+			"Cascades from libraries, and from a parent artist want to " +
+			"the album wants it derived. Unlike download_requests these " +
+			"are not history — a want outlives every attempt made on it " +
+			"and is only removed by the user or by the library coming " +
+			"to own what it names.",
+	},
+	{
+		Name: "explore_champion_fts", Kind: Cache, Lifetime: Retained, FTS: true,
+		Note: "Full-text index over the champion entities of the " +
+			"MusicBrainz dump. Rebuilt only by a full index build.",
+	},
+	{
+		Name: "explore_index", Kind: Cache, Lifetime: Retained,
+		Note: "The offline MusicBrainz search index. Rebuilding costs a " +
+			"~205GB dump stream, so it is never swept automatically.",
+	},
+	{
+		Name: "explore_index_fts", Kind: Cache, Lifetime: Retained, FTS: true,
+		Note: "Full-text index over explore_index.",
+	},
+	{
+		Name: "explore_index_meta", Kind: Cache, Lifetime: Retained,
+		Note: "Build metadata for explore_index: dump version, coverage " +
+			"tiers, last refresh.",
+	},
+	{
+		Name: "file_types", Kind: Derived, Lifetime: Retained,
+		Note: "Static lookup rows seeded from code, not user data.",
+	},
+	{
+		Name: "genres", Kind: Owned, Lifetime: Swept,
+		Note: "Genres parsed from file tags. Orphan-swept.",
+	},
+	{
+		Name: "http_cache", Kind: Cache, Lifetime: Swept,
+		Note: "Cached HTTP responses with a TTL. Reads filter on " +
+			"expires_at; the janitor deletes expired rows.",
+	},
+	{
+		Name: "job_state", Kind: Authored, Lifetime: Retained,
+		Note: "Persisted background-job state, including scans the user " +
+			"paused. Represents user intent, so it survives restarts.",
+	},
+	{
+		Name: "libraries", Kind: Authored, Lifetime: Retained,
+		Note: "The directories the user chose. Removed only by explicit " +
+			"user action via RemoveLibrary.",
+	},
+	{
+		Name: "lyrics_index", Kind: Derived, Lifetime: Retained, FTS: true,
+		Note: "Full-text index over embedded and fetched lyrics. Rebuilt " +
+			"from owned files plus the LRCLIB backfill.",
+	},
+	{
+		Name: "play_history", Kind: Authored, Lifetime: Cascade,
+		Note: "Listening history. Authored, but intentionally cascades " +
+			"with its track — history for a file no longer in the library " +
+			"has nothing to point at.",
+	},
+	{
+		Name: "player_state", Kind: Authored, Lifetime: Retained,
+		Note: "Volume, repeat and shuffle modes, last position.",
+	},
+	{
+		Name: "playlist_tracks", Kind: Authored, Lifetime: SetNull,
+		Note: "Playlist membership. Deliberately survives its track: " +
+			"audio_file_id is nulled and phantom_* columns preserve the " +
+			"entry so a rescan can re-link it.",
+	},
+	{
+		Name: "playlists", Kind: Authored, Lifetime: Retained,
+		Note: "User-created playlists, including smart playlist rules.",
+	},
+	{
+		Name: "queue", Kind: Authored, Lifetime: Retained,
+		Note: "The play queue's own state (current position, source).",
+	},
+	{
+		Name: "queue_tracks", Kind: Authored, Lifetime: Cascade,
+		Note: "Queue entries. Cascade with their track; the queue is " +
+			"compacted afterwards.",
+	},
+	{
+		Name: "recording_genres", Kind: Owned, Lifetime: Swept,
+		Note: "Join table between recordings and genres.",
+	},
+	{
+		Name: "recordings", Kind: Owned, Lifetime: Swept,
+		Note: "Tracks as parsed from file tags. Orphan-swept.",
+	},
+	{
+		Name: "release_group_recordings", Kind: Owned, Lifetime: Swept,
+		Note: "Join table between release groups and recordings.",
+	},
+	{
+		Name: "release_groups", Kind: Owned, Lifetime: Swept,
+		Note: "Albums as parsed from file tags. Orphan-swept.",
+	},
+	{
+		Name: "release_to_rg", Kind: Cache, Lifetime: Retained,
+		Note: "Release to release-group mapping from the dump.",
+	},
+	{
+		Name: "search_clicks", Kind: Authored, Lifetime: Retained,
+		Note: "Which results the user picked, used to rank future " +
+			"searches. Behavioural but unrecoverable if dropped.",
+	},
+	{
+		Name: "search_index", Kind: Derived, Lifetime: Retained, FTS: true,
+		Note: "Contentless FTS5 over the library. Individual rows cannot " +
+			"be deleted, so stale entries are tolerated and filtered by " +
+			"joining track_metadata; a full rescan rebuilds it.",
+	},
+	{
+		Name: "similar_artist_map", Kind: Cache, Lifetime: Retained,
+		Note: "Artist similarity edges derived from the dump.",
+	},
+	{
+		Name: "tagging_candidates", Kind: Derived, Lifetime: Cascade,
+		Note: "Scored MusicBrainz candidates for a tagging group. " +
+			"Cascades with its tagging_items row.",
+	},
+	{
+		Name: "tagging_items", Kind: Derived, Lifetime: Swept,
+		Note: "MIXED KIND. The grouping is derived from folder layout, " +
+			"but status and cleared_at record the user's review " +
+			"decisions. Its library_id foreign key is NO ACTION, so " +
+			"RemoveLibrary must delete these rows explicitly or the " +
+			"library cannot be removed at all.",
+	},
+	{
+		Name: "track_metadata", Kind: Derived, Lifetime: Retained,
+		Note: "A view joining audio_files to its entity chain. Holds no " +
+			"rows of its own.",
+	},
+}
+
+// directories are the on-disk asset trees under the user data directory.
+// Files there have no foreign keys, so nothing removes them implicitly —
+// each needs a sweeper in the janitor.
+var directories = []Directory{
+	{
+		Name: "covers", Kind: Derived,
+		Note: "Extracted cover art plus generated _sm/_md/_lg variants. " +
+			"Live set is cover_art.file_path expanded to its variants.",
+	},
+	{
+		Name: "artist-images", Kind: Cache,
+		Note: "Per-artist-MBID directories of fetched photos, a " +
+			"primary.jpg with thumbnails, and a .miss marker for artists " +
+			"known to have no art. Live set is artist_images.file_path.",
+	},
+	{
+		Name: "cover-art-cache", Kind: Cache,
+		Note: "Cover Art Archive images fetched for Explore browsing, " +
+			"keyed by release-group MBID. Not tied to owned data at all.",
+	},
+}
+
+// Directory is an on-disk asset tree owned by the application.
+type Directory struct {
+	// Name is the directory name under the user data directory.
+	Name string
+	// Kind is what the files cost to lose.
+	Kind Kind
+	// Note records what lives there and how the live set is determined.
+	Note string
+}
+
+// Tables returns the full catalog.
+func Tables() []Table {
+	out := make([]Table, len(tables))
+	copy(out, tables)
+
+	return out
+}
+
+// Directories returns the catalogued asset directories.
+func Directories() []Directory {
+	out := make([]Directory, len(directories))
+	copy(out, directories)
+
+	return out
+}
+
+// Lookup returns the catalog entry for a table name, resolving FTS
+// shadow tables to their parent.  Reports false for SQLite's internal
+// tables and for anything not catalogued.
+func Lookup(name string) (Table, bool) {
+	if internalTables[name] {
+		return Table{}, false
+	}
+
+	for _, t := range tables {
+		if t.Name == name {
+			return t, true
+		}
+	}
+
+	if parent, ok := ftsParent(name); ok {
+		return Lookup(parent)
+	}
+
+	return Table{}, false
+}
+
+// IsInternal reports whether a table is SQLite's own bookkeeping.
+func IsInternal(name string) bool {
+	return internalTables[name] || strings.HasPrefix(name, "sqlite_")
+}
+
+// ftsParent maps an FTS5 shadow table to the virtual table that owns it.
+func ftsParent(name string) (string, bool) {
+	for _, suffix := range ftsShadowSuffixes {
+		if !strings.HasSuffix(name, suffix) {
+			continue
+		}
+
+		parent := strings.TrimSuffix(name, suffix)
+
+		for _, t := range tables {
+			if t.Name == parent && t.FTS {
+				return parent, true
+			}
+		}
+	}
+
+	return "", false
+}
+
+// ByKind returns every catalogued table of a given kind.
+func ByKind(k Kind) []Table {
+	var out []Table
+
+	for _, t := range tables {
+		if t.Kind == k {
+			out = append(out, t)
+		}
+	}
+
+	return out
+}

backend/datamap/datamap_test.go

@@ -0,0 +1,287 @@
+package datamap_test
+
+import (
+	"fmt"
+	"testing"
+
+	"yellowjacket/backend/database"
+	"yellowjacket/backend/datamap"
+)
+
+// liveTables returns every table and view in a freshly migrated schema.
+func liveTables(t *testing.T, db *database.DB) []string {
+	t.Helper()
+
+	rows, err := db.QueryContext(
+		`SELECT name FROM sqlite_master
+		 WHERE type IN ('table', 'view')
+		 ORDER BY name`,
+	)
+	if err != nil {
+		t.Fatalf("read sqlite_master: %v", err)
+	}
+
+	defer func() { _ = rows.Close() }()
+
+	var names []string
+
+	for rows.Next() {
+		var name string
+
+		if err := rows.Scan(&name); err != nil {
+			t.Fatalf("scan table name: %v", err)
+		}
+
+		names = append(names, name)
+	}
+
+	return names
+}
+
+// Every table in the schema must be claimed by exactly one catalog
+// entry.  This is the mechanism that stops a new table from silently
+// having no deletion policy — the failure mode that made libraries
+// unremovable when tagging_items was added.
+func TestCatalogCoversSchema(t *testing.T) {
+	t.Parallel()
+
+	db := database.NewTestDB(t)
+
+	for _, name := range liveTables(t, db) {
+		if datamap.IsInternal(name) {
+			continue
+		}
+
+		if _, ok := datamap.Lookup(name); !ok {
+			t.Errorf(
+				"table %q exists in the schema but is not in the datamap "+
+					"catalog — add an entry stating its Kind and Lifetime",
+				name,
+			)
+		}
+	}
+}
+
+// The reverse direction: a catalog entry naming a table that no longer
+// exists means the catalog has drifted.
+func TestCatalogHasNoStaleEntries(t *testing.T) {
+	t.Parallel()
+
+	db := database.NewTestDB(t)
+
+	live := make(map[string]bool)
+	for _, name := range liveTables(t, db) {
+		live[name] = true
+	}
+
+	for _, entry := range datamap.Tables() {
+		if !live[entry.Name] {
+			t.Errorf(
+				"catalog lists %q but it is not in the schema",
+				entry.Name,
+			)
+		}
+	}
+}
+
+type foreignKey struct {
+	child    string
+	from     string
+	parent   string
+	onDelete string
+}
+
+// liveForeignKeys reads every foreign key in the schema.
+func liveForeignKeys(t *testing.T, db *database.DB) []foreignKey {
+	t.Helper()
+
+	var out []foreignKey
+
+	for _, table := range liveTables(t, db) {
+		if datamap.IsInternal(table) {
+			continue
+		}
+
+		rows, err := db.QueryContext(
+			fmt.Sprintf("PRAGMA foreign_key_list(%q)", table),
+		)
+		if err != nil {
+			continue // views have none
+		}
+
+		for rows.Next() {
+			var (
+				id, seq                                 int
+				parent, from, to, onUpd, onDel, matchOn string
+			)
+
+			if err := rows.Scan(
+				&id, &seq, &parent, &from, &to, &onUpd, &onDel, &matchOn,
+			); err != nil {
+				continue
+			}
+
+			out = append(out, foreignKey{
+				child:    table,
+				from:     from,
+				parent:   parent,
+				onDelete: onDel,
+			})
+		}
+
+		_ = rows.Close()
+	}
+
+	return out
+}
+
+// A foreign key with NO ACTION blocks its parent's deletion until
+// application code clears the child rows.  Any table with such a key
+// must therefore declare Lifetime "swept" — an assertion that some
+// removal path or janitor actually deletes them.  Declaring "retained"
+// or "cascade" while holding a NO ACTION key is the exact shape of the
+// tagging_items bug.
+func TestNoActionForeignKeysAreDeclaredSwept(t *testing.T) {
+	t.Parallel()
+
+	db := database.NewTestDB(t)
+
+	for _, fk := range liveForeignKeys(t, db) {
+		if fk.onDelete != "NO ACTION" {
+			continue
+		}
+
+		entry, ok := datamap.Lookup(fk.child)
+		if !ok {
+			continue // TestCatalogCoversSchema reports this
+		}
+
+		if entry.Lifetime != datamap.Swept {
+			t.Errorf(
+				"%s.%s references %s with ON DELETE NO ACTION, so it "+
+					"blocks deletion of %s — but the catalog declares "+
+					"Lifetime %q. Either declare it %q and delete the rows "+
+					"explicitly, or give the key an ON DELETE action.",
+				fk.child, fk.from, fk.parent, fk.parent,
+				entry.Lifetime, datamap.Swept,
+			)
+		}
+	}
+}
+
+// Declared cascade/set-null lifetimes must match the actual schema, so
+// the catalog cannot quietly drift from what SQLite enforces.
+func TestLifetimesMatchSchema(t *testing.T) {
+	t.Parallel()
+
+	db := database.NewTestDB(t)
+
+	actual := make(map[string]map[string]bool)
+
+	for _, fk := range liveForeignKeys(t, db) {
+		if actual[fk.child] == nil {
+			actual[fk.child] = make(map[string]bool)
+		}
+
+		actual[fk.child][fk.onDelete] = true
+	}
+
+	for _, entry := range datamap.Tables() {
+		switch entry.Lifetime {
+		case datamap.Cascade:
+			if !actual[entry.Name]["CASCADE"] {
+				t.Errorf(
+					"%s declares Lifetime cascade but has no "+
+						"ON DELETE CASCADE foreign key",
+					entry.Name,
+				)
+			}
+		case datamap.SetNull:
+			if !actual[entry.Name]["SET NULL"] {
+				t.Errorf(
+					"%s declares Lifetime set-null but has no "+
+						"ON DELETE SET NULL foreign key",
+					entry.Name,
+				)
+			}
+		case datamap.Swept, datamap.Retained:
+			// No schema-level obligation.
+		}
+	}
+}
+
+// Authored data is unrecoverable, so it must never be removed as a side
+// effect of deleting owned data.  Cascade is allowed only where the
+// catalog explains why (play_history, queue_tracks); this test pins the
+// set so a new cascade onto authored data is a deliberate decision.
+func TestAuthoredCascadesAreDeliberate(t *testing.T) {
+	t.Parallel()
+
+	allowed := map[string]bool{
+		"play_history": true,
+		"queue_tracks": true,
+
+		// Download history is scoped to the library it imported into.
+		// When that library is removed the files it acquired go with
+		// it, so a request describing "fetch this into library 3" has
+		// nothing left to mean. Keeping the rows would leave history
+		// pointing at a library the user deleted.
+		"download_requests": true,
+
+		// Items belong to their request and have no independent
+		// meaning; they cascade with it.
+		"download_items": true,
+
+		// A want says "put this in library 3". Delete that library and
+		// there is no longer anywhere for it to go, so the want has
+		// nothing left to mean — the same reasoning as its requests.
+		// The second cascade, artist want to derived album wants, is
+		// the point of the subscription: unsubscribing from an artist
+		// must stop the albums it queued on the user's behalf.
+		"download_wants": true,
+	}
+
+	for _, entry := range datamap.ByKind(datamap.Authored) {
+		if entry.Lifetime == datamap.Cascade && !allowed[entry.Name] {
+			t.Errorf(
+				"authored table %q cascades on delete — authored data is "+
+					"unrecoverable, so this needs an explicit exemption "+
+					"and a note explaining it",
+				entry.Name,
+			)
+		}
+	}
+}
+
+// Every catalogued table needs a note; the classification is only useful
+// if the reasoning is written down.
+func TestEveryEntryHasANote(t *testing.T) {
+	t.Parallel()
+
+	for _, entry := range datamap.Tables() {
+		if entry.Note == "" {
+			t.Errorf("catalog entry %q has no Note", entry.Name)
+		}
+	}
+
+	for _, dir := range datamap.Directories() {
+		if dir.Note == "" {
+			t.Errorf("catalog directory %q has no Note", dir.Name)
+		}
+	}
+}
+
+// FTS shadow tables must resolve to their parent entry rather than
+// needing catalogue entries of their own.
+func TestFTSShadowResolution(t *testing.T) {
+	t.Parallel()
+
+	entry, ok := datamap.Lookup("search_index_data")
+	if !ok {
+		t.Fatal("search_index_data did not resolve to a catalog entry")
+	}
+
+	if entry.Name != "search_index" {
+		t.Errorf("resolved to %q, want search_index", entry.Name)
+	}
+}