feat: data lifecycle rewrite, download clients, wanted list, and central catalog index
Ships the fresh-start schema cleanup: rebuilt explore catalog index pipeline (dump import, artifact fetch/build, incremental listen-count refresh), a new download subsystem (Lidarr/Prowlarr/qBittorrent/SABnzbd/ slskd/yt-dlp providers, staging, reconciliation, wanted list), and the supporting schema/query/store changes across backend and frontend. Also includes two smaller follow-ups: bump the central index's rebuild-after cadence from 90 to 180 days, and remove the Explore "library only" online/offline toggle entirely (frontend-only, no backend counterpart) rather than carry unused UI/state. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Y2Agd9af5hE7qzti2ackiS
This commit is contained in:
Claude Sonnet 5
@@ -0,0 +1,444 @@
|
||||
package explore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"yellowjacket/backend/database"
|
||||
)
|
||||
|
||||
// artifactRow is one catalog row written into a test artifact.
|
||||
type artifactRow struct {
|
||||
entityType string
|
||||
mbid string
|
||||
title string
|
||||
artistName string
|
||||
artistMBID string
|
||||
popularity int
|
||||
}
|
||||
|
||||
// writeTestArtifact builds an artifact file matching what cmd/indexexport
|
||||
// produces: catalog columns only, no FTS, no triggers.
|
||||
func writeTestArtifact(
|
||||
t *testing.T, meta map[string]string, rows []artifactRow,
|
||||
) string {
|
||||
t.Helper()
|
||||
|
||||
path := filepath.Join(t.TempDir(), "core-index.db")
|
||||
|
||||
db, err := sql.Open("sqlite", "file:"+path)
|
||||
if err != nil {
|
||||
t.Fatalf("open artifact: %v", err)
|
||||
}
|
||||
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
for _, stmt := range []string{
|
||||
`CREATE TABLE explore_index (
|
||||
entity_type TEXT NOT NULL,
|
||||
mbid TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
artist_name TEXT NOT NULL,
|
||||
artist_mbid TEXT NOT NULL,
|
||||
aliases TEXT NOT NULL DEFAULT '',
|
||||
popularity INTEGER NOT NULL DEFAULT 0,
|
||||
listener_count INTEGER NOT NULL DEFAULT 0,
|
||||
duration INTEGER NOT NULL DEFAULT 0,
|
||||
caa_release_mbid TEXT NOT NULL DEFAULT '',
|
||||
release_name TEXT NOT NULL DEFAULT '',
|
||||
primary_type TEXT NOT NULL DEFAULT '',
|
||||
secondary_types TEXT NOT NULL DEFAULT '',
|
||||
release_date TEXT NOT NULL DEFAULT '',
|
||||
artist_type TEXT NOT NULL DEFAULT '',
|
||||
country TEXT NOT NULL DEFAULT '',
|
||||
disambiguation TEXT NOT NULL DEFAULT '',
|
||||
sort_name TEXT NOT NULL DEFAULT '',
|
||||
discog_fetched INTEGER NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (mbid)
|
||||
) WITHOUT ROWID`,
|
||||
`CREATE TABLE artifact_meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
)`,
|
||||
} {
|
||||
if _, err := db.Exec(stmt); err != nil {
|
||||
t.Fatalf("create artifact schema: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
for k, v := range meta {
|
||||
if _, err := db.Exec(
|
||||
`INSERT INTO artifact_meta (key, value) VALUES (?, ?)`, k, v,
|
||||
); err != nil {
|
||||
t.Fatalf("stamp artifact meta: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, r := range rows {
|
||||
if _, err := db.Exec(`
|
||||
INSERT INTO explore_index
|
||||
(entity_type, mbid, title, artist_name, artist_mbid, popularity)
|
||||
VALUES (?, ?, ?, ?, ?, ?)`,
|
||||
r.entityType, r.mbid, r.title, r.artistName, r.artistMBID, r.popularity,
|
||||
); err != nil {
|
||||
t.Fatalf("insert artifact row: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
return path
|
||||
}
|
||||
|
||||
// validMeta is the artifact_meta a well-formed artifact carries.
|
||||
func validMeta() map[string]string {
|
||||
return map[string]string{
|
||||
"artifact_version": supportedArtifactVersion,
|
||||
"built_at": "2026-07-17T03:53:43Z",
|
||||
"listens_applied_series": "2593",
|
||||
"source_rows": "2052168",
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportCoreArtifactMergesCatalog(t *testing.T) {
|
||||
db := database.NewTestDB(t)
|
||||
si := NewSearchIndex(db, nil, nil, testLogger())
|
||||
|
||||
path := writeTestArtifact(t, validMeta(), []artifactRow{
|
||||
{"artist", artA, "Artist A", "Artist A", artA, 5000},
|
||||
{"artist", artB, "Artist B", "Artist B", artB, 4000},
|
||||
{"release_group", rgA, "Album A", "Artist A", artA, 3000},
|
||||
{"recording", recA, "Song A", "Artist A", artA, 2000},
|
||||
})
|
||||
|
||||
if err := si.importCoreArtifact(context.Background(), path); err != nil {
|
||||
t.Fatalf("importCoreArtifact: %v", err)
|
||||
}
|
||||
|
||||
var got int
|
||||
if err := db.QueryRowWriter(
|
||||
"SELECT COUNT(*) FROM explore_index",
|
||||
).Scan(&got); err != nil {
|
||||
t.Fatalf("count rows: %v", err)
|
||||
}
|
||||
|
||||
if got != 4 {
|
||||
t.Errorf("merged %d rows, want 4", got)
|
||||
}
|
||||
|
||||
// The merge must leave the index searchable: the FTS rebuild that
|
||||
// closes the bulk-load window is the only thing populating it, since
|
||||
// the triggers were dropped for the duration.
|
||||
var hits int
|
||||
if err := db.QueryRowWriter(
|
||||
`SELECT COUNT(*) FROM explore_index_fts WHERE explore_index_fts MATCH ?`,
|
||||
"Song",
|
||||
).Scan(&hits); err != nil {
|
||||
t.Fatalf("query fts: %v", err)
|
||||
}
|
||||
|
||||
if hits == 0 {
|
||||
t.Error("FTS index is empty after merge; search would return nothing")
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportCoreArtifactStampsMeta(t *testing.T) {
|
||||
db := database.NewTestDB(t)
|
||||
si := NewSearchIndex(db, nil, nil, testLogger())
|
||||
|
||||
path := writeTestArtifact(t, validMeta(), []artifactRow{
|
||||
{"artist", artA, "Artist A", "Artist A", artA, 5000},
|
||||
})
|
||||
|
||||
if err := si.importCoreArtifact(context.Background(), path); err != nil {
|
||||
t.Fatalf("importCoreArtifact: %v", err)
|
||||
}
|
||||
|
||||
if !si.hasMeta(dumpImportDoneKey) {
|
||||
t.Error("dump_import_done not stamped; a full dump import would run anyway")
|
||||
}
|
||||
|
||||
// Without this the incremental refresh refuses to run and the
|
||||
// shipped popularity never updates again.
|
||||
if series, ok := si.metaInt(listensAppliedSeriesKey); !ok || series != 2593 {
|
||||
t.Errorf("listens_applied_series = %d (ok=%v), want 2593", series, ok)
|
||||
}
|
||||
|
||||
if !si.hasMeta(coreArtifactVersionKey) {
|
||||
t.Error("core_artifact_version not stamped")
|
||||
}
|
||||
}
|
||||
|
||||
// A merge must never downgrade what the index already holds, because a
|
||||
// user's own library rows and any lazily-fetched detail predate it.
|
||||
func TestImportCoreArtifactPreservesLocalData(t *testing.T) {
|
||||
db := database.NewTestDB(t)
|
||||
si := NewSearchIndex(db, nil, nil, testLogger())
|
||||
|
||||
si.upsertBatch([]SearchIndexResult{{
|
||||
EntityType: "recording",
|
||||
MBID: recA,
|
||||
Title: "Song A",
|
||||
ArtistName: "Artist A",
|
||||
ArtistMBID: artA,
|
||||
Popularity: 9999,
|
||||
Duration: 210000,
|
||||
InLibrary: true,
|
||||
DiscogFetched: true,
|
||||
}})
|
||||
|
||||
path := writeTestArtifact(t, validMeta(), []artifactRow{
|
||||
// Lower popularity and no duration: both must lose.
|
||||
{"recording", recA, "Song A", "Artist A", artA, 10},
|
||||
})
|
||||
|
||||
if err := si.importCoreArtifact(context.Background(), path); err != nil {
|
||||
t.Fatalf("importCoreArtifact: %v", err)
|
||||
}
|
||||
|
||||
var popularity, duration, inLibrary, discogFetched int
|
||||
|
||||
if err := db.QueryRowWriter(`
|
||||
SELECT popularity, duration, in_library, discog_fetched
|
||||
FROM explore_index WHERE mbid = ?`, recA,
|
||||
).Scan(&popularity, &duration, &inLibrary, &discogFetched); err != nil {
|
||||
t.Fatalf("read merged row: %v", err)
|
||||
}
|
||||
|
||||
if popularity != 9999 {
|
||||
t.Errorf("popularity = %d, want 9999 (higher must win)", popularity)
|
||||
}
|
||||
|
||||
if duration != 210000 {
|
||||
t.Errorf("duration = %d, want 210000 (artifact carries none)", duration)
|
||||
}
|
||||
|
||||
if inLibrary != 1 {
|
||||
t.Error("in_library was cleared; the artifact must not touch personal columns")
|
||||
}
|
||||
|
||||
if discogFetched != 1 {
|
||||
t.Error("discog_fetched was cleared by the merge")
|
||||
}
|
||||
}
|
||||
|
||||
// The batch walk is the part most likely to drop or duplicate rows, so
|
||||
// it is exercised across many batch boundaries rather than one.
|
||||
func TestImportCoreArtifactBatchWalkCoversAllRows(t *testing.T) {
|
||||
db := database.NewTestDB(t)
|
||||
si := NewSearchIndex(db, nil, nil, testLogger())
|
||||
|
||||
// Shrink the batch rather than growing the fixture: what matters is
|
||||
// crossing several boundaries, including a short final one.
|
||||
original := artifactMergeBatch
|
||||
artifactMergeBatch = 100
|
||||
|
||||
t.Cleanup(func() { artifactMergeBatch = original })
|
||||
|
||||
const total = 337
|
||||
|
||||
rows := make([]artifactRow, 0, total)
|
||||
for i := range total {
|
||||
rows = append(rows, artifactRow{
|
||||
entityType: "recording",
|
||||
mbid: syntheticMBID(i),
|
||||
title: "Song",
|
||||
artistName: "Artist",
|
||||
artistMBID: artA,
|
||||
popularity: i,
|
||||
})
|
||||
}
|
||||
|
||||
path := writeTestArtifact(t, validMeta(), rows)
|
||||
|
||||
if err := si.importCoreArtifact(context.Background(), path); err != nil {
|
||||
t.Fatalf("importCoreArtifact: %v", err)
|
||||
}
|
||||
|
||||
var got int
|
||||
if err := db.QueryRowWriter(
|
||||
"SELECT COUNT(*) FROM explore_index",
|
||||
).Scan(&got); err != nil {
|
||||
t.Fatalf("count rows: %v", err)
|
||||
}
|
||||
|
||||
if got != total {
|
||||
t.Errorf("merged %d rows, want %d", got, total)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportCoreArtifactRejectsBadArtifacts(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
meta map[string]string
|
||||
rows []artifactRow
|
||||
want error
|
||||
}{
|
||||
{
|
||||
name: "version mismatch",
|
||||
meta: map[string]string{"artifact_version": "99"},
|
||||
rows: []artifactRow{{"artist", artA, "A", "A", artA, 1}},
|
||||
want: ErrArtifactVersion,
|
||||
},
|
||||
{
|
||||
name: "no version",
|
||||
meta: map[string]string{},
|
||||
rows: []artifactRow{{"artist", artA, "A", "A", artA, 1}},
|
||||
want: ErrArtifactVersion,
|
||||
},
|
||||
{
|
||||
name: "empty catalog",
|
||||
meta: validMeta(),
|
||||
rows: nil,
|
||||
want: ErrArtifactUnusable,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
db := database.NewTestDB(t)
|
||||
si := NewSearchIndex(db, nil, nil, testLogger())
|
||||
|
||||
path := writeTestArtifact(t, tt.meta, tt.rows)
|
||||
|
||||
err := si.importCoreArtifact(context.Background(), path)
|
||||
if err == nil {
|
||||
t.Fatal("expected rejection, got nil")
|
||||
}
|
||||
|
||||
if !strings.Contains(err.Error(), tt.want.Error()) {
|
||||
t.Errorf("error = %v, want it to wrap %v", err, tt.want)
|
||||
}
|
||||
|
||||
// A rejected artifact must not leave the index claiming it
|
||||
// has a catalog, or the real build would never run.
|
||||
if si.hasMeta(dumpImportDoneKey) {
|
||||
t.Error("rejected artifact still stamped dump_import_done")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectArtifactRejectsNonArtifactFile(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "garbage.db")
|
||||
|
||||
if err := os.WriteFile(path, []byte("not a database"), 0o644); err != nil {
|
||||
t.Fatalf("write file: %v", err)
|
||||
}
|
||||
|
||||
if _, err := inspectArtifact(path); err == nil {
|
||||
t.Error("expected rejection of a non-artifact file")
|
||||
}
|
||||
}
|
||||
|
||||
// syntheticMBID produces distinct, well-formed MBIDs for bulk fixtures.
|
||||
func syntheticMBID(i int) string {
|
||||
const hex = "0123456789abcdef"
|
||||
|
||||
buf := []byte("00000000-0000-0000-0000-000000000000")
|
||||
|
||||
for pos := len(buf) - 1; pos >= 0 && i > 0; pos-- {
|
||||
if buf[pos] == '-' {
|
||||
continue
|
||||
}
|
||||
|
||||
buf[pos] = hex[i%16]
|
||||
i /= 16
|
||||
}
|
||||
|
||||
return string(buf)
|
||||
}
|
||||
|
||||
// resolveArtistName falls back to the MBID when no name is available.
|
||||
// That value must never reach the index: the upsert no longer defends
|
||||
// against it, so the writer is the only thing standing in the way.
|
||||
func TestAddFromCacheNeverStoresMBIDAsName(t *testing.T) {
|
||||
db := database.NewTestDB(t)
|
||||
si := NewSearchIndex(db, nil, nil, testLogger())
|
||||
|
||||
si.AddFromCache(artA, artA, []MBReleaseGroup{
|
||||
{MBID: rgA, Title: "Album A", PrimaryType: "Album"},
|
||||
})
|
||||
|
||||
var artistRows int
|
||||
if err := db.QueryRowWriter(
|
||||
`SELECT COUNT(*) FROM explore_index
|
||||
WHERE entity_type = 'artist' AND title = mbid`,
|
||||
).Scan(&artistRows); err != nil {
|
||||
t.Fatalf("count artist rows: %v", err)
|
||||
}
|
||||
|
||||
if artistRows != 0 {
|
||||
t.Errorf("%d artist rows stored the MBID as their title", artistRows)
|
||||
}
|
||||
|
||||
var artistName string
|
||||
if err := db.QueryRowWriter(
|
||||
`SELECT artist_name FROM explore_index WHERE mbid = ?`, rgA,
|
||||
).Scan(&artistName); err != nil {
|
||||
t.Fatalf("read release group: %v", err)
|
||||
}
|
||||
|
||||
if artistName == artA {
|
||||
t.Error("release group stored the artist MBID as its artist_name")
|
||||
}
|
||||
|
||||
// A real name arriving later must still win over the empty one.
|
||||
si.AddFromCache("Real Artist", artA, []MBReleaseGroup{
|
||||
{MBID: rgA, Title: "Album A", PrimaryType: "Album"},
|
||||
})
|
||||
|
||||
if err := db.QueryRowWriter(
|
||||
`SELECT artist_name FROM explore_index WHERE mbid = ?`, rgA,
|
||||
).Scan(&artistName); err != nil {
|
||||
t.Fatalf("re-read release group: %v", err)
|
||||
}
|
||||
|
||||
if artistName != "Real Artist" {
|
||||
t.Errorf("artist_name = %q, want it filled in once known", artistName)
|
||||
}
|
||||
}
|
||||
|
||||
// The importer names the artifact's columns independently of the
|
||||
// exporter that writes them. If the two lists drift, the merge either
|
||||
// fails outright or silently shifts values into the wrong columns, so
|
||||
// they are compared directly against cmd/indexexport's source.
|
||||
func TestArtifactColumnsMatchExporter(t *testing.T) {
|
||||
src, err := os.ReadFile("../../cmd/indexexport/main.go")
|
||||
if err != nil {
|
||||
t.Fatalf("read exporter: %v", err)
|
||||
}
|
||||
|
||||
const marker = "const catalogColumns = `"
|
||||
|
||||
i := strings.Index(string(src), marker)
|
||||
if i < 0 {
|
||||
t.Fatalf("catalogColumns not found in cmd/indexexport/main.go")
|
||||
}
|
||||
|
||||
rest := string(src)[i+len(marker):]
|
||||
|
||||
j := strings.Index(rest, "`")
|
||||
if j < 0 {
|
||||
t.Fatal("unterminated catalogColumns literal")
|
||||
}
|
||||
|
||||
normalise := func(s string) string {
|
||||
out := make([]string, 0, 32)
|
||||
for _, f := range strings.Split(s, ",") {
|
||||
out = append(out, strings.Join(strings.Fields(f), ""))
|
||||
}
|
||||
|
||||
return strings.Join(out, ",")
|
||||
}
|
||||
|
||||
exporter := normalise(rest[:j])
|
||||
importer := normalise(artifactCatalogColumns)
|
||||
|
||||
if exporter != importer {
|
||||
t.Errorf("column lists have drifted:\n exporter: %s\n importer: %s",
|
||||
exporter, importer)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user