The post-scan backfills share MusicBrainz's rate limiters with every page the user can open, and both were FIFO — so a thousand-artist enrichment put an album page behind an hour of queued work. WithBackgroundLane/WithBackgroundPriority add a slower second lane: a marked wait takes no token while any interactive wait is outstanding. It is a context marker rather than a parameter because a backfill calls the same client methods a detail page does. A long backfill also has to be visible and stoppable, so jobs.KindCatalogEnrich registers both with progress and cancel — after the work is counted, since these passes are a no-op on every launch once the library is covered. What it does not fetch is the point. It ran for hours against a 900-artist library and marked nothing, because three of the four things it did per artist were work nobody asked for: similar artists, which the artist page already resolves on view, and a full GetArtistImage (fanart.tv, TheAudioDB, Wikidata, Wikipedia, ten portraits) reached only to warm the MB artist lookup EnsureArtistRels does alone. It was also serial across artists while every limiter is per-host and idle. The marks are a table rather than more explore_index columns, because artifactimport merges by column list and a flag added there is a second place to remember. BrowseReleaseGroupsAll pages to exhaustion, where the old call silently cut a prolific artist at 100 release groups. One portrait is downloaded now; the rest are remembered as URLs. resolveAllSources downloaded every candidate, up to ten, full size, while nothing reads anything but primary.jpg — 5.3 GB measured on a real cache, 4.1 GB of it unreachable. OrphanedArtistImagesJob is why that survived: it joined the bare MBID onto the images directory, but artist directories are sharded under a two-character prefix, so it named a path that never existed and deleted the rows that were the only record of the files it left behind. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UDCbcCZQepnpSQYJ6SxxZm
141 lines
3.9 KiB
Go
141 lines
3.9 KiB
Go
package explore
|
|
|
|
import (
|
|
"log/slog"
|
|
"testing"
|
|
|
|
"yellowjacket/backend/database"
|
|
)
|
|
|
|
// seedIndexArtist inserts an explore_index artist row, optionally
|
|
// already marked as having had its ListenBrainz discography fetched.
|
|
func seedIndexArtist(t *testing.T, db *database.DB, mbid string, discogFetched int) {
|
|
t.Helper()
|
|
|
|
if _, err := db.ExecContext(
|
|
upsertIndexSQL,
|
|
"artist", mbid, "Seeded Artist", "Seeded Artist", mbid, "",
|
|
0, 0,
|
|
0, "", "",
|
|
"", "", "",
|
|
"", "", "", "",
|
|
1, 0,
|
|
0, 0, 0,
|
|
discogFetched,
|
|
); err != nil {
|
|
t.Fatalf("seed explore_index row for %q: %v", mbid, err)
|
|
}
|
|
}
|
|
|
|
// TestArtistEnrichmentMarksAreIndependent is the reason these are two
|
|
// columns rather than one boolean: each fetch fails on its own, and one
|
|
// failing must not claim the other as done.
|
|
func TestArtistEnrichmentMarksAreIndependent(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
db := database.NewTestDB(t)
|
|
si := NewSearchIndex(db, nil, nil, slog.Default())
|
|
|
|
const mbid = "11111111-1111-1111-1111-111111111111"
|
|
|
|
if mark := si.enrichmentFor(mbid); mark.Browsed || mark.Similar {
|
|
t.Fatalf("unmarked artist reported as enriched: %+v", mark)
|
|
}
|
|
|
|
si.markArtistBrowsed(mbid)
|
|
|
|
mark := si.enrichmentFor(mbid)
|
|
if !mark.Browsed {
|
|
t.Error("browsed_at was not recorded")
|
|
}
|
|
|
|
if mark.Similar {
|
|
t.Error("marking browsed also marked similar")
|
|
}
|
|
|
|
// The second mark upserts onto the same row rather than replacing it.
|
|
si.markArtistSimilar(mbid)
|
|
|
|
mark = si.enrichmentFor(mbid)
|
|
if !mark.Browsed || !mark.Similar {
|
|
t.Errorf("marking similar lost the browsed mark: %+v", mark)
|
|
}
|
|
|
|
if !si.artistBrowsed(mbid) {
|
|
t.Error("artistBrowsed disagreed with enrichmentFor")
|
|
}
|
|
}
|
|
|
|
// TestUnenrichedIncludesBrowsedGap is the query change 011 turns on: an
|
|
// artist the catalog artifact already covers (discog_fetched = 1) has
|
|
// still never been browsed, and the artifact's per-artist coverage is
|
|
// graded — so "the artifact knows them" is not "we have their
|
|
// discography", and the backfill must still pick them up.
|
|
func TestUnenrichedIncludesBrowsedGap(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
db := database.NewTestDB(t)
|
|
si := NewSearchIndex(db, nil, nil, slog.Default())
|
|
|
|
const mbid = "22222222-2222-2222-2222-222222222222"
|
|
|
|
if _, err := db.ExecContext(
|
|
"INSERT INTO artists (name, mbid) VALUES (?, ?)", "Owned Artist", mbid,
|
|
); err != nil {
|
|
t.Fatalf("seed library artist: %v", err)
|
|
}
|
|
|
|
seedIndexArtist(t, db, mbid, 1)
|
|
|
|
mbids := si.unenrichedLibraryArtistMBIDs(10)
|
|
if !containsMBID(mbids, mbid) {
|
|
t.Fatalf("artist with discog_fetched=1 but no browse was skipped: %v", mbids)
|
|
}
|
|
|
|
// Both marks set: nothing left to do, so it drops out entirely.
|
|
si.markArtistBrowsed(mbid)
|
|
si.markArtistSimilar(mbid)
|
|
|
|
if mbids := si.unenrichedLibraryArtistMBIDs(10); containsMBID(mbids, mbid) {
|
|
t.Errorf("fully enriched artist was returned again: %v", mbids)
|
|
}
|
|
}
|
|
|
|
// TestUnenrichedIgnoresSimilarMark pins the other half of that query:
|
|
// the backfill no longer fetches similar artists (the artist page does,
|
|
// on view), so an artist it has finished with must drop out even though
|
|
// similar_at is still NULL. Testing a mark nothing in the pass sets
|
|
// makes every owned artist a candidate on every run, forever — which is
|
|
// a backfill that reports progress and never converges.
|
|
func TestUnenrichedIgnoresSimilarMark(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
db := database.NewTestDB(t)
|
|
si := NewSearchIndex(db, nil, nil, slog.Default())
|
|
|
|
const mbid = "33333333-3333-3333-3333-333333333333"
|
|
|
|
if _, err := db.ExecContext(
|
|
"INSERT INTO artists (name, mbid) VALUES (?, ?)", "Owned Artist", mbid,
|
|
); err != nil {
|
|
t.Fatalf("seed library artist: %v", err)
|
|
}
|
|
|
|
seedIndexArtist(t, db, mbid, 1)
|
|
si.markArtistBrowsed(mbid)
|
|
|
|
if mbids := si.unenrichedLibraryArtistMBIDs(10); containsMBID(mbids, mbid) {
|
|
t.Errorf("artist with no similar_at was returned again: %v", mbids)
|
|
}
|
|
}
|
|
|
|
func containsMBID(mbids []string, want string) bool {
|
|
for _, m := range mbids {
|
|
if m == want {
|
|
return true
|
|
}
|
|
}
|
|
|
|
return false
|
|
}
|