Files
yellowjacket/backend/explore/browseall_test.go
yonluandClaude Opus 5 20fbf28f2a perf(explore): make the owned-artist backfill yield, mark, and stop
The post-scan backfills share MusicBrainz's rate limiters with every
page the user can open, and both were FIFO — so a thousand-artist
enrichment put an album page behind an hour of queued work.
WithBackgroundLane/WithBackgroundPriority add a slower second lane: a
marked wait takes no token while any interactive wait is outstanding.
It is a context marker rather than a parameter because a backfill calls
the same client methods a detail page does. A long backfill also has to
be visible and stoppable, so jobs.KindCatalogEnrich registers both with
progress and cancel — after the work is counted, since these passes are
a no-op on every launch once the library is covered.

What it does not fetch is the point. It ran for hours against a
900-artist library and marked nothing, because three of the four things
it did per artist were work nobody asked for: similar artists, which
the artist page already resolves on view, and a full GetArtistImage
(fanart.tv, TheAudioDB, Wikidata, Wikipedia, ten portraits) reached
only to warm the MB artist lookup EnsureArtistRels does alone. It was
also serial across artists while every limiter is per-host and idle.

The marks are a table rather than more explore_index columns, because
artifactimport merges by column list and a flag added there is a second
place to remember. BrowseReleaseGroupsAll pages to exhaustion, where
the old call silently cut a prolific artist at 100 release groups.

One portrait is downloaded now; the rest are remembered as URLs.
resolveAllSources downloaded every candidate, up to ten, full size,
while nothing reads anything but primary.jpg — 5.3 GB measured on a
real cache, 4.1 GB of it unreachable. OrphanedArtistImagesJob is why
that survived: it joined the bare MBID onto the images directory, but
artist directories are sharded under a two-character prefix, so it
named a path that never existed and deleted the rows that were the only
record of the files it left behind.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UDCbcCZQepnpSQYJ6SxxZm
2026-08-14 13:33:54 -04:00

157 lines
4.1 KiB
Go

package explore
import (
"encoding/json"
"fmt"
"log/slog"
"net/http"
"net/http/httptest"
"strconv"
"strings"
"testing"
"yellowjacket/backend/database"
)
// browseServer serves a paged release-group browse for an artist with
// `total` release groups, and records how many requests it received.
func browseServer(t *testing.T, total int) (*httptest.Server, *int) {
t.Helper()
requests := 0
srv := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
requests++
offset, _ := strconv.Atoi(r.URL.Query().Get("offset"))
limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
if limit <= 0 {
limit = 25
}
end := min(offset+limit, total)
groups := make([]map[string]any, 0, max(0, end-offset))
for i := offset; i < end; i++ {
groups = append(groups, map[string]any{
"id": fmt.Sprintf("rg-%03d", i),
"title": "Release " + strconv.Itoa(i),
"primary-type": "Album",
"secondary-types": []string{"Live"},
"first-release-da": "",
})
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{
"release-group-count": total,
"release-group-offset": offset,
"release-groups": groups,
})
},
))
t.Cleanup(srv.Close)
return srv, &requests
}
// browseClient points a real MusicBrainzClient at a test server.
func browseClient(t *testing.T, srv *httptest.Server) *MusicBrainzClient {
t.Helper()
db := database.NewTestDB(t)
// Fast limiter: this test is about paging, not pacing.
c := NewMusicBrainzClient(
NewCache(db, slog.Default()), NewRateLimiterN(1000), slog.Default(),
)
c.mb.SetBaseURL(strings.TrimSuffix(srv.URL, "/") + "/ws/2/")
return c
}
// TestBrowseReleaseGroupsAllPages is the bug 011 is built on: the
// single-page browse asks for MaxLimit and takes what comes back, so a
// prolific artist's discography was silently cut at 100 — and a hundred
// albums looks like a complete answer unless you count.
func TestBrowseReleaseGroupsAllPages(t *testing.T) {
t.Parallel()
const total = 237
srv, requests := browseServer(t, total)
c := browseClient(t, srv)
all, err := c.BrowseReleaseGroupsAll(t.Context(), "artist-mbid")
if err != nil {
t.Fatalf("BrowseReleaseGroupsAll: %v", err)
}
if len(all) != total {
t.Errorf("got %d release groups, want %d", len(all), total)
}
// 100 + 100 + 37: the short third page ends it, with no fourth
// request to discover that it is over.
if *requests != 3 {
t.Errorf("made %d requests, want 3", *requests)
}
}
// TestBrowseReleaseGroupsAllExactMultiple covers the boundary the
// short-page terminator exists for: a total that is an exact multiple
// of the page size needs one more (empty) request to know it is done,
// and must not loop past it.
func TestBrowseReleaseGroupsAllExactMultiple(t *testing.T) {
t.Parallel()
srv, requests := browseServer(t, 200)
c := browseClient(t, srv)
all, err := c.BrowseReleaseGroupsAll(t.Context(), "artist-mbid")
if err != nil {
t.Fatalf("BrowseReleaseGroupsAll: %v", err)
}
if len(all) != 200 {
t.Errorf("got %d release groups, want 200", len(all))
}
if *requests != 3 {
t.Errorf("made %d requests, want 3 (two full pages and an empty one)", *requests)
}
}
// TestBrowseReleaseGroupsAllCachesForSinglePageReader checks the half
// that makes this worth doing interactively: the complete list is
// written under the key the single-page browse reads, so the next
// ordinary browse is served all of it without a request.
func TestBrowseReleaseGroupsAllCachesForSinglePageReader(t *testing.T) {
t.Parallel()
srv, requests := browseServer(t, 150)
c := browseClient(t, srv)
if _, err := c.BrowseReleaseGroupsAll(t.Context(), "artist-mbid"); err != nil {
t.Fatalf("BrowseReleaseGroupsAll: %v", err)
}
before := *requests
cached, err := c.BrowseReleaseGroups(t.Context(), "artist-mbid")
if err != nil {
t.Fatalf("BrowseReleaseGroups: %v", err)
}
if len(cached) != 150 {
t.Errorf("cached read got %d release groups, want 150", len(cached))
}
if *requests != before {
t.Errorf("cached read made %d extra requests, want 0", *requests-before)
}
}