yellowjacket/backend/explore/dumpcredits_test.go

//go:build indexbuild

package explore

import (
	"archive/tar"
	"bytes"
	"context"
	"errors"
	"strings"
	"testing"

	"yellowjacket/backend/database"
)

// tarOf builds an uncompressed tar of the named members, in the order
// given.  Order is the point of several of these tests: the real dump's
// members are alphabetical, which is what lets one pass resolve an
// entity's credit without buffering 35M recordings.
func tarOf(t *testing.T, members ...[2]string) *tar.Reader {
	t.Helper()

	var buf bytes.Buffer

	tw := tar.NewWriter(&buf)

	for _, m := range members {
		body := []byte(m[1])

		if err := tw.WriteHeader(&tar.Header{
			Name:     "mbdump/" + m[0],
			Mode:     0o644,
			Size:     int64(len(body)),
			Typeflag: tar.TypeReg,
		}); err != nil {
			t.Fatalf("tar header: %v", err)
		}

		if _, err := tw.Write(body); err != nil {
			t.Fatalf("tar write: %v", err)
		}
	}

	if err := tw.Close(); err != nil {
		t.Fatalf("tar close: %v", err)
	}

	return tar.NewReader(&buf)
}

func tsv(rows ...[]string) string {
	var b strings.Builder

	for _, r := range rows {
		b.WriteString(strings.Join(r, "\t"))
		b.WriteByte('\n')
	}

	return b.String()
}

// mustMBID is testMBID in the packed form the catalog stores.
func mustMBID(label string) uuid16 {
	var u uuid16

	if !parseUUID(testMBID(label), u[:]) {
		panic("testMBID did not produce a UUID for " + label)
	}

	return u
}

// The two artists of the worked example, and the entities they credit.
var (
	creditRecMBID = mustMBID("recording-1")
	creditRGMBID  = mustMBID("release-group-1")
)

// sampleDump is the shape verified against the 20260815 export:
// artist(id, gid, ...), artist_credit(id, name, artist_count, ...),
// artist_credit_name(credit, position, artist, name, join_phrase),
// recording/release_group(id, gid, name, artist_credit, ...).
func sampleDump(t *testing.T) *tar.Reader {
	t.Helper()

	return tarOf(t,
		[2]string{"artist", tsv(
			[]string{"11", testMBID("artist-a"), "Snoop Doggy Dogg", "Snoop Doggy Dogg"},
			[]string{"22", testMBID("artist-b"), "2Pac", "2Pac"},
		)},
		[2]string{"artist_credit", tsv(
			[]string{"900", "2Pac feat. Snoop Dogg", "2", "1", "", "0", ""},
			[]string{"901", "Solo Artist", "1", "1", "", "0", ""},
		)},
		[2]string{"artist_credit_name", tsv(
			// Deliberately out of position order: the dump is not
			// obliged to emit them sorted and the credit's meaning is
			// the order, not the file's.
			[]string{"900", "1", "11", "Snoop Dogg", ""},
			[]string{"900", "0", "22", "2Pac", " feat. "},
			[]string{"901", "0", "11", "Solo Artist", ""},
		)},
		[2]string{"recording", tsv(
			[]string{"1", testMBID("recording-1"), "Some Song", "900", "180000"},
			[]string{"2", testMBID("not-kept"), "Other", "900", "1"},
			[]string{"3", testMBID("solo"), "Solo", "901", "1"},
		)},
		[2]string{"release_group", tsv(
			[]string{"5", testMBID("release-group-1"), "Some Album", "900", "1"},
		)},
	)
}

func creditTestImporter(t *testing.T) *dumpImporter {
	t.Helper()

	db := database.NewTestDB(t)

	return &dumpImporter{
		si:     NewSearchIndex(db, nil, nil, testLogger()),
		logger: testLogger(),
	}
}

// TestScanCreditDumpDecomposes is the worked example end to end: the
// credit's parts come back in position order, with the *credited*
// names and the join phrase between them.
func TestScanCreditDumpDecomposes(t *testing.T) {
	imp := creditTestImporter(t)

	kept := map[uuid16]struct{}{
		creditRecMBID: {},
		creditRGMBID:  {},
	}

	scan, err := imp.scanCreditTar(context.Background(), sampleDump(t), kept)
	if err != nil {
		t.Fatalf("scan: %v", err)
	}

	if got := len(scan.refs); got != 2 {
		t.Fatalf("refs = %d, want 2 (the recording and the release group)", got)
	}

	if scan.refs[creditRecMBID] != 900 {
		t.Errorf("recording credit = %d, want 900", scan.refs[creditRecMBID])
	}

	parts := scan.parts[900]
	if len(parts) != 2 {
		t.Fatalf("parts = %d, want 2", len(parts))
	}

	// Sorting happens on write, so assert the pieces are all present
	// and let the render test below check the order.
	byPos := map[int]creditPart{}
	for _, p := range parts {
		byPos[p.position] = p
	}

	if byPos[0].name != "2Pac" || byPos[0].join != " feat. " {
		t.Errorf("position 0 = %q/%q, want \"2Pac\"/\" feat. \"",
			byPos[0].name, byPos[0].join)
	}

	// The credited name, not the artist's own name: this is the whole
	// reason credited_name is stored per row.
	if byPos[1].name != "Snoop Dogg" {
		t.Errorf("position 1 credited name = %q, want \"Snoop Dogg\"", byPos[1].name)
	}
}

// TestSingleArtistCreditsAreNotStored: a one-artist credit is already
// described by explore_index's artist_name/artist_mbid, and storing it
// would roughly triple the table to say nothing new.
func TestSingleArtistCreditsAreNotStored(t *testing.T) {
	imp := creditTestImporter(t)

	solo := mustMBID("solo")
	kept := map[uuid16]struct{}{solo: {}}

	scan, err := imp.scanCreditTar(context.Background(), sampleDump(t), kept)
	if err != nil {
		t.Fatalf("scan: %v", err)
	}

	if len(scan.refs) != 0 {
		t.Fatalf("a single-artist credit was referenced: %v", scan.refs)
	}

	if _, ok := scan.multiCredits[901]; ok {
		t.Error("credit 901 has artist_count 1 and should not be multi")
	}
}

// TestOnlyKeptEntitiesAreReferenced: the catalog's popularity filter
// decides what is worth carrying credits for, and an entity outside it
// must not produce a row pointing at nothing.
func TestOnlyKeptEntitiesAreReferenced(t *testing.T) {
	imp := creditTestImporter(t)

	kept := map[uuid16]struct{}{creditRecMBID: {}}

	scan, err := imp.scanCreditTar(context.Background(), sampleDump(t), kept)
	if err != nil {
		t.Fatalf("scan: %v", err)
	}

	if _, ok := scan.refs[mustMBID("not-kept")]; ok {
		t.Error("an entity outside the catalog was referenced")
	}

	if len(scan.used) != 1 {
		t.Errorf("used credits = %d, want 1", len(scan.used))
	}
}

// TestWriteCreditsRoundTrips checks what the frontend will actually
// read: parts in position order, dashed MBIDs out of the 16 raw bytes,
// and a rendered credit that reassembles to the tagged string.
func TestWriteCreditsRoundTrips(t *testing.T) {
	imp := creditTestImporter(t)

	kept := map[uuid16]struct{}{creditRecMBID: {}, creditRGMBID: {}}

	scan, err := imp.scanCreditTar(context.Background(), sampleDump(t), kept)
	if err != nil {
		t.Fatalf("scan: %v", err)
	}

	if err := imp.writeCredits(context.Background(), scan); err != nil {
		t.Fatalf("writeCredits: %v", err)
	}

	rows, err := imp.si.db.QueryContext(
		`SELECT p.position, p.artist_mbid, p.credited_name, p.join_phrase
		 FROM artist_credit_ref r
		 JOIN artist_credit_part p ON p.credit_id = r.credit_id
		 WHERE r.mbid = ?
		 ORDER BY p.position`,
		creditRecMBID[:],
	)
	if err != nil {
		t.Fatalf("query: %v", err)
	}

	defer func() { _ = rows.Close() }()

	var rendered strings.Builder

	names := []string{}

	for rows.Next() {
		var (
			pos  int
			mbid []byte
			name string
			join string
		)

		if err := rows.Scan(&pos, &mbid, &name, &join); err != nil {
			t.Fatalf("scan row: %v", err)
		}

		if len(mbid) != 16 {
			t.Fatalf("artist_mbid is %d bytes, want 16", len(mbid))
		}

		names = append(names, name)

		rendered.WriteString(name)
		rendered.WriteString(join)
	}

	if err := rows.Err(); err != nil {
		t.Fatalf("rows: %v", err)
	}

	// Concatenation is the contract: names in order, join phrases
	// between them, and no searching a name inside a credit string.
	if got := rendered.String(); got != "2Pac feat. Snoop Dogg" {
		t.Errorf("rendered credit = %q, want %q", got, "2Pac feat. Snoop Dogg")
	}

	if len(names) != 2 || names[0] != "2Pac" {
		t.Errorf("parts came back out of position order: %v", names)
	}
}

// TestCreditRefsNeverDangle: a ref whose parts were not stored renders
// as a credit with no artists at all, which is worse than the
// single-artist fallback it replaced.
func TestCreditRefsNeverDangle(t *testing.T) {
	imp := creditTestImporter(t)

	kept := map[uuid16]struct{}{creditRecMBID: {}}

	scan, err := imp.scanCreditTar(context.Background(), sampleDump(t), kept)
	if err != nil {
		t.Fatalf("scan: %v", err)
	}

	// An artist the dump never named: the credit cannot be navigated to
	// and must be dropped whole, taking its ref with it.
	scan.artistGIDs = map[int32]uuid16{}

	if err := imp.writeCredits(context.Background(), scan); err != nil {
		t.Fatalf("writeCredits: %v", err)
	}

	var refs, parts int

	if err := imp.si.db.QueryRowWriter(
		"SELECT COUNT(*) FROM artist_credit_ref",
	).Scan(&refs); err != nil {
		t.Fatalf("count refs: %v", err)
	}

	if err := imp.si.db.QueryRowWriter(
		"SELECT COUNT(*) FROM artist_credit_part",
	).Scan(&parts); err != nil {
		t.Fatalf("count parts: %v", err)
	}

	if refs != 0 || parts != 0 {
		t.Fatalf("refs=%d parts=%d, want 0/0 when the artists are unknown", refs, parts)
	}
}

// TestCreditDumpShapeIsAsserted: the dump has no header row, so a
// column that moved would be read as its neighbour and produce a
// catalog that is quietly wrong.  Loud is the requirement.
func TestCreditDumpShapeIsAsserted(t *testing.T) {
	imp := creditTestImporter(t)

	short := tarOf(t, [2]string{"artist", tsv([]string{"11", "only-two-columns"})})

	_, err := imp.scanCreditTar(context.Background(), short, map[uuid16]struct{}{})
	if err == nil {
		t.Fatal("a member with a non-UUID gid was accepted")
	}

	if !errors.Is(err, ErrDumpShape) {
		t.Errorf("error = %v, want ErrDumpShape", err)
	}
}

// TestUnescapeCopy covers Postgres COPY's text escaping, which reaches
// artist names routinely -- a tab or backslash in a name would
// otherwise shift every field after it.
func TestUnescapeCopy(t *testing.T) {
	tests := []struct{ in, want string }{
		{`plain`, `plain`},
		{`\N`, ``},
		{`a\tb`, "a\tb"},
		{`a\nb`, "a\nb"},
		{`back\\slash`, `back\slash`},
		{`AC\/DC`, `AC\/DC`},
		{`trailing\`, `trailing\`},
	}

	for _, tt := range tests {
		if got := unescapeCopy(tt.in); got != tt.want {
			t.Errorf("unescapeCopy(%q) = %q, want %q", tt.in, got, tt.want)
		}
	}
}

// TestEnsureArtistCreditsIsIdempotent pins what the index job depends
// on to decide whether to publish.
//
// The pass runs on every mode, including the `refresh` that a complete
// catalog always chooses — so it must be free when there is nothing to
// do, and it must say so.  A `true` here republishes the artifact; a
// `true` on every run would republish an identical one weekly, and a
// permanent `false` would mean a catalog that never gains credits at
// all.
func TestEnsureArtistCreditsIsIdempotent(t *testing.T) {
	imp := creditTestImporter(t)

	// The marker is what "already done" means; with it set, the pass
	// must not reach the network or report a change.
	imp.si.setMeta(creditsImportDoneKey, "1")

	if imp.ensureArtistCredits(context.Background()) {
		t.Fatal("a second run reported new credits; the artifact would republish forever")
	}
}

// TestEnsureArtistCreditsReportsFailureAsNoChange: a dump that cannot be
// reached leaves the catalog exactly as it was, and must not claim
// otherwise — publishing on it would ship an artifact with no credits
// and mark the work done.
func TestEnsureArtistCreditsReportsFailureAsNoChange(t *testing.T) {
	imp := creditTestImporter(t)
	imp.httpClient = newDumpHTTPClient()
	imp.mbdumpBaseURL = "http://127.0.0.1:1/nonexistent/"

	if imp.ensureArtistCredits(context.Background()) {
		t.Fatal("an unreachable dump reported new credits")
	}

	if imp.si.hasMeta(creditsImportDoneKey) {
		t.Error("a failed pass marked itself done; it would never retry")
	}
}