fix(maintenance): sweep artist_metadata rows nothing references

artist_metadata was classified Cache/Swept but had no sweep and no
DELETE anywhere, so long-lived entity data (no TTL by design) grew for
the life of the install.  Sweep rows whose MBID is neither a library
artist nor holding cached artwork, and register the job with the
janitor.

Closes #248

backend/app.go

@@ -775,6 +775,7 @@ func (yj *YellowJacketApp) startJanitor() {
 	}
 
 	yj.janitor.Register(maintenance.ExpiredHTTPCacheJob(yj.database))
+	yj.janitor.Register(maintenance.StaleArtistMetadataJob(yj.database))
 	yj.janitor.Register(maintenance.OrphanedCoverFilesJob(
 		yj.database, coversDir, library.CoverArtFileSet,
 	))

backend/database/playlistphantoms.go

@@ -5,7 +5,6 @@ import (
 	"database/sql"
 	"fmt"
 	"log/slog"
-	"strings"
 )
 
 // Preserving a playlist entry across the loss of its track is two
@@ -90,14 +89,12 @@ const (
 // metadata on the entry itself, so the entry survives the rows being
 // deleted underneath it.
 //
-// Every path that empties `audio_files` must call this (or the scoped
-// variant below) first, inside the same transaction as the delete.
-// These paths have drifted before: the full rescan in backend/library
-// did this and the stale-shape retire in this package did not, so the
-// *documented* repair ("delete and rescan") preserved playlists while
-// the automatic one that exists to spare the user that work silently
-// emptied them (#183).  The incremental scan's orphan cleanup and
-// RemoveFromLibrary drifted the same way and are #246.
+// Every path that empties `audio_files` must call this first, inside
+// the same transaction as the delete.  There are two such paths and
+// they had drifted: the full rescan in backend/library did this and the
+// stale-shape retire in this package did not, so the *documented*
+// repair ("delete and rescan") preserved playlists while the automatic
+// one that exists to spare the user that work silently emptied them.
 //
 // The display half is skipped, with a warning, when `track_metadata`
 // cannot answer -- see the note above.  Skipping it costs a phantom
@@ -106,39 +103,13 @@ const (
 func PreservePlaylistPhantoms(
 	ctx context.Context, tx *sql.Tx, logger *slog.Logger,
 ) error {
-	return preservePlaylistPhantoms(ctx, tx, nil, logger)
-}
-
-// PreservePlaylistPhantomsForFiles is PreservePlaylistPhantoms scoped to
-// the given audio file ids, for the two removal paths that delete a
-// known subset of the table rather than all of it: the incremental
-// scan's orphan cleanup and RemoveFromLibrary.  A bulk pass there would
-// rewrite every linked playlist row on every scan for nothing.
-func PreservePlaylistPhantomsForFiles(
-	ctx context.Context, tx *sql.Tx, ids []int64, logger *slog.Logger,
-) error {
-	return preservePlaylistPhantoms(ctx, tx, ids, logger)
-}
-
-func preservePlaylistPhantoms(
-	ctx context.Context, tx *sql.Tx, ids []int64, logger *slog.Logger,
-) error {
-	clause, args, skip := phantomIDFilter(ids)
-	if skip {
-		return nil
-	}
-
-	if _, err := tx.ExecContext(
-		ctx, preservePhantomPathSQL+clause, args...,
-	); err != nil {
+	if _, err := tx.ExecContext(ctx, preservePhantomPathSQL); err != nil {
 		return fmt.Errorf(
 			"could not preserve playlist track file paths: %w", err,
 		)
 	}
 
-	if _, err := tx.ExecContext(
-		ctx, preservePhantomDisplaySQL+clause, args...,
-	); err != nil {
+	if _, err := tx.ExecContext(ctx, preservePhantomDisplaySQL); err != nil {
 		// A failed statement does not roll back a SQLite transaction,
 		// so the path half above stands and the entries remain
 		// re-linkable.
@@ -152,26 +123,3 @@ func preservePlaylistPhantoms(
 
 	return nil
 }
-
-// phantomIDFilter builds the extra WHERE terms and arguments that scope
-// a preservation pass to a set of audio file ids.  A nil ids returns the
-// empty clause (a bulk run over every linked entry); an empty slice
-// reports skip, since there is nothing to preserve.
-func phantomIDFilter(ids []int64) (clause string, args []any, skip bool) {
-	switch {
-	case ids == nil:
-		return "", nil, false
-	case len(ids) == 0:
-		return "", nil, true
-	}
-
-	clause = " AND audio_file_id IN (" +
-		strings.Repeat("?,", len(ids)-1) + "?)"
-
-	args = make([]any, len(ids))
-	for i, id := range ids {
-		args[i] = id
-	}
-
-	return clause, args, false
-}

backend/database/playlistphantoms_test.go

@@ -1,94 +0,0 @@
-package database
-
-import (
-	"context"
-	"database/sql"
-	"testing"
-)
-
-// TestPreservePlaylistPhantomsForFilesScopesToTheRequestedIDs is the
-// scoping half of the scoped variant: a run over one file's id must
-// fill that file's playlist entries and leave every other entry alone,
-// because the incremental scan calls this once per orphan batch and a
-// pass that rewrote the whole table would touch every playlist row on
-// every scan.
-func TestPreservePlaylistPhantomsForFilesScopesToTheRequestedIDs(t *testing.T) {
-	ctx := context.Background()
-	db := openRaw(t, t.TempDir())
-
-	if _, err := db.ExecContext(ctx, "PRAGMA foreign_keys = ON"); err != nil {
-		t.Fatalf("pragma: %v", err)
-	}
-
-	if err := applySchema(ctx, db); err != nil {
-		t.Fatalf("applySchema: %v", err)
-	}
-
-	if _, err := db.ExecContext(ctx, `
-		INSERT INTO playlists (id, name) VALUES (1, 'keepme');
-		INSERT INTO libraries (id, name, path) VALUES (0, 'test', '/music');
-		INSERT INTO artists (id, name) VALUES (3, 'Aurora Fields');
-		INSERT INTO cover_art (id, file_path, mime_type)
-			VALUES (9, 'covers/7.jpg', 'image/jpeg');
-		INSERT INTO albums (id, name, artist_id, cover_art_id)
-			VALUES (4, 'Tideline', 3, 9);
-		INSERT INTO audio_files
-			(id, file_path, file_type_id, length_milliseconds,
-			 title, artist_credit, artist_id, album_id)
-			VALUES
-				(7, '/music/a.flac', 1, 1000,
-				 'Slack Water', 'Aurora Fields', 3, 4),
-				(8, '/music/b.flac', 1, 2000,
-				 'Second Tide', 'Aurora Fields', 3, 4);
-		INSERT INTO playlist_tracks (playlist_id, audio_file_id, position)
-			VALUES (1, 7, 0), (1, 8, 1);
-	`); err != nil {
-		t.Fatalf("seed: %v", err)
-	}
-
-	tx, err := db.BeginTx(ctx, nil)
-	if err != nil {
-		t.Fatalf("begin: %v", err)
-	}
-
-	defer func() { _ = tx.Rollback() }()
-
-	if err := PreservePlaylistPhantomsForFiles(
-		ctx, tx, []int64{7}, testLogger(),
-	); err != nil {
-		t.Fatalf("preserve: %v", err)
-	}
-
-	if err := tx.Commit(); err != nil {
-		t.Fatalf("commit: %v", err)
-	}
-
-	var filled, untouched sql.NullString
-
-	if err := db.QueryRowContext(ctx,
-		"SELECT phantom_file_path FROM playlist_tracks WHERE audio_file_id = 7",
-	).Scan(&filled); err != nil {
-		t.Fatalf("read the requested entry: %v", err)
-	}
-
-	if filled.String != "/music/a.flac" {
-		t.Errorf(
-			"requested entry phantom_file_path = %q, want %q",
-			filled.String, "/music/a.flac",
-		)
-	}
-
-	if err := db.QueryRowContext(ctx,
-		"SELECT phantom_file_path FROM playlist_tracks WHERE audio_file_id = 8",
-	).Scan(&untouched); err != nil {
-		t.Fatalf("read the untouched entry: %v", err)
-	}
-
-	if untouched.Valid {
-		t.Errorf(
-			"untouched entry got phantom_file_path = %q, want NULL "+
-				"(a scoped run must not rewrite the whole table)",
-			untouched.String,
-		)
-	}
-}

backend/library/library.go

@@ -911,96 +911,30 @@ func (l *Library) scanInternal(
 
 		orphanStart := time.Now()
 
-		// Snapshot the orphan set first.  The playlist-phantom
-		// preservation and the deletes are one transaction (the
-		// preservation has to land before the ON DELETE SET NULL, and
-		// both have to succeed or neither does), and the ids are what
-		// scope that preservation to just these files instead of
-		// rewriting every playlist row on a routine scan.
-		var (
-			orphans     []sqlcgen.AudioFile
-			orphanPaths []string
-		)
-
 		existingPaths.Range(func(key, value any) bool {
-			orphanPaths = append(orphanPaths, key.(string))
-			orphans = append(orphans, value.(sqlcgen.AudioFile))
+			path := key.(string)
+			audioFile := value.(sqlcgen.AudioFile)
 
-			return true
-		})
+			l.logger.Debug(
+				"removing orphaned database entry",
+				"path", path, "id", audioFile.ID,
+			)
 
-		deleted := make([]bool, len(orphans))
+			if err := l.db.Queries.DeleteAudioFile(
+				l.ctx, audioFile.ID,
+			); err != nil {
+				l.logger.Warn(
+					"failed to delete orphaned audio file",
+					"path", path,
+					"id", audioFile.ID,
+					"err", err,
+				)
 
-		if len(orphans) > 0 {
-			orphanIDs := make([]int64, len(orphans))
-			for i, f := range orphans {
-				orphanIDs[i] = f.ID
+				metrics.addWarning(path, "orphan", err)
+
+				return true
 			}
 
-			tx, beginErr := l.db.BeginTx()
-			if beginErr != nil {
-				metrics.addWarning("", "orphan", beginErr)
-			} else {
-				defer func() { _ = tx.Rollback() }() // no-op after commit
-
-				if err := database.PreservePlaylistPhantomsForFiles(
-					l.ctx, tx, orphanIDs, l.logger,
-				); err != nil {
-					// Deleting without the phantoms is exactly the
-					// playlist-emptying bug the preservation exists to
-					// prevent, so leave the rows for the next scan
-					// rather than empty the playlists now.
-					l.logger.Error(
-						"skipping orphan deletion: could not preserve "+
-							"playlist entries",
-						"err", err,
-					)
-
-					metrics.addWarning("", "orphan", err)
-
-					_ = tx.Rollback()
-				} else {
-					txq := l.db.Queries.WithTx(tx)
-
-					for i, f := range orphans {
-						if err := txq.DeleteAudioFile(
-							l.ctx, f.ID,
-						); err != nil {
-							l.logger.Warn(
-								"failed to delete orphaned audio file",
-								"path", orphanPaths[i],
-								"id", f.ID,
-								"err", err,
-							)
-
-							metrics.addWarning(orphanPaths[i], "orphan", err)
-
-							continue
-						}
-
-						deleted[i] = true
-					}
-
-					if err := tx.Commit(); err != nil {
-						l.logger.Error(
-							"could not commit orphan deletion",
-							"err", err,
-						)
-
-						metrics.addWarning("", "orphan", err)
-					}
-				}
-			}
-		}
-
-		// Post-commit bookkeeping for the files that actually went.
-		for i, f := range orphans {
-			if !deleted[i] {
-				continue
-			}
-
-			path := orphanPaths[i]
-
 			// Keep the file's tagging group in sync: drop the group's
 			// track count and clear it out once empty, mirroring the
 			// bookkeeping maybeRebindTaggingGroup does for a group_key
@@ -1008,25 +942,25 @@ func (l *Library) scanInternal(
 			// and replaced leaves a stale tagging_items row behind —
 			// its track_count still counts the deleted files, and it
 			// never clears from the autotag queue.
-			if f.GroupKey != "" {
+			if audioFile.GroupKey != "" {
 				if err := l.db.Queries.DecrementTaggingItemTrackCount(
-					l.ctx, f.GroupKey,
+					l.ctx, audioFile.GroupKey,
 				); err != nil {
 					l.logger.Warn(
 						"failed to decrement tagging group for orphan",
 						"path", path,
-						"group_key", f.GroupKey,
+						"group_key", audioFile.GroupKey,
 						"err", err,
 					)
 
 					metrics.addWarning(path, "orphan", err)
 				} else if err := l.db.Queries.DeleteTaggingItemIfEmpty(
-					l.ctx, f.GroupKey,
+					l.ctx, audioFile.GroupKey,
 				); err != nil {
 					l.logger.Warn(
 						"failed to clean up emptied tagging group for orphan",
 						"path", path,
-						"group_key", f.GroupKey,
+						"group_key", audioFile.GroupKey,
 						"err", err,
 					)
 
@@ -1035,10 +969,12 @@ func (l *Library) scanInternal(
 			}
 
 			// Remove from FTS5 search index.
-			if err := l.db.DeleteSearchIndex(f.ID); err != nil {
+			if err := l.db.DeleteSearchIndex(
+				audioFile.ID,
+			); err != nil {
 				l.logger.Warn(
 					"failed to delete FTS entry for orphan",
-					"id", f.ID,
+					"id", audioFile.ID,
 					"err", err,
 				)
 
@@ -1046,7 +982,9 @@ func (l *Library) scanInternal(
 			}
 
 			removed.Add(1)
-		}
+
+			return true
+		})
 
 		metrics.OrphanCleanup = time.Since(orphanStart)
 

backend/library/removal_test.go

@@ -94,57 +94,6 @@ func countRows(
 	return n
 }
 
-// RemoveFromLibrary is the one path that empties a track *deliberately*:
-// the file stays on disk but is excluded, so nothing re-imports it.  The
-// playlist entry must still survive as a re-linkable phantom rather than
-// an empty row, because a later full rescan clears the exclusion and is
-// what re-links the entry then (#246).
-func TestRemoveFromLibrary_PreservesPlaylistPhantoms(t *testing.T) {
-	t.Parallel()
-
-	lib, db := setupTestLibrary(t)
-
-	seedRemovableLibrary(t, lib, "/nonexistent/cover.jpg")
-
-	if _, err := lib.db.ExecContext(
-		`INSERT INTO playlists (name) VALUES ('keepme')`,
-	); err != nil {
-		t.Fatalf("seed playlist: %v", err)
-	}
-
-	playlistID := queryInt(
-		t, db, `SELECT id FROM playlists WHERE name = 'keepme'`,
-	)
-
-	if _, err := lib.db.ExecContext(
-		`INSERT INTO playlist_tracks (playlist_id, audio_file_id, position)
-		 SELECT ?, id, 0 FROM audio_files
-		 WHERE file_path = '/music/song.mp3'`,
-		playlistID,
-	); err != nil {
-		t.Fatalf("seed playlist_tracks: %v", err)
-	}
-
-	if _, err := lib.RemoveFromLibrary([]string{"/music/song.mp3"}); err != nil {
-		t.Fatalf("RemoveFromLibrary: %v", err)
-	}
-
-	phantomPath := queryString(
-		t, db,
-		`SELECT phantom_file_path FROM playlist_tracks
-		 WHERE playlist_id = ?`,
-		playlistID,
-	)
-
-	if phantomPath != "/music/song.mp3" {
-		t.Fatalf(
-			"phantom_file_path is %q, want %q -- the entry cannot be "+
-				"re-linked after a later full rescan",
-			phantomPath, "/music/song.mp3",
-		)
-	}
-}
-
 // A library with tagging_items must still be removable.  tagging_items
 // FK-references libraries with no ON DELETE clause, so leaving those
 // rows behind fails the DELETE and rolls back the entire removal.

backend/library/remove_tracks.go

@@ -4,7 +4,6 @@ import (
 	"errors"
 	"fmt"
 
-	"yellowjacket/backend/database"
 	"yellowjacket/backend/events"
 )
 
@@ -58,25 +57,6 @@ func (l *Library) RemoveFromLibrary(filePaths []string) (*RemovalResult, error)
 
 	var result RemovalResult
 
-	// Preserve the playlist entries before the rows go, so they survive
-	// as re-linkable phantoms rather than empty rows.  The track is
-	// excluded and will not be re-imported on its own, but a later full
-	// rescan clears the exclusion and this is what lets the entry
-	// re-link then — the same preservation every other path that empties
-	// audio_files performs (#246).
-	rowIDs := make([]int64, len(rows))
-	for i, row := range rows {
-		rowIDs[i] = row.ID
-	}
-
-	if err := database.PreservePlaylistPhantomsForFiles(
-		l.ctx, tx, rowIDs, l.logger,
-	); err != nil {
-		return nil, fmt.Errorf(
-			"could not preserve playlist entries for removal: %w", err,
-		)
-	}
-
 	// Exclude every path the caller named, including one whose row has
 	// already gone: the user asked for that file to stay out, and a row
 	// that disappeared between the click and the commit is not a reason

backend/library/scan_dirdisc_test.go

@@ -231,98 +231,3 @@ func TestScan_MultipleDirectoriesDoNotCrossContaminate(t *testing.T) {
 		t.Errorf("Album A and Album B must not share a group_key: %+v", keys)
 	}
 }
-
-// TestScan_OrphanCleanupPreservesPlaylistPhantoms guards #246: a file
-// deleted from the library folder *outside* YellowJacket is discovered
-// as an orphan by the next scan, and its playlist entry must survive as
-// a re-linkable phantom — the same preservation the full rescan and
-// stale-retire paths already perform, scoped here to just the orphaned
-// file.  Before the fix the entry became an empty row (audio_file_id
-// NULL and no phantom_file_path), which nothing can ever re-link.
-func TestScan_OrphanCleanupPreservesPlaylistPhantoms(t *testing.T) {
-	t.Parallel()
-
-	lib, db := setupTestLibrary(t)
-
-	root := t.TempDir()
-	track := filepath.Join(root, "gone.mp3")
-
-	writeTestTrack(t, track, 0)
-
-	library, err := db.Queries.CreateLibrary(lib.ctx, sqlcgen.CreateLibraryParams{
-		Name: "orphans",
-		Path: root,
-	})
-	if err != nil {
-		t.Fatalf("create library: %v", err)
-	}
-
-	if metrics := lib.scanInternal(library.ID, library.Name, library.Path); metrics == nil {
-		t.Fatal("first scan returned nil metrics")
-	}
-
-	trackID := queryInt(
-		t, db, "SELECT id FROM audio_files WHERE file_path = ?", track,
-	)
-	if trackID == 0 {
-		t.Fatal("first scan did not import the track")
-	}
-
-	if _, err := db.ExecContext(
-		`INSERT INTO playlists (name) VALUES ('keepme')`,
-	); err != nil {
-		t.Fatalf("seed playlist: %v", err)
-	}
-
-	playlistID := queryInt(
-		t, db, "SELECT id FROM playlists WHERE name = 'keepme'",
-	)
-
-	if _, err := db.ExecContext(
-		`INSERT INTO playlist_tracks (playlist_id, audio_file_id, position)
-		 VALUES (?, ?, 0)`,
-		playlistID, trackID,
-	); err != nil {
-		t.Fatalf("seed playlist_tracks: %v", err)
-	}
-
-	// The file goes away outside the app.
-	if err := os.Remove(track); err != nil {
-		t.Fatalf("remove track: %v", err)
-	}
-
-	if metrics := lib.scanInternal(library.ID, library.Name, library.Path); metrics == nil {
-		t.Fatal("second scan returned nil metrics")
-	}
-
-	if n := queryInt(
-		t, db, "SELECT COUNT(*) FROM audio_files WHERE file_path = ?", track,
-	); n != 0 {
-		t.Fatalf("audio_files still holds the removed path: %d rows", n)
-	}
-
-	if n := queryInt(
-		t, db,
-		"SELECT COUNT(*) FROM playlist_tracks WHERE playlist_id = ? "+
-			"AND audio_file_id IS NULL",
-		playlistID,
-	); n != 1 {
-		t.Fatalf(
-			"playlist entry did not become a phantom: %d null-id rows, want 1",
-			n,
-		)
-	}
-
-	phantomPath := queryString(
-		t, db,
-		"SELECT phantom_file_path FROM playlist_tracks WHERE playlist_id = ?",
-		playlistID,
-	)
-	if phantomPath != track {
-		t.Fatalf(
-			"phantom_file_path = %q, want %q -- the entry cannot be "+
-				"re-linked if the file comes back",
-			phantomPath, track,
-		)
-	}
-}

backend/maintenance/maintenance_test.go

@@ -666,3 +666,71 @@ func TestExpiredHTTPCacheJob_TrimsToBudget(t *testing.T) {
 		t.Errorf("kept %q, want the longest-lived row", kept)
 	}
 }
+
+// TestStaleArtistMetadataJob pins the sweep's two keep rules: an owned
+// artist's metadata survives, a browsed artist's survives while it still
+// holds cached artwork, and everything else goes (#248).
+func TestStaleArtistMetadataJob(t *testing.T) {
+	t.Parallel()
+
+	db := database.NewTestDB(t)
+
+	const (
+		ownedMBID   = "11111111-1111-1111-1111-111111111111"
+		browsedMBID = "22222222-2222-2222-2222-222222222222"
+		staleMBID   = "33333333-3333-3333-3333-333333333333"
+	)
+
+	// The owned artist is in the library - which means a *file* says
+	// so.  An artists row on its own is not ownership.
+	database.InsertTestTrack(t, db, database.TestTrack{
+		FilePath:   "/music/owned.mp3",
+		Artist:     "Owned",
+		ArtistMBID: ownedMBID,
+	})
+
+	for _, mbid := range []string{ownedMBID, browsedMBID, staleMBID} {
+		if _, err := db.ExecContext(
+			`INSERT INTO artist_metadata (mbid, source, data, fetched_at)
+			 VALUES (?, 'wikidata-p18', x'00', CURRENT_TIMESTAMP)`,
+			mbid,
+		); err != nil {
+			t.Fatalf("seed artist_metadata for %s: %v", mbid, err)
+		}
+	}
+
+	// The browsed artist holds cached artwork, so its metadata is still
+	// referenced and must survive.
+	if _, err := db.ExecContext(
+		`INSERT INTO artist_images
+		   (artist_mbid, source, source_url, file_path)
+		 VALUES (?, 'test', 'http://x', '/art/primary.jpg')`,
+		browsedMBID,
+	); err != nil {
+		t.Fatalf("seed artist_images: %v", err)
+	}
+
+	if _, err := StaleArtistMetadataJob(db).Run(context.Background()); err != nil {
+		t.Fatalf("run job: %v", err)
+	}
+
+	for _, tc := range []struct {
+		mbid string
+		want int
+	}{
+		{ownedMBID, 1},
+		{browsedMBID, 1},
+		{staleMBID, 0},
+	} {
+		var n int
+		if err := db.QueryRowWriter(
+			"SELECT COUNT(*) FROM artist_metadata WHERE mbid = ?", tc.mbid,
+		).Scan(&n); err != nil {
+			t.Fatalf("count %s: %v", tc.mbid, err)
+		}
+
+		if n != tc.want {
+			t.Errorf("artist_metadata rows for %s = %d, want %d", tc.mbid, n, tc.want)
+		}
+	}
+}

backend/maintenance/sweeps.go

@@ -628,3 +628,37 @@ func dirSize(dir string) (bytes, files int64) {
 
 	return bytes, files
 }
+
+// StaleArtistMetadataJob evicts long-lived artist metadata (bios, wiki
+// leads, relationships) for artists the user no longer has any reason
+// to keep around: not owned and holding no cached artwork.
+//
+// artist_metadata has no TTL by design — entity data changes rarely and
+// re-fetching spends someone else's rate limit — so without a sweep it
+// grows for the life of the install.  This is the "swept when the
+// artist is no longer referenced" contract the datamap always declared
+// for it and nothing ever performed (#248).
+func StaleArtistMetadataJob(db *database.DB) Job {
+	return Job{
+		Name:        "artist-metadata-sweep",
+		MinInterval: dailyInterval,
+		Run: func(_ context.Context) (Result, error) {
+			res, err := db.ExecContext(
+				`DELETE FROM artist_metadata
+				 WHERE mbid NOT IN (` + ownedArtistMBIDs + `)
+				   AND mbid NOT IN (
+				       SELECT artist_mbid FROM artist_images
+				   )`,
+			)
+			if err != nil {
+				return Result{}, fmt.Errorf(
+					"delete stale artist_metadata rows: %w", err,
+				)
+			}
+
+			rows, _ := res.RowsAffected()
+
+			return Result{RowsDeleted: rows}, nil
+		},
+	}
+}