yellowjacket/backend/maintenance/sweeps.go

package maintenance

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"time"

	"yellowjacket/backend/database"
)

// Retention windows for cache data.  Cached artwork for artists the user
// actually owns is kept indefinitely; art fetched while browsing Explore
// is transient and ages out.
const (
	// browsedArtRetention is how long artwork for a non-library artist
	// survives after it was fetched.
	browsedArtRetention = 90 * 24 * time.Hour

	// browsedArtBudget bounds what browsing costs on disk.  Age alone
	// is not a ceiling: a real install had portraits for 5,770 artists
	// in a 1,301-artist library - 1.2 GB - because every artist page
	// opened in Explore fetches one and nothing was counting.  Art for
	// artists the user owns is not in this budget and is never evicted.
	browsedArtBudget = 256 << 20 // 256 MB

	// proxyCacheRetention is how long an Explore cover-art thumbnail
	// survives after it was last written.
	proxyCacheRetention = 30 * 24 * time.Hour

	// httpCacheBudget bounds the response cache.  Entity answers are
	// kept for a year, so expiry no longer bounds anything.
	httpCacheBudget = 128 << 20 // 128 MB
)

// Default intervals.  These are minimums, not schedules — the runner
// skips a job that ran more recently.
const (
	frequentInterval = 6 * time.Hour
	dailyInterval    = 24 * time.Hour
)

// ExpiredHTTPCacheJob deletes HTTP cache rows past their TTL, then
// enforces a size ceiling on what is left.
//
// Reads already filter on expires_at, so expired rows are inert — but
// nothing was deleting them, so the table grew without bound for the
// life of the install.
//
// The ceiling is the other half, and it is what makes a long TTL safe:
// MusicBrainz entity data is cached for a year now, because it does not
// change and re-fetching it spends someone else's rate limit for
// nothing. Expiry therefore stops being a bound, and something has to
// be.
func ExpiredHTTPCacheJob(db *database.DB) Job {
	return Job{
		Name:        "http-cache-evict",
		MinInterval: frequentInterval,
		Run: func(_ context.Context) (Result, error) {
			res, err := db.ExecContext(
				"DELETE FROM http_cache WHERE expires_at < datetime('now')",
			)
			if err != nil {
				return Result{}, fmt.Errorf(
					"delete expired http_cache rows: %w", err,
				)
			}

			rows, _ := res.RowsAffected()

			trimmed, err := trimHTTPCache(db)
			if err != nil {
				return Result{RowsDeleted: rows}, err
			}

			return Result{RowsDeleted: rows + trimmed}, nil
		},
	}
}

// trimHTTPCache evicts the oldest responses until the cache fits
// httpCacheBudget, and returns how many rows it removed.
//
// "Oldest" is by expiry, which orders by fetch time within a TTL class
// and puts the shortest-lived answers first across classes — a search
// result before an entity lookup, which is the right order to lose them
// in.
func trimHTTPCache(db *database.DB) (int64, error) {
	var total int64

	if err := db.QueryRowWriter(
		"SELECT COALESCE(SUM(LENGTH(response)), 0) FROM http_cache",
	).Scan(&total); err != nil {
		return 0, fmt.Errorf("measure http_cache: %w", err)
	}

	if total <= httpCacheBudget {
		return 0, nil
	}

	res, err := db.ExecContext(`
		DELETE FROM http_cache WHERE url_key IN (
			SELECT url_key FROM (
				SELECT url_key,
				       SUM(LENGTH(response)) OVER (
				           ORDER BY expires_at DESC
				           ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
				       ) AS running
				FROM http_cache
			) WHERE running > ?
		)`, httpCacheBudget)
	if err != nil {
		return 0, fmt.Errorf("trim http_cache: %w", err)
	}

	rows, _ := res.RowsAffected()

	return rows, nil
}

// OrphanedCoverFilesJob removes files from the covers directory that no
// cover_art row references.
//
// Cover art is derived data, so the live set is authoritative: every
// file that is not the original named by a cover_art row, or one of that
// original's derived size variants, is garbage.  This reclaims art left
// behind by earlier versions that deleted only the original and left its
// thumbnails.
func OrphanedCoverFilesJob(
	db *database.DB,
	coversDir string,
	expandVariants func(originalPath string) []string,
) Job {
	return Job{
		Name:        "covers-sweep",
		MinInterval: dailyInterval,
		Run: func(ctx context.Context) (Result, error) {
			live, err := liveCoverFiles(db, coversDir, expandVariants)
			if err != nil {
				return Result{}, err
			}

			// A covers directory with no live entries almost certainly
			// means the query failed to see the real table rather than
			// that every cover is garbage.  Refuse to empty the
			// directory on that basis.
			if len(live) == 0 {
				return Result{}, nil
			}

			return sweepDir(ctx, coversDir, func(name string) bool {
				return !live[name]
			})
		},
	}
}

// liveCoverFiles returns the basenames of every file the covers
// directory is supposed to contain.
func liveCoverFiles(
	db *database.DB,
	coversDir string,
	expandVariants func(string) []string,
) (map[string]bool, error) {
	rows, err := db.QueryContext("SELECT file_path FROM cover_art")
	if err != nil {
		return nil, fmt.Errorf("read cover_art paths: %w", err)
	}

	defer func() { _ = rows.Close() }()

	live := make(map[string]bool)

	for rows.Next() {
		var path string

		if err := rows.Scan(&path); err != nil {
			return nil, fmt.Errorf("scan cover_art path: %w", err)
		}

		// Rows may store an absolute path from a previous install
		// location, so compare by basename within the covers directory.
		original := filepath.Join(coversDir, filepath.Base(path))

		for _, variant := range expandVariants(original) {
			live[filepath.Base(variant)] = true
		}
	}

	if err := rows.Err(); err != nil {
		return nil, fmt.Errorf("iterate cover_art paths: %w", err)
	}

	return live, nil
}

// OrphanedArtistImagesJob evicts cached artist artwork.
//
// Artist images are cache data with no owner to compare against — they
// are fetched for any artist the user browses in Explore, most of whom
// are not in the library. The policy is therefore twofold: artwork for
// an artist the user owns is kept indefinitely, and everything else ages
// out. Rows whose file has vanished are dropped so the table matches
// what is actually on disk.
//
// dirFor maps an artist MBID to its directory.  It is a parameter, as
// OrphanedCoverFilesJob's expandVariants is, because that layout is the
// image provider's business and this job had been guessing it: artist
// directories are sharded under a two-character prefix, so joining the
// bare MBID named a path that has never existed.  RemoveAll succeeds on
// a missing path, so the job reported success, freed nothing, and
// deleted the rows that were the only record of the files it left
// behind.
func OrphanedArtistImagesJob(
	db *database.DB,
	artistImagesDir string,
	dirFor func(baseDir, mbid string) string,
) Job {
	return Job{
		Name:        "artist-images-sweep",
		MinInterval: dailyInterval,
		Run: func(ctx context.Context) (Result, error) {
			var result Result

			cutoff := time.Now().Add(-browsedArtRetention)

			// Collect the directories to remove before deleting rows, so
			// a failure partway leaves rows pointing at real files
			// rather than the reverse.
			stale, err := staleArtistMBIDs(db, cutoff)
			if err != nil {
				return Result{}, err
			}

			for _, mbid := range stale {
				if ctx.Err() != nil {
					return result, nil
				}

				dir := dirFor(artistImagesDir, mbid)

				freed, files := dirSize(dir)

				if err := os.RemoveAll(dir); err != nil && !os.IsNotExist(err) {
					continue
				}

				result.FilesDeleted += files
				result.BytesFreed += freed
			}

			if len(stale) > 0 {
				rows, delErr := deleteArtistImageRows(db, stale)
				if delErr != nil {
					return result, delErr
				}

				result.RowsDeleted += rows
			}

			// Then the ceiling.  Age alone bounds nothing: a browsing
			// session can fetch hundreds of portraits in an afternoon,
			// and all of them are within the retention window.
			evicted, err := evictOverBudget(ctx, db, artistImagesDir, dirFor, &result)
			if err != nil {
				return result, err
			}

			if len(evicted) > 0 {
				rows, delErr := deleteArtistImageRows(db, evicted)
				if delErr != nil {
					return result, delErr
				}

				result.RowsDeleted += rows
			}

			return result, nil
		},
	}
}

// evictOverBudget removes the least recently fetched non-library artist
// art until what is left fits browsedArtBudget, and returns the MBIDs it
// removed so their rows can go too.
func evictOverBudget(
	ctx context.Context,
	db *database.DB,
	artistImagesDir string,
	dirFor func(baseDir, mbid string) string,
	result *Result,
) ([]string, error) {
	browsed, err := browsedArtistsByAge(db)
	if err != nil {
		return nil, err
	}

	sizes := make([]int64, len(browsed))

	var total int64

	for i, mbid := range browsed {
		bytes, _ := dirSize(dirFor(artistImagesDir, mbid))
		sizes[i] = bytes
		total += bytes
	}

	if total <= browsedArtBudget {
		return nil, nil
	}

	var evicted []string

	// browsed is oldest first, so this drops the least recently wanted.
	for i, mbid := range browsed {
		if total <= browsedArtBudget {
			break
		}

		if ctx.Err() != nil {
			return evicted, nil
		}

		dir := dirFor(artistImagesDir, mbid)

		bytes, files := dirSize(dir)
		if err := os.RemoveAll(dir); err != nil && !os.IsNotExist(err) {
			continue
		}

		total -= sizes[i]
		result.FilesDeleted += files
		result.BytesFreed += bytes

		evicted = append(evicted, mbid)
	}

	return evicted, nil
}

// StrayArtistImageFilesJob removes downloaded image candidates that
// were never the artist's portrait.
//
// The image provider used to download every candidate an upstream
// offered — up to ten, full size — and keep them all, while nothing in
// the app has ever read anything but primary.jpg and its three size
// tiers.  It now downloads candidates in priority order until one
// succeeds and records the rest as URLs, so nothing new lands here;
// this reclaims what earlier versions left, which on a real cache was
// about four fifths of it.
//
// keep is the set of names an artist directory is allowed to hold.  It
// is a parameter for the same reason dirFor above is: the provider owns
// the naming, and a sweep that guesses it deletes the wrong files.
func StrayArtistImageFilesJob(artistImagesDir string, keep map[string]bool) Job {
	return Job{
		Name:        "artist-images-strays",
		MinInterval: dailyInterval,
		Run: func(ctx context.Context) (Result, error) {
			// An empty keep set would mean "every file is garbage",
			// which is never the intent — refuse, as the covers sweep
			// refuses an empty live set.
			if len(keep) == 0 {
				return Result{}, nil
			}

			var result Result

			shards, err := os.ReadDir(artistImagesDir)
			if err != nil {
				if os.IsNotExist(err) {
					return Result{}, nil
				}

				return Result{}, fmt.Errorf("read %s: %w", artistImagesDir, err)
			}

			for _, shard := range shards {
				if !shard.IsDir() {
					continue
				}

				shardDir := filepath.Join(artistImagesDir, shard.Name())

				artists, err := os.ReadDir(shardDir)
				if err != nil {
					continue
				}

				for _, artist := range artists {
					if ctx.Err() != nil {
						return result, nil
					}

					if !artist.IsDir() {
						continue
					}

					swept, err := sweepDir(ctx,
						filepath.Join(shardDir, artist.Name()),
						func(name string) bool { return !keep[name] },
					)
					if err != nil {
						continue
					}

					result.FilesDeleted += swept.FilesDeleted
					result.BytesFreed += swept.BytesFreed
				}
			}

			return result, nil
		},
	}
}

// staleArtistMBIDs returns artist MBIDs whose cached artwork may be
// evicted: fetched before the cutoff and not an artist in the library.
// ownedArtistMBIDs is the ownership test, asked the way every other one
// is: an artist is the user's if a *file* says so.  An artists row on
// its own is not ownership - that was the bug the file-shaped schema
// removed, and this is the same rule one table over.
const ownedArtistMBIDs = `
	SELECT a.mbid FROM artists a
	WHERE a.mbid IS NOT NULL AND a.mbid != ''
	  AND (
	    EXISTS (SELECT 1 FROM audio_files af WHERE af.artist_id = a.id)
	    OR EXISTS (
	      SELECT 1 FROM albums al
	      JOIN audio_files af2 ON af2.album_id = al.id
	      WHERE al.artist_id = a.id
	    )
	  )`

// browsedArtistsByAge lists non-library artists with cached art, oldest
// first, so the budget pass evicts the least recently wanted.
func browsedArtistsByAge(db *database.DB) ([]string, error) {
	rows, err := db.QueryContext(
		`SELECT artist_mbid FROM artist_images
		 WHERE artist_mbid NOT IN (` + ownedArtistMBIDs + `)
		 GROUP BY artist_mbid
		 ORDER BY MAX(created_at) ASC`,
	)
	if err != nil {
		return nil, fmt.Errorf("query browsed artist images: %w", err)
	}

	defer func() { _ = rows.Close() }()

	var mbids []string

	for rows.Next() {
		var mbid string

		if err := rows.Scan(&mbid); err != nil {
			return nil, fmt.Errorf("scan artist mbid: %w", err)
		}

		mbids = append(mbids, mbid)
	}

	if err := rows.Err(); err != nil {
		return nil, fmt.Errorf("iterate browsed artist images: %w", err)
	}

	return mbids, nil
}

func staleArtistMBIDs(
	db *database.DB,
	cutoff time.Time,
) ([]string, error) {
	rows, err := db.QueryContext(
		`SELECT DISTINCT artist_mbid FROM artist_images
		 WHERE created_at < ?
		   AND artist_mbid NOT IN (`+ownedArtistMBIDs+`)`,
		cutoff,
	)
	if err != nil {
		return nil, fmt.Errorf("query stale artist images: %w", err)
	}

	defer func() { _ = rows.Close() }()

	var mbids []string

	for rows.Next() {
		var mbid string

		if err := rows.Scan(&mbid); err != nil {
			return nil, fmt.Errorf("scan artist mbid: %w", err)
		}

		if mbid != "" {
			mbids = append(mbids, mbid)
		}
	}

	if err := rows.Err(); err != nil {
		return nil, fmt.Errorf("iterate stale artist images: %w", err)
	}

	return mbids, nil
}

// deleteArtistImageRows removes the rows for the given artist MBIDs.
func deleteArtistImageRows(
	db *database.DB,
	mbids []string,
) (int64, error) {
	var total int64

	for _, mbid := range mbids {
		res, err := db.ExecContext(
			"DELETE FROM artist_images WHERE artist_mbid = ?", mbid,
		)
		if err != nil {
			return total, fmt.Errorf(
				"delete artist_images rows for %s: %w", mbid, err,
			)
		}

		n, _ := res.RowsAffected()
		total += n
	}

	return total, nil
}

// ExpiredProxyCacheJob evicts Explore cover-art thumbnails that have not
// been rewritten within the retention window.
//
// This cache has no database table at all — it is keyed by release-group
// MBID on the filesystem — so age is the only signal available.
func ExpiredProxyCacheJob(proxyCacheDir string) Job {
	return Job{
		Name:        "cover-art-proxy-sweep",
		MinInterval: dailyInterval,
		Run: func(ctx context.Context) (Result, error) {
			cutoff := time.Now().Add(-proxyCacheRetention)

			return sweepDirFunc(ctx, proxyCacheDir,
				func(_ string, info os.FileInfo) bool {
					return info.ModTime().Before(cutoff)
				},
			)
		},
	}
}

// sweepDir removes every file in dir for which shouldDelete reports true.
func sweepDir(
	ctx context.Context,
	dir string,
	shouldDelete func(name string) bool,
) (Result, error) {
	return sweepDirFunc(ctx, dir, func(name string, _ os.FileInfo) bool {
		return shouldDelete(name)
	})
}

// sweepDirFunc removes files from a flat directory based on a predicate
// over the name and stat info.  Subdirectories are left alone; sweeps
// that own directory trees handle them explicitly.
func sweepDirFunc(
	ctx context.Context,
	dir string,
	shouldDelete func(name string, info os.FileInfo) bool,
) (Result, error) {
	entries, err := os.ReadDir(dir)
	if err != nil {
		if os.IsNotExist(err) {
			return Result{}, nil
		}

		return Result{}, fmt.Errorf("read %s: %w", dir, err)
	}

	var result Result

	for _, entry := range entries {
		if ctx.Err() != nil {
			return result, nil
		}

		if entry.IsDir() {
			continue
		}

		info, infoErr := entry.Info()
		if infoErr != nil {
			continue
		}

		if !shouldDelete(entry.Name(), info) {
			continue
		}

		if err := os.Remove(filepath.Join(dir, entry.Name())); err != nil {
			continue
		}

		result.FilesDeleted++
		result.BytesFreed += info.Size()
	}

	return result, nil
}

// dirSize totals the files in a directory tree.
func dirSize(dir string) (bytes, files int64) {
	_ = filepath.WalkDir(dir, func(_ string, d os.DirEntry, err error) error {
		if err != nil || d.IsDir() {
			return nil //nolint:nilerr // best-effort accounting
		}

		info, infoErr := d.Info()
		if infoErr != nil {
			return nil
		}

		bytes += info.Size()
		files++

		return nil
	})

	return bytes, files
}

// StaleArtistMetadataJob evicts long-lived artist metadata (bios, wiki
// leads, relationships) for artists the user no longer has any reason
// to keep around: not owned and holding no cached artwork.
//
// artist_metadata has no TTL by design — entity data changes rarely and
// re-fetching spends someone else's rate limit — so without a sweep it
// grows for the life of the install.  This is the "swept when the
// artist is no longer referenced" contract the datamap always declared
// for it and nothing ever performed (#248).
func StaleArtistMetadataJob(db *database.DB) Job {
	return Job{
		Name:        "artist-metadata-sweep",
		MinInterval: dailyInterval,
		Run: func(_ context.Context) (Result, error) {
			res, err := db.ExecContext(
				`DELETE FROM artist_metadata
				 WHERE mbid NOT IN (` + ownedArtistMBIDs + `)
				   AND mbid NOT IN (
				       SELECT artist_mbid FROM artist_images
				   )`,
			)
			if err != nil {
				return Result{}, fmt.Errorf(
					"delete stale artist_metadata rows: %w", err,
				)
			}

			rows, _ := res.RowsAffected()

			return Result{RowsDeleted: rows}, nil
		},
	}
}