feat: data lifecycle rewrite, download clients, wanted list, and central catalog index

Ships the fresh-start schema cleanup: rebuilt explore catalog index pipeline (dump import, artifact fetch/build, incremental listen-count refresh), a new download subsystem (Lidarr/Prowlarr/qBittorrent/SABnzbd/ slskd/yt-dlp providers, staging, reconciliation, wanted list), and the supporting schema/query/store changes across backend and frontend. Also includes two smaller follow-ups: bump the central index's rebuild-after cadence from 90 to 180 days, and remove the Explore "library only" online/offline toggle entirely (frontend-only, no backend counterpart) rather than carry unused UI/state. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Y2Agd9af5hE7qzti2ackiS
2026-08-06 17:12:01 -04:00
parent d0d86f85d5
commit e190fd75b9
165 changed files with 31088 additions and 5192 deletions
@@ -44,6 +44,13 @@ type TrackMetadata struct {
 	// Format info
 	TagFormat  string // "ID3v2.3", "VORBIS", etc.
 	FileFormat string // "MP3", "FLAC", etc.
+
+	// TagReadWarning is set when the tag could not be read cleanly:
+	// ErrTagsRecovered if the lenient parser salvaged the fields above,
+	// ErrTagsUnreadable if they are empty because nothing could read the
+	// tag.  Nil on a clean read.  Either way the file is still usable —
+	// callers should surface the warning, not discard the track.
+	TagReadWarning error
 }

 // PictureData holds embedded artwork.
@@ -74,7 +81,11 @@ func ExtractTagsFromReader(r io.ReadSeeker) (*TrackMetadata, error) {
 			return &TrackMetadata{}, nil
 		}

-		return nil, fmt.Errorf("could not read tags: %w", err)
+		// A single malformed frame must not cost us the whole file: retry
+		// with a more forgiving parser and, failing that, hand back empty
+		// metadata carrying a warning.  The audio is still playable and
+		// the caller can fall back to the filename.
+		return recoverTags(r, err), nil
 	}

 	trackNum, totalTracks := m.Track()
@@ -0,0 +1,253 @@
+package metadata
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+
+	"github.com/bogem/id3v2/v2"
+)
+
+// Sentinel errors describing a degraded tag read.  Both travel on
+// TrackMetadata.TagReadWarning rather than being returned, so that one
+// malformed frame never costs the caller the entire file.
+var (
+	// ErrTagsRecovered means the strict parser rejected the tag but the
+	// lenient ID3v2 fallback read it.  The metadata is populated.
+	ErrTagsRecovered = errors.New("tags recovered by lenient parser")
+
+	// ErrTagsUnreadable means no parser could read the tag.  The
+	// metadata is empty and callers should fall back to the filename.
+	ErrTagsUnreadable = errors.New("tags could not be parsed")
+)
+
+// recoverTags retries a failed tag read with bogem/id3v2, which
+// tolerates frames that dhowden/tag rejects outright — a stray NUL
+// inside a UTF-16 TXXX frame, for example.  It always returns usable
+// metadata: when nothing can be salvaged the metadata is empty and only
+// TagReadWarning is set.
+func recoverTags(r io.ReadSeeker, cause error) *TrackMetadata {
+	if _, err := r.Seek(0, io.SeekStart); err != nil {
+		return &TrackMetadata{
+			TagReadWarning: fmt.Errorf("%w: %w", ErrTagsUnreadable, cause),
+		}
+	}
+
+	meta, err := extractID3v2Lenient(r)
+	if err != nil {
+		return &TrackMetadata{
+			TagReadWarning: fmt.Errorf("%w: %w", ErrTagsUnreadable, cause),
+		}
+	}
+
+	meta.TagReadWarning = fmt.Errorf("%w: %w", ErrTagsRecovered, cause)
+
+	return meta
+}
+
+// extractID3v2Lenient reads an ID3v2 tag with bogem/id3v2.  Only MP3
+// (and other ID3v2-carrying containers) can be recovered this way;
+// for anything else the tag has no frames and the read fails.
+func extractID3v2Lenient(r io.Reader) (*TrackMetadata, error) {
+	// The tag is not Closed here: it wraps a reader the caller owns.
+	t, err := id3v2.ParseReader(r, id3v2.Options{Parse: true})
+	if err != nil {
+		return nil, fmt.Errorf("lenient id3v2 parse: %w", err)
+	}
+
+	if !t.HasFrames() {
+		return nil, ErrTagsUnreadable
+	}
+
+	meta := &TrackMetadata{
+		Title:       t.Title(),
+		Artist:      t.Artist(),
+		Album:       t.Album(),
+		AlbumArtist: id3Text(t, "Band/Orchestra/Accompaniment"),
+		Composer:    id3Text(t, "Composer"),
+		Genre:       t.Genre(),
+		Year:        parseLeadingInt(t.Year()),
+		Lyrics:      id3Lyrics(t),
+		Comment:     id3Comment(t),
+		TagFormat:   fmt.Sprintf("ID3v2.%d", t.Version()),
+		FileFormat:  strings.ToUpper(strings.TrimPrefix(string(MP3), ".")),
+	}
+
+	meta.TrackNumber, meta.TotalTracks = parsePosition(
+		id3Text(t, "Track number/Position in set"),
+	)
+	meta.DiscNumber, meta.TotalDiscs = parsePosition(
+		id3Text(t, "Part of a set"),
+	)
+
+	extractMBIDsID3v2(t, meta)
+
+	meta.Picture = id3Picture(t)
+
+	return meta, nil
+}
+
+// id3Text returns the text of the frame registered under the given
+// common description, empty if the frame is absent.
+func id3Text(t *id3v2.Tag, description string) string {
+	return strings.TrimRight(
+		t.GetTextFrame(t.CommonID(description)).Text, "\x00 \t\n\r",
+	)
+}
+
+// id3Lyrics returns the first non-empty USLT frame.
+func id3Lyrics(t *id3v2.Tag) string {
+	for _, f := range t.GetFrames(t.CommonID("Unsynchronised lyrics/text transcription")) {
+		if uslf, ok := f.(id3v2.UnsynchronisedLyricsFrame); ok && uslf.Lyrics != "" {
+			return uslf.Lyrics
+		}
+	}
+
+	return ""
+}
+
+// id3Comment returns the first non-empty COMM frame, skipping the
+// machine-written iTunes frames that carry no user comment.
+func id3Comment(t *id3v2.Tag) string {
+	for _, f := range t.GetFrames(t.CommonID("Comments")) {
+		cf, ok := f.(id3v2.CommentFrame)
+		if !ok || cf.Text == "" {
+			continue
+		}
+
+		if strings.HasPrefix(cf.Description, "iTun") {
+			continue
+		}
+
+		return cf.Text
+	}
+
+	return ""
+}
+
+// id3Picture returns the front cover if one is attached, otherwise the
+// first attached picture of any type.
+func id3Picture(t *id3v2.Tag) *PictureData {
+	var first *PictureData
+
+	for _, f := range t.GetFrames(t.CommonID("Attached picture")) {
+		pf, ok := f.(id3v2.PictureFrame)
+		if !ok || len(pf.Picture) == 0 {
+			continue
+		}
+
+		pic := &PictureData{
+			Data:     pf.Picture,
+			MIMEType: pf.MimeType,
+			Ext:      imageExtFromMIME(pf.MimeType),
+		}
+
+		if pf.PictureType == id3v2.PTFrontCover {
+			return pic
+		}
+
+		if first == nil {
+			first = pic
+		}
+	}
+
+	return first
+}
+
+// extractMBIDsID3v2 populates the MBID fields of meta from TXXX and
+// UFID frames, mirroring extractMBIDs for the lenient parser.
+func extractMBIDsID3v2(t *id3v2.Tag, meta *TrackMetadata) {
+	normalized := make(map[string]string)
+
+	for _, f := range t.GetFrames(t.CommonID("User defined text information frame")) {
+		if udtf, ok := f.(id3v2.UserDefinedTextFrame); ok && udtf.Description != "" {
+			normalized[strings.ToLower(udtf.Description)] = strings.TrimRight(
+				udtf.Value, "\x00 \t\n\r",
+			)
+		}
+	}
+
+	for _, f := range t.GetFrames(t.CommonID("Unique file identifier")) {
+		if ufid, ok := f.(id3v2.UFIDFrame); ok &&
+			ufid.OwnerIdentifier == "http://musicbrainz.org" {
+			meta.RecordingMBID = strings.TrimRight(
+				string(ufid.Identifier), "\x00 \t\n\r",
+			)
+		}
+	}
+
+	targets := map[string]*string{
+		"ArtistMBID":       &meta.ArtistMBID,
+		"AlbumArtistMBID":  &meta.AlbumArtistMBID,
+		"ReleaseGroupMBID": &meta.ReleaseGroupMBID,
+		"ReleaseMBID":      &meta.ReleaseMBID,
+		"RecordingMBID":    &meta.RecordingMBID,
+	}
+
+	for field, keys := range mbidTagKeys {
+		for _, key := range keys {
+			if val, ok := normalized[key]; ok && val != "" {
+				*targets[field] = val
+
+				break
+			}
+		}
+	}
+}
+
+// parsePosition splits an ID3 "n/total" position string such as the
+// TRCK or TPOS payload.  Missing parts come back as zero.
+func parsePosition(raw string) (int, int) {
+	raw = strings.TrimSpace(raw)
+	if raw == "" {
+		return 0, 0
+	}
+
+	number, total, found := strings.Cut(raw, "/")
+	if !found {
+		return parseLeadingInt(number), 0
+	}
+
+	return parseLeadingInt(number), parseLeadingInt(total)
+}
+
+// parseLeadingInt reads the leading run of digits from s, returning
+// zero when there is none.  Tolerates values like "2021-06-11" (a
+// TDRC date) and "3 " (a padded track number).
+func parseLeadingInt(s string) int {
+	s = strings.TrimSpace(s)
+
+	end := 0
+	for end < len(s) && s[end] >= '0' && s[end] <= '9' {
+		end++
+	}
+
+	if end == 0 {
+		return 0
+	}
+
+	n, err := strconv.Atoi(s[:end])
+	if err != nil {
+		return 0
+	}
+
+	return n
+}
+
+// imageExtFromMIME returns a file extension for common image MIME types.
+func imageExtFromMIME(mimeType string) string {
+	switch strings.ToLower(strings.TrimSpace(mimeType)) {
+	case "image/png":
+		return "png"
+	case "image/gif":
+		return "gif"
+	case "image/webp":
+		return "webp"
+	case "image/bmp":
+		return "bmp"
+	default:
+		return "jpg"
+	}
+}
@@ -0,0 +1,214 @@
+package metadata
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"strings"
+	"testing"
+)
+
+// errTestParseFailure stands in for the strict parser's error when
+// exercising the fallback directly.
+var errTestParseFailure = errors.New("original parse failure")
+
+// synchsafe encodes n as a 4-byte ID3v2.4 synchsafe integer.
+func synchsafe(n int) []byte {
+	return []byte{
+		byte(n>>21) & 0x7f,
+		byte(n>>14) & 0x7f,
+		byte(n>>7) & 0x7f,
+		byte(n) & 0x7f,
+	}
+}
+
+// id3Frame builds a single ID3v2.4 frame from a raw body.
+func id3Frame(id string, body []byte) []byte {
+	var buf bytes.Buffer
+
+	buf.WriteString(id)
+	buf.Write(synchsafe(len(body)))
+	buf.Write([]byte{0, 0}) // Flags.
+	buf.Write(body)
+
+	return buf.Bytes()
+}
+
+// utf8TextBody builds a UTF-8 text frame body (encoding byte $03).
+func utf8TextBody(s string) []byte {
+	return append([]byte{3}, []byte(s)...)
+}
+
+// malformedUTF16TXXX reproduces the real-world frame that motivated the
+// lenient fallback: a UTF-16BE TXXX frame carrying two stray NUL bytes,
+// one after the description terminator and one at the very end.  Both
+// the description and the value end up an odd number of bytes, which
+// dhowden/tag rejects outright.
+func malformedUTF16TXXX(description, value string) []byte {
+	var buf bytes.Buffer
+
+	buf.WriteByte(1) // Encoding: UTF-16 with BOM.
+
+	writeUTF16BE := func(s string) {
+		buf.Write([]byte{0xfe, 0xff}) // Big-endian BOM.
+
+		for _, r := range s {
+			_ = binary.Write(&buf, binary.BigEndian, uint16(r))
+		}
+	}
+
+	writeUTF16BE(description)
+	buf.Write([]byte{0, 0}) // Terminator.
+	buf.WriteByte(0)        // Stray NUL.
+	writeUTF16BE(value)
+	buf.WriteByte(0) // Stray NUL.
+
+	return buf.Bytes()
+}
+
+// buildID3v24 assembles a tag from frames and appends a stub of MPEG
+// audio so the result looks like a real file to a parser.
+func buildID3v24(frames ...[]byte) []byte {
+	body := bytes.Join(frames, nil)
+
+	var buf bytes.Buffer
+
+	buf.WriteString("ID3")
+	buf.Write([]byte{4, 0}) // Version 2.4.0.
+	buf.WriteByte(0)        // Flags.
+	buf.Write(synchsafe(len(body)))
+	buf.Write(body)
+	buf.Write([]byte{0xff, 0xfb, 0x90, 0x00}) // MPEG frame header stub.
+
+	return buf.Bytes()
+}
+
+func TestExtractTagsFromReaderRecoversMalformedFrame(t *testing.T) {
+	data := buildID3v24(
+		id3Frame("TIT2", utf8TextBody("Evolution's a Lie")),
+		id3Frame("TPE1", utf8TextBody("Ariel Pink")),
+		id3Frame("TALB", utf8TextBody("Sit n' Spin")),
+		id3Frame("TRCK", utf8TextBody("1/17")),
+		id3Frame("TXXX", malformedUTF16TXXX("LABEL", "Mexican Summer")),
+	)
+
+	meta, err := ExtractTagsFromReader(bytes.NewReader(data))
+	if err != nil {
+		t.Fatalf("ExtractTagsFromReader returned a hard error: %v", err)
+	}
+
+	if !errors.Is(meta.TagReadWarning, ErrTagsRecovered) {
+		t.Errorf(
+			"TagReadWarning = %v, want it to wrap ErrTagsRecovered",
+			meta.TagReadWarning,
+		)
+	}
+
+	if meta.Title != "Evolution's a Lie" {
+		t.Errorf("Title = %q, want %q", meta.Title, "Evolution's a Lie")
+	}
+
+	if meta.Artist != "Ariel Pink" {
+		t.Errorf("Artist = %q, want %q", meta.Artist, "Ariel Pink")
+	}
+
+	if meta.Album != "Sit n' Spin" {
+		t.Errorf("Album = %q, want %q", meta.Album, "Sit n' Spin")
+	}
+
+	if meta.TrackNumber != 1 || meta.TotalTracks != 17 {
+		t.Errorf(
+			"track = %d/%d, want 1/17",
+			meta.TrackNumber, meta.TotalTracks,
+		)
+	}
+}
+
+// TestExtractTagsFromReaderCleanTagHasNoWarning guards against the
+// fallback firing on tags the strict parser handles.
+func TestExtractTagsFromReaderCleanTagHasNoWarning(t *testing.T) {
+	data := buildID3v24(
+		id3Frame("TIT2", utf8TextBody("Clean Title")),
+		id3Frame("TXXX", utf8TextBody("LABEL\x00Mexican Summer")),
+	)
+
+	meta, err := ExtractTagsFromReader(bytes.NewReader(data))
+	if err != nil {
+		t.Fatalf("ExtractTagsFromReader: %v", err)
+	}
+
+	if meta.TagReadWarning != nil {
+		t.Errorf("TagReadWarning = %v, want nil", meta.TagReadWarning)
+	}
+
+	if meta.Title != "Clean Title" {
+		t.Errorf("Title = %q, want %q", meta.Title, "Clean Title")
+	}
+}
+
+// TestRecoverTagsUnsalvageable covers the case where the fallback finds
+// no ID3v2 frames either: metadata comes back empty but usable, with a
+// warning that tells the caller to fall back to the filename.
+func TestRecoverTagsUnsalvageable(t *testing.T) {
+	r := strings.NewReader("not an audio file at all")
+
+	meta := recoverTags(r, errTestParseFailure)
+
+	if !errors.Is(meta.TagReadWarning, ErrTagsUnreadable) {
+		t.Errorf(
+			"TagReadWarning = %v, want it to wrap ErrTagsUnreadable",
+			meta.TagReadWarning,
+		)
+	}
+
+	if !errors.Is(meta.TagReadWarning, errTestParseFailure) {
+		t.Errorf("TagReadWarning = %v, want it to wrap the cause", meta.TagReadWarning)
+	}
+
+	if meta.Title != "" || meta.Artist != "" {
+		t.Errorf("expected empty metadata, got %+v", meta)
+	}
+}
+
+func TestParsePosition(t *testing.T) {
+	tests := []struct {
+		raw          string
+		want, wantOf int
+	}{
+		{"", 0, 0},
+		{"3", 3, 0},
+		{"3/17", 3, 17},
+		{" 3 / 17 ", 3, 17},
+		{"03/17", 3, 17},
+		{"A/B", 0, 0},
+		{"1/", 1, 0},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.raw, func(t *testing.T) {
+			got, gotOf := parsePosition(tt.raw)
+			if got != tt.want || gotOf != tt.wantOf {
+				t.Errorf(
+					"parsePosition(%q) = %d/%d, want %d/%d",
+					tt.raw, got, gotOf, tt.want, tt.wantOf,
+				)
+			}
+		})
+	}
+}
+
+func TestParseLeadingInt(t *testing.T) {
+	tests := map[string]int{
+		"":           0,
+		"2021":       2021,
+		"2021-06-11": 2021,
+		"1995\t":     1995,
+		"none":       0,
+	}
+
+	for raw, want := range tests {
+		if got := parseLeadingInt(raw); got != want {
+			t.Errorf("parseLeadingInt(%q) = %d, want %d", raw, got, want)
+		}
+	}
+}