feat: autotag scoring overhaul, dump-based explore index, and lyrics search
Consolidates in-progress work across autotag, explore, and library: - autotag: beets/Picard-informed scoring engine — ID-first matching, VA handling, recommendation tiers, and a merged distance/rank cascade, with an eval harness for regression tracking. - explore: offline MusicBrainz dump import/incremental refresh replaces the legacy tier crawl; index-first local search with fuzzy matching and a dedicated ranker; disk-free guards for dump downloads. - library: artist-credit extraction and matching. - lyrics: owned-library lyric search (FTS) with LRCLIB backfill. Also: rewrite README to be user-focused, and migrate upstream to git.ljones.me/yonlu/yellowjacket. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
+327
-51
@@ -32,6 +32,25 @@ type MBRelease struct {
|
||||
Tracks []CandidateTrack
|
||||
}
|
||||
|
||||
// MBRecordingHit is the minimal projection of a MusicBrainz recording
|
||||
// search result — used by the in-app search path (singletons).
|
||||
type MBRecordingHit struct {
|
||||
MBID string
|
||||
Title string
|
||||
ArtistCredit string
|
||||
LengthMillis int64
|
||||
}
|
||||
|
||||
// MBReleaseRef is a slim reference to one release a recording appears
|
||||
// on. The resolver ranks these to pick a representative release and
|
||||
// then resolves it in full via ResolveOneReleaseMBID.
|
||||
type MBReleaseRef struct {
|
||||
MBID string
|
||||
Title string
|
||||
Status string
|
||||
Date string
|
||||
}
|
||||
|
||||
// MBClient is the subset of the explore.MusicBrainzClient surface
|
||||
// the autotagger depends on. Implementations must be cache-first
|
||||
// — repeated calls with the same inputs must not repeat network
|
||||
@@ -42,16 +61,42 @@ type MBClient interface {
|
||||
query string,
|
||||
limit int,
|
||||
) ([]MBReleaseGroupHit, int, error)
|
||||
SearchRecordings(
|
||||
ctx context.Context,
|
||||
query string,
|
||||
limit int,
|
||||
) ([]MBRecordingHit, int, error)
|
||||
LookupRecordingReleases(ctx context.Context, recordingMBID string) ([]MBReleaseRef, error)
|
||||
BrowseReleases(ctx context.Context, releaseGroupMBID string) ([]MBRelease, error)
|
||||
LookupRelease(ctx context.Context, releaseMBID string) (MBRelease, error)
|
||||
LookupReleaseGroup(ctx context.Context, releaseGroupMBID string) (MBReleaseGroupHit, error)
|
||||
LookupArtist(ctx context.Context, mbid string) (string, error) // returns sort name or name
|
||||
}
|
||||
|
||||
// mbidVariousArtists is the MusicBrainz artist MBID for the special
|
||||
// "Various Artists" entity — used as an arid filter when a group
|
||||
// looks like a compilation.
|
||||
const mbidVariousArtists = "89ad4ac3-39f7-470e-963a-56509c546377"
|
||||
|
||||
// cascadeSufficient is the merged-best score at which the cascade
|
||||
// stops issuing looser queries. "First step with any hits" is the
|
||||
// wrong stop condition — a strict query can return plausible-but-
|
||||
// wrong release groups and starve the looser steps of the chance to
|
||||
// surface the right one. Scoring is free; searches and browses are
|
||||
// rate-limited network calls, so the cascade pays for another step
|
||||
// only while the best candidate so far is still mediocre.
|
||||
const cascadeSufficient = 0.70
|
||||
|
||||
// hitBrowseFloor is the minimum title-or-artist similarity a search
|
||||
// hit needs before the resolver pays a rate-limited BrowseReleases
|
||||
// call for it. Hits failing both checks are junk from MB's fuzzy
|
||||
// tokenizer.
|
||||
const hitBrowseFloor = 0.30
|
||||
|
||||
// MBResolver orchestrates MusicBrainz lookups for a tagging group.
|
||||
// Strategy: normalize the user-provided album/artist first, then
|
||||
// issue a cascade of progressively looser Lucene queries, stopping
|
||||
// at the first one that yields enough candidates.
|
||||
// Strategy: use recording MBIDs already present in local tags when
|
||||
// possible (exact, cheap); otherwise issue a cascade of
|
||||
// progressively looser Lucene queries, merging results until a
|
||||
// candidate scores well enough to stop.
|
||||
type MBResolver struct {
|
||||
client MBClient
|
||||
logger *slog.Logger
|
||||
@@ -73,27 +118,21 @@ type mbQueryStep struct {
|
||||
}
|
||||
|
||||
// ResolveMB returns MB-sourced candidates for a tagging group.
|
||||
// Runs a cascade of Lucene queries, returning at the first step
|
||||
// that produces results. Each search hit fans out to
|
||||
// BrowseReleases (one per release-group) for track-level data.
|
||||
func (r *MBResolver) ResolveMB(
|
||||
ctx context.Context,
|
||||
albumName, albumArtist string,
|
||||
trackCount int,
|
||||
knownArtistMBID string,
|
||||
) ([]Candidate, error) {
|
||||
if albumName == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
nAlbum := Normalize(albumName)
|
||||
nArtist := Normalize(albumArtist)
|
||||
|
||||
// Runs the Lucene query cascade, accumulating deduplicated
|
||||
// candidates across steps and stopping once the best merged
|
||||
// candidate scores at least cascadeSufficient against the group.
|
||||
func (r *MBResolver) ResolveMB(ctx context.Context, g Group) ([]Candidate, error) {
|
||||
nAlbum := Normalize(g.AlbumName)
|
||||
if nAlbum == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
steps := buildMBQueryCascade(nAlbum, nArtist, trackCount, knownArtistMBID)
|
||||
nArtist := Normalize(groupArtist(g))
|
||||
steps := buildMBQueryCascade(nAlbum, nArtist, len(g.Tracks), vaLikely(g))
|
||||
|
||||
seen := make(map[string]bool)
|
||||
|
||||
var merged []Candidate
|
||||
|
||||
for _, step := range steps {
|
||||
hits, _, err := r.client.SearchReleaseGroups(ctx, step.query, r.limit)
|
||||
@@ -106,35 +145,65 @@ func (r *MBResolver) ResolveMB(
|
||||
continue
|
||||
}
|
||||
|
||||
if len(hits) == 0 {
|
||||
added := r.fanOutBrowse(ctx, g, hits, step.label, seen, &merged)
|
||||
|
||||
r.logger.Debug(
|
||||
"MB search step done",
|
||||
"step", step.label, "hits", len(hits), "new_candidates", added,
|
||||
)
|
||||
|
||||
if added == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// Score what we have so far; good enough means the looser
|
||||
// (noisier, costlier) steps aren't needed.
|
||||
ranked := RankCandidates(g, merged)
|
||||
if len(ranked) > 0 && ranked[0].Score >= cascadeSufficient {
|
||||
r.logger.Info(
|
||||
"MB cascade stopped — sufficient candidate",
|
||||
"step", step.label, "score", ranked[0].Score,
|
||||
)
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return merged, nil
|
||||
}
|
||||
|
||||
// fanOutBrowse iterates search hits, fetches each plausible
|
||||
// release-group's releases, and appends previously-unseen ones to
|
||||
// merged as Candidates. Returns how many candidates were added.
|
||||
// Errors on individual browses are logged and skipped.
|
||||
func (r *MBResolver) fanOutBrowse(
|
||||
ctx context.Context,
|
||||
g Group,
|
||||
hits []MBReleaseGroupHit,
|
||||
step string,
|
||||
seen map[string]bool,
|
||||
merged *[]Candidate,
|
||||
) int {
|
||||
added := 0
|
||||
|
||||
for _, h := range hits {
|
||||
if seen["rg:"+h.MBID] {
|
||||
continue
|
||||
}
|
||||
|
||||
seen["rg:"+h.MBID] = true
|
||||
|
||||
// Don't pay a rate-limited browse for a hit that resembles
|
||||
// neither the folder's album name nor its artist.
|
||||
if !hitPlausible(g, h) {
|
||||
r.logger.Debug(
|
||||
"MB search step empty — trying next",
|
||||
"step", step.label, "query", step.query,
|
||||
"skipping implausible search hit",
|
||||
"title", h.Title, "artist", h.ArtistCredit,
|
||||
)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
r.logger.Info(
|
||||
"MB search step succeeded",
|
||||
"step", step.label, "hits", len(hits),
|
||||
)
|
||||
|
||||
return r.fanOutBrowse(ctx, hits, step.label), nil
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// fanOutBrowse iterates search hits, fetches each release-group's
|
||||
// releases, and returns them as Candidates. Errors on individual
|
||||
// browses are logged and skipped.
|
||||
func (r *MBResolver) fanOutBrowse(
|
||||
ctx context.Context, hits []MBReleaseGroupHit, step string,
|
||||
) []Candidate {
|
||||
var out []Candidate
|
||||
|
||||
for _, h := range hits {
|
||||
releases, err := r.client.BrowseReleases(ctx, h.MBID)
|
||||
if err != nil {
|
||||
r.logger.Warn(
|
||||
@@ -146,11 +215,105 @@ func (r *MBResolver) fanOutBrowse(
|
||||
}
|
||||
|
||||
for _, rel := range releases {
|
||||
out = append(out, mkCandidate(h, rel, step))
|
||||
if rel.MBID != "" && seen[rel.MBID] {
|
||||
continue
|
||||
}
|
||||
|
||||
seen[rel.MBID] = true
|
||||
|
||||
*merged = append(*merged, mkCandidate(h, rel, step))
|
||||
added++
|
||||
}
|
||||
}
|
||||
|
||||
return out
|
||||
return added
|
||||
}
|
||||
|
||||
// hitPlausible reports whether a release-group search hit is worth
|
||||
// a BrowseReleases round-trip: its title or artist must bear at
|
||||
// least a loose resemblance to the group's. Unknown local fields
|
||||
// never disqualify a hit.
|
||||
func hitPlausible(g Group, h MBReleaseGroupHit) bool {
|
||||
if g.AlbumName != "" && h.Title != "" &&
|
||||
titleSimilarity(g.AlbumName, h.Title) >= hitBrowseFloor {
|
||||
return true
|
||||
}
|
||||
|
||||
artist := groupArtist(g)
|
||||
if artist != "" && h.ArtistCredit != "" &&
|
||||
titleSimilarity(artist, h.ArtistCredit) >= hitBrowseFloor {
|
||||
return true
|
||||
}
|
||||
|
||||
// Nothing to compare against (or a VA credit): stay permissive.
|
||||
return g.AlbumName == "" || h.Title == "" || isVAName(h.ArtistCredit)
|
||||
}
|
||||
|
||||
// ResolveByRecordingMBIDs resolves candidates from recording MBIDs
|
||||
// already present in the local tags — the highest-precision signal
|
||||
// available, and the reason previously-tagged files should never
|
||||
// need a fuzzy search. Each recording is looked up, releases are
|
||||
// counted as votes, and the best-voted release (Official and
|
||||
// earliest among ties) is resolved in full with provenance "id".
|
||||
// Returns nil when no recording resolves to any release.
|
||||
func (r *MBResolver) ResolveByRecordingMBIDs(
|
||||
ctx context.Context, recordingMBIDs []string,
|
||||
) ([]Candidate, error) {
|
||||
votes := make(map[string]int)
|
||||
refs := make(map[string]MBReleaseRef)
|
||||
|
||||
for _, id := range recordingMBIDs {
|
||||
rels, err := r.client.LookupRecordingReleases(ctx, id)
|
||||
if err != nil {
|
||||
r.logger.Warn(
|
||||
"recording lookup failed — skipping",
|
||||
"recording_mbid", id, "err", err,
|
||||
)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
counted := make(map[string]bool, len(rels))
|
||||
|
||||
for _, ref := range rels {
|
||||
if ref.MBID == "" || counted[ref.MBID] {
|
||||
continue
|
||||
}
|
||||
|
||||
counted[ref.MBID] = true
|
||||
votes[ref.MBID]++
|
||||
refs[ref.MBID] = ref
|
||||
}
|
||||
}
|
||||
|
||||
if len(votes) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Highest vote count wins; betterRelease breaks ties so the
|
||||
// pick is deterministic and favours Official + earliest.
|
||||
var (
|
||||
bestMBID string
|
||||
bestVotes int
|
||||
)
|
||||
|
||||
for mbid, n := range votes {
|
||||
switch {
|
||||
case n > bestVotes:
|
||||
bestMBID, bestVotes = mbid, n
|
||||
case n == bestVotes && betterRelease(refs[mbid], refs[bestMBID]):
|
||||
bestMBID = mbid
|
||||
}
|
||||
}
|
||||
|
||||
cand, err := r.ResolveOneReleaseMBID(ctx, bestMBID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve voted release %s: %w", bestMBID, err)
|
||||
}
|
||||
|
||||
cand.Provenance = "id"
|
||||
|
||||
return []Candidate{cand}, nil
|
||||
}
|
||||
|
||||
// ResolveOneReleaseMBID fetches a single release by MBID and
|
||||
@@ -208,6 +371,7 @@ func mkCandidate(h MBReleaseGroupHit, rel MBRelease, step string) Candidate {
|
||||
OriginalDate: h.FirstDate,
|
||||
Country: rel.Country,
|
||||
Status: rel.Status,
|
||||
PrimaryType: h.PrimaryType,
|
||||
TrackCount: len(rel.Tracks),
|
||||
Tracks: rel.Tracks,
|
||||
Source: SourceMusicBrainz,
|
||||
@@ -215,19 +379,131 @@ func mkCandidate(h MBReleaseGroupHit, rel MBRelease, step string) Candidate {
|
||||
}
|
||||
}
|
||||
|
||||
// SearchReleaseGroupHits runs a single release-group search from a
|
||||
// user-supplied album + artist (the in-app "suggest a candidate"
|
||||
// path). Both fields are normalized and phrase-quoted; artist is
|
||||
// dropped from the query when empty.
|
||||
func (r *MBResolver) SearchReleaseGroupHits(
|
||||
ctx context.Context, album, artist string,
|
||||
) ([]MBReleaseGroupHit, error) {
|
||||
query := "release:" + luceneQuote(Normalize(album))
|
||||
if a := Normalize(artist); a != "" {
|
||||
query += " AND artist:" + luceneQuote(a)
|
||||
}
|
||||
|
||||
hits, _, err := r.client.SearchReleaseGroups(ctx, query, r.limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("search release groups: %w", err)
|
||||
}
|
||||
|
||||
return hits, nil
|
||||
}
|
||||
|
||||
// SearchRecordingHits runs a single recording search from a user-
|
||||
// supplied title + artist — the singleton path, where the folder has
|
||||
// one track and release-group search is too coarse.
|
||||
func (r *MBResolver) SearchRecordingHits(
|
||||
ctx context.Context, title, artist string,
|
||||
) ([]MBRecordingHit, error) {
|
||||
query := "recording:" + luceneQuote(Normalize(title))
|
||||
if a := Normalize(artist); a != "" {
|
||||
query += " AND artist:" + luceneQuote(a)
|
||||
}
|
||||
|
||||
hits, _, err := r.client.SearchRecordings(ctx, query, r.limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("search recordings: %w", err)
|
||||
}
|
||||
|
||||
return hits, nil
|
||||
}
|
||||
|
||||
// ResolveOneRecordingMBID turns a picked recording into a fully-scored
|
||||
// Candidate by resolving it to a representative release (so the
|
||||
// existing release-based diff + Apply pipeline works unchanged).
|
||||
// Picks the release the same way a human would default: prefer an
|
||||
// Official status, then the earliest date. Provenance is
|
||||
// "search-recording" so the UI can label where it came from.
|
||||
func (r *MBResolver) ResolveOneRecordingMBID(
|
||||
ctx context.Context, recordingMBID string,
|
||||
) (Candidate, error) {
|
||||
refs, err := r.client.LookupRecordingReleases(ctx, recordingMBID)
|
||||
if err != nil {
|
||||
return Candidate{}, fmt.Errorf("lookup recording releases: %w", err)
|
||||
}
|
||||
|
||||
best := pickRepresentativeRelease(refs)
|
||||
if best.MBID == "" {
|
||||
return Candidate{}, fmt.Errorf("%w: %s", errNoReleasesForRecording, recordingMBID)
|
||||
}
|
||||
|
||||
cand, err := r.ResolveOneReleaseMBID(ctx, best.MBID)
|
||||
if err != nil {
|
||||
return Candidate{}, err
|
||||
}
|
||||
|
||||
cand.Provenance = "search-recording"
|
||||
|
||||
return cand, nil
|
||||
}
|
||||
|
||||
// pickRepresentativeRelease chooses the release most likely to be the
|
||||
// one the user means: an Official release beats a non-Official one,
|
||||
// and among equals the earliest date wins (favouring the original
|
||||
// over later reissues). Returns the zero value for an empty slice.
|
||||
func pickRepresentativeRelease(refs []MBReleaseRef) MBReleaseRef {
|
||||
var best MBReleaseRef
|
||||
|
||||
for _, ref := range refs {
|
||||
if best.MBID == "" || betterRelease(ref, best) {
|
||||
best = ref
|
||||
}
|
||||
}
|
||||
|
||||
return best
|
||||
}
|
||||
|
||||
// betterRelease reports whether a should be preferred over b.
|
||||
func betterRelease(a, b MBReleaseRef) bool {
|
||||
aOfficial := strings.EqualFold(a.Status, "Official")
|
||||
bOfficial := strings.EqualFold(b.Status, "Official")
|
||||
|
||||
if aOfficial != bOfficial {
|
||||
return aOfficial
|
||||
}
|
||||
|
||||
// Same official-ness: earlier date wins. Empty dates sort last
|
||||
// so a dated release beats an undated one.
|
||||
switch {
|
||||
case a.Date == "":
|
||||
return false
|
||||
case b.Date == "":
|
||||
return true
|
||||
default:
|
||||
return a.Date < b.Date
|
||||
}
|
||||
}
|
||||
|
||||
// errNoReleasesForRecording signals a recording that resolved to zero
|
||||
// releases — nothing to diff or apply against.
|
||||
var errNoReleasesForRecording = errors.New("autotag: recording has no releases")
|
||||
|
||||
// buildMBQueryCascade returns the Lucene queries to try in order.
|
||||
// Cascade:
|
||||
//
|
||||
// 1. Full: release + arid/artist + tracks:N
|
||||
// 1. Full: release + artist (or VA arid) + tracks:N
|
||||
// 2. Drop tracks:N (bonus tracks, live editions, etc.)
|
||||
// 3. Drop artist entirely (wrong artist tag is common)
|
||||
// 4. Fuzzy title (unquoted; Lucene does token/prefix match)
|
||||
//
|
||||
// Each step is only added when it would differ from the previous.
|
||||
// VA-likely groups filter on the Various Artists arid instead of an
|
||||
// artist name — per-track artists on a compilation say nothing
|
||||
// about the release's artist credit. Each step is only added when
|
||||
// it would differ from the previous.
|
||||
func buildMBQueryCascade(
|
||||
normAlbum, normArtist string,
|
||||
trackCount int,
|
||||
artistMBID string,
|
||||
va bool,
|
||||
) []mbQueryStep {
|
||||
var steps []mbQueryStep
|
||||
|
||||
@@ -235,8 +511,8 @@ func buildMBQueryCascade(
|
||||
artistClause := ""
|
||||
|
||||
switch {
|
||||
case artistMBID != "":
|
||||
artistClause = "arid:" + artistMBID
|
||||
case va:
|
||||
artistClause = "arid:" + mbidVariousArtists
|
||||
case normArtist != "":
|
||||
artistClause = "artist:" + luceneQuote(normArtist)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user