feat: autotag scoring overhaul, dump-based explore index, and lyrics search
Consolidates in-progress work across autotag, explore, and library: - autotag: beets/Picard-informed scoring engine — ID-first matching, VA handling, recommendation tiers, and a merged distance/rank cascade, with an eval harness for regression tracking. - explore: offline MusicBrainz dump import/incremental refresh replaces the legacy tier crawl; index-first local search with fuzzy matching and a dedicated ranker; disk-free guards for dump downloads. - library: artist-credit extraction and matching. - lyrics: owned-library lyric search (FTS) with LRCLIB backfill. Also: rewrite README to be user-focused, and migrate upstream to git.ljones.me/yonlu/yellowjacket. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
@@ -0,0 +1,108 @@
|
||||
// Package eval is the autotag candidate-scoring evaluation harness.
|
||||
// It turns "this match feels wrong" into a number that goes up or
|
||||
// down, so a scoring change can be validated against a frozen set of
|
||||
// labelled cases instead of tuned by anecdote.
|
||||
//
|
||||
// A case describes a local album-group (the files on disk) plus a set
|
||||
// of candidate releases, and pins expectations: which candidate must
|
||||
// rank first, and per-candidate score floors/ceilings. The harness
|
||||
// is decoupled from the scorer: a caller adapts whatever ranking
|
||||
// function it wants to measure to the Ranker interface (the autotag
|
||||
// package wires autotag.RankCandidates to it in eval_harness_test.go).
|
||||
//
|
||||
// The point of the ceiling assertions is negative testing: a known
|
||||
// wrong candidate (same title, different artist) must stay BELOW a
|
||||
// confidence bar, which is exactly the false-positive class the
|
||||
// artist term + evidence scaling exist to suppress.
|
||||
package eval
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
// ErrNoCases is returned when a fixture file contains zero cases.
|
||||
var ErrNoCases = errors.New("eval: fixture set is empty")
|
||||
|
||||
// LocalTrackFixture is one on-disk track, in the shape a case author
|
||||
// hand-writes. Millisecond durations match the scorer's units.
|
||||
type LocalTrackFixture struct {
|
||||
Title string `json:"title"`
|
||||
Artist string `json:"artist,omitempty"`
|
||||
Track int `json:"track,omitempty"`
|
||||
Disc int `json:"disc,omitempty"`
|
||||
LengthMs int64 `json:"lengthMs,omitempty"`
|
||||
}
|
||||
|
||||
// CandidateTrackFixture is one track inside a candidate release.
|
||||
type CandidateTrackFixture struct {
|
||||
Pos int `json:"pos"`
|
||||
Disc int `json:"disc,omitempty"`
|
||||
Title string `json:"title"`
|
||||
LengthMs int64 `json:"lengthMs,omitempty"`
|
||||
}
|
||||
|
||||
// CandidateFixture is one candidate release the scorer must rank.
|
||||
// Source is "local" or "musicbrainz" (default) — it drives
|
||||
// evidence scaling, so it matters for singleton cases.
|
||||
type CandidateFixture struct {
|
||||
MBID string `json:"mbid"`
|
||||
Title string `json:"title,omitempty"`
|
||||
ArtistCredit string `json:"artistCredit,omitempty"`
|
||||
Status string `json:"status,omitempty"`
|
||||
Country string `json:"country,omitempty"`
|
||||
PrimaryType string `json:"primaryType,omitempty"`
|
||||
Source string `json:"source,omitempty"`
|
||||
Tracks []CandidateTrackFixture `json:"tracks"`
|
||||
}
|
||||
|
||||
// Case is a single labelled scoring scenario. Every real-world
|
||||
// mismatch worth guarding against belongs here so it can never
|
||||
// silently regress.
|
||||
//
|
||||
// - AlbumName / AlbumArtist mirror the tagging item's fields —
|
||||
// leave them empty to keep the album-title / artist terms
|
||||
// neutral for the case.
|
||||
// - ExpectTop, when set, is the MBID that must rank first.
|
||||
// - MaxScore pins per-MBID ceilings (candidate must score <= value).
|
||||
// - MinScore pins per-MBID floors (candidate must score >= value).
|
||||
type Case struct {
|
||||
Note string `json:"note,omitempty"`
|
||||
AlbumName string `json:"albumName,omitempty"`
|
||||
AlbumArtist string `json:"albumArtist,omitempty"`
|
||||
Local []LocalTrackFixture `json:"local"`
|
||||
Candidates []CandidateFixture `json:"candidates"`
|
||||
ExpectTop string `json:"expectTop,omitempty"`
|
||||
MaxScore map[string]float64 `json:"maxScore,omitempty"`
|
||||
MinScore map[string]float64 `json:"minScore,omitempty"`
|
||||
}
|
||||
|
||||
// LoadCases reads a JSON fixture file from disk.
|
||||
func LoadCases(path string) ([]Case, error) {
|
||||
f, err := os.Open(path) //nolint:gosec // path is a test fixture, not user input
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("eval: open cases: %w", err)
|
||||
}
|
||||
|
||||
defer func() { _ = f.Close() }()
|
||||
|
||||
return ParseCases(f)
|
||||
}
|
||||
|
||||
// ParseCases decodes a JSON case set from a reader.
|
||||
func ParseCases(r io.Reader) ([]Case, error) {
|
||||
var cases []Case
|
||||
|
||||
if err := json.NewDecoder(r).Decode(&cases); err != nil {
|
||||
return nil, fmt.Errorf("eval: decode cases: %w", err)
|
||||
}
|
||||
|
||||
if len(cases) == 0 {
|
||||
return nil, ErrNoCases
|
||||
}
|
||||
|
||||
return cases, nil
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
package eval
|
||||
|
||||
import "fmt"
|
||||
|
||||
// ScoredCandidate is one ranked candidate reduced to what the harness
|
||||
// checks: its MBID and final score. The slice a Ranker returns must
|
||||
// be ordered best-first.
|
||||
type ScoredCandidate struct {
|
||||
MBID string
|
||||
Score float64
|
||||
}
|
||||
|
||||
// Ranker scores and orders the candidates of a single case. Best
|
||||
// candidate first. Implemented by adapting the real scorer (see
|
||||
// eval_harness_test.go), which is why the eval package itself never
|
||||
// imports autotag.
|
||||
type Ranker interface {
|
||||
Rank(c Case) []ScoredCandidate
|
||||
}
|
||||
|
||||
// RankerFunc adapts a plain function to the Ranker interface.
|
||||
type RankerFunc func(c Case) []ScoredCandidate
|
||||
|
||||
// Rank calls the underlying function.
|
||||
func (f RankerFunc) Rank(c Case) []ScoredCandidate {
|
||||
return f(c)
|
||||
}
|
||||
|
||||
// CaseResult records how one case fared. Failures is empty when the
|
||||
// case passed every pinned expectation.
|
||||
type CaseResult struct {
|
||||
Case Case
|
||||
Failures []string
|
||||
}
|
||||
|
||||
// Passed reports whether the case met every expectation.
|
||||
func (r CaseResult) Passed() bool {
|
||||
return len(r.Failures) == 0
|
||||
}
|
||||
|
||||
// Report aggregates results across a case set.
|
||||
type Report struct {
|
||||
Results []CaseResult
|
||||
}
|
||||
|
||||
// Passed counts cases that met every expectation.
|
||||
func (r Report) Passed() int {
|
||||
n := 0
|
||||
|
||||
for _, c := range r.Results {
|
||||
if c.Passed() {
|
||||
n++
|
||||
}
|
||||
}
|
||||
|
||||
return n
|
||||
}
|
||||
|
||||
// Accuracy is the fraction of cases that passed, in [0, 1].
|
||||
func (r Report) Accuracy() float64 {
|
||||
if len(r.Results) == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
return float64(r.Passed()) / float64(len(r.Results))
|
||||
}
|
||||
|
||||
// Evaluate runs every case through the ranker and checks its pinned
|
||||
// expectations (ExpectTop, MaxScore ceilings, MinScore floors),
|
||||
// returning a Report the caller can assert on and print.
|
||||
func Evaluate(cases []Case, ranker Ranker) Report {
|
||||
rep := Report{Results: make([]CaseResult, 0, len(cases))}
|
||||
|
||||
for _, c := range cases {
|
||||
rep.Results = append(rep.Results, evaluateCase(c, ranker))
|
||||
}
|
||||
|
||||
return rep
|
||||
}
|
||||
|
||||
func evaluateCase(c Case, ranker Ranker) CaseResult {
|
||||
ranked := ranker.Rank(c)
|
||||
|
||||
res := CaseResult{Case: c}
|
||||
|
||||
byMBID := make(map[string]float64, len(ranked))
|
||||
for _, r := range ranked {
|
||||
byMBID[r.MBID] = r.Score
|
||||
}
|
||||
|
||||
if c.ExpectTop != "" {
|
||||
switch {
|
||||
case len(ranked) == 0:
|
||||
res.Failures = append(
|
||||
res.Failures,
|
||||
"expected top "+c.ExpectTop+" but ranking was empty",
|
||||
)
|
||||
case ranked[0].MBID != c.ExpectTop:
|
||||
res.Failures = append(res.Failures, fmt.Sprintf(
|
||||
"top = %q (%.3f), want %q (%.3f)",
|
||||
ranked[0].MBID, ranked[0].Score, c.ExpectTop, byMBID[c.ExpectTop],
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
for mbid, ceil := range c.MaxScore {
|
||||
if got, ok := byMBID[mbid]; ok && got > ceil {
|
||||
res.Failures = append(res.Failures, fmt.Sprintf(
|
||||
"%s scored %.3f, want <= %.3f", mbid, got, ceil,
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
for mbid, floor := range c.MinScore {
|
||||
if got, ok := byMBID[mbid]; ok && got < floor {
|
||||
res.Failures = append(res.Failures, fmt.Sprintf(
|
||||
"%s scored %.3f, want >= %.3f", mbid, got, floor,
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
+225
@@ -0,0 +1,225 @@
|
||||
[
|
||||
{
|
||||
"note": "seed: single with a generic title must reject a same-title, different-artist, wrong-length MB hit and prefer the correct artist. This is the reported 92%-false-positive class.",
|
||||
"local": [
|
||||
{ "title": "Intro", "artist": "Real Artist", "track": 1, "lengthMs": 90000 }
|
||||
],
|
||||
"candidates": [
|
||||
{
|
||||
"mbid": "right",
|
||||
"artistCredit": "Real Artist",
|
||||
"status": "Official",
|
||||
"source": "musicbrainz",
|
||||
"tracks": [{ "pos": 1, "title": "Intro", "lengthMs": 90000 }]
|
||||
},
|
||||
{
|
||||
"mbid": "wrong-artist",
|
||||
"artistCredit": "Some Other Band",
|
||||
"status": "Official",
|
||||
"source": "musicbrainz",
|
||||
"tracks": [{ "pos": 1, "title": "Intro", "lengthMs": 240000 }]
|
||||
}
|
||||
],
|
||||
"expectTop": "right",
|
||||
"maxScore": { "wrong-artist": 0.75 }
|
||||
},
|
||||
{
|
||||
"note": "seed: full-album exact match should still read as a confident, near-perfect match (evidence scaling must not punish real albums).",
|
||||
"local": [
|
||||
{ "title": "Song A", "artist": "The Band", "track": 1, "lengthMs": 200000 },
|
||||
{ "title": "Song B", "artist": "The Band", "track": 2, "lengthMs": 210000 },
|
||||
{ "title": "Song C", "artist": "The Band", "track": 3, "lengthMs": 195000 },
|
||||
{ "title": "Song D", "artist": "The Band", "track": 4, "lengthMs": 220000 }
|
||||
],
|
||||
"candidates": [
|
||||
{
|
||||
"mbid": "album",
|
||||
"artistCredit": "The Band",
|
||||
"status": "Official",
|
||||
"source": "musicbrainz",
|
||||
"tracks": [
|
||||
{ "pos": 1, "title": "Song A", "lengthMs": 200000 },
|
||||
{ "pos": 2, "title": "Song B", "lengthMs": 210000 },
|
||||
{ "pos": 3, "title": "Song C", "lengthMs": 195000 },
|
||||
{ "pos": 4, "title": "Song D", "lengthMs": 220000 }
|
||||
]
|
||||
}
|
||||
],
|
||||
"expectTop": "album",
|
||||
"minScore": { "album": 0.9 }
|
||||
},
|
||||
{
|
||||
"note": "seed: near-right artist (accent/spelling) must stay a strong match; soft artist term, not a gate.",
|
||||
"local": [
|
||||
{ "title": "Jolene", "artist": "Beyonce", "track": 1, "lengthMs": 200000 },
|
||||
{ "title": "Halo", "artist": "Beyonce", "track": 2, "lengthMs": 220000 }
|
||||
],
|
||||
"candidates": [
|
||||
{
|
||||
"mbid": "accented",
|
||||
"artistCredit": "Beyoncé",
|
||||
"status": "Official",
|
||||
"source": "musicbrainz",
|
||||
"tracks": [
|
||||
{ "pos": 1, "title": "Jolene", "lengthMs": 200000 },
|
||||
{ "pos": 2, "title": "Halo", "lengthMs": 220000 }
|
||||
]
|
||||
}
|
||||
],
|
||||
"expectTop": "accented",
|
||||
"minScore": { "accented": 0.85 }
|
||||
},
|
||||
{
|
||||
"note": "VA compilation: per-track artists all differ, album-artist says Various Artists. The VA candidate must not be penalised for its artist credit and must read as a strong match.",
|
||||
"albumName": "Now That's What I Call Music! 60",
|
||||
"albumArtist": "Various Artists",
|
||||
"local": [
|
||||
{ "title": "Song One", "artist": "Artist A", "track": 1, "lengthMs": 200000 },
|
||||
{ "title": "Song Two", "artist": "Artist B", "track": 2, "lengthMs": 210000 },
|
||||
{ "title": "Song Three", "artist": "Artist C", "track": 3, "lengthMs": 195000 },
|
||||
{ "title": "Song Four", "artist": "Artist D", "track": 4, "lengthMs": 205000 }
|
||||
],
|
||||
"candidates": [
|
||||
{
|
||||
"mbid": "va-comp",
|
||||
"title": "Now That's What I Call Music! 60",
|
||||
"artistCredit": "Various Artists",
|
||||
"status": "Official",
|
||||
"primaryType": "Compilation",
|
||||
"source": "musicbrainz",
|
||||
"tracks": [
|
||||
{ "pos": 1, "title": "Song One", "lengthMs": 200000 },
|
||||
{ "pos": 2, "title": "Song Two", "lengthMs": 210000 },
|
||||
{ "pos": 3, "title": "Song Three", "lengthMs": 195000 },
|
||||
{ "pos": 4, "title": "Song Four", "lengthMs": 205000 }
|
||||
]
|
||||
}
|
||||
],
|
||||
"expectTop": "va-comp",
|
||||
"minScore": { "va-comp": 0.85 }
|
||||
},
|
||||
{
|
||||
"note": "same recordings, two release groups: folder album name must pull the studio album above the greatest-hits comp with an identical tracklist.",
|
||||
"albumName": "The Studio Album",
|
||||
"albumArtist": "The Band",
|
||||
"local": [
|
||||
{ "title": "Song A", "artist": "The Band", "track": 1, "lengthMs": 200000 },
|
||||
{ "title": "Song B", "artist": "The Band", "track": 2, "lengthMs": 210000 },
|
||||
{ "title": "Song C", "artist": "The Band", "track": 3, "lengthMs": 195000 }
|
||||
],
|
||||
"candidates": [
|
||||
{
|
||||
"mbid": "studio",
|
||||
"title": "The Studio Album",
|
||||
"artistCredit": "The Band",
|
||||
"status": "Official",
|
||||
"primaryType": "Album",
|
||||
"source": "musicbrainz",
|
||||
"tracks": [
|
||||
{ "pos": 1, "title": "Song A", "lengthMs": 200000 },
|
||||
{ "pos": 2, "title": "Song B", "lengthMs": 210000 },
|
||||
{ "pos": 3, "title": "Song C", "lengthMs": 195000 }
|
||||
]
|
||||
},
|
||||
{
|
||||
"mbid": "hits-comp",
|
||||
"title": "Greatest Hits",
|
||||
"artistCredit": "The Band",
|
||||
"status": "Official",
|
||||
"primaryType": "Compilation",
|
||||
"source": "musicbrainz",
|
||||
"tracks": [
|
||||
{ "pos": 1, "title": "Song A", "lengthMs": 200000 },
|
||||
{ "pos": 2, "title": "Song B", "lengthMs": 210000 },
|
||||
{ "pos": 3, "title": "Song C", "lengthMs": 195000 }
|
||||
]
|
||||
}
|
||||
],
|
||||
"expectTop": "studio",
|
||||
"minScore": { "studio": 0.9 }
|
||||
},
|
||||
{
|
||||
"note": "disc 1 of a 2-disc release: per-disc group must not be punished for 'missing' disc 2, and must not lose to a random single-disc release with fewer matching tracks.",
|
||||
"albumName": "The Double Album",
|
||||
"albumArtist": "The Band",
|
||||
"local": [
|
||||
{ "title": "D1 Track 1", "artist": "The Band", "track": 1, "disc": 1, "lengthMs": 200000 },
|
||||
{ "title": "D1 Track 2", "artist": "The Band", "track": 2, "disc": 1, "lengthMs": 210000 },
|
||||
{ "title": "D1 Track 3", "artist": "The Band", "track": 3, "disc": 1, "lengthMs": 195000 }
|
||||
],
|
||||
"candidates": [
|
||||
{
|
||||
"mbid": "double",
|
||||
"title": "The Double Album",
|
||||
"artistCredit": "The Band",
|
||||
"status": "Official",
|
||||
"primaryType": "Album",
|
||||
"source": "musicbrainz",
|
||||
"tracks": [
|
||||
{ "pos": 1, "disc": 1, "title": "D1 Track 1", "lengthMs": 200000 },
|
||||
{ "pos": 2, "disc": 1, "title": "D1 Track 2", "lengthMs": 210000 },
|
||||
{ "pos": 3, "disc": 1, "title": "D1 Track 3", "lengthMs": 195000 },
|
||||
{ "pos": 1, "disc": 2, "title": "D2 Track 1", "lengthMs": 220000 },
|
||||
{ "pos": 2, "disc": 2, "title": "D2 Track 2", "lengthMs": 230000 },
|
||||
{ "pos": 3, "disc": 2, "title": "D2 Track 3", "lengthMs": 240000 }
|
||||
]
|
||||
}
|
||||
],
|
||||
"expectTop": "double",
|
||||
"minScore": { "double": 0.9 }
|
||||
},
|
||||
{
|
||||
"note": "streaming-style dash qualifiers: '- 2011 Remaster' suffixes on every local title must not cost title score against the clean MB tracklist.",
|
||||
"albumName": "Classic Album",
|
||||
"albumArtist": "The Band",
|
||||
"local": [
|
||||
{ "title": "Opener - 2011 Remaster", "artist": "The Band", "track": 1, "lengthMs": 200000 },
|
||||
{ "title": "Middle Cut - 2011 Remaster", "artist": "The Band", "track": 2, "lengthMs": 210000 },
|
||||
{ "title": "Closer - 2011 Remaster", "artist": "The Band", "track": 3, "lengthMs": 195000 }
|
||||
],
|
||||
"candidates": [
|
||||
{
|
||||
"mbid": "clean-titles",
|
||||
"title": "Classic Album",
|
||||
"artistCredit": "The Band",
|
||||
"status": "Official",
|
||||
"primaryType": "Album",
|
||||
"source": "musicbrainz",
|
||||
"tracks": [
|
||||
{ "pos": 1, "title": "Opener", "lengthMs": 200000 },
|
||||
{ "pos": 2, "title": "Middle Cut", "lengthMs": 210000 },
|
||||
{ "pos": 3, "title": "Closer", "lengthMs": 195000 }
|
||||
]
|
||||
}
|
||||
],
|
||||
"expectTop": "clean-titles",
|
||||
"minScore": { "clean-titles": 0.9 }
|
||||
},
|
||||
{
|
||||
"note": "sorted-artist tag: 'Beatles, The' must match 'The Beatles' releases at full strength (article rotation).",
|
||||
"albumName": "Abbey Road",
|
||||
"albumArtist": "Beatles, The",
|
||||
"local": [
|
||||
{ "title": "Come Together", "artist": "Beatles, The", "track": 1, "lengthMs": 259000 },
|
||||
{ "title": "Something", "artist": "Beatles, The", "track": 2, "lengthMs": 182000 },
|
||||
{ "title": "Octopus's Garden", "artist": "Beatles, The", "track": 3, "lengthMs": 170000 }
|
||||
],
|
||||
"candidates": [
|
||||
{
|
||||
"mbid": "abbey",
|
||||
"title": "Abbey Road",
|
||||
"artistCredit": "The Beatles",
|
||||
"status": "Official",
|
||||
"primaryType": "Album",
|
||||
"source": "musicbrainz",
|
||||
"tracks": [
|
||||
{ "pos": 1, "title": "Come Together", "lengthMs": 259000 },
|
||||
{ "pos": 2, "title": "Something", "lengthMs": 182000 },
|
||||
{ "pos": 3, "title": "Octopus's Garden", "lengthMs": 170000 }
|
||||
]
|
||||
}
|
||||
],
|
||||
"expectTop": "abbey",
|
||||
"minScore": { "abbey": 0.9 }
|
||||
}
|
||||
]
|
||||
Reference in New Issue
Block a user