feat: autotag scoring overhaul, dump-based explore index, and lyrics search
Consolidates in-progress work across autotag, explore, and library: - autotag: beets/Picard-informed scoring engine — ID-first matching, VA handling, recommendation tiers, and a merged distance/rank cascade, with an eval harness for regression tracking. - explore: offline MusicBrainz dump import/incremental refresh replaces the legacy tier crawl; index-first local search with fuzzy matching and a dedicated ranker; disk-free guards for dump downloads. - library: artist-credit extraction and matching. - lyrics: owned-library lyric search (FTS) with LRCLIB backfill. Also: rewrite README to be user-focused, and migrate upstream to git.ljones.me/yonlu/yellowjacket. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
@@ -0,0 +1,115 @@
|
||||
// Package eval is the search-ranking evaluation harness. It turns
|
||||
// "this query feels wrong" into a number that goes up or down, so a
|
||||
// ranking change can be validated against a frozen set of labelled
|
||||
// queries instead of tuned by anecdote.
|
||||
//
|
||||
// The harness is deliberately decoupled from the explore package: it
|
||||
// knows nothing about MusicBrainz, ListenBrainz, or the search index.
|
||||
// A caller adapts whatever ranking function it wants to measure to the
|
||||
// Ranker interface, loads a fixture set, and runs Evaluate. The
|
||||
// explore package wires its real index Search to this in an
|
||||
// integration test (see explore/eval_harness_test.go).
|
||||
package eval
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ErrNoFixtures is returned when a fixture file contains zero queries.
|
||||
var ErrNoFixtures = errors.New("eval: fixture set is empty")
|
||||
|
||||
// Result is one ranked search hit, reduced to the only two fields the
|
||||
// harness needs to decide whether it matches an expectation.
|
||||
type Result struct {
|
||||
EntityType string `json:"entityType"`
|
||||
MBID string `json:"mbid"`
|
||||
}
|
||||
|
||||
// Ranker produces an ordered result list for a query. Best result
|
||||
// first. Implemented by adapting a real search function.
|
||||
type Ranker interface {
|
||||
Rank(query string, limit int) []Result
|
||||
}
|
||||
|
||||
// RankerFunc adapts a plain function to the Ranker interface.
|
||||
type RankerFunc func(query string, limit int) []Result
|
||||
|
||||
// Rank calls the underlying function.
|
||||
func (f RankerFunc) Rank(query string, limit int) []Result {
|
||||
return f(query, limit)
|
||||
}
|
||||
|
||||
// Expected is one acceptable result for a fixture query. Grade is the
|
||||
// graded-relevance weight used by nDCG (higher = more relevant); it
|
||||
// defaults to 1 when omitted. Type is optional — when set, a ranked
|
||||
// result must match both MBID and entity type to count as a hit.
|
||||
type Expected struct {
|
||||
Type string `json:"type,omitempty"`
|
||||
MBID string `json:"mbid"`
|
||||
Grade int `json:"grade,omitempty"`
|
||||
}
|
||||
|
||||
// Fixture is a single labelled query: the input plus the result(s) a
|
||||
// user should get. Every edge case ever hand-fixed in the ranker
|
||||
// belongs here so it can never silently regress.
|
||||
type Fixture struct {
|
||||
Query string `json:"query"`
|
||||
Note string `json:"note,omitempty"`
|
||||
Expect []Expected `json:"expect"`
|
||||
}
|
||||
|
||||
// LoadFixtures reads a JSON fixture file from disk.
|
||||
func LoadFixtures(path string) ([]Fixture, error) {
|
||||
f, err := os.Open(path) //nolint:gosec // path is a test fixture, not user input
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("eval: open fixtures: %w", err)
|
||||
}
|
||||
|
||||
defer func() { _ = f.Close() }()
|
||||
|
||||
return ParseFixtures(f)
|
||||
}
|
||||
|
||||
// ParseFixtures decodes a JSON fixture set from a reader.
|
||||
func ParseFixtures(r io.Reader) ([]Fixture, error) {
|
||||
var fixtures []Fixture
|
||||
|
||||
if err := json.NewDecoder(r).Decode(&fixtures); err != nil {
|
||||
return nil, fmt.Errorf("eval: decode fixtures: %w", err)
|
||||
}
|
||||
|
||||
if len(fixtures) == 0 {
|
||||
return nil, ErrNoFixtures
|
||||
}
|
||||
|
||||
return fixtures, nil
|
||||
}
|
||||
|
||||
// matches reports whether a ranked result satisfies an expectation.
|
||||
// MBID match is required; entity type is checked only when the
|
||||
// expectation pins one.
|
||||
func (e Expected) matches(r Result) bool {
|
||||
if !strings.EqualFold(e.MBID, r.MBID) {
|
||||
return false
|
||||
}
|
||||
|
||||
if e.Type != "" && !strings.EqualFold(e.Type, r.EntityType) {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// grade returns the graded-relevance weight, defaulting to 1.
|
||||
func (e Expected) grade() int {
|
||||
if e.Grade <= 0 {
|
||||
return 1
|
||||
}
|
||||
|
||||
return e.Grade
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
package eval
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// QueryScore holds the per-query metrics for one fixture.
|
||||
type QueryScore struct {
|
||||
Query string
|
||||
Note string
|
||||
|
||||
// BestRank is the 1-based rank of the highest-placed expected
|
||||
// result, or 0 if none of the expected results appear in topK.
|
||||
BestRank int
|
||||
|
||||
ReciprocalRank float64
|
||||
PrecisionAtK float64
|
||||
NDCGAtK float64
|
||||
}
|
||||
|
||||
// Hit reports whether any expected result landed in topK.
|
||||
func (q QueryScore) Hit() bool {
|
||||
return q.BestRank > 0
|
||||
}
|
||||
|
||||
// Report aggregates per-query scores into the numbers you watch across
|
||||
// a ranking change: mean reciprocal rank, mean precision@k, mean
|
||||
// nDCG@k, plus the list of queries that missed entirely.
|
||||
type Report struct {
|
||||
K int
|
||||
NumQueries int
|
||||
|
||||
MRR float64
|
||||
MeanPAtK float64
|
||||
MeanNDCG float64
|
||||
HitRate float64 // fraction of queries with any expected result in topK
|
||||
Top1Rate float64 // fraction whose best expected result is rank 1
|
||||
PerQuery []QueryScore
|
||||
}
|
||||
|
||||
// Evaluate runs every fixture through the ranker and aggregates the
|
||||
// results into a Report. topK bounds how deep a result can be and
|
||||
// still count (a result at rank 20 helps no one).
|
||||
func Evaluate(r Ranker, fixtures []Fixture, topK int) Report {
|
||||
if topK <= 0 {
|
||||
topK = 5
|
||||
}
|
||||
|
||||
report := Report{K: topK, NumQueries: len(fixtures)}
|
||||
|
||||
for _, fx := range fixtures {
|
||||
ranked := r.Rank(fx.Query, topK)
|
||||
report.PerQuery = append(report.PerQuery, scoreQuery(fx, ranked, topK))
|
||||
}
|
||||
|
||||
for _, q := range report.PerQuery {
|
||||
report.MRR += q.ReciprocalRank
|
||||
report.MeanPAtK += q.PrecisionAtK
|
||||
report.MeanNDCG += q.NDCGAtK
|
||||
|
||||
if q.Hit() {
|
||||
report.HitRate++
|
||||
}
|
||||
|
||||
if q.BestRank == 1 {
|
||||
report.Top1Rate++
|
||||
}
|
||||
}
|
||||
|
||||
if n := float64(len(fixtures)); n > 0 {
|
||||
report.MRR /= n
|
||||
report.MeanPAtK /= n
|
||||
report.MeanNDCG /= n
|
||||
report.HitRate /= n
|
||||
report.Top1Rate /= n
|
||||
}
|
||||
|
||||
return report
|
||||
}
|
||||
|
||||
// scoreQuery computes the metrics for a single fixture against a ranked
|
||||
// result list.
|
||||
func scoreQuery(fx Fixture, ranked []Result, topK int) QueryScore {
|
||||
score := QueryScore{Query: fx.Query, Note: fx.Note}
|
||||
|
||||
limit := min(topK, len(ranked))
|
||||
|
||||
relevantInK := 0
|
||||
|
||||
for i := range limit {
|
||||
if !anyMatch(fx.Expect, ranked[i]) {
|
||||
continue
|
||||
}
|
||||
|
||||
relevantInK++
|
||||
|
||||
if score.BestRank == 0 {
|
||||
score.BestRank = i + 1
|
||||
score.ReciprocalRank = 1.0 / float64(i+1)
|
||||
}
|
||||
}
|
||||
|
||||
score.PrecisionAtK = float64(relevantInK) / float64(topK)
|
||||
score.NDCGAtK = ndcg(fx.Expect, ranked, topK)
|
||||
|
||||
return score
|
||||
}
|
||||
|
||||
// anyMatch reports whether a result satisfies any expectation.
|
||||
func anyMatch(expected []Expected, r Result) bool {
|
||||
for _, e := range expected {
|
||||
if e.matches(r) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// ndcg computes normalized discounted cumulative gain at k using graded
|
||||
// relevance. Returns 0 when there are no expected results.
|
||||
func ndcg(expected []Expected, ranked []Result, k int) float64 {
|
||||
ideal := idealDCG(expected, k)
|
||||
if ideal == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
limit := min(k, len(ranked))
|
||||
|
||||
dcg := 0.0
|
||||
|
||||
for i := range limit {
|
||||
g := matchedGrade(expected, ranked[i])
|
||||
if g == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
dcg += gain(g, i)
|
||||
}
|
||||
|
||||
return dcg / ideal
|
||||
}
|
||||
|
||||
// matchedGrade returns the relevance grade for a result, or 0 if it
|
||||
// matches no expectation.
|
||||
func matchedGrade(expected []Expected, r Result) int {
|
||||
for _, e := range expected {
|
||||
if e.matches(r) {
|
||||
return e.grade()
|
||||
}
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
// idealDCG is the DCG of the best possible ordering: every expected
|
||||
// result, sorted by grade descending, placed at the front.
|
||||
func idealDCG(expected []Expected, k int) float64 {
|
||||
grades := make([]int, 0, len(expected))
|
||||
for _, e := range expected {
|
||||
grades = append(grades, e.grade())
|
||||
}
|
||||
|
||||
sort.Sort(sort.Reverse(sort.IntSlice(grades)))
|
||||
|
||||
limit := min(k, len(grades))
|
||||
|
||||
ideal := 0.0
|
||||
for i := range limit {
|
||||
ideal += gain(grades[i], i)
|
||||
}
|
||||
|
||||
return ideal
|
||||
}
|
||||
|
||||
// gain is the discounted gain of a grade at 0-based position i.
|
||||
func gain(grade, i int) float64 {
|
||||
return (math.Pow(2, float64(grade)) - 1) / math.Log2(float64(i+2))
|
||||
}
|
||||
|
||||
// Format renders a Report as a human-readable table for test output.
|
||||
func (r Report) Format() string {
|
||||
var b strings.Builder
|
||||
|
||||
fmt.Fprintf(&b, "ranking eval — %d queries @k=%d\n", r.NumQueries, r.K)
|
||||
fmt.Fprintf(&b, " MRR %.3f\n", r.MRR)
|
||||
fmt.Fprintf(&b, " P@%d %.3f\n", r.K, r.MeanPAtK)
|
||||
fmt.Fprintf(&b, " nDCG@%d %.3f\n", r.K, r.MeanNDCG)
|
||||
fmt.Fprintf(&b, " hit rate %.3f\n", r.HitRate)
|
||||
fmt.Fprintf(&b, " top-1 rate %.3f\n", r.Top1Rate)
|
||||
|
||||
misses := r.Misses()
|
||||
if len(misses) > 0 {
|
||||
b.WriteString(" misses:\n")
|
||||
|
||||
for _, m := range misses {
|
||||
fmt.Fprintf(&b, " %-40q rank=%s\n", m.Query, rankLabel(m.BestRank))
|
||||
}
|
||||
}
|
||||
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// Misses returns the queries whose best expected result was absent
|
||||
// from topK or buried below rank 1 — the regression watch-list.
|
||||
func (r Report) Misses() []QueryScore {
|
||||
var out []QueryScore
|
||||
|
||||
for _, q := range r.PerQuery {
|
||||
if q.BestRank != 1 {
|
||||
out = append(out, q)
|
||||
}
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func rankLabel(rank int) string {
|
||||
if rank == 0 {
|
||||
return "absent"
|
||||
}
|
||||
|
||||
return strconv.Itoa(rank)
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
package eval
|
||||
|
||||
import (
|
||||
"math"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// rankerFromIDs builds a Ranker that returns a fixed ordering keyed by
|
||||
// query, for deterministic metric tests.
|
||||
func rankerFromIDs(table map[string][]Result) Ranker {
|
||||
return RankerFunc(func(query string, limit int) []Result {
|
||||
out := table[query]
|
||||
if limit < len(out) {
|
||||
out = out[:limit]
|
||||
}
|
||||
|
||||
return out
|
||||
})
|
||||
}
|
||||
|
||||
func approx(a, b float64) bool {
|
||||
return math.Abs(a-b) < 1e-9
|
||||
}
|
||||
|
||||
func TestReciprocalRank(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
ranked []Result
|
||||
expect []Expected
|
||||
wantRR float64
|
||||
wantPos int
|
||||
}{
|
||||
{
|
||||
name: "top result",
|
||||
ranked: []Result{{MBID: "a"}, {MBID: "b"}},
|
||||
expect: []Expected{{MBID: "a"}},
|
||||
wantRR: 1.0,
|
||||
wantPos: 1,
|
||||
},
|
||||
{
|
||||
name: "third result",
|
||||
ranked: []Result{{MBID: "x"}, {MBID: "y"}, {MBID: "a"}},
|
||||
expect: []Expected{{MBID: "a"}},
|
||||
wantRR: 1.0 / 3.0,
|
||||
wantPos: 3,
|
||||
},
|
||||
{
|
||||
name: "absent",
|
||||
ranked: []Result{{MBID: "x"}, {MBID: "y"}},
|
||||
expect: []Expected{{MBID: "a"}},
|
||||
wantRR: 0,
|
||||
wantPos: 0,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
fx := Fixture{Query: "q", Expect: tt.expect}
|
||||
got := scoreQuery(fx, tt.ranked, 5)
|
||||
|
||||
if !approx(got.ReciprocalRank, tt.wantRR) {
|
||||
t.Errorf("RR = %v, want %v", got.ReciprocalRank, tt.wantRR)
|
||||
}
|
||||
|
||||
if got.BestRank != tt.wantPos {
|
||||
t.Errorf("BestRank = %d, want %d", got.BestRank, tt.wantPos)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrecisionAtK(t *testing.T) {
|
||||
fx := Fixture{
|
||||
Query: "q",
|
||||
Expect: []Expected{{MBID: "a"}, {MBID: "b"}},
|
||||
}
|
||||
ranked := []Result{{MBID: "a"}, {MBID: "x"}, {MBID: "b"}, {MBID: "y"}}
|
||||
|
||||
got := scoreQuery(fx, ranked, 4)
|
||||
|
||||
// 2 relevant out of k=4.
|
||||
if !approx(got.PrecisionAtK, 0.5) {
|
||||
t.Errorf("P@4 = %v, want 0.5", got.PrecisionAtK)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNDCGRespectsOrdering(t *testing.T) {
|
||||
expect := []Expected{{MBID: "a", Grade: 3}, {MBID: "b", Grade: 1}}
|
||||
|
||||
// Ideal ordering: high-grade result first.
|
||||
good := scoreQuery(
|
||||
Fixture{Query: "q", Expect: expect},
|
||||
[]Result{{MBID: "a"}, {MBID: "b"}, {MBID: "z"}},
|
||||
5,
|
||||
)
|
||||
|
||||
// Worse ordering: high-grade result buried below an irrelevant one.
|
||||
bad := scoreQuery(
|
||||
Fixture{Query: "q", Expect: expect},
|
||||
[]Result{{MBID: "z"}, {MBID: "b"}, {MBID: "a"}},
|
||||
5,
|
||||
)
|
||||
|
||||
if !approx(good.NDCGAtK, 1.0) {
|
||||
t.Errorf("ideal ordering nDCG = %v, want 1.0", good.NDCGAtK)
|
||||
}
|
||||
|
||||
if bad.NDCGAtK >= good.NDCGAtK {
|
||||
t.Errorf("worse ordering nDCG %v should be < ideal %v", bad.NDCGAtK, good.NDCGAtK)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTypeMustMatchWhenPinned(t *testing.T) {
|
||||
fx := Fixture{
|
||||
Query: "q",
|
||||
Expect: []Expected{{Type: "artist", MBID: "a"}},
|
||||
}
|
||||
|
||||
// Same MBID but wrong entity type — must not count.
|
||||
wrongType := scoreQuery(fx, []Result{{EntityType: "recording", MBID: "a"}}, 5)
|
||||
if wrongType.Hit() {
|
||||
t.Error("result with wrong entity type counted as a hit")
|
||||
}
|
||||
|
||||
rightType := scoreQuery(fx, []Result{{EntityType: "artist", MBID: "a"}}, 5)
|
||||
if !rightType.Hit() {
|
||||
t.Error("result with matching entity type did not count")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluateAggregates(t *testing.T) {
|
||||
fixtures := []Fixture{
|
||||
{Query: "hit-top", Expect: []Expected{{MBID: "a"}}},
|
||||
{Query: "hit-second", Expect: []Expected{{MBID: "a"}}},
|
||||
{Query: "miss", Expect: []Expected{{MBID: "a"}}},
|
||||
}
|
||||
|
||||
r := rankerFromIDs(map[string][]Result{
|
||||
"hit-top": {{MBID: "a"}},
|
||||
"hit-second": {{MBID: "x"}, {MBID: "a"}},
|
||||
"miss": {{MBID: "x"}, {MBID: "y"}},
|
||||
})
|
||||
|
||||
report := Evaluate(r, fixtures, 5)
|
||||
|
||||
// MRR = (1 + 1/2 + 0) / 3.
|
||||
wantMRR := (1.0 + 0.5 + 0.0) / 3.0
|
||||
if !approx(report.MRR, wantMRR) {
|
||||
t.Errorf("MRR = %v, want %v", report.MRR, wantMRR)
|
||||
}
|
||||
|
||||
// 2 of 3 queries surfaced the result somewhere in topK.
|
||||
if !approx(report.HitRate, 2.0/3.0) {
|
||||
t.Errorf("HitRate = %v, want %v", report.HitRate, 2.0/3.0)
|
||||
}
|
||||
|
||||
// Only 1 of 3 had it at rank 1.
|
||||
if !approx(report.Top1Rate, 1.0/3.0) {
|
||||
t.Errorf("Top1Rate = %v, want %v", report.Top1Rate, 1.0/3.0)
|
||||
}
|
||||
|
||||
if len(report.Misses()) != 2 {
|
||||
t.Errorf("Misses = %d, want 2", len(report.Misses()))
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFixtures(t *testing.T) {
|
||||
const doc = `[
|
||||
{"query": "radiohead", "expect": [{"type": "artist", "mbid": "abc"}]},
|
||||
{"query": "ok computer", "note": "album not band", "expect": [{"mbid": "def", "grade": 2}]}
|
||||
]`
|
||||
|
||||
fixtures, err := ParseFixtures(strings.NewReader(doc))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFixtures: %v", err)
|
||||
}
|
||||
|
||||
if len(fixtures) != 2 {
|
||||
t.Fatalf("got %d fixtures, want 2", len(fixtures))
|
||||
}
|
||||
|
||||
if fixtures[0].Expect[0].MBID != "abc" {
|
||||
t.Errorf("MBID = %q, want abc", fixtures[0].Expect[0].MBID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFixturesEmpty(t *testing.T) {
|
||||
_, err := ParseFixtures(strings.NewReader(`[]`))
|
||||
if err == nil {
|
||||
t.Fatal("expected ErrNoFixtures, got nil")
|
||||
}
|
||||
}
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
[
|
||||
{
|
||||
"query": "radiohead",
|
||||
"note": "single-word artist name — should resolve to the artist, not an album titled similarly",
|
||||
"expect": [{ "type": "artist", "mbid": "a74b1b7f-71a5-4011-9441-d0b5e4122711" }]
|
||||
},
|
||||
{
|
||||
"query": "the beatles",
|
||||
"note": "common-word prefix must not let the article dominate",
|
||||
"expect": [{ "type": "artist", "mbid": "b10bbbfc-cf9e-42e0-be17-e2c3e1d2600d" }]
|
||||
},
|
||||
{
|
||||
"query": "abbey road",
|
||||
"note": "album title — should rank the release group above any track of the same name",
|
||||
"expect": [{ "type": "release_group", "mbid": "" }]
|
||||
},
|
||||
{
|
||||
"query": "calling you blue october",
|
||||
"note": "composite title+artist query — recording should win even if the artist is unindexed",
|
||||
"expect": [{ "type": "recording", "mbid": "" }]
|
||||
},
|
||||
{
|
||||
"query": "the teenagers",
|
||||
"note": "regression: must rank The Teenagers above The Beatles despite far lower popularity",
|
||||
"expect": [{ "type": "artist", "mbid": "" }]
|
||||
},
|
||||
{
|
||||
"query": "beyonce",
|
||||
"note": "diacritic folding (migration 37): unaccented query must find the accented artist Beyoncé",
|
||||
"expect": [{ "type": "artist", "mbid": "859d0860-d480-4efd-970c-c05d5f1776b8" }]
|
||||
}
|
||||
]
|
||||
Reference in New Issue
Block a user