feat: autotag scoring overhaul, dump-based explore index, and lyrics search
Consolidates in-progress work across autotag, explore, and library: - autotag: beets/Picard-informed scoring engine — ID-first matching, VA handling, recommendation tiers, and a merged distance/rank cascade, with an eval harness for regression tracking. - explore: offline MusicBrainz dump import/incremental refresh replaces the legacy tier crawl; index-first local search with fuzzy matching and a dedicated ranker; disk-free guards for dump downloads. - library: artist-credit extraction and matching. - lyrics: owned-library lyric search (FTS) with LRCLIB backfill. Also: rewrite README to be user-focused, and migrate upstream to git.ljones.me/yonlu/yellowjacket. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
@@ -95,22 +95,52 @@ func (c *AutotagClient) LookupReleaseGroup(
|
||||
}, nil
|
||||
}
|
||||
|
||||
// LookupArtist returns the artist's sort name when available,
|
||||
// falling back to the display name. Used by the resolver when
|
||||
// constructing Lucene-style fallback queries.
|
||||
func (c *AutotagClient) LookupArtist(
|
||||
ctx context.Context, mbid string,
|
||||
) (string, error) {
|
||||
a, err := c.inner.LookupArtist(ctx, mbid)
|
||||
// SearchRecordings delegates to the wrapped client and projects hits
|
||||
// into autotag's minimal recording shape. Length is millisecond-
|
||||
// aligned to match local audio_files.
|
||||
func (c *AutotagClient) SearchRecordings(
|
||||
ctx context.Context, query string, limit int,
|
||||
) ([]autotag.MBRecordingHit, int, error) {
|
||||
recs, total, err := c.inner.SearchRecordings(ctx, query, limit)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
if a.SortName != "" {
|
||||
return a.SortName, nil
|
||||
out := make([]autotag.MBRecordingHit, 0, len(recs))
|
||||
for _, rec := range recs {
|
||||
out = append(out, autotag.MBRecordingHit{
|
||||
MBID: rec.MBID,
|
||||
Title: rec.Title,
|
||||
ArtistCredit: rec.ArtistCredit,
|
||||
LengthMillis: int64(rec.Length),
|
||||
})
|
||||
}
|
||||
|
||||
return a.Name, nil
|
||||
return out, total, nil
|
||||
}
|
||||
|
||||
// LookupRecordingReleases returns the releases a recording appears on,
|
||||
// as slim references the resolver ranks to pick a representative
|
||||
// release.
|
||||
func (c *AutotagClient) LookupRecordingReleases(
|
||||
ctx context.Context, recordingMBID string,
|
||||
) ([]autotag.MBReleaseRef, error) {
|
||||
refs, err := c.inner.LookupRecordingReleases(ctx, recordingMBID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
out := make([]autotag.MBReleaseRef, 0, len(refs))
|
||||
for _, r := range refs {
|
||||
out = append(out, autotag.MBReleaseRef{
|
||||
MBID: r.MBID,
|
||||
Title: r.Title,
|
||||
Status: r.Status,
|
||||
Date: r.Date,
|
||||
})
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func exploreToAutotagRelease(rel MBRelease) autotag.MBRelease {
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
//go:build unix
|
||||
|
||||
package explore
|
||||
|
||||
import "golang.org/x/sys/unix"
|
||||
|
||||
// diskFreeBytes returns the free bytes available to the current user
|
||||
// on the filesystem containing path. ok is false when unknown.
|
||||
func diskFreeBytes(path string) (free uint64, ok bool) {
|
||||
var st unix.Statfs_t
|
||||
|
||||
if err := unix.Statfs(path, &st); err != nil {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// Bavail/Bsize integer types vary by platform.
|
||||
//nolint:unconvert,gosec
|
||||
return st.Bavail * uint64(st.Bsize), true
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
//go:build windows
|
||||
|
||||
package explore
|
||||
|
||||
import "golang.org/x/sys/windows"
|
||||
|
||||
// diskFreeBytes returns the free bytes available to the current user
|
||||
// on the volume containing path. ok is false when unknown.
|
||||
func diskFreeBytes(path string) (free uint64, ok bool) {
|
||||
p, err := windows.UTF16PtrFromString(path)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
var freeToCaller, total, totalFree uint64
|
||||
|
||||
if err := windows.GetDiskFreeSpaceEx(p, &freeToCaller, &total, &totalFree); err != nil {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
return freeToCaller, true
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,584 @@
|
||||
package explore
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/parquet-go/parquet-go"
|
||||
)
|
||||
|
||||
// Stage 1 of the dump import: stream the ListenBrainz spark listens
|
||||
// dump (a plain tar of ~128MB parquet files) and aggregate listen
|
||||
// counts per recording, release, and artist MBID. Nothing is written
|
||||
// to disk except counts.bin — each parquet member is buffered in RAM,
|
||||
// parsed, and discarded. The aggregate map lives in RAM (~40M entities
|
||||
// ≈ 2GB) and is flushed atomically with the stream byte offset so an
|
||||
// interrupted import resumes without re-downloading processed data.
|
||||
|
||||
const (
|
||||
// countKindRecording etc. tag entries in the counts map/file.
|
||||
countKindRecording = byte(1)
|
||||
countKindRelease = byte(2)
|
||||
countKindArtist = byte(3)
|
||||
|
||||
// countsFlushEveryMembers controls checkpoint frequency. Each
|
||||
// flush rewrites counts.bin (~1GB by the end), so this trades
|
||||
// checkpoint I/O against re-download on crash (~150 members ≈
|
||||
// 19GB of stream progress).
|
||||
countsFlushEveryMembers = 150
|
||||
|
||||
// countsProgressEveryMembers controls progress log frequency.
|
||||
countsProgressEveryMembers = 50
|
||||
|
||||
// parquetParseWorkers is the number of concurrent parquet
|
||||
// decoders. Bounded to limit RAM: each worker holds one
|
||||
// ~128MB member buffer.
|
||||
parquetParseWorkers = 3
|
||||
|
||||
// maxParquetMemberSize guards against unexpected dump format
|
||||
// changes blowing out RAM.
|
||||
maxParquetMemberSize = 1 << 30
|
||||
|
||||
// countsFileMagic identifies + versions the counts file format.
|
||||
countsFileMagic = "YJCNTS01"
|
||||
)
|
||||
|
||||
// ErrDumpFormat is returned when dump contents don't match the
|
||||
// expected format.
|
||||
var ErrDumpFormat = errors.New("unexpected dump format")
|
||||
|
||||
// mbidKey is a parsed UUID plus an entity-kind tag, used as the counts
|
||||
// map key. 17 bytes instead of a 36-byte string keeps the ~40M-entry
|
||||
// map around 2GB.
|
||||
type mbidKey [17]byte
|
||||
|
||||
func makeMBIDKey(kind byte, mbid string) (mbidKey, bool) {
|
||||
var k mbidKey
|
||||
|
||||
k[0] = kind
|
||||
|
||||
if !parseUUID(mbid, k[1:]) {
|
||||
return k, false
|
||||
}
|
||||
|
||||
return k, true
|
||||
}
|
||||
|
||||
// parseUUID parses a canonical 36-char UUID string into 16 bytes.
|
||||
// Returns false for anything malformed.
|
||||
func parseUUID(s string, out []byte) bool {
|
||||
if len(s) != 36 || s[8] != '-' || s[13] != '-' || s[18] != '-' || s[23] != '-' {
|
||||
return false
|
||||
}
|
||||
|
||||
j := 0
|
||||
|
||||
for i := 0; i < 36; i++ {
|
||||
if i == 8 || i == 13 || i == 18 || i == 23 {
|
||||
continue
|
||||
}
|
||||
|
||||
hi := hexNibble(s[i])
|
||||
i++
|
||||
|
||||
lo := hexNibble(s[i])
|
||||
if hi == 0xFF || lo == 0xFF {
|
||||
return false
|
||||
}
|
||||
|
||||
out[j] = hi<<4 | lo
|
||||
j++
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func hexNibble(c byte) byte {
|
||||
switch {
|
||||
case c >= '0' && c <= '9':
|
||||
return c - '0'
|
||||
case c >= 'a' && c <= 'f':
|
||||
return c - 'a' + 10
|
||||
case c >= 'A' && c <= 'F':
|
||||
return c - 'A' + 10
|
||||
default:
|
||||
return 0xFF
|
||||
}
|
||||
}
|
||||
|
||||
func formatUUID(b []byte) string {
|
||||
const hexdigits = "0123456789abcdef"
|
||||
|
||||
out := make([]byte, 36)
|
||||
j := 0
|
||||
|
||||
for i := range 16 {
|
||||
if i == 4 || i == 6 || i == 8 || i == 10 {
|
||||
out[j] = '-'
|
||||
j++
|
||||
}
|
||||
|
||||
out[j] = hexdigits[b[i]>>4]
|
||||
out[j+1] = hexdigits[b[i]&0x0F]
|
||||
j += 2
|
||||
}
|
||||
|
||||
return string(out)
|
||||
}
|
||||
|
||||
// countsState is the checkpointed stage-1 state: the counts map plus
|
||||
// the stream position it corresponds to.
|
||||
type countsState struct {
|
||||
// SparkURL pins the dump being processed so a resume never mixes
|
||||
// two different dumps.
|
||||
SparkURL string `json:"sparkUrl"`
|
||||
|
||||
// Offset is the byte offset of the next unprocessed tar member
|
||||
// header (exact — includes the padding of the previous member).
|
||||
Offset int64 `json:"offset"`
|
||||
|
||||
// MemberIdx is the index of the next unprocessed parquet member
|
||||
// (logging only; Offset is authoritative for resume).
|
||||
MemberIdx int `json:"memberIdx"`
|
||||
|
||||
// Done marks stage 1 complete.
|
||||
Done bool `json:"done"`
|
||||
|
||||
counts map[mbidKey]uint32
|
||||
}
|
||||
|
||||
// sparkListenRow is the projection of the spark listens parquet schema
|
||||
// that the aggregator reads. All other columns are skipped.
|
||||
type sparkListenRow struct {
|
||||
RecordingMBID string `parquet:"recording_mbid,optional"`
|
||||
ReleaseMBID string `parquet:"release_mbid,optional"`
|
||||
ArtistMBIDs []string `parquet:"artist_credit_mbids,optional,list"`
|
||||
}
|
||||
|
||||
type countParseJob struct {
|
||||
idx int
|
||||
endOffset int64 // exact offset of the next member header
|
||||
buf []byte
|
||||
}
|
||||
|
||||
type countParseResult struct {
|
||||
idx int
|
||||
endOffset int64
|
||||
deltas map[mbidKey]uint32
|
||||
err error
|
||||
}
|
||||
|
||||
// aggregateListenCounts runs stage 1 to completion (or ctx cancel),
|
||||
// checkpointing to the staging counts file as it goes.
|
||||
func (imp *dumpImporter) aggregateListenCounts(ctx context.Context, st *countsState) error {
|
||||
if st.counts == nil {
|
||||
st.counts = make(map[mbidKey]uint32, 1<<20)
|
||||
}
|
||||
|
||||
stream := newResumableReader(ctx, imp.httpClient, st.SparkURL, st.Offset)
|
||||
|
||||
defer func() { _ = stream.Close() }()
|
||||
|
||||
buffered := bufio.NewReaderSize(stream, 1<<20)
|
||||
tr := tar.NewReader(buffered)
|
||||
|
||||
// consumedOffset is the absolute stream position of everything the
|
||||
// tar reader has consumed: bytes delivered by HTTP minus bytes
|
||||
// still sitting in the bufio buffer.
|
||||
consumedOffset := func() int64 {
|
||||
return stream.Offset - int64(buffered.Buffered())
|
||||
}
|
||||
|
||||
jobs := make(chan countParseJob)
|
||||
results := make(chan countParseResult, parquetParseWorkers)
|
||||
applierDone := make(chan struct{})
|
||||
bufPool := sync.Pool{New: func() any { return []byte(nil) }}
|
||||
|
||||
var workerWG sync.WaitGroup
|
||||
|
||||
for range parquetParseWorkers {
|
||||
workerWG.Add(1)
|
||||
|
||||
go func() {
|
||||
defer workerWG.Done()
|
||||
|
||||
for job := range jobs {
|
||||
deltas, err := parseListenParquet(job.buf)
|
||||
// Buffer reuse across members is intentional.
|
||||
bufPool.Put(job.buf[:0]) //nolint:staticcheck
|
||||
|
||||
results <- countParseResult{
|
||||
idx: job.idx,
|
||||
endOffset: job.endOffset,
|
||||
deltas: deltas,
|
||||
err: err,
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// The applier merges results into st in member order, so every
|
||||
// checkpoint is a contiguous prefix of the stream. It owns
|
||||
// st.counts, st.Offset, and st.MemberIdx until applierDone closes;
|
||||
// on error it keeps draining results so nothing deadlocks.
|
||||
var applyErr error
|
||||
|
||||
go func() {
|
||||
defer close(applierDone)
|
||||
|
||||
pending := make(map[int]countParseResult)
|
||||
next := st.MemberIdx
|
||||
lastFlushed := st.MemberIdx
|
||||
|
||||
for res := range results {
|
||||
if applyErr != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
pending[res.idx] = res
|
||||
|
||||
for {
|
||||
r, ok := pending[next]
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
|
||||
delete(pending, next)
|
||||
|
||||
if r.err != nil {
|
||||
applyErr = r.err
|
||||
|
||||
break
|
||||
}
|
||||
|
||||
for k, v := range r.deltas {
|
||||
st.counts[k] += v
|
||||
}
|
||||
|
||||
next++
|
||||
st.MemberIdx = next
|
||||
st.Offset = r.endOffset
|
||||
|
||||
if next-lastFlushed >= countsFlushEveryMembers {
|
||||
if err := imp.writeCountsFile(st); err != nil {
|
||||
applyErr = err
|
||||
|
||||
break
|
||||
}
|
||||
|
||||
lastFlushed = next
|
||||
|
||||
imp.logCountsProgress(next, r.endOffset, stream.Size, len(st.counts))
|
||||
|
||||
if err := imp.checkDiskHeadroom(); err != nil {
|
||||
applyErr = err
|
||||
|
||||
break
|
||||
}
|
||||
} else if next%countsProgressEveryMembers == 0 {
|
||||
imp.logCountsProgress(next, r.endOffset, stream.Size, len(st.counts))
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
memberIdx := st.MemberIdx
|
||||
readErr := error(nil)
|
||||
|
||||
readLoop:
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
readErr = err
|
||||
|
||||
break
|
||||
}
|
||||
|
||||
hdr, err := tr.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
readErr = fmt.Errorf("listens tar: %w", err)
|
||||
|
||||
break
|
||||
}
|
||||
|
||||
if hdr.Typeflag != tar.TypeReg || !strings.HasSuffix(hdr.Name, ".parquet") {
|
||||
continue
|
||||
}
|
||||
|
||||
if hdr.Size > maxParquetMemberSize {
|
||||
readErr = fmt.Errorf("%w: parquet member %s is %d bytes", ErrDumpFormat, hdr.Name, hdr.Size)
|
||||
|
||||
break
|
||||
}
|
||||
|
||||
buf, _ := bufPool.Get().([]byte)
|
||||
if cap(buf) < int(hdr.Size) {
|
||||
buf = make([]byte, hdr.Size)
|
||||
}
|
||||
|
||||
buf = buf[:hdr.Size]
|
||||
|
||||
if _, err := io.ReadFull(tr, buf); err != nil {
|
||||
readErr = fmt.Errorf("listens tar member read: %w", err)
|
||||
|
||||
break
|
||||
}
|
||||
|
||||
// Exact next-header offset: position after the entry data plus
|
||||
// the entry's block padding. Correct even when the next member
|
||||
// uses PAX extension headers (those start at its header offset).
|
||||
endOffset := consumedOffset() + tarPadding(hdr.Size)
|
||||
|
||||
select {
|
||||
case jobs <- countParseJob{idx: memberIdx, endOffset: endOffset, buf: buf}:
|
||||
case <-ctx.Done():
|
||||
readErr = ctx.Err()
|
||||
|
||||
break readLoop
|
||||
}
|
||||
|
||||
memberIdx++
|
||||
}
|
||||
|
||||
close(jobs)
|
||||
workerWG.Wait()
|
||||
close(results)
|
||||
<-applierDone
|
||||
|
||||
if readErr == nil {
|
||||
readErr = applyErr
|
||||
}
|
||||
|
||||
if readErr != nil {
|
||||
// Best-effort checkpoint of applied progress before bailing,
|
||||
// so even a cancelled run resumes where it left off.
|
||||
_ = imp.writeCountsFile(st)
|
||||
|
||||
return readErr
|
||||
}
|
||||
|
||||
st.Done = true
|
||||
|
||||
if err := imp.writeCountsFile(st); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
imp.logger.Info("dump import: listen counts complete",
|
||||
"members", st.MemberIdx,
|
||||
"entities", len(st.counts),
|
||||
)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// tarPadding returns the number of zero bytes following a tar entry of
|
||||
// the given size (entries are padded to 512-byte blocks).
|
||||
func tarPadding(size int64) int64 {
|
||||
const block = 512
|
||||
|
||||
return (block - size%block) % block
|
||||
}
|
||||
|
||||
// parseListenParquet decodes one parquet member and returns the
|
||||
// per-entity listen-count deltas.
|
||||
func parseListenParquet(buf []byte) (map[mbidKey]uint32, error) {
|
||||
reader := parquet.NewGenericReader[sparkListenRow](bytes.NewReader(buf))
|
||||
|
||||
defer func() { _ = reader.Close() }()
|
||||
|
||||
deltas := make(map[mbidKey]uint32, 1<<18)
|
||||
rows := make([]sparkListenRow, 4096)
|
||||
|
||||
for {
|
||||
n, err := reader.Read(rows)
|
||||
|
||||
for _, row := range rows[:n] {
|
||||
key, ok := makeMBIDKey(countKindRecording, row.RecordingMBID)
|
||||
if !ok {
|
||||
// Unmapped listen — no usable recording MBID.
|
||||
continue
|
||||
}
|
||||
|
||||
deltas[key]++
|
||||
|
||||
if relKey, relOK := makeMBIDKey(countKindRelease, row.ReleaseMBID); relOK {
|
||||
deltas[relKey]++
|
||||
}
|
||||
|
||||
for _, artist := range row.ArtistMBIDs {
|
||||
if artKey, artOK := makeMBIDKey(countKindArtist, artist); artOK {
|
||||
deltas[artKey]++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parquet read: %w", err)
|
||||
}
|
||||
|
||||
if n == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return deltas, nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// counts.bin persistence
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// writeCountsFile atomically persists the counts map + stream position
|
||||
// (write to temp file, fsync, rename).
|
||||
func (imp *dumpImporter) writeCountsFile(st *countsState) error {
|
||||
tmp := imp.countsPath() + ".tmp"
|
||||
|
||||
f, err := os.Create(tmp)
|
||||
if err != nil {
|
||||
return fmt.Errorf("counts file create: %w", err)
|
||||
}
|
||||
|
||||
w := bufio.NewWriterSize(f, 1<<20)
|
||||
|
||||
meta, err := json.Marshal(st)
|
||||
if err != nil {
|
||||
_ = f.Close()
|
||||
|
||||
return fmt.Errorf("counts meta marshal: %w", err)
|
||||
}
|
||||
|
||||
_, _ = w.WriteString(countsFileMagic)
|
||||
|
||||
var lenBuf [4]byte
|
||||
|
||||
binary.LittleEndian.PutUint32(lenBuf[:], uint32(len(meta)))
|
||||
_, _ = w.Write(lenBuf[:])
|
||||
_, _ = w.Write(meta)
|
||||
|
||||
var rec [21]byte
|
||||
|
||||
for k, v := range st.counts {
|
||||
copy(rec[:17], k[:])
|
||||
binary.LittleEndian.PutUint32(rec[17:], v)
|
||||
|
||||
if _, err := w.Write(rec[:]); err != nil {
|
||||
_ = f.Close()
|
||||
|
||||
return fmt.Errorf("counts file write: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := w.Flush(); err != nil {
|
||||
_ = f.Close()
|
||||
|
||||
return fmt.Errorf("counts file flush: %w", err)
|
||||
}
|
||||
|
||||
if err := f.Sync(); err != nil {
|
||||
_ = f.Close()
|
||||
|
||||
return fmt.Errorf("counts file sync: %w", err)
|
||||
}
|
||||
|
||||
if err := f.Close(); err != nil {
|
||||
return fmt.Errorf("counts file close: %w", err)
|
||||
}
|
||||
|
||||
if err := os.Rename(tmp, imp.countsPath()); err != nil {
|
||||
return fmt.Errorf("counts file rename: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// readCountsFile loads a previously checkpointed counts file. Returns
|
||||
// (nil, nil) when no checkpoint exists.
|
||||
func (imp *dumpImporter) readCountsFile() (*countsState, error) {
|
||||
f, err := os.Open(imp.countsPath())
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil, nil //nolint:nilnil // no checkpoint is a valid, non-error state
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("counts file open: %w", err)
|
||||
}
|
||||
|
||||
defer func() { _ = f.Close() }()
|
||||
|
||||
r := bufio.NewReaderSize(f, 1<<20)
|
||||
|
||||
magic := make([]byte, len(countsFileMagic))
|
||||
if _, err := io.ReadFull(r, magic); err != nil || string(magic) != countsFileMagic {
|
||||
return nil, fmt.Errorf("%w: bad counts file header", ErrDumpFormat)
|
||||
}
|
||||
|
||||
var lenBuf [4]byte
|
||||
|
||||
if _, err := io.ReadFull(r, lenBuf[:]); err != nil {
|
||||
return nil, fmt.Errorf("counts meta length: %w", err)
|
||||
}
|
||||
|
||||
meta := make([]byte, binary.LittleEndian.Uint32(lenBuf[:]))
|
||||
if _, err := io.ReadFull(r, meta); err != nil {
|
||||
return nil, fmt.Errorf("counts meta read: %w", err)
|
||||
}
|
||||
|
||||
st := &countsState{}
|
||||
if err := json.Unmarshal(meta, st); err != nil {
|
||||
return nil, fmt.Errorf("counts meta unmarshal: %w", err)
|
||||
}
|
||||
|
||||
st.counts = make(map[mbidKey]uint32, 1<<20)
|
||||
|
||||
var rec [21]byte
|
||||
|
||||
for {
|
||||
if _, err := io.ReadFull(r, rec[:]); err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("counts record read: %w", err)
|
||||
}
|
||||
|
||||
var k mbidKey
|
||||
|
||||
copy(k[:], rec[:17])
|
||||
st.counts[k] = binary.LittleEndian.Uint32(rec[17:])
|
||||
}
|
||||
|
||||
return st, nil
|
||||
}
|
||||
|
||||
func (imp *dumpImporter) logCountsProgress(members int, offset, size int64, entities int) {
|
||||
pct := float64(0)
|
||||
if size > 0 {
|
||||
pct = float64(offset) / float64(size) * 100
|
||||
}
|
||||
|
||||
imp.logger.Info("dump import: listen counts progress",
|
||||
"members", members,
|
||||
"gb", fmt.Sprintf("%.1f", float64(offset)/(1<<30)),
|
||||
"pct", fmt.Sprintf("%.1f", pct),
|
||||
"entities", entities,
|
||||
)
|
||||
|
||||
imp.setStageProgress(dumpStageCounts, int(pct), 100)
|
||||
}
|
||||
@@ -0,0 +1,574 @@
|
||||
package explore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"yellowjacket/backend/system"
|
||||
)
|
||||
|
||||
// Dump-based index population. Instead of crawling the ListenBrainz
|
||||
// API artist-by-artist, the index is built from two MetaBrainz dumps:
|
||||
//
|
||||
// 1. The spark listens dump (~170GB, streamed, never stored) yields
|
||||
// listen counts for every recording/release/artist MBID.
|
||||
// 2. The MusicBrainz canonical dump (~2GB, streamed) yields names and
|
||||
// MBIDs, filtered to entities above a popularity floor.
|
||||
//
|
||||
// A short API patch pass then fills listener counts (top rows only),
|
||||
// artist metadata, and the similar-artist map. Total temporary disk
|
||||
// is one ~1GB counts file, deleted on completion. The pipeline
|
||||
// resumes from checkpoints after interruption.
|
||||
|
||||
const (
|
||||
// dumpImportDoneKey marks a completed import in explore_index_meta.
|
||||
dumpImportDoneKey = "dump_import_done"
|
||||
|
||||
// listensAppliedSeriesKey stores the dump series number whose listen
|
||||
// counts are folded into popularity (the high-water-mark for the
|
||||
// incremental refresh). Set to the full dump's series at import, then
|
||||
// advanced by each applied incremental.
|
||||
listensAppliedSeriesKey = "listens_applied_series"
|
||||
|
||||
// releaseToRGInsertBatch bounds how many rows are written per
|
||||
// transaction when persisting the release→release-group map.
|
||||
releaseToRGInsertBatch = 10_000
|
||||
|
||||
// dumpMinStartFreeBytes is the free-disk requirement to begin an
|
||||
// import (counts file + index growth + headroom).
|
||||
dumpMinStartFreeBytes = 6 << 30
|
||||
|
||||
// dumpAbortFreeBytes aborts a running import when free disk
|
||||
// drops below it.
|
||||
dumpAbortFreeBytes = 2 << 30
|
||||
|
||||
// dumpStageAssembled in state.json means the index rows are
|
||||
// written and only patch passes remain.
|
||||
dumpStageAssembled = "assembled"
|
||||
)
|
||||
|
||||
// ErrDiskSpace is returned when free disk falls below the safety floor.
|
||||
var ErrDiskSpace = errors.New("insufficient free disk space")
|
||||
|
||||
// Dump import stages, mapped to status names shown in the UI.
|
||||
const (
|
||||
dumpStageCounts = iota
|
||||
dumpStageCatalog
|
||||
dumpStagePatch
|
||||
dumpStageListeners
|
||||
)
|
||||
|
||||
var dumpStageNames = [...]string{
|
||||
"Listen Counts",
|
||||
"Catalog Import",
|
||||
"Metadata Patch",
|
||||
"Listener Counts",
|
||||
}
|
||||
|
||||
// Dump discovery patterns.
|
||||
var (
|
||||
canonicalDirRe = regexp.MustCompile(`^musicbrainz-canonical-dump-\d{8}-\d+$`)
|
||||
canonicalFileRe = regexp.MustCompile(`^musicbrainz-canonical-dump-.*\.tar\.zst$`)
|
||||
listensDirRe = regexp.MustCompile(`^listenbrainz-dump-\d+-\d{8}-\d+-full$`)
|
||||
sparkFileRe = regexp.MustCompile(`^listenbrainz-spark-dump-.*-full\.tar$`)
|
||||
)
|
||||
|
||||
// Production dump locations (overridable for tests).
|
||||
const (
|
||||
defaultCanonicalBaseURL = "https://data.metabrainz.org/pub/musicbrainz/canonical_data/"
|
||||
defaultListensBaseURL = "https://data.metabrainz.org/pub/musicbrainz/listenbrainz/fullexport/"
|
||||
)
|
||||
|
||||
// dumpImportState is the small persistent state file (staging dir).
|
||||
// The heavyweight stage-1 checkpoint lives in counts.bin.
|
||||
type dumpImportState struct {
|
||||
SparkURL string `json:"sparkUrl"`
|
||||
CanonicalURL string `json:"canonicalUrl"`
|
||||
Stage string `json:"stage"`
|
||||
}
|
||||
|
||||
// dumpImporter runs the dump import pipeline.
|
||||
type dumpImporter struct {
|
||||
si *SearchIndex
|
||||
lb *ListenBrainzClient
|
||||
logger *slog.Logger
|
||||
|
||||
httpClient *http.Client
|
||||
stagingDir string
|
||||
|
||||
canonicalBaseURL string
|
||||
listensBaseURL string
|
||||
|
||||
// Disk safety floors (fields so tests can relax them).
|
||||
minStartFreeBytes uint64
|
||||
abortFreeBytes uint64
|
||||
|
||||
// pendingArtists are kept artists whose names weren't derivable
|
||||
// from the canonical dump; the metadata patch pass resolves them.
|
||||
pendingArtists []string
|
||||
}
|
||||
|
||||
func newDumpImporter(si *SearchIndex, lb *ListenBrainzClient) (*dumpImporter, error) {
|
||||
dataDir, err := system.GetUserDataDirPath()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dump import: data dir: %w", err)
|
||||
}
|
||||
|
||||
stagingDir := filepath.Join(dataDir, "explore-staging")
|
||||
if err := os.MkdirAll(stagingDir, 0o755); err != nil {
|
||||
return nil, fmt.Errorf("dump import: staging dir: %w", err)
|
||||
}
|
||||
|
||||
return &dumpImporter{
|
||||
si: si,
|
||||
lb: lb,
|
||||
logger: si.logger,
|
||||
// No client-level timeout: the listens stream runs for hours.
|
||||
// Discovery requests use per-request context timeouts, and
|
||||
// resumableReader recovers from stalled connections.
|
||||
httpClient: &http.Client{},
|
||||
stagingDir: stagingDir,
|
||||
canonicalBaseURL: defaultCanonicalBaseURL,
|
||||
listensBaseURL: defaultListensBaseURL,
|
||||
minStartFreeBytes: dumpMinStartFreeBytes,
|
||||
abortFreeBytes: dumpAbortFreeBytes,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (imp *dumpImporter) countsPath() string {
|
||||
return filepath.Join(imp.stagingDir, "counts.bin")
|
||||
}
|
||||
|
||||
func (imp *dumpImporter) statePath() string {
|
||||
return filepath.Join(imp.stagingDir, "state.json")
|
||||
}
|
||||
|
||||
// run executes the pipeline, resuming from any prior checkpoint.
|
||||
func (imp *dumpImporter) run(ctx context.Context) error {
|
||||
if err := checkFreeDisk(imp.stagingDir, imp.minStartFreeBytes); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
state, err := imp.readState()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Fast path: rows already assembled, only patch passes remain.
|
||||
if state.Stage == dumpStageAssembled {
|
||||
imp.si.MarkReadyIfPopulated()
|
||||
imp.runPatchPasses(ctx)
|
||||
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return imp.finalize()
|
||||
}
|
||||
|
||||
// Stage 1: listen counts from the spark listens dump.
|
||||
counts, err := imp.readCountsFile()
|
||||
if err != nil {
|
||||
imp.logger.Warn("dump import: discarding unreadable counts checkpoint", "error", err)
|
||||
|
||||
counts = nil
|
||||
}
|
||||
|
||||
if counts == nil {
|
||||
sparkURL, err := discoverDumpFile(
|
||||
ctx, imp.httpClient, imp.listensBaseURL, listensDirRe, sparkFileRe,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
imp.logger.Info("dump import: starting", "listensDump", sparkURL)
|
||||
|
||||
counts = &countsState{SparkURL: sparkURL}
|
||||
} else if !counts.Done {
|
||||
imp.logger.Info("dump import: resuming listen counts",
|
||||
"offset", counts.Offset,
|
||||
"members", counts.MemberIdx,
|
||||
"entities", len(counts.counts),
|
||||
)
|
||||
}
|
||||
|
||||
// Record which dump series this import is baselined on, so the
|
||||
// incremental refresh knows where to resume applying daily deltas.
|
||||
// Written early (before assembly) so it survives a crash-and-resume;
|
||||
// incrementals only apply once dumpImportDoneKey confirms completion.
|
||||
imp.recordDumpSeries(counts.SparkURL)
|
||||
|
||||
imp.setStageProgress(dumpStageCounts, 0, 100)
|
||||
|
||||
if !counts.Done {
|
||||
if err := imp.aggregateListenCounts(ctx, counts); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
imp.si.setTierStatus(dumpStageNames[dumpStageCounts], "complete", 100, 100)
|
||||
|
||||
// Stage 2: popularity thresholds (in RAM, deterministic).
|
||||
kept := imp.computeThreshold(counts.counts)
|
||||
|
||||
// Free the full counts map; only the kept sets are needed now.
|
||||
counts.counts = nil
|
||||
|
||||
// Stage 3: canonical dump scan + assembly. Restartable: the
|
||||
// scan is a cheap 2GB stream and assembly is an idempotent
|
||||
// upsert, so no intra-stage checkpoint is needed.
|
||||
if state.CanonicalURL == "" {
|
||||
state.CanonicalURL, err = discoverDumpFile(
|
||||
ctx, imp.httpClient, imp.canonicalBaseURL, canonicalDirRe, canonicalFileRe,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := imp.writeState(state); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
imp.setStageProgress(dumpStageCatalog, 0, 0)
|
||||
|
||||
scan, err := imp.scanCanonicalDump(ctx, state.CanonicalURL, kept)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := imp.checkDiskHeadroom(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// One-time reset when migrating from the legacy API-crawled
|
||||
// index: its popularity values are on a different scale (the LB
|
||||
// API includes MLHD+ history) and would permanently outrank
|
||||
// dump-derived counts via the highest-wins upsert. Re-imports
|
||||
// (dump→dump) skip this — listen counts only grow.
|
||||
if !imp.si.hasMeta(dumpImportDoneKey) {
|
||||
if _, err := imp.si.db.ExecContext("DELETE FROM explore_index"); err == nil {
|
||||
imp.logger.Info("dump import: cleared legacy index for consistent popularity scale")
|
||||
}
|
||||
}
|
||||
|
||||
if err := imp.assembleIndex(ctx, kept, scan); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Persist the release→release-group map (otherwise in-memory only)
|
||||
// so incremental dumps can roll per-release listen deltas up to their
|
||||
// album without an API call. Kept in lockstep with the index it was
|
||||
// just built from.
|
||||
imp.persistReleaseToRG(ctx, scan.releaseToRG)
|
||||
|
||||
imp.si.setTierStatus(dumpStageNames[dumpStageCatalog], "complete", 0, 0)
|
||||
|
||||
// Artists that need names from the metadata patch pass.
|
||||
for mbid := range kept.artists {
|
||||
if _, ok := scan.artistNames[mbid]; !ok {
|
||||
imp.pendingArtists = append(imp.pendingArtists, formatUUID(mbid[:]))
|
||||
}
|
||||
}
|
||||
|
||||
state.Stage = dumpStageAssembled
|
||||
if err := imp.writeState(state); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
imp.si.MarkReadyIfPopulated()
|
||||
imp.si.refreshStatusCounts()
|
||||
|
||||
// Stage 4: API patch passes (idempotent).
|
||||
imp.runPatchPasses(ctx)
|
||||
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return imp.finalize()
|
||||
}
|
||||
|
||||
// finalize records completion and removes all staging data.
|
||||
func (imp *dumpImporter) finalize() error {
|
||||
imp.si.setMeta(dumpImportDoneKey, time.Now().UTC().Format(time.RFC3339))
|
||||
|
||||
// Retire the legacy tier-crawl freshness keys.
|
||||
_, _ = imp.si.db.ExecContext(
|
||||
`DELETE FROM explore_index_meta
|
||||
WHERE key IN ('tier1_built', 'tier2_built', 'tier3_built', 'tier4_built')`,
|
||||
)
|
||||
|
||||
if err := os.RemoveAll(imp.stagingDir); err != nil {
|
||||
imp.logger.Warn("dump import: staging cleanup failed", "error", err)
|
||||
}
|
||||
|
||||
imp.si.setTierStatus(dumpStageNames[dumpStagePatch], "complete", 0, 0)
|
||||
imp.si.setTierStatus(dumpStageNames[dumpStageListeners], "complete", 0, 0)
|
||||
imp.si.refreshStatusCounts()
|
||||
|
||||
// The imported catalog changed which rows are popular, so refresh the
|
||||
// champion tier used for generic short-prefix searches.
|
||||
imp.si.scheduleChampionRebuild()
|
||||
|
||||
imp.logger.Info("dump import: complete")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// dumpSeriesRe extracts the monotonic series number NNNN from a dump
|
||||
// URL or directory name (e.g. "listenbrainz-spark-dump-2593-…").
|
||||
var dumpSeriesRe = regexp.MustCompile(`listenbrainz-(?:spark-)?dump-(\d+)-`)
|
||||
|
||||
// parseDumpSeries pulls the series number out of a dump URL/name.
|
||||
func parseDumpSeries(url string) (int, bool) {
|
||||
m := dumpSeriesRe.FindStringSubmatch(url)
|
||||
if m == nil {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
n, err := strconv.Atoi(m[1])
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
return n, true
|
||||
}
|
||||
|
||||
// recordDumpSeries stores the baseline series number for this import.
|
||||
func (imp *dumpImporter) recordDumpSeries(sparkURL string) {
|
||||
series, ok := parseDumpSeries(sparkURL)
|
||||
if !ok {
|
||||
imp.logger.Warn("dump import: could not parse dump series", "url", sparkURL)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
imp.si.setMeta(listensAppliedSeriesKey, strconv.Itoa(series))
|
||||
}
|
||||
|
||||
// persistReleaseToRG replaces the release_to_rg table with the mapping
|
||||
// captured during this import, so it always reflects the just-built
|
||||
// index. Idempotent: a full rebuild clears and repopulates it.
|
||||
func (imp *dumpImporter) persistReleaseToRG(ctx context.Context, m map[uuid16]rgTarget) {
|
||||
if len(m) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
if _, err := imp.si.db.ExecContext("DELETE FROM release_to_rg"); err != nil {
|
||||
imp.logger.Warn("dump import: clear release_to_rg failed", "error", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
written := 0
|
||||
pending := 0
|
||||
|
||||
tx, err := imp.si.db.BeginTx()
|
||||
if err != nil {
|
||||
imp.logger.Warn("dump import: begin release_to_rg tx failed", "error", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
for rel, target := range m {
|
||||
if ctx.Err() != nil {
|
||||
_ = tx.Rollback()
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(
|
||||
"INSERT OR REPLACE INTO release_to_rg (release_mbid, rg_mbid) VALUES (?, ?)",
|
||||
formatUUID(rel[:]), formatUUID(target.rg[:]),
|
||||
); err != nil {
|
||||
imp.logger.Warn("dump import: insert release_to_rg failed", "error", err)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
written++
|
||||
pending++
|
||||
|
||||
if pending >= releaseToRGInsertBatch {
|
||||
if err := tx.Commit(); err != nil {
|
||||
imp.logger.Warn("dump import: commit release_to_rg batch failed", "error", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
pending = 0
|
||||
|
||||
tx, err = imp.si.db.BeginTx()
|
||||
if err != nil {
|
||||
imp.logger.Warn("dump import: begin release_to_rg tx failed", "error", err)
|
||||
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
imp.logger.Warn("dump import: commit release_to_rg failed", "error", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
imp.logger.Info("dump import: persisted release→release-group map", "rows", written)
|
||||
}
|
||||
|
||||
func (imp *dumpImporter) readState() (*dumpImportState, error) {
|
||||
state := &dumpImportState{}
|
||||
|
||||
data, err := os.ReadFile(imp.statePath())
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return state, nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dump import state read: %w", err)
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(data, state); err != nil {
|
||||
// Corrupt state: start over rather than fail permanently.
|
||||
return &dumpImportState{}, nil
|
||||
}
|
||||
|
||||
return state, nil
|
||||
}
|
||||
|
||||
func (imp *dumpImporter) writeState(state *dumpImportState) error {
|
||||
data, err := json.Marshal(state)
|
||||
if err != nil {
|
||||
return fmt.Errorf("dump import state marshal: %w", err)
|
||||
}
|
||||
|
||||
tmp := imp.statePath() + ".tmp"
|
||||
if err := os.WriteFile(tmp, data, 0o644); err != nil {
|
||||
return fmt.Errorf("dump import state write: %w", err)
|
||||
}
|
||||
|
||||
if err := os.Rename(tmp, imp.statePath()); err != nil {
|
||||
return fmt.Errorf("dump import state rename: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// setStageProgress reports stage progress to the UI status feed.
|
||||
func (imp *dumpImporter) setStageProgress(stage, completed, total int) {
|
||||
imp.si.setTierStatus(dumpStageNames[stage], "running", total, completed)
|
||||
}
|
||||
|
||||
// checkDiskHeadroom aborts the import when free disk is critically low.
|
||||
func (imp *dumpImporter) checkDiskHeadroom() error {
|
||||
return checkFreeDisk(imp.stagingDir, imp.abortFreeBytes)
|
||||
}
|
||||
|
||||
// checkFreeDisk returns ErrDiskSpace when the volume holding path has
|
||||
// less than minBytes free. Unknown free space (unsupported platform)
|
||||
// passes.
|
||||
func checkFreeDisk(path string, minBytes uint64) error {
|
||||
free, ok := diskFreeBytes(path)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
if free < minBytes {
|
||||
return fmt.Errorf("%w: %d MB free, need %d MB",
|
||||
ErrDiskSpace, free>>20, minBytes>>20)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// SearchIndex integration
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// runDumpBuild is the build entrypoint called from StartBuild's
|
||||
// goroutine. It replaces the legacy tier crawl.
|
||||
func (si *SearchIndex) runDumpBuild(ctx context.Context) {
|
||||
si.MarkReadyIfPopulated()
|
||||
|
||||
// The catalog dump is authoritative and only grows; it is imported
|
||||
// once and never re-crawled on a timer. Popularity freshness comes
|
||||
// from incremental dumps, and new releases from lazy per-artist
|
||||
// fetches — not from re-running this multi-GB import.
|
||||
if si.hasMeta(dumpImportDoneKey) {
|
||||
si.logger.Info("search index: dump import already complete, skipping")
|
||||
si.refreshStatusCounts()
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
si.mu.Lock()
|
||||
si.buildStatus = IndexStatus{
|
||||
Building: true,
|
||||
Tiers: []TierStatus{
|
||||
{Name: dumpStageNames[dumpStageCounts], State: "pending"},
|
||||
{Name: dumpStageNames[dumpStageCatalog], State: "pending"},
|
||||
{Name: dumpStageNames[dumpStagePatch], State: "pending"},
|
||||
{Name: dumpStageNames[dumpStageListeners], State: "pending"},
|
||||
},
|
||||
}
|
||||
si.mu.Unlock()
|
||||
si.refreshStatusCounts()
|
||||
|
||||
// Patch passes use a dedicated rate limiter so background API
|
||||
// calls never compete with interactive search/browse requests.
|
||||
var indexLB *ListenBrainzClient
|
||||
|
||||
if si.lb != nil {
|
||||
indexLB = NewListenBrainzClient(
|
||||
NewRateLimiterN(indexerRate), si.lb.cache, si.logger.WithGroup("indexer"),
|
||||
)
|
||||
}
|
||||
|
||||
imp, err := newDumpImporter(si, indexLB)
|
||||
if err != nil {
|
||||
si.logger.Error("search index: dump import init failed", "error", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
|
||||
if err := imp.run(ctx); err != nil {
|
||||
if errors.Is(err, context.Canceled) || ctx.Err() != nil {
|
||||
si.logger.Info("search index: dump import paused (will resume)",
|
||||
"elapsed", time.Since(start).Round(time.Second),
|
||||
)
|
||||
} else {
|
||||
si.logger.Error("search index: dump import failed", "error", err)
|
||||
si.setTierError(dumpStageNames[dumpStageCounts], err.Error())
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
si.mu.Lock()
|
||||
si.buildStatus.Building = false
|
||||
si.mu.Unlock()
|
||||
|
||||
// Fold the local library into the freshly-imported catalog: owned
|
||||
// entities below the dump's popularity floor are inserted, and
|
||||
// dump-seeded rows that match the library are flagged in_library.
|
||||
// Deep discographies stay lazy (fetched when an artist page opens).
|
||||
si.PopulateLocalCrossReferences()
|
||||
|
||||
si.refreshStatusCounts()
|
||||
si.logger.Info("search index: dump import finished",
|
||||
"elapsed", time.Since(start).Round(time.Second),
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,925 @@
|
||||
package explore
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/csv"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/parquet-go/parquet-go"
|
||||
|
||||
"yellowjacket/backend/database"
|
||||
)
|
||||
|
||||
// Fixed MBIDs for fixtures.
|
||||
const (
|
||||
recA = "11111111-1111-1111-1111-111111111111"
|
||||
recB = "22222222-2222-2222-2222-222222222222"
|
||||
recC = "33333333-3333-3333-3333-333333333333"
|
||||
relA = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"
|
||||
relB = "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"
|
||||
rgA = "cccccccc-cccc-cccc-cccc-cccccccccccc"
|
||||
rgB = "dddddddd-dddd-dddd-dddd-dddddddddddd"
|
||||
artA = "eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee"
|
||||
artB = "ffffffff-ffff-ffff-ffff-ffffffffffff"
|
||||
)
|
||||
|
||||
func testLogger() *slog.Logger {
|
||||
return slog.New(slog.DiscardHandler)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Unit tests: parsing helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func TestParseUUIDRoundTrip(t *testing.T) {
|
||||
var buf [16]byte
|
||||
|
||||
if !parseUUID(recA, buf[:]) {
|
||||
t.Fatalf("parseUUID rejected valid UUID %s", recA)
|
||||
}
|
||||
|
||||
if got := formatUUID(buf[:]); got != recA {
|
||||
t.Fatalf("round trip = %q, want %q", got, recA)
|
||||
}
|
||||
|
||||
invalid := []string{
|
||||
"", "not-a-uuid",
|
||||
"11111111-1111-1111-1111-11111111111", // too short
|
||||
"11111111-1111-1111-1111-1111111111111", // too long
|
||||
"1111111101111-1111-1111-111111111111", // bad dash
|
||||
"gggggggg-1111-1111-1111-111111111111", // bad hex
|
||||
}
|
||||
|
||||
for _, s := range invalid {
|
||||
if parseUUID(s, buf[:]) {
|
||||
t.Errorf("parseUUID accepted invalid input %q", s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParsePGStringArray(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
want []string
|
||||
}{
|
||||
{"{" + artA + "}", []string{artA}},
|
||||
{"{" + artA + "," + artB + "}", []string{artA, artB}},
|
||||
{`{"` + artA + `","` + artB + `"}`, []string{artA, artB}},
|
||||
{"['" + artA + "', '" + artB + "']", []string{artA, artB}},
|
||||
{artA, []string{artA}},
|
||||
{"", nil},
|
||||
{"{}", nil},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
got := parsePGStringArray(c.in)
|
||||
if len(got) != len(c.want) {
|
||||
t.Errorf("parsePGStringArray(%q) = %v, want %v", c.in, got, c.want)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
for i := range got {
|
||||
if got[i] != c.want[i] {
|
||||
t.Errorf("parsePGStringArray(%q)[%d] = %q, want %q", c.in, i, got[i], c.want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFloorForBudget(t *testing.T) {
|
||||
vals := []uint32{100, 90, 80, 70, 60, 50, 40, 30, 20, 5}
|
||||
|
||||
if got := floorForBudget(vals, 3, 10); got != 80 {
|
||||
t.Errorf("budget 3: floor = %d, want 80", got)
|
||||
}
|
||||
|
||||
// Budget larger than data → min floor.
|
||||
if got := floorForBudget(vals, 100, 10); got != 10 {
|
||||
t.Errorf("budget 100: floor = %d, want 10", got)
|
||||
}
|
||||
|
||||
// Floor clamped up to minFloor.
|
||||
if got := floorForBudget(vals, 10, 10); got != 10 {
|
||||
t.Errorf("clamp: floor = %d, want 10", got)
|
||||
}
|
||||
|
||||
if got := floorForBudget(nil, 5, 7); got != 7 {
|
||||
t.Errorf("empty: floor = %d, want 7", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRankFloor(t *testing.T) {
|
||||
desc := []uint32{100, 90, 80, 70, 60}
|
||||
|
||||
cases := []struct {
|
||||
rank int
|
||||
want uint32
|
||||
}{
|
||||
{1, 100},
|
||||
{3, 80},
|
||||
{5, 60},
|
||||
{99, 60}, // clamped to the last element
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
if got := rankFloor(desc, c.rank); got != c.want {
|
||||
t.Errorf("rankFloor(rank=%d) = %d, want %d", c.rank, got, c.want)
|
||||
}
|
||||
}
|
||||
|
||||
if got := rankFloor(nil, 3); got != 0 {
|
||||
t.Errorf("rankFloor(empty) = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTierBudget(t *testing.T) {
|
||||
const aFloor, bFloor = uint32(1000), uint32(100)
|
||||
|
||||
cases := []struct {
|
||||
listens uint32
|
||||
wantTrack, wantRG int
|
||||
}{
|
||||
{2000, perArtistTierATrack, perArtistTierARG}, // tier A
|
||||
{1000, perArtistTierATrack, perArtistTierARG}, // exactly on A floor
|
||||
{500, perArtistTierBTrack, perArtistTierBRG}, // tier B
|
||||
{100, perArtistTierBTrack, perArtistTierBRG}, // exactly on B floor
|
||||
{10, perArtistTierCTrack, perArtistTierCRG}, // tier C
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
gotTrack, gotRG := tierBudget(c.listens, aFloor, bFloor)
|
||||
if gotTrack != c.wantTrack || gotRG != c.wantRG {
|
||||
t.Errorf("tierBudget(%d) = (%d, %d), want (%d, %d)",
|
||||
c.listens, gotTrack, gotRG, c.wantTrack, c.wantRG)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// mkMBID builds a distinct uuid16 from a single byte, for heap tests.
|
||||
func mkMBID(b byte) uuid16 {
|
||||
var id uuid16
|
||||
|
||||
id[0] = b
|
||||
|
||||
return id
|
||||
}
|
||||
|
||||
func TestArtistTopNBoundedAndDeduped(t *testing.T) {
|
||||
a := &artistTopN{n: 3, inSet: make(map[uuid16]struct{})}
|
||||
|
||||
// Add five distinct recordings; only the top 3 by listens survive.
|
||||
for i, listens := range []uint32{10, 50, 30, 5, 40} {
|
||||
a.add(keptRecordingRow{mbid: mkMBID(byte(i + 1)), listens: listens})
|
||||
}
|
||||
|
||||
if len(a.rows) != 3 {
|
||||
t.Fatalf("len = %d, want 3 (bounded)", len(a.rows))
|
||||
}
|
||||
|
||||
got := map[uint32]bool{}
|
||||
for _, r := range a.rows {
|
||||
got[r.listens] = true
|
||||
}
|
||||
|
||||
for _, want := range []uint32{50, 40, 30} {
|
||||
if !got[want] {
|
||||
t.Errorf("expected top listens %d retained, have %v", want, got)
|
||||
}
|
||||
}
|
||||
|
||||
if got[10] || got[5] {
|
||||
t.Errorf("evicted entries survived: %v", got)
|
||||
}
|
||||
|
||||
// Re-adding an existing MBID is a no-op, even with a higher count.
|
||||
before := len(a.rows)
|
||||
|
||||
a.add(keptRecordingRow{mbid: mkMBID(2), listens: 9999})
|
||||
|
||||
if len(a.rows) != before {
|
||||
t.Errorf("duplicate MBID grew the set: %d != %d", len(a.rows), before)
|
||||
}
|
||||
|
||||
if _, dupHigh := got[9999]; dupHigh {
|
||||
t.Error("duplicate MBID should not have been re-ranked")
|
||||
}
|
||||
}
|
||||
|
||||
func TestArtistTopRGBoundedAndDeduped(t *testing.T) {
|
||||
a := &artistTopRG{n: 2, inSet: make(map[uuid16]struct{})}
|
||||
|
||||
a.add(mkMBID(1), 100)
|
||||
a.add(mkMBID(2), 200)
|
||||
a.add(mkMBID(3), 50) // below both, dropped
|
||||
a.add(mkMBID(1), 999) // duplicate, ignored
|
||||
|
||||
if len(a.rgs) != 2 {
|
||||
t.Fatalf("len = %d, want 2", len(a.rgs))
|
||||
}
|
||||
|
||||
for _, c := range a.rgs {
|
||||
if c.listens == 50 {
|
||||
t.Error("sub-threshold RG was kept")
|
||||
}
|
||||
|
||||
if c.rg == mkMBID(1) && c.listens != 100 {
|
||||
t.Errorf("duplicate RG re-ranked to %d", c.listens)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Fixture builders
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// sparkFixtureRow mimics the real spark listens schema: the aggregator
|
||||
// must project just recording/release/artist MBIDs out of it.
|
||||
type sparkFixtureRow struct {
|
||||
ListenedAt int64 `parquet:"listened_at"`
|
||||
UserID int64 `parquet:"user_id"`
|
||||
ArtistName string `parquet:"artist_name,optional"`
|
||||
RecordingMBID string `parquet:"recording_mbid,optional"`
|
||||
ReleaseMBID string `parquet:"release_mbid,optional"`
|
||||
ArtistMBIDs []string `parquet:"artist_credit_mbids,optional,list"`
|
||||
}
|
||||
|
||||
func makeParquet(t *testing.T, rows []sparkFixtureRow) []byte {
|
||||
t.Helper()
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
w := parquet.NewGenericWriter[sparkFixtureRow](&buf)
|
||||
|
||||
if _, err := w.Write(rows); err != nil {
|
||||
t.Fatalf("parquet write: %v", err)
|
||||
}
|
||||
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatalf("parquet close: %v", err)
|
||||
}
|
||||
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func makeTar(t *testing.T, members map[string][]byte, order []string) []byte {
|
||||
t.Helper()
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
tw := tar.NewWriter(&buf)
|
||||
|
||||
for _, name := range order {
|
||||
data := members[name]
|
||||
hdr := &tar.Header{
|
||||
Name: name,
|
||||
Mode: 0o644,
|
||||
Size: int64(len(data)),
|
||||
Typeflag: tar.TypeReg,
|
||||
}
|
||||
|
||||
if err := tw.WriteHeader(hdr); err != nil {
|
||||
t.Fatalf("tar header: %v", err)
|
||||
}
|
||||
|
||||
if _, err := tw.Write(data); err != nil {
|
||||
t.Fatalf("tar write: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := tw.Close(); err != nil {
|
||||
t.Fatalf("tar close: %v", err)
|
||||
}
|
||||
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func zstdCompress(t *testing.T, data []byte) []byte {
|
||||
t.Helper()
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
zw, err := zstd.NewWriter(&buf)
|
||||
if err != nil {
|
||||
t.Fatalf("zstd writer: %v", err)
|
||||
}
|
||||
|
||||
if _, err := zw.Write(data); err != nil {
|
||||
t.Fatalf("zstd write: %v", err)
|
||||
}
|
||||
|
||||
if err := zw.Close(); err != nil {
|
||||
t.Fatalf("zstd close: %v", err)
|
||||
}
|
||||
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func csvBytes(t *testing.T, rows [][]string) []byte {
|
||||
t.Helper()
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
w := csv.NewWriter(&buf)
|
||||
if err := w.WriteAll(rows); err != nil {
|
||||
t.Fatalf("csv write: %v", err)
|
||||
}
|
||||
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// listensOf builds n identical listen rows for a recording.
|
||||
func listensOf(n int, recording, release string, artists []string) []sparkFixtureRow {
|
||||
rows := make([]sparkFixtureRow, n)
|
||||
for i := range rows {
|
||||
rows[i] = sparkFixtureRow{
|
||||
ListenedAt: 1700000000 + int64(i),
|
||||
UserID: int64(i),
|
||||
ArtistName: "Fixture Artist",
|
||||
RecordingMBID: recording,
|
||||
ReleaseMBID: release,
|
||||
ArtistMBIDs: artists,
|
||||
}
|
||||
}
|
||||
|
||||
return rows
|
||||
}
|
||||
|
||||
// canonicalDataCSV builds a canonical_musicbrainz_data.csv fixture.
|
||||
func canonicalDataCSV(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
|
||||
rows := [][]string{
|
||||
{
|
||||
"id", "artist_credit_id", "artist_mbids", "artist_credit_name",
|
||||
"release_mbid", "release_name", "recording_mbid", "recording_name",
|
||||
"combined_lookup", "score",
|
||||
},
|
||||
{"1", "10", "{" + artA + "}", "Solo Star", relA, "Big Album", recA, "Hit Song", "x", "1"},
|
||||
{"2", "10", "{" + artA + "}", "Solo Star", relA, "Big Album", recB, "Deep Cut", "x", "1"},
|
||||
{
|
||||
"3", "11", "{" + artA + "," + artB + "}", "Solo Star feat. Guest",
|
||||
relB, "Duet Album", recC, "Duet Song", "x", "1",
|
||||
},
|
||||
}
|
||||
|
||||
return csvBytes(t, rows)
|
||||
}
|
||||
|
||||
// canonicalRedirectCSV builds a canonical_release_redirect.csv fixture.
|
||||
func canonicalRedirectCSV(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
|
||||
rows := [][]string{
|
||||
{"release_mbid", "canonical_release_mbid", "release_group_mbid"},
|
||||
{relA, relA, rgA},
|
||||
{relB, relB, rgB},
|
||||
}
|
||||
|
||||
return csvBytes(t, rows)
|
||||
}
|
||||
|
||||
// serveDumps returns an httptest server presenting MetaBrainz-style
|
||||
// listing pages and Range-capable dump files.
|
||||
func serveDumps(t *testing.T, sparkTar, canonicalTarZst []byte) *httptest.Server {
|
||||
t.Helper()
|
||||
|
||||
const (
|
||||
listensDir = "listenbrainz-dump-1-20260101-000003-full"
|
||||
sparkFile = "listenbrainz-spark-dump-1-20260101-000003-full.tar"
|
||||
canonicalDir = "musicbrainz-canonical-dump-20260101-080003"
|
||||
canonicalTar = "musicbrainz-canonical-dump-20260101-080003.tar.zst"
|
||||
)
|
||||
|
||||
modTime := time.Now()
|
||||
mux := http.NewServeMux()
|
||||
|
||||
mux.HandleFunc("/listens/", func(w http.ResponseWriter, r *http.Request) {
|
||||
switch strings.TrimPrefix(r.URL.Path, "/listens/") {
|
||||
case "":
|
||||
_, _ = fmt.Fprintf(w, `<a href="%s/">%s/</a>`, listensDir, listensDir)
|
||||
case listensDir + "/":
|
||||
_, _ = fmt.Fprintf(w, `<a href="%s">%s</a>`, sparkFile, sparkFile)
|
||||
case listensDir + "/" + sparkFile:
|
||||
http.ServeContent(w, r, sparkFile, modTime, bytes.NewReader(sparkTar))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
})
|
||||
|
||||
mux.HandleFunc("/canonical/", func(w http.ResponseWriter, r *http.Request) {
|
||||
switch strings.TrimPrefix(r.URL.Path, "/canonical/") {
|
||||
case "":
|
||||
_, _ = fmt.Fprintf(w, `<a href="%s/">%s/</a>`, canonicalDir, canonicalDir)
|
||||
case canonicalDir + "/":
|
||||
_, _ = fmt.Fprintf(w, `<a href="%s">%s</a>`, canonicalTar, canonicalTar)
|
||||
case canonicalDir + "/" + canonicalTar:
|
||||
http.ServeContent(w, r, canonicalTar, modTime, bytes.NewReader(canonicalTarZst))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
})
|
||||
|
||||
srv := httptest.NewServer(mux)
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
return srv
|
||||
}
|
||||
|
||||
func testImporter(t *testing.T, si *SearchIndex, srv *httptest.Server) *dumpImporter {
|
||||
t.Helper()
|
||||
|
||||
return &dumpImporter{
|
||||
si: si,
|
||||
lb: nil, // patch passes skipped in tests
|
||||
logger: testLogger(),
|
||||
httpClient: srv.Client(),
|
||||
stagingDir: t.TempDir(),
|
||||
canonicalBaseURL: srv.URL + "/canonical/",
|
||||
listensBaseURL: srv.URL + "/listens/",
|
||||
}
|
||||
}
|
||||
|
||||
func fixtureSparkTar(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
|
||||
// Member 1: recA is popular (12 listens). Member 2: recB has 11,
|
||||
// recC has 12 (multi-artist credit). Totals: artA = 35, artB = 12.
|
||||
member1 := makeParquet(t, listensOf(12, recA, relA, []string{artA}))
|
||||
member2 := makeParquet(t, append(
|
||||
listensOf(11, recB, relA, []string{artA}),
|
||||
listensOf(12, recC, relB, []string{artA, artB})...,
|
||||
))
|
||||
|
||||
prefix := "listenbrainz-spark-dump-1-20260101-000003-full/listens/"
|
||||
|
||||
return makeTar(t,
|
||||
map[string][]byte{
|
||||
prefix + "1.parquet": member1,
|
||||
prefix + "2.parquet": member2,
|
||||
},
|
||||
[]string{prefix + "1.parquet", prefix + "2.parquet"},
|
||||
)
|
||||
}
|
||||
|
||||
func fixtureCanonicalTarZst(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
|
||||
prefix := "musicbrainz-canonical-dump-20260101-080003/"
|
||||
|
||||
raw := makeTar(t,
|
||||
map[string][]byte{
|
||||
prefix + "canonical_musicbrainz_data.csv": canonicalDataCSV(t),
|
||||
prefix + "canonical_release_redirect.csv": canonicalRedirectCSV(t),
|
||||
prefix + "canonical_recording_redirect.csv": {},
|
||||
},
|
||||
[]string{
|
||||
prefix + "canonical_release_redirect.csv",
|
||||
prefix + "canonical_musicbrainz_data.csv",
|
||||
prefix + "canonical_recording_redirect.csv",
|
||||
},
|
||||
)
|
||||
|
||||
return zstdCompress(t, raw)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Stage tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func TestAggregateListenCounts(t *testing.T) {
|
||||
srv := serveDumps(t, fixtureSparkTar(t), nil)
|
||||
|
||||
db := database.NewTestDB(t)
|
||||
si := NewSearchIndex(db, nil, nil, testLogger())
|
||||
imp := testImporter(t, si, srv)
|
||||
|
||||
sparkURL, err := discoverDumpFile(
|
||||
context.Background(), imp.httpClient, imp.listensBaseURL, listensDirRe, sparkFileRe,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("discover: %v", err)
|
||||
}
|
||||
|
||||
st := &countsState{SparkURL: sparkURL}
|
||||
if err := imp.aggregateListenCounts(context.Background(), st); err != nil {
|
||||
t.Fatalf("aggregate: %v", err)
|
||||
}
|
||||
|
||||
assertCount := func(kind byte, mbid string, want uint32) {
|
||||
t.Helper()
|
||||
|
||||
key, ok := makeMBIDKey(kind, mbid)
|
||||
if !ok {
|
||||
t.Fatalf("bad fixture mbid %s", mbid)
|
||||
}
|
||||
|
||||
if got := st.counts[key]; got != want {
|
||||
t.Errorf("count(kind=%d, %s) = %d, want %d", kind, mbid, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
assertCount(countKindRecording, recA, 12)
|
||||
assertCount(countKindRecording, recB, 11)
|
||||
assertCount(countKindRecording, recC, 12)
|
||||
assertCount(countKindRelease, relA, 23)
|
||||
assertCount(countKindRelease, relB, 12)
|
||||
assertCount(countKindArtist, artA, 35)
|
||||
assertCount(countKindArtist, artB, 12)
|
||||
|
||||
if !st.Done {
|
||||
t.Error("state not marked done")
|
||||
}
|
||||
|
||||
// The checkpoint file round-trips.
|
||||
loaded, err := imp.readCountsFile()
|
||||
if err != nil {
|
||||
t.Fatalf("read counts file: %v", err)
|
||||
}
|
||||
|
||||
if loaded == nil || !loaded.Done || len(loaded.counts) != len(st.counts) {
|
||||
t.Fatalf("checkpoint mismatch: %+v", loaded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAggregateResumeFromOffset(t *testing.T) {
|
||||
sparkTar := fixtureSparkTar(t)
|
||||
srv := serveDumps(t, sparkTar, nil)
|
||||
|
||||
db := database.NewTestDB(t)
|
||||
si := NewSearchIndex(db, nil, nil, testLogger())
|
||||
imp := testImporter(t, si, srv)
|
||||
|
||||
sparkURL := srv.URL + "/listens/listenbrainz-dump-1-20260101-000003-full/listenbrainz-spark-dump-1-20260101-000003-full.tar"
|
||||
|
||||
// Full run for reference.
|
||||
full := &countsState{SparkURL: sparkURL}
|
||||
if err := imp.aggregateListenCounts(context.Background(), full); err != nil {
|
||||
t.Fatalf("full aggregate: %v", err)
|
||||
}
|
||||
|
||||
// Simulate a checkpoint taken after member 1: offset = header
|
||||
// block + padded member-1 size (fixture names are short, so the
|
||||
// header is a single 512-byte block).
|
||||
member1 := makeParquet(t, listensOf(12, recA, relA, []string{artA}))
|
||||
offset := int64(512) + (int64(len(member1))+511)/512*512
|
||||
|
||||
key, _ := makeMBIDKey(countKindRecording, recA)
|
||||
relKey, _ := makeMBIDKey(countKindRelease, relA)
|
||||
artKey, _ := makeMBIDKey(countKindArtist, artA)
|
||||
|
||||
resumed := &countsState{
|
||||
SparkURL: sparkURL,
|
||||
Offset: offset,
|
||||
MemberIdx: 1,
|
||||
counts: map[mbidKey]uint32{
|
||||
key: 12,
|
||||
relKey: 12,
|
||||
artKey: 12,
|
||||
},
|
||||
}
|
||||
|
||||
if err := imp.aggregateListenCounts(context.Background(), resumed); err != nil {
|
||||
t.Fatalf("resumed aggregate: %v", err)
|
||||
}
|
||||
|
||||
if len(resumed.counts) != len(full.counts) {
|
||||
t.Fatalf("resumed entities = %d, want %d", len(resumed.counts), len(full.counts))
|
||||
}
|
||||
|
||||
for k, want := range full.counts {
|
||||
if got := resumed.counts[k]; got != want {
|
||||
t.Errorf("resumed count %s = %d, want %d (double count?)", formatUUID(k[1:]), got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResumableReaderReconnects(t *testing.T) {
|
||||
payload := bytes.Repeat([]byte("0123456789abcdef"), 4096) // 64KB
|
||||
|
||||
// A flaky server that truncates every response to 10KB, forcing
|
||||
// the reader to reconnect with Range requests.
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
offset := int64(0)
|
||||
if rng := r.Header.Get("Range"); rng != "" {
|
||||
_, _ = fmt.Sscanf(rng, "bytes=%d-", &offset)
|
||||
}
|
||||
|
||||
chunk := payload[offset:min(offset+10240, int64(len(payload)))]
|
||||
|
||||
w.Header().Set("Content-Range",
|
||||
fmt.Sprintf("bytes %d-%d/%d", offset, offset+int64(len(chunk))-1, len(payload)))
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
_, _ = w.Write(chunk)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
r := newResumableReader(context.Background(), srv.Client(), srv.URL, 0)
|
||||
|
||||
got, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
t.Fatalf("read: %v", err)
|
||||
}
|
||||
|
||||
if !bytes.Equal(got, payload) {
|
||||
t.Fatalf("payload mismatch: got %d bytes, want %d", len(got), len(payload))
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// End-to-end
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func TestDumpImportEndToEnd(t *testing.T) {
|
||||
srv := serveDumps(t, fixtureSparkTar(t), fixtureCanonicalTarZst(t))
|
||||
|
||||
db := database.NewTestDB(t)
|
||||
si := NewSearchIndex(db, nil, nil, testLogger())
|
||||
imp := testImporter(t, si, srv)
|
||||
stagingDir := imp.stagingDir
|
||||
|
||||
// A legacy API-crawled row with inflated popularity must be
|
||||
// cleared by the first dump import (scale consistency).
|
||||
legacyMBID := "99999999-9999-9999-9999-999999999999"
|
||||
|
||||
si.upsertBatch([]SearchIndexResult{{
|
||||
EntityType: "recording",
|
||||
MBID: legacyMBID,
|
||||
Title: "Legacy Row",
|
||||
ArtistName: "Old Crawl",
|
||||
ArtistMBID: artA,
|
||||
Popularity: 123_456_789,
|
||||
}})
|
||||
|
||||
if err := imp.run(context.Background()); err != nil {
|
||||
t.Fatalf("run: %v", err)
|
||||
}
|
||||
|
||||
legacyRows, err := db.QueryContext(
|
||||
"SELECT COUNT(*) FROM explore_index WHERE mbid = ?", legacyMBID,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("legacy query: %v", err)
|
||||
}
|
||||
|
||||
if legacyRows.Next() {
|
||||
var n int
|
||||
|
||||
_ = legacyRows.Scan(&n)
|
||||
|
||||
if n != 0 {
|
||||
t.Error("legacy API-crawled row survived the first dump import")
|
||||
}
|
||||
}
|
||||
|
||||
_ = legacyRows.Close()
|
||||
|
||||
// Index rows landed with dump-derived popularity.
|
||||
assertRow := func(mbid, entityType, title string, popularity int) {
|
||||
t.Helper()
|
||||
|
||||
rows, err := db.QueryContext(
|
||||
"SELECT title, popularity FROM explore_index WHERE mbid = ? AND entity_type = ?",
|
||||
mbid, entityType,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("query: %v", err)
|
||||
}
|
||||
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
if !rows.Next() {
|
||||
t.Fatalf("no %s row for %s", entityType, mbid)
|
||||
}
|
||||
|
||||
var gotTitle string
|
||||
|
||||
var gotPop int
|
||||
|
||||
if err := rows.Scan(&gotTitle, &gotPop); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
|
||||
if gotTitle != title || gotPop != popularity {
|
||||
t.Errorf("%s %s = (%q, %d), want (%q, %d)",
|
||||
entityType, mbid, gotTitle, gotPop, title, popularity)
|
||||
}
|
||||
}
|
||||
|
||||
assertRow(recA, "recording", "Hit Song", 12)
|
||||
assertRow(recB, "recording", "Deep Cut", 11)
|
||||
assertRow(recC, "recording", "Duet Song", 12)
|
||||
assertRow(rgA, "release_group", "Big Album", 23)
|
||||
assertRow(rgB, "release_group", "Duet Album", 12)
|
||||
assertRow(artA, "artist", "Solo Star", 35)
|
||||
|
||||
// artB only ever appears in a multi-artist credit: no name is
|
||||
// derivable from the dump, so it must be queued for the API
|
||||
// metadata patch instead of being written nameless.
|
||||
rows, err := db.QueryContext(
|
||||
"SELECT COUNT(*) FROM explore_index WHERE mbid = ?", artB,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("query artB: %v", err)
|
||||
}
|
||||
|
||||
if rows.Next() {
|
||||
var n int
|
||||
|
||||
_ = rows.Scan(&n)
|
||||
|
||||
if n != 0 {
|
||||
t.Errorf("artB row written without a name source")
|
||||
}
|
||||
}
|
||||
|
||||
_ = rows.Close()
|
||||
|
||||
found := false
|
||||
|
||||
for _, mbid := range imp.pendingArtists {
|
||||
if mbid == artB {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
|
||||
if !found {
|
||||
t.Errorf("artB not queued for metadata patch: %v", imp.pendingArtists)
|
||||
}
|
||||
|
||||
// FTS search works end to end.
|
||||
si.MarkReadyIfPopulated()
|
||||
|
||||
results := si.Search(context.Background(), "hit song", 10)
|
||||
if len(results) == 0 || results[0].MBID != recA {
|
||||
t.Fatalf("search for indexed recording failed: %+v", results)
|
||||
}
|
||||
|
||||
// Completion recorded; staging cleaned up.
|
||||
if !si.hasMeta(dumpImportDoneKey) {
|
||||
t.Error("dump_import_done not recorded")
|
||||
}
|
||||
|
||||
// The incremental refresh baseline was recorded, and the
|
||||
// release→release-group map was persisted for future rollups.
|
||||
if _, ok := si.metaInt(listensAppliedSeriesKey); !ok {
|
||||
t.Error("listens_applied_series baseline not recorded")
|
||||
}
|
||||
|
||||
var relToRGRows int
|
||||
|
||||
rtrRows, err := db.QueryContext("SELECT COUNT(*) FROM release_to_rg")
|
||||
if err != nil {
|
||||
t.Fatalf("query release_to_rg: %v", err)
|
||||
}
|
||||
|
||||
if rtrRows.Next() {
|
||||
_ = rtrRows.Scan(&relToRGRows)
|
||||
}
|
||||
|
||||
_ = rtrRows.Close()
|
||||
|
||||
if relToRGRows == 0 {
|
||||
t.Error("release_to_rg not populated after import")
|
||||
}
|
||||
|
||||
if _, err := os.Stat(stagingDir); !os.IsNotExist(err) {
|
||||
t.Errorf("staging dir not cleaned up: %v", err)
|
||||
}
|
||||
|
||||
// Re-running is a cheap no-op that doesn't error.
|
||||
imp2 := testImporter(t, si, srv)
|
||||
if err := imp2.run(context.Background()); err != nil {
|
||||
t.Fatalf("second run: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDumpImportResumesAfterCancel(t *testing.T) {
|
||||
srv := serveDumps(t, fixtureSparkTar(t), fixtureCanonicalTarZst(t))
|
||||
|
||||
db := database.NewTestDB(t)
|
||||
si := NewSearchIndex(db, nil, nil, testLogger())
|
||||
imp := testImporter(t, si, srv)
|
||||
|
||||
// Cancelled before it can start streaming: no partial state may
|
||||
// break the follow-up run.
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
if err := imp.run(ctx); err == nil {
|
||||
t.Fatal("cancelled run should return an error")
|
||||
}
|
||||
|
||||
if err := imp.run(context.Background()); err != nil {
|
||||
t.Fatalf("rerun after cancel: %v", err)
|
||||
}
|
||||
|
||||
results := si.Search(context.Background(), "hit song", 10)
|
||||
if len(results) == 0 {
|
||||
t.Fatal("index empty after resumed run")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckFreeDisk(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
|
||||
if err := checkFreeDisk(dir, 1); err != nil {
|
||||
t.Errorf("1 byte requirement should pass: %v", err)
|
||||
}
|
||||
|
||||
if err := checkFreeDisk(dir, 1<<62); err == nil {
|
||||
t.Error("absurd requirement should fail")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoverDumpFilePickNewest(t *testing.T) {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/":
|
||||
_, _ = io.WriteString(w, `
|
||||
<a href="musicbrainz-canonical-dump-20260101-080003/">old</a>
|
||||
<a href="musicbrainz-canonical-dump-20260615-080003/">new</a>
|
||||
<a href="unrelated-dir/">x</a>`)
|
||||
case "/musicbrainz-canonical-dump-20260615-080003/":
|
||||
_, _ = io.WriteString(w,
|
||||
`<a href="musicbrainz-canonical-dump-20260615-080003.tar.zst">f</a>`)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
})
|
||||
|
||||
srv := httptest.NewServer(mux)
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
url, err := discoverDumpFile(
|
||||
context.Background(), srv.Client(), srv.URL+"/", canonicalDirRe, canonicalFileRe,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("discover: %v", err)
|
||||
}
|
||||
|
||||
want := srv.URL + "/musicbrainz-canonical-dump-20260615-080003/musicbrainz-canonical-dump-20260615-080003.tar.zst"
|
||||
if url != want {
|
||||
t.Errorf("url = %s, want %s", url, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListenerCountUpdateDoesNotTouchPopularity(t *testing.T) {
|
||||
db := database.NewTestDB(t)
|
||||
si := NewSearchIndex(db, nil, nil, testLogger())
|
||||
|
||||
si.upsertBatch([]SearchIndexResult{{
|
||||
EntityType: "recording",
|
||||
MBID: recA,
|
||||
Title: "Hit Song",
|
||||
ArtistName: "Solo Star",
|
||||
ArtistMBID: artA,
|
||||
Popularity: 12,
|
||||
}})
|
||||
|
||||
updated := si.updateListenerCounts(map[string]PopularityData{
|
||||
recA: {ListenCount: 999_999, ListenerCount: 42},
|
||||
})
|
||||
if updated != 1 {
|
||||
t.Fatalf("updated = %d, want 1", updated)
|
||||
}
|
||||
|
||||
rows, err := db.QueryContext(
|
||||
"SELECT popularity, listener_count FROM explore_index WHERE mbid = ?", recA,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("query: %v", err)
|
||||
}
|
||||
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
if !rows.Next() {
|
||||
t.Fatal("row missing")
|
||||
}
|
||||
|
||||
var pop, listeners int
|
||||
|
||||
if err := rows.Scan(&pop, &listeners); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
|
||||
if pop != 12 {
|
||||
t.Errorf("popularity = %d, want 12 (dump scale must stay authoritative)", pop)
|
||||
}
|
||||
|
||||
if listeners != 42 {
|
||||
t.Errorf("listener_count = %d, want 42", listeners)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,522 @@
|
||||
package explore
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bufio"
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Incremental listen-count refresh. ListenBrainz publishes a small
|
||||
// (~250MB) incremental spark dump every day containing only the listens
|
||||
// submitted since the previous dump, in the same parquet-in-tar format
|
||||
// as the full dump. This folds those daily deltas into the index's
|
||||
// popularity numbers additively, so recordings, artists, and albums stay
|
||||
// fresh without re-streaming the ~170GB full dump and without a single
|
||||
// ListenBrainz API call.
|
||||
//
|
||||
// Album (release-group) popularity is derived locally: the incremental
|
||||
// gives per-release listen counts, which are rolled up to their release
|
||||
// group via the release_to_rg map captured during the full import.
|
||||
//
|
||||
// Correctness: each dump's deltas plus the high-water-mark advance are
|
||||
// applied in one transaction, so a crash can't half-apply a dump, and
|
||||
// the high-water-mark guarantees each dump is applied at most once. The
|
||||
// sum of the full dump plus every incremental is therefore the exact
|
||||
// cumulative listen count — the additive model does not drift.
|
||||
|
||||
const (
|
||||
// defaultIncrementalBaseURL is where ListenBrainz publishes daily
|
||||
// incremental listen dumps.
|
||||
defaultIncrementalBaseURL = "https://data.metabrainz.org/pub/musicbrainz/listenbrainz/incremental/"
|
||||
|
||||
// listensCatchupTsKey records when the incremental refresh last ran
|
||||
// (RFC3339), gating how often it re-checks for new dumps.
|
||||
listensCatchupTsKey = "listens_last_catchup"
|
||||
|
||||
// listensCatchupInterval is the default minimum time between refresh
|
||||
// checks. Album/track popularity is slow-moving and each dump is a
|
||||
// ~250MB download, so a weekly cadence keeps the index current while
|
||||
// bounding background network use.
|
||||
listensCatchupInterval = 7 * 24 * time.Hour
|
||||
|
||||
// deltaInsertBatch bounds rows per multi-row INSERT into the temp
|
||||
// delta table during a single incremental apply.
|
||||
deltaInsertBatch = 500
|
||||
|
||||
// releaseLookupBatch bounds release MBIDs per release_to_rg lookup.
|
||||
releaseLookupBatch = 500
|
||||
)
|
||||
|
||||
var (
|
||||
incrementalDirRe = regexp.MustCompile(`^listenbrainz-dump-\d+-\d{8}-\d+-incremental$`)
|
||||
incrementalFileRe = regexp.MustCompile(`^listenbrainz-spark-dump-.*-incremental\.tar$`)
|
||||
)
|
||||
|
||||
// incrementalDump identifies one daily incremental dump.
|
||||
type incrementalDump struct {
|
||||
series int
|
||||
url string
|
||||
}
|
||||
|
||||
// RefreshListenCounts folds any incremental dumps newer than the current
|
||||
// high-water-mark into the index's popularity numbers. It is a no-op
|
||||
// when there is no completed baseline import, when a full build is
|
||||
// running, when the last refresh was within minInterval (pass 0 to
|
||||
// force), or when offline. Runs synchronously — callers wanting the
|
||||
// background behaviour should invoke it in a goroutine.
|
||||
func (si *SearchIndex) RefreshListenCounts(ctx context.Context, minInterval time.Duration) {
|
||||
if !si.hasMeta(dumpImportDoneKey) {
|
||||
si.logger.Info("incremental refresh: no baseline import yet, skipping")
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
si.mu.RLock()
|
||||
building := si.cancel != nil
|
||||
si.mu.RUnlock()
|
||||
|
||||
if building {
|
||||
si.logger.Info("incremental refresh: full build running, skipping")
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if minInterval > 0 && si.refreshedWithin(minInterval) {
|
||||
si.logger.Info("incremental refresh: checked recently, skipping")
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
hwm, ok := si.metaInt(listensAppliedSeriesKey)
|
||||
if !ok {
|
||||
si.logger.Warn("incremental refresh: no baseline series recorded, skipping")
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
client := &http.Client{}
|
||||
|
||||
dumps, err := discoverIncrementalDumps(ctx, client, defaultIncrementalBaseURL, hwm)
|
||||
if err != nil {
|
||||
si.logger.Warn("incremental refresh: discovery failed", "error", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Record the check even when there is nothing to apply, so the
|
||||
// cadence gate holds regardless of outcome.
|
||||
defer si.setMeta(listensCatchupTsKey, time.Now().UTC().Format(time.RFC3339))
|
||||
|
||||
if len(dumps) == 0 {
|
||||
si.logger.Info("incremental refresh: up to date", "throughSeries", hwm)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
si.logger.Info("incremental refresh: applying dumps",
|
||||
"count", len(dumps), "fromSeries", hwm+1, "toSeries", dumps[len(dumps)-1].series,
|
||||
)
|
||||
|
||||
applied := 0
|
||||
through := hwm
|
||||
|
||||
for _, d := range dumps {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
|
||||
if err := si.applyIncremental(ctx, client, d); err != nil {
|
||||
// Stop at the first failure; the high-water-mark is only
|
||||
// advanced on success, so the next run resumes from here.
|
||||
si.logger.Warn("incremental refresh: apply failed, stopping",
|
||||
"series", d.series, "error", err)
|
||||
|
||||
break
|
||||
}
|
||||
|
||||
applied++
|
||||
through = d.series
|
||||
}
|
||||
|
||||
if applied > 0 {
|
||||
// Popularity changed, so refresh the champion tier that backs
|
||||
// generic short-prefix searches.
|
||||
si.scheduleChampionRebuild()
|
||||
}
|
||||
|
||||
si.logger.Info("incremental refresh: complete", "applied", applied, "throughSeries", through)
|
||||
}
|
||||
|
||||
// applyIncremental downloads one incremental dump, aggregates its listen
|
||||
// counts, rolls release counts up to release groups, and applies the
|
||||
// deltas atomically together with the high-water-mark advance.
|
||||
func (si *SearchIndex) applyIncremental(
|
||||
ctx context.Context, client *http.Client, dump incrementalDump,
|
||||
) error {
|
||||
start := time.Now()
|
||||
|
||||
stream := newResumableReader(ctx, client, dump.url, 0)
|
||||
|
||||
defer func() { _ = stream.Close() }()
|
||||
|
||||
counts, err := aggregateTarListens(ctx, stream)
|
||||
if err != nil {
|
||||
return fmt.Errorf("aggregate incremental %d: %w", dump.series, err)
|
||||
}
|
||||
|
||||
rec, art, rel := splitCountsByKind(counts)
|
||||
rg := si.rollupReleaseDeltas(rel)
|
||||
|
||||
if err := si.commitListenDeltas(dump.series, rec, art, rg); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
si.logger.Info("incremental refresh: applied dump",
|
||||
"series", dump.series,
|
||||
"recordings", len(rec),
|
||||
"artists", len(art),
|
||||
"releaseGroups", len(rg),
|
||||
"elapsed", time.Since(start).Round(time.Millisecond),
|
||||
)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// splitCountsByKind separates an aggregated counts map into per-kind
|
||||
// maps of canonical MBID string → delta.
|
||||
func splitCountsByKind(counts map[mbidKey]uint32) (rec, art, rel map[string]uint32) {
|
||||
rec = make(map[string]uint32)
|
||||
art = make(map[string]uint32)
|
||||
rel = make(map[string]uint32)
|
||||
|
||||
for k, v := range counts {
|
||||
mbid := formatUUID(k[1:])
|
||||
|
||||
switch k[0] {
|
||||
case countKindRecording:
|
||||
rec[mbid] += v
|
||||
case countKindArtist:
|
||||
art[mbid] += v
|
||||
case countKindRelease:
|
||||
rel[mbid] += v
|
||||
}
|
||||
}
|
||||
|
||||
return rec, art, rel
|
||||
}
|
||||
|
||||
// rollupReleaseDeltas maps per-release listen deltas to their release
|
||||
// group via the release_to_rg table and sums per group. Releases not in
|
||||
// the table (below the index floor, or unknown) contribute nothing.
|
||||
func (si *SearchIndex) rollupReleaseDeltas(rel map[string]uint32) map[string]uint32 {
|
||||
rg := make(map[string]uint32)
|
||||
if len(rel) == 0 {
|
||||
return rg
|
||||
}
|
||||
|
||||
mbids := make([]string, 0, len(rel))
|
||||
for m := range rel {
|
||||
mbids = append(mbids, m)
|
||||
}
|
||||
|
||||
for i := 0; i < len(mbids); i += releaseLookupBatch {
|
||||
end := min(i+releaseLookupBatch, len(mbids))
|
||||
batch := mbids[i:end]
|
||||
|
||||
placeholders := strings.TrimSuffix(strings.Repeat("?,", len(batch)), ",")
|
||||
|
||||
args := make([]any, len(batch))
|
||||
for j, m := range batch {
|
||||
args[j] = m
|
||||
}
|
||||
|
||||
rows, err := si.db.QueryContext(
|
||||
"SELECT release_mbid, rg_mbid FROM release_to_rg WHERE release_mbid IN ("+placeholders+")",
|
||||
args...,
|
||||
)
|
||||
if err != nil {
|
||||
si.logger.Warn("incremental refresh: release_to_rg lookup failed", "error", err)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
for rows.Next() {
|
||||
var relMBID, rgMBID string
|
||||
if err := rows.Scan(&relMBID, &rgMBID); err == nil {
|
||||
rg[rgMBID] += rel[relMBID]
|
||||
}
|
||||
}
|
||||
|
||||
_ = rows.Close()
|
||||
}
|
||||
|
||||
return rg
|
||||
}
|
||||
|
||||
// commitListenDeltas applies recording, artist, and release-group deltas
|
||||
// to explore_index and advances the high-water-mark to series, all in a
|
||||
// single transaction so the apply is crash-atomic and exactly-once.
|
||||
func (si *SearchIndex) commitListenDeltas(
|
||||
series int, rec, art, rg map[string]uint32,
|
||||
) error {
|
||||
tx, err := si.db.BeginTx()
|
||||
if err != nil {
|
||||
return fmt.Errorf("incremental tx: %w", err)
|
||||
}
|
||||
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
if _, err := tx.Exec(
|
||||
"CREATE TEMP TABLE IF NOT EXISTS incr_delta (mbid TEXT, kind TEXT, delta INTEGER)",
|
||||
); err != nil {
|
||||
return fmt.Errorf("incremental temp table: %w", err)
|
||||
}
|
||||
|
||||
if _, err := tx.Exec("DELETE FROM incr_delta"); err != nil {
|
||||
return fmt.Errorf("incremental temp reset: %w", err)
|
||||
}
|
||||
|
||||
for _, kd := range []struct {
|
||||
kind string
|
||||
deltas map[string]uint32
|
||||
}{
|
||||
{"recording", rec},
|
||||
{"artist", art},
|
||||
{"release_group", rg},
|
||||
} {
|
||||
if err := insertDeltas(tx, kd.kind, kd.deltas); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Additive apply: bump popularity for every index row that has a
|
||||
// matching delta. Rows with no delta are untouched; deltas with no
|
||||
// matching row (entity not indexed) are ignored.
|
||||
if _, err := tx.Exec(`
|
||||
UPDATE explore_index
|
||||
SET popularity = popularity + d.delta
|
||||
FROM incr_delta d
|
||||
WHERE d.mbid = explore_index.mbid
|
||||
AND d.kind = explore_index.entity_type
|
||||
`); err != nil {
|
||||
return fmt.Errorf("incremental apply: %w", err)
|
||||
}
|
||||
|
||||
if _, err := tx.Exec("DELETE FROM incr_delta"); err != nil {
|
||||
return fmt.Errorf("incremental temp cleanup: %w", err)
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(
|
||||
"INSERT OR REPLACE INTO explore_index_meta (key, value) VALUES (?, ?)",
|
||||
listensAppliedSeriesKey, strconv.Itoa(series),
|
||||
); err != nil {
|
||||
return fmt.Errorf("incremental advance high-water-mark: %w", err)
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("incremental commit: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// insertDeltas bulk-inserts a kind's deltas into the temp table.
|
||||
func insertDeltas(tx *sql.Tx, kind string, deltas map[string]uint32) error {
|
||||
if len(deltas) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
rowArgs := make([]any, 0, deltaInsertBatch*3)
|
||||
pending := 0
|
||||
|
||||
flush := func() error {
|
||||
if pending == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
values := strings.TrimSuffix(strings.Repeat("(?,?,?),", pending), ",")
|
||||
query := "INSERT INTO incr_delta (mbid, kind, delta) VALUES " + values
|
||||
|
||||
if _, err := tx.Exec(query, rowArgs...); err != nil {
|
||||
return fmt.Errorf("incremental insert deltas: %w", err)
|
||||
}
|
||||
|
||||
rowArgs = rowArgs[:0]
|
||||
pending = 0
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
for mbid, d := range deltas {
|
||||
rowArgs = append(rowArgs, mbid, kind, int64(d))
|
||||
pending++
|
||||
|
||||
if pending >= deltaInsertBatch {
|
||||
if err := flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return flush()
|
||||
}
|
||||
|
||||
// aggregateTarListens streams a tar of parquet listen members and sums
|
||||
// the per-entity listen counts. Unlike the full-dump path, the whole
|
||||
// (small) incremental is aggregated in RAM with no checkpointing.
|
||||
func aggregateTarListens(ctx context.Context, r io.Reader) (map[mbidKey]uint32, error) {
|
||||
buffered := bufio.NewReaderSize(r, 1<<20)
|
||||
tr := tar.NewReader(buffered)
|
||||
counts := make(map[mbidKey]uint32, 1<<18)
|
||||
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
hdr, err := tr.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("incremental tar: %w", err)
|
||||
}
|
||||
|
||||
if hdr.Typeflag != tar.TypeReg || !strings.HasSuffix(hdr.Name, ".parquet") {
|
||||
continue
|
||||
}
|
||||
|
||||
if hdr.Size > maxParquetMemberSize {
|
||||
return nil, fmt.Errorf(
|
||||
"%w: parquet member %s is %d bytes", ErrDumpFormat, hdr.Name, hdr.Size,
|
||||
)
|
||||
}
|
||||
|
||||
buf := make([]byte, hdr.Size)
|
||||
if _, err := io.ReadFull(tr, buf); err != nil {
|
||||
return nil, fmt.Errorf("incremental member read: %w", err)
|
||||
}
|
||||
|
||||
deltas, err := parseListenParquet(buf)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("incremental parse: %w", err)
|
||||
}
|
||||
|
||||
for k, v := range deltas {
|
||||
counts[k] += v
|
||||
}
|
||||
}
|
||||
|
||||
return counts, nil
|
||||
}
|
||||
|
||||
// discoverIncrementalDumps lists the incremental directory and returns
|
||||
// the spark-dump URLs for every dump with a series greater than
|
||||
// sinceSeries, sorted ascending so they apply in chronological order.
|
||||
func discoverIncrementalDumps(
|
||||
ctx context.Context, client *http.Client, baseURL string, sinceSeries int,
|
||||
) ([]incrementalDump, error) {
|
||||
hrefs, err := listHrefs(ctx, client, baseURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var dumps []incrementalDump
|
||||
|
||||
for _, h := range hrefs {
|
||||
dir := trimTrailingSlash(h)
|
||||
if !incrementalDirRe.MatchString(dir) {
|
||||
continue
|
||||
}
|
||||
|
||||
series, ok := parseDumpSeries(dir)
|
||||
if !ok || series <= sinceSeries {
|
||||
continue
|
||||
}
|
||||
|
||||
dirURL := baseURL + dir + "/"
|
||||
|
||||
files, err := listHrefs(ctx, client, dirURL)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, f := range files {
|
||||
if incrementalFileRe.MatchString(f) {
|
||||
dumps = append(dumps, incrementalDump{series: series, url: dirURL + f})
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sort.Slice(dumps, func(i, j int) bool { return dumps[i].series < dumps[j].series })
|
||||
|
||||
return dumps, nil
|
||||
}
|
||||
|
||||
// metaInt reads an integer-valued explore_index_meta key.
|
||||
func (si *SearchIndex) metaInt(key string) (int, bool) {
|
||||
rows, err := si.db.QueryContext("SELECT value FROM explore_index_meta WHERE key = ?", key)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
if !rows.Next() {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
var v string
|
||||
if err := rows.Scan(&v); err != nil {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
n, err := strconv.Atoi(v)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
return n, true
|
||||
}
|
||||
|
||||
// refreshedWithin reports whether the incremental refresh last ran less
|
||||
// than d ago.
|
||||
func (si *SearchIndex) refreshedWithin(d time.Duration) bool {
|
||||
rows, err := si.db.QueryContext(
|
||||
"SELECT value FROM explore_index_meta WHERE key = ?", listensCatchupTsKey,
|
||||
)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
if !rows.Next() {
|
||||
return false
|
||||
}
|
||||
|
||||
var v string
|
||||
if err := rows.Scan(&v); err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
t, err := time.Parse(time.RFC3339, v)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
return time.Since(t) < d
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
package explore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"yellowjacket/backend/database"
|
||||
)
|
||||
|
||||
func TestParseDumpSeries(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
want int
|
||||
ok bool
|
||||
}{
|
||||
{"https://x/fullexport/listenbrainz-dump-2593-20260712-000004-full/y.tar", 2593, true},
|
||||
{"listenbrainz-dump-2594-20260713-000003-incremental", 2594, true},
|
||||
{"listenbrainz-spark-dump-2603-20260722-000003-incremental.tar", 2603, true},
|
||||
{"nothing-here", 0, false},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
got, ok := parseDumpSeries(c.in)
|
||||
if ok != c.ok || (ok && got != c.want) {
|
||||
t.Errorf("parseDumpSeries(%q) = (%d, %v); want (%d, %v)", c.in, got, ok, c.want, c.ok)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func popularityOf(t *testing.T, db *database.DB, mbid string) (int, bool) {
|
||||
t.Helper()
|
||||
|
||||
rows, err := db.QueryContext(
|
||||
"SELECT popularity FROM explore_index WHERE mbid = ?", mbid,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("query popularity: %v", err)
|
||||
}
|
||||
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
if !rows.Next() {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
var pop int
|
||||
if err := rows.Scan(&pop); err != nil {
|
||||
t.Fatalf("scan popularity: %v", err)
|
||||
}
|
||||
|
||||
return pop, true
|
||||
}
|
||||
|
||||
// applyTar runs the incremental split → rollup → commit path against an
|
||||
// in-memory tar, mirroring applyIncremental without the HTTP stream.
|
||||
func applyTar(t *testing.T, si *SearchIndex, series int, tarBytes []byte) {
|
||||
t.Helper()
|
||||
|
||||
counts, err := aggregateTarListens(context.Background(), bytes.NewReader(tarBytes))
|
||||
if err != nil {
|
||||
t.Fatalf("aggregate: %v", err)
|
||||
}
|
||||
|
||||
rec, art, rel := splitCountsByKind(counts)
|
||||
rg := si.rollupReleaseDeltas(rel)
|
||||
|
||||
if err := si.commitListenDeltas(series, rec, art, rg); err != nil {
|
||||
t.Fatalf("commit series %d: %v", series, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIncrementalApplyAdditive(t *testing.T) {
|
||||
db := database.NewTestDB(t)
|
||||
si := NewSearchIndex(db, nil, nil, testLogger())
|
||||
|
||||
si.upsertBatch([]SearchIndexResult{
|
||||
{EntityType: "recording", MBID: recA, Title: "Rec A", Popularity: 100},
|
||||
{EntityType: "artist", MBID: artA, Title: "Art A", Popularity: 100},
|
||||
{EntityType: "release_group", MBID: rgA, Title: "RG A", Popularity: 100},
|
||||
})
|
||||
|
||||
if _, err := db.ExecContext(
|
||||
"INSERT INTO release_to_rg (release_mbid, rg_mbid) VALUES (?, ?)", relA, rgA,
|
||||
); err != nil {
|
||||
t.Fatalf("seed release_to_rg: %v", err)
|
||||
}
|
||||
|
||||
// 5 listens of recA on relA credited to artA.
|
||||
tarBytes := makeTar(t,
|
||||
map[string][]byte{
|
||||
"listens/1.parquet": makeParquet(t, listensOf(5, recA, relA, []string{artA})),
|
||||
},
|
||||
[]string{"listens/1.parquet"},
|
||||
)
|
||||
|
||||
applyTar(t, si, 2, tarBytes)
|
||||
|
||||
for _, c := range []struct {
|
||||
mbid string
|
||||
want int
|
||||
}{{recA, 105}, {artA, 105}, {rgA, 105}} {
|
||||
if got, ok := popularityOf(t, db, c.mbid); !ok || got != c.want {
|
||||
t.Errorf("popularity(%s) = %d (present=%v); want %d", c.mbid, got, ok, c.want)
|
||||
}
|
||||
}
|
||||
|
||||
if hwm, ok := si.metaInt(listensAppliedSeriesKey); !ok || hwm != 2 {
|
||||
t.Errorf("high-water-mark = %d (present=%v); want 2", hwm, ok)
|
||||
}
|
||||
|
||||
// A second, later dump accumulates additively.
|
||||
applyTar(t, si, 3, tarBytes)
|
||||
|
||||
if got, _ := popularityOf(t, db, recA); got != 110 {
|
||||
t.Errorf("popularity(recA) after second dump = %d; want 110", got)
|
||||
}
|
||||
|
||||
if hwm, _ := si.metaInt(listensAppliedSeriesKey); hwm != 3 {
|
||||
t.Errorf("high-water-mark after second dump = %d; want 3", hwm)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIncrementalIgnoresUnknownAndUnmapped(t *testing.T) {
|
||||
db := database.NewTestDB(t)
|
||||
si := NewSearchIndex(db, nil, nil, testLogger())
|
||||
|
||||
// Only recA is indexed; relA has no release_to_rg mapping.
|
||||
si.upsertBatch([]SearchIndexResult{
|
||||
{EntityType: "recording", MBID: recA, Title: "Rec A", Popularity: 100},
|
||||
})
|
||||
|
||||
// Listens for recB (not indexed) on relB (unmapped) credited to
|
||||
// artB (not indexed). Nothing should change and no row created.
|
||||
tarBytes := makeTar(t,
|
||||
map[string][]byte{
|
||||
"listens/1.parquet": makeParquet(t, listensOf(7, recB, relB, []string{artB})),
|
||||
},
|
||||
[]string{"listens/1.parquet"},
|
||||
)
|
||||
|
||||
applyTar(t, si, 5, tarBytes)
|
||||
|
||||
if _, ok := popularityOf(t, db, recB); ok {
|
||||
t.Error("recB should not have been inserted into the index")
|
||||
}
|
||||
|
||||
if got, _ := popularityOf(t, db, recA); got != 100 {
|
||||
t.Errorf("popularity(recA) = %d; want 100 (untouched)", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoverIncrementalDumps(t *testing.T) {
|
||||
series := []string{"2594", "2595", "2596"}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
|
||||
mux.HandleFunc("/incremental/", func(w http.ResponseWriter, r *http.Request) {
|
||||
// Only the base listing (exact path); dir listings are handled
|
||||
// by their own more-specific patterns below.
|
||||
if r.URL.Path != "/incremental/" {
|
||||
http.NotFound(w, r)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
for _, s := range series {
|
||||
_, _ = fmt.Fprintf(w,
|
||||
`<a href="listenbrainz-dump-%s-20260713-000003-incremental/">dir</a>`+"\n", s,
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
for _, s := range series {
|
||||
dir := fmt.Sprintf("/incremental/listenbrainz-dump-%s-20260713-000003-incremental/", s)
|
||||
file := fmt.Sprintf("listenbrainz-spark-dump-%s-20260713-000003-incremental.tar", s)
|
||||
|
||||
mux.HandleFunc(dir, func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = fmt.Fprintf(w, `<a href="%s">file</a>`, file)
|
||||
})
|
||||
}
|
||||
|
||||
srv := httptest.NewServer(mux)
|
||||
defer srv.Close()
|
||||
|
||||
dumps, err := discoverIncrementalDumps(
|
||||
context.Background(), srv.Client(), srv.URL+"/incremental/", 2594,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("discover: %v", err)
|
||||
}
|
||||
|
||||
// Only 2595 and 2596 are newer than the 2594 high-water-mark.
|
||||
if len(dumps) != 2 {
|
||||
t.Fatalf("got %d dumps; want 2: %+v", len(dumps), dumps)
|
||||
}
|
||||
|
||||
if dumps[0].series != 2595 || dumps[1].series != 2596 {
|
||||
t.Errorf("series order wrong: %d, %d; want 2595, 2596", dumps[0].series, dumps[1].series)
|
||||
}
|
||||
|
||||
if !strings.Contains(dumps[0].url, "spark-dump-2595") {
|
||||
t.Errorf("unexpected url: %s", dumps[0].url)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
package explore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Stage 4 of the dump import: small, idempotent API patch passes that
|
||||
// fill in what the dumps can't provide. All calls go through the
|
||||
// shared rate-limited ListenBrainz client and its HTTP cache, so
|
||||
// re-running after an interruption is cheap.
|
||||
|
||||
const (
|
||||
// Listener-count patch budgets: only the most popular rows get
|
||||
// listener counts (a secondary ranking signal). 1000 MBIDs per
|
||||
// batched POST call → ~350 API calls total.
|
||||
listenerPatchArtists = 50_000
|
||||
listenerPatchRGs = 100_000
|
||||
listenerPatchRecordings = 200_000
|
||||
|
||||
// popularityBatchSize is the number of MBIDs per LB popularity
|
||||
// or metadata request. LB accepts up to 1000 per call.
|
||||
popularityBatchSize = 1000
|
||||
)
|
||||
|
||||
// runPatchPasses fills listener counts, artist metadata, and the
|
||||
// similar-artist map from the ListenBrainz API.
|
||||
func (imp *dumpImporter) runPatchPasses(ctx context.Context) {
|
||||
if imp.lb == nil {
|
||||
return
|
||||
}
|
||||
|
||||
imp.patchArtistMetadata(ctx)
|
||||
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
|
||||
imp.patchSimilarArtists(ctx)
|
||||
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
|
||||
imp.patchListenerCounts(ctx)
|
||||
}
|
||||
|
||||
// patchArtistMetadata batch-fetches type/country/name for indexed
|
||||
// artists that are missing them. Also creates rows for kept artists
|
||||
// whose name wasn't derivable from the canonical dump (multi-artist
|
||||
// credits only).
|
||||
func (imp *dumpImporter) patchArtistMetadata(ctx context.Context) {
|
||||
rows, err := imp.si.db.QueryContext(`
|
||||
SELECT mbid FROM explore_index
|
||||
WHERE entity_type = 'artist'
|
||||
AND (artist_type = '' OR country = '' OR title = '' OR title = mbid)
|
||||
`)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
var mbids []string
|
||||
|
||||
for rows.Next() {
|
||||
var m string
|
||||
if err := rows.Scan(&m); err == nil {
|
||||
mbids = append(mbids, m)
|
||||
}
|
||||
}
|
||||
|
||||
_ = rows.Close()
|
||||
|
||||
if len(imp.pendingArtists) > 0 {
|
||||
mbids = append(mbids, imp.pendingArtists...)
|
||||
}
|
||||
|
||||
if len(mbids) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
imp.logger.Info("dump import: patching artist metadata", "artists", len(mbids))
|
||||
|
||||
batches := chunkStrings(mbids, popularityBatchSize)
|
||||
patched := 0
|
||||
|
||||
for i, batch := range batches {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
|
||||
meta, err := imp.lb.BatchArtistMetadata(ctx, batch)
|
||||
if err != nil || len(meta) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
entries := make([]SearchIndexResult, 0, len(meta))
|
||||
|
||||
for mbid, m := range meta {
|
||||
if m.Name == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
entries = append(entries, SearchIndexResult{
|
||||
EntityType: "artist",
|
||||
MBID: mbid,
|
||||
Title: m.Name,
|
||||
ArtistName: m.Name,
|
||||
ArtistMBID: mbid,
|
||||
ArtistType: m.Type,
|
||||
Country: m.Country,
|
||||
})
|
||||
|
||||
// Pre-populate the MB rels cache so on-demand artist
|
||||
// image resolution skips a MusicBrainz call.
|
||||
if imp.si.artistImg != nil {
|
||||
imp.si.artistImg.PreloadArtistRels(mbid, m)
|
||||
}
|
||||
}
|
||||
|
||||
imp.si.upsertBatch(entries)
|
||||
|
||||
patched += len(entries)
|
||||
|
||||
imp.setStageProgress(dumpStagePatch, i+1, len(batches))
|
||||
}
|
||||
|
||||
imp.logger.Info("dump import: artist metadata patched", "artists", patched)
|
||||
}
|
||||
|
||||
// patchSimilarArtists refreshes the similar-artist map for library
|
||||
// artists (one API call per library artist, cached for a week).
|
||||
func (imp *dumpImporter) patchSimilarArtists(ctx context.Context) {
|
||||
libraryMBIDs := imp.si.getLibraryArtistMBIDs()
|
||||
if len(libraryMBIDs) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
for i := 0; i < len(libraryMBIDs); i += similarArtistsBatchSize {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
|
||||
end := min(i+similarArtistsBatchSize, len(libraryMBIDs))
|
||||
|
||||
grouped := imp.si.fetchSimilarArtistsBatch(ctx, imp.lb, libraryMBIDs[i:end])
|
||||
for seed, similar := range grouped {
|
||||
imp.si.storeSimilarArtists(seed, similar)
|
||||
|
||||
// Flag indexed similar artists for personalized ranking.
|
||||
for _, s := range similar {
|
||||
_, _ = imp.si.db.ExecContext(
|
||||
"UPDATE explore_index SET is_similar = 1 WHERE artist_mbid = ?",
|
||||
s.ArtistMBID,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
imp.logger.Info("dump import: similar artists patched", "libraryArtists", len(libraryMBIDs))
|
||||
}
|
||||
|
||||
// patchListenerCounts fills listener_count for the most popular rows
|
||||
// of each entity type. Popularity (listen count) is NOT overwritten —
|
||||
// the dump-derived counts stay authoritative so the ranking scale is
|
||||
// consistent across the whole index.
|
||||
func (imp *dumpImporter) patchListenerCounts(ctx context.Context) {
|
||||
kinds := []struct {
|
||||
entityType string
|
||||
limit int
|
||||
fetch func(context.Context, []string) (map[string]PopularityData, error)
|
||||
}{
|
||||
{"artist", listenerPatchArtists, imp.lb.ArtistPopularity},
|
||||
{"release_group", listenerPatchRGs, imp.lb.ReleaseGroupPopularity},
|
||||
{"recording", listenerPatchRecordings, imp.lb.RecordingPopularity},
|
||||
}
|
||||
|
||||
for _, kind := range kinds {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
|
||||
mbids := imp.topMBIDs(kind.entityType, kind.limit)
|
||||
if len(mbids) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
batches := chunkStrings(mbids, popularityBatchSize)
|
||||
filled := 0
|
||||
|
||||
for i, batch := range batches {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
|
||||
pops, err := kind.fetch(ctx, batch)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
filled += imp.si.updateListenerCounts(pops)
|
||||
|
||||
imp.setStageProgress(dumpStageListeners, i+1, len(batches))
|
||||
}
|
||||
|
||||
imp.logger.Info("dump import: listener counts patched",
|
||||
"entityType", kind.entityType,
|
||||
"rows", filled,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// topMBIDs returns the most popular index MBIDs for an entity type
|
||||
// that don't have listener counts yet.
|
||||
func (imp *dumpImporter) topMBIDs(entityType string, limit int) []string {
|
||||
rows, err := imp.si.db.QueryContext(`
|
||||
SELECT mbid FROM explore_index
|
||||
WHERE entity_type = ? AND listener_count = 0
|
||||
ORDER BY popularity DESC
|
||||
LIMIT ?
|
||||
`, entityType, limit)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
var mbids []string
|
||||
|
||||
for rows.Next() {
|
||||
var m string
|
||||
if err := rows.Scan(&m); err == nil {
|
||||
mbids = append(mbids, m)
|
||||
}
|
||||
}
|
||||
|
||||
return mbids
|
||||
}
|
||||
|
||||
// updateListenerCounts writes listener counts only (never popularity),
|
||||
// keeping the dump-derived popularity scale consistent. Returns the
|
||||
// number of rows updated.
|
||||
func (si *SearchIndex) updateListenerCounts(updates map[string]PopularityData) int {
|
||||
if len(updates) == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
tx, err := si.db.BeginTx()
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
updated := 0
|
||||
|
||||
for mbid, data := range updates {
|
||||
if data.ListenerCount <= 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
res, err := tx.Exec(
|
||||
`UPDATE explore_index
|
||||
SET listener_count = ?
|
||||
WHERE mbid = ? AND listener_count < ?`,
|
||||
data.ListenerCount, strings.ToLower(mbid), data.ListenerCount,
|
||||
)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if n, err := res.RowsAffected(); err == nil {
|
||||
updated += int(n)
|
||||
}
|
||||
}
|
||||
|
||||
_ = tx.Commit()
|
||||
|
||||
return updated
|
||||
}
|
||||
@@ -0,0 +1,329 @@
|
||||
package explore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Streaming helpers for the MetaBrainz dump imports. Dumps are never
|
||||
// written to disk: the HTTP body is decoded (tar / zstd+tar) in flight.
|
||||
// resumableReader reconnects with HTTP Range requests on transient
|
||||
// failures, which also lets the listens import resume across app
|
||||
// restarts from a checkpointed byte offset.
|
||||
|
||||
const (
|
||||
// maxStreamRetries is the number of consecutive failed reconnect
|
||||
// attempts before a stream read gives up. The counter resets
|
||||
// whenever bytes are successfully delivered.
|
||||
maxStreamRetries = 8
|
||||
|
||||
// streamRetryBaseDelay is the initial reconnect backoff; it
|
||||
// doubles per consecutive failure.
|
||||
streamRetryBaseDelay = 2 * time.Second
|
||||
|
||||
// dumpDiscoveryTimeout bounds the small directory-listing
|
||||
// requests (not the multi-hour stream requests).
|
||||
dumpDiscoveryTimeout = 30 * time.Second
|
||||
)
|
||||
|
||||
// ErrDumpDiscovery is returned when a dump directory listing does not
|
||||
// contain the expected entries.
|
||||
var ErrDumpDiscovery = errors.New("dump discovery failed")
|
||||
|
||||
// ErrDumpStream is returned when a dump stream fails permanently.
|
||||
var ErrDumpStream = errors.New("dump stream failed")
|
||||
|
||||
var hrefRe = regexp.MustCompile(`href="([^"?/][^"?]*)"`)
|
||||
|
||||
// listHrefs fetches an Apache-style index page and returns the href
|
||||
// values (directory entries end with a trailing slash).
|
||||
func listHrefs(ctx context.Context, client *http.Client, url string) ([]string, error) {
|
||||
reqCtx, cancel := context.WithTimeout(ctx, dumpDiscoveryTimeout)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dump listing request: %w", err)
|
||||
}
|
||||
|
||||
req.Header.Set("User-Agent", lbUserAgent)
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dump listing fetch: %w", err)
|
||||
}
|
||||
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf(
|
||||
"%w: listing %s returned HTTP %d", ErrDumpDiscovery, url, resp.StatusCode,
|
||||
)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dump listing read: %w", err)
|
||||
}
|
||||
|
||||
var hrefs []string
|
||||
|
||||
for _, m := range hrefRe.FindAllStringSubmatch(string(body), -1) {
|
||||
hrefs = append(hrefs, m[1])
|
||||
}
|
||||
|
||||
return hrefs, nil
|
||||
}
|
||||
|
||||
// discoverDumpFile walks a dump base directory, finds subdirectories
|
||||
// matching dirRe (newest first, lexicographically — MetaBrainz dump
|
||||
// directory names embed sortable timestamps), and returns the full URL
|
||||
// of the first file inside matching fileRe. Directories that don't
|
||||
// contain a matching file (e.g. partial uploads) are skipped.
|
||||
func discoverDumpFile(
|
||||
ctx context.Context,
|
||||
client *http.Client,
|
||||
baseURL string,
|
||||
dirRe, fileRe *regexp.Regexp,
|
||||
) (string, error) {
|
||||
hrefs, err := listHrefs(ctx, client, baseURL)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var dirs []string
|
||||
|
||||
for _, h := range hrefs {
|
||||
trimmed := trimTrailingSlash(h)
|
||||
if dirRe.MatchString(trimmed) {
|
||||
dirs = append(dirs, trimmed)
|
||||
}
|
||||
}
|
||||
|
||||
if len(dirs) == 0 {
|
||||
return "", fmt.Errorf("%w: no dump directories under %s", ErrDumpDiscovery, baseURL)
|
||||
}
|
||||
|
||||
sort.Sort(sort.Reverse(sort.StringSlice(dirs)))
|
||||
|
||||
for _, dir := range dirs {
|
||||
dirURL := baseURL + dir + "/"
|
||||
|
||||
files, err := listHrefs(ctx, client, dirURL)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, f := range files {
|
||||
if fileRe.MatchString(f) {
|
||||
return dirURL + f, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("%w: no matching dump file under %s", ErrDumpDiscovery, baseURL)
|
||||
}
|
||||
|
||||
func trimTrailingSlash(s string) string {
|
||||
if len(s) > 0 && s[len(s)-1] == '/' {
|
||||
return s[:len(s)-1]
|
||||
}
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
// resumableReader is an io.Reader over an HTTP resource that survives
|
||||
// connection failures by reconnecting with a Range request at the
|
||||
// current offset. Offset is the absolute position of the next byte to
|
||||
// deliver, so callers can checkpoint it and construct a new
|
||||
// resumableReader later to resume a partially-processed stream.
|
||||
type resumableReader struct {
|
||||
ctx context.Context
|
||||
client *http.Client
|
||||
url string
|
||||
|
||||
// Offset is the absolute byte position of the next read.
|
||||
Offset int64
|
||||
|
||||
// Size is the total resource size, learned from the first
|
||||
// response. -1 until known.
|
||||
Size int64
|
||||
|
||||
body io.ReadCloser
|
||||
retries int
|
||||
}
|
||||
|
||||
func newResumableReader(
|
||||
ctx context.Context, client *http.Client, url string, offset int64,
|
||||
) *resumableReader {
|
||||
return &resumableReader{
|
||||
ctx: ctx,
|
||||
client: client,
|
||||
url: url,
|
||||
Offset: offset,
|
||||
Size: -1,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *resumableReader) Read(p []byte) (int, error) {
|
||||
for {
|
||||
if err := r.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if r.body == nil {
|
||||
if err := r.connect(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
|
||||
n, err := r.body.Read(p)
|
||||
r.Offset += int64(n)
|
||||
|
||||
if n > 0 {
|
||||
r.retries = 0
|
||||
}
|
||||
|
||||
switch {
|
||||
case err == nil:
|
||||
return n, nil
|
||||
case errors.Is(err, io.EOF):
|
||||
// A server that closes early looks like EOF; only
|
||||
// trust it when we've seen the advertised size.
|
||||
if r.Size >= 0 && r.Offset < r.Size {
|
||||
r.closeBody()
|
||||
|
||||
if retryErr := r.backoff(err); retryErr != nil {
|
||||
return n, retryErr
|
||||
}
|
||||
|
||||
if n > 0 {
|
||||
return n, nil
|
||||
}
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
return n, io.EOF
|
||||
default:
|
||||
r.closeBody()
|
||||
|
||||
if retryErr := r.backoff(err); retryErr != nil {
|
||||
return n, retryErr
|
||||
}
|
||||
|
||||
if n > 0 {
|
||||
return n, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// backoff sleeps with exponential backoff, or returns a terminal error
|
||||
// once the retry budget is exhausted.
|
||||
func (r *resumableReader) backoff(cause error) error {
|
||||
r.retries++
|
||||
if r.retries > maxStreamRetries {
|
||||
return fmt.Errorf(
|
||||
"%w: %s after %d retries: %w", ErrDumpStream, r.url, maxStreamRetries, cause,
|
||||
)
|
||||
}
|
||||
|
||||
delay := streamRetryBaseDelay << (r.retries - 1)
|
||||
|
||||
select {
|
||||
case <-r.ctx.Done():
|
||||
return r.ctx.Err()
|
||||
case <-time.After(delay):
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (r *resumableReader) connect() error {
|
||||
req, err := http.NewRequestWithContext(r.ctx, http.MethodGet, r.url, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("dump stream request: %w", err)
|
||||
}
|
||||
|
||||
req.Header.Set("User-Agent", lbUserAgent)
|
||||
|
||||
if r.Offset > 0 {
|
||||
req.Header.Set("Range", "bytes="+strconv.FormatInt(r.Offset, 10)+"-")
|
||||
}
|
||||
|
||||
resp, err := r.client.Do(req)
|
||||
if err != nil {
|
||||
return r.backoff(err)
|
||||
}
|
||||
|
||||
switch resp.StatusCode {
|
||||
case http.StatusPartialContent:
|
||||
if r.Size < 0 {
|
||||
r.Size = parseContentRangeTotal(resp.Header.Get("Content-Range"))
|
||||
}
|
||||
|
||||
r.body = resp.Body
|
||||
|
||||
return nil
|
||||
case http.StatusOK:
|
||||
if r.Size < 0 && resp.ContentLength > 0 {
|
||||
r.Size = resp.ContentLength
|
||||
}
|
||||
|
||||
// Server ignored the Range header: discard the prefix so
|
||||
// the caller still reads from the requested offset.
|
||||
if r.Offset > 0 {
|
||||
if _, err := io.CopyN(io.Discard, resp.Body, r.Offset); err != nil {
|
||||
_ = resp.Body.Close()
|
||||
|
||||
return r.backoff(err)
|
||||
}
|
||||
}
|
||||
|
||||
r.body = resp.Body
|
||||
|
||||
return nil
|
||||
default:
|
||||
_ = resp.Body.Close()
|
||||
|
||||
return r.backoff(fmt.Errorf("%w: HTTP %d from %s", ErrDumpStream, resp.StatusCode, r.url))
|
||||
}
|
||||
}
|
||||
|
||||
func (r *resumableReader) closeBody() {
|
||||
if r.body != nil {
|
||||
_ = r.body.Close()
|
||||
r.body = nil
|
||||
}
|
||||
}
|
||||
|
||||
// Close releases the underlying HTTP body, if any.
|
||||
func (r *resumableReader) Close() error {
|
||||
r.closeBody()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// parseContentRangeTotal extracts the total size from a Content-Range
|
||||
// header ("bytes 100-199/12345"). Returns -1 if unavailable.
|
||||
func parseContentRangeTotal(v string) int64 {
|
||||
for i := len(v) - 1; i >= 0; i-- {
|
||||
if v[i] == '/' {
|
||||
total, err := strconv.ParseInt(v[i+1:], 10, 64)
|
||||
if err != nil {
|
||||
return -1
|
||||
}
|
||||
|
||||
return total
|
||||
}
|
||||
}
|
||||
|
||||
return -1
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
// Package eval is the search-ranking evaluation harness. It turns
|
||||
// "this query feels wrong" into a number that goes up or down, so a
|
||||
// ranking change can be validated against a frozen set of labelled
|
||||
// queries instead of tuned by anecdote.
|
||||
//
|
||||
// The harness is deliberately decoupled from the explore package: it
|
||||
// knows nothing about MusicBrainz, ListenBrainz, or the search index.
|
||||
// A caller adapts whatever ranking function it wants to measure to the
|
||||
// Ranker interface, loads a fixture set, and runs Evaluate. The
|
||||
// explore package wires its real index Search to this in an
|
||||
// integration test (see explore/eval_harness_test.go).
|
||||
package eval
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ErrNoFixtures is returned when a fixture file contains zero queries.
|
||||
var ErrNoFixtures = errors.New("eval: fixture set is empty")
|
||||
|
||||
// Result is one ranked search hit, reduced to the only two fields the
|
||||
// harness needs to decide whether it matches an expectation.
|
||||
type Result struct {
|
||||
EntityType string `json:"entityType"`
|
||||
MBID string `json:"mbid"`
|
||||
}
|
||||
|
||||
// Ranker produces an ordered result list for a query. Best result
|
||||
// first. Implemented by adapting a real search function.
|
||||
type Ranker interface {
|
||||
Rank(query string, limit int) []Result
|
||||
}
|
||||
|
||||
// RankerFunc adapts a plain function to the Ranker interface.
|
||||
type RankerFunc func(query string, limit int) []Result
|
||||
|
||||
// Rank calls the underlying function.
|
||||
func (f RankerFunc) Rank(query string, limit int) []Result {
|
||||
return f(query, limit)
|
||||
}
|
||||
|
||||
// Expected is one acceptable result for a fixture query. Grade is the
|
||||
// graded-relevance weight used by nDCG (higher = more relevant); it
|
||||
// defaults to 1 when omitted. Type is optional — when set, a ranked
|
||||
// result must match both MBID and entity type to count as a hit.
|
||||
type Expected struct {
|
||||
Type string `json:"type,omitempty"`
|
||||
MBID string `json:"mbid"`
|
||||
Grade int `json:"grade,omitempty"`
|
||||
}
|
||||
|
||||
// Fixture is a single labelled query: the input plus the result(s) a
|
||||
// user should get. Every edge case ever hand-fixed in the ranker
|
||||
// belongs here so it can never silently regress.
|
||||
type Fixture struct {
|
||||
Query string `json:"query"`
|
||||
Note string `json:"note,omitempty"`
|
||||
Expect []Expected `json:"expect"`
|
||||
}
|
||||
|
||||
// LoadFixtures reads a JSON fixture file from disk.
|
||||
func LoadFixtures(path string) ([]Fixture, error) {
|
||||
f, err := os.Open(path) //nolint:gosec // path is a test fixture, not user input
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("eval: open fixtures: %w", err)
|
||||
}
|
||||
|
||||
defer func() { _ = f.Close() }()
|
||||
|
||||
return ParseFixtures(f)
|
||||
}
|
||||
|
||||
// ParseFixtures decodes a JSON fixture set from a reader.
|
||||
func ParseFixtures(r io.Reader) ([]Fixture, error) {
|
||||
var fixtures []Fixture
|
||||
|
||||
if err := json.NewDecoder(r).Decode(&fixtures); err != nil {
|
||||
return nil, fmt.Errorf("eval: decode fixtures: %w", err)
|
||||
}
|
||||
|
||||
if len(fixtures) == 0 {
|
||||
return nil, ErrNoFixtures
|
||||
}
|
||||
|
||||
return fixtures, nil
|
||||
}
|
||||
|
||||
// matches reports whether a ranked result satisfies an expectation.
|
||||
// MBID match is required; entity type is checked only when the
|
||||
// expectation pins one.
|
||||
func (e Expected) matches(r Result) bool {
|
||||
if !strings.EqualFold(e.MBID, r.MBID) {
|
||||
return false
|
||||
}
|
||||
|
||||
if e.Type != "" && !strings.EqualFold(e.Type, r.EntityType) {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// grade returns the graded-relevance weight, defaulting to 1.
|
||||
func (e Expected) grade() int {
|
||||
if e.Grade <= 0 {
|
||||
return 1
|
||||
}
|
||||
|
||||
return e.Grade
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
package eval
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// QueryScore holds the per-query metrics for one fixture.
|
||||
type QueryScore struct {
|
||||
Query string
|
||||
Note string
|
||||
|
||||
// BestRank is the 1-based rank of the highest-placed expected
|
||||
// result, or 0 if none of the expected results appear in topK.
|
||||
BestRank int
|
||||
|
||||
ReciprocalRank float64
|
||||
PrecisionAtK float64
|
||||
NDCGAtK float64
|
||||
}
|
||||
|
||||
// Hit reports whether any expected result landed in topK.
|
||||
func (q QueryScore) Hit() bool {
|
||||
return q.BestRank > 0
|
||||
}
|
||||
|
||||
// Report aggregates per-query scores into the numbers you watch across
|
||||
// a ranking change: mean reciprocal rank, mean precision@k, mean
|
||||
// nDCG@k, plus the list of queries that missed entirely.
|
||||
type Report struct {
|
||||
K int
|
||||
NumQueries int
|
||||
|
||||
MRR float64
|
||||
MeanPAtK float64
|
||||
MeanNDCG float64
|
||||
HitRate float64 // fraction of queries with any expected result in topK
|
||||
Top1Rate float64 // fraction whose best expected result is rank 1
|
||||
PerQuery []QueryScore
|
||||
}
|
||||
|
||||
// Evaluate runs every fixture through the ranker and aggregates the
|
||||
// results into a Report. topK bounds how deep a result can be and
|
||||
// still count (a result at rank 20 helps no one).
|
||||
func Evaluate(r Ranker, fixtures []Fixture, topK int) Report {
|
||||
if topK <= 0 {
|
||||
topK = 5
|
||||
}
|
||||
|
||||
report := Report{K: topK, NumQueries: len(fixtures)}
|
||||
|
||||
for _, fx := range fixtures {
|
||||
ranked := r.Rank(fx.Query, topK)
|
||||
report.PerQuery = append(report.PerQuery, scoreQuery(fx, ranked, topK))
|
||||
}
|
||||
|
||||
for _, q := range report.PerQuery {
|
||||
report.MRR += q.ReciprocalRank
|
||||
report.MeanPAtK += q.PrecisionAtK
|
||||
report.MeanNDCG += q.NDCGAtK
|
||||
|
||||
if q.Hit() {
|
||||
report.HitRate++
|
||||
}
|
||||
|
||||
if q.BestRank == 1 {
|
||||
report.Top1Rate++
|
||||
}
|
||||
}
|
||||
|
||||
if n := float64(len(fixtures)); n > 0 {
|
||||
report.MRR /= n
|
||||
report.MeanPAtK /= n
|
||||
report.MeanNDCG /= n
|
||||
report.HitRate /= n
|
||||
report.Top1Rate /= n
|
||||
}
|
||||
|
||||
return report
|
||||
}
|
||||
|
||||
// scoreQuery computes the metrics for a single fixture against a ranked
|
||||
// result list.
|
||||
func scoreQuery(fx Fixture, ranked []Result, topK int) QueryScore {
|
||||
score := QueryScore{Query: fx.Query, Note: fx.Note}
|
||||
|
||||
limit := min(topK, len(ranked))
|
||||
|
||||
relevantInK := 0
|
||||
|
||||
for i := range limit {
|
||||
if !anyMatch(fx.Expect, ranked[i]) {
|
||||
continue
|
||||
}
|
||||
|
||||
relevantInK++
|
||||
|
||||
if score.BestRank == 0 {
|
||||
score.BestRank = i + 1
|
||||
score.ReciprocalRank = 1.0 / float64(i+1)
|
||||
}
|
||||
}
|
||||
|
||||
score.PrecisionAtK = float64(relevantInK) / float64(topK)
|
||||
score.NDCGAtK = ndcg(fx.Expect, ranked, topK)
|
||||
|
||||
return score
|
||||
}
|
||||
|
||||
// anyMatch reports whether a result satisfies any expectation.
|
||||
func anyMatch(expected []Expected, r Result) bool {
|
||||
for _, e := range expected {
|
||||
if e.matches(r) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// ndcg computes normalized discounted cumulative gain at k using graded
|
||||
// relevance. Returns 0 when there are no expected results.
|
||||
func ndcg(expected []Expected, ranked []Result, k int) float64 {
|
||||
ideal := idealDCG(expected, k)
|
||||
if ideal == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
limit := min(k, len(ranked))
|
||||
|
||||
dcg := 0.0
|
||||
|
||||
for i := range limit {
|
||||
g := matchedGrade(expected, ranked[i])
|
||||
if g == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
dcg += gain(g, i)
|
||||
}
|
||||
|
||||
return dcg / ideal
|
||||
}
|
||||
|
||||
// matchedGrade returns the relevance grade for a result, or 0 if it
|
||||
// matches no expectation.
|
||||
func matchedGrade(expected []Expected, r Result) int {
|
||||
for _, e := range expected {
|
||||
if e.matches(r) {
|
||||
return e.grade()
|
||||
}
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
// idealDCG is the DCG of the best possible ordering: every expected
|
||||
// result, sorted by grade descending, placed at the front.
|
||||
func idealDCG(expected []Expected, k int) float64 {
|
||||
grades := make([]int, 0, len(expected))
|
||||
for _, e := range expected {
|
||||
grades = append(grades, e.grade())
|
||||
}
|
||||
|
||||
sort.Sort(sort.Reverse(sort.IntSlice(grades)))
|
||||
|
||||
limit := min(k, len(grades))
|
||||
|
||||
ideal := 0.0
|
||||
for i := range limit {
|
||||
ideal += gain(grades[i], i)
|
||||
}
|
||||
|
||||
return ideal
|
||||
}
|
||||
|
||||
// gain is the discounted gain of a grade at 0-based position i.
|
||||
func gain(grade, i int) float64 {
|
||||
return (math.Pow(2, float64(grade)) - 1) / math.Log2(float64(i+2))
|
||||
}
|
||||
|
||||
// Format renders a Report as a human-readable table for test output.
|
||||
func (r Report) Format() string {
|
||||
var b strings.Builder
|
||||
|
||||
fmt.Fprintf(&b, "ranking eval — %d queries @k=%d\n", r.NumQueries, r.K)
|
||||
fmt.Fprintf(&b, " MRR %.3f\n", r.MRR)
|
||||
fmt.Fprintf(&b, " P@%d %.3f\n", r.K, r.MeanPAtK)
|
||||
fmt.Fprintf(&b, " nDCG@%d %.3f\n", r.K, r.MeanNDCG)
|
||||
fmt.Fprintf(&b, " hit rate %.3f\n", r.HitRate)
|
||||
fmt.Fprintf(&b, " top-1 rate %.3f\n", r.Top1Rate)
|
||||
|
||||
misses := r.Misses()
|
||||
if len(misses) > 0 {
|
||||
b.WriteString(" misses:\n")
|
||||
|
||||
for _, m := range misses {
|
||||
fmt.Fprintf(&b, " %-40q rank=%s\n", m.Query, rankLabel(m.BestRank))
|
||||
}
|
||||
}
|
||||
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// Misses returns the queries whose best expected result was absent
|
||||
// from topK or buried below rank 1 — the regression watch-list.
|
||||
func (r Report) Misses() []QueryScore {
|
||||
var out []QueryScore
|
||||
|
||||
for _, q := range r.PerQuery {
|
||||
if q.BestRank != 1 {
|
||||
out = append(out, q)
|
||||
}
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func rankLabel(rank int) string {
|
||||
if rank == 0 {
|
||||
return "absent"
|
||||
}
|
||||
|
||||
return strconv.Itoa(rank)
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
package eval
|
||||
|
||||
import (
|
||||
"math"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// rankerFromIDs builds a Ranker that returns a fixed ordering keyed by
|
||||
// query, for deterministic metric tests.
|
||||
func rankerFromIDs(table map[string][]Result) Ranker {
|
||||
return RankerFunc(func(query string, limit int) []Result {
|
||||
out := table[query]
|
||||
if limit < len(out) {
|
||||
out = out[:limit]
|
||||
}
|
||||
|
||||
return out
|
||||
})
|
||||
}
|
||||
|
||||
func approx(a, b float64) bool {
|
||||
return math.Abs(a-b) < 1e-9
|
||||
}
|
||||
|
||||
func TestReciprocalRank(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
ranked []Result
|
||||
expect []Expected
|
||||
wantRR float64
|
||||
wantPos int
|
||||
}{
|
||||
{
|
||||
name: "top result",
|
||||
ranked: []Result{{MBID: "a"}, {MBID: "b"}},
|
||||
expect: []Expected{{MBID: "a"}},
|
||||
wantRR: 1.0,
|
||||
wantPos: 1,
|
||||
},
|
||||
{
|
||||
name: "third result",
|
||||
ranked: []Result{{MBID: "x"}, {MBID: "y"}, {MBID: "a"}},
|
||||
expect: []Expected{{MBID: "a"}},
|
||||
wantRR: 1.0 / 3.0,
|
||||
wantPos: 3,
|
||||
},
|
||||
{
|
||||
name: "absent",
|
||||
ranked: []Result{{MBID: "x"}, {MBID: "y"}},
|
||||
expect: []Expected{{MBID: "a"}},
|
||||
wantRR: 0,
|
||||
wantPos: 0,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
fx := Fixture{Query: "q", Expect: tt.expect}
|
||||
got := scoreQuery(fx, tt.ranked, 5)
|
||||
|
||||
if !approx(got.ReciprocalRank, tt.wantRR) {
|
||||
t.Errorf("RR = %v, want %v", got.ReciprocalRank, tt.wantRR)
|
||||
}
|
||||
|
||||
if got.BestRank != tt.wantPos {
|
||||
t.Errorf("BestRank = %d, want %d", got.BestRank, tt.wantPos)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrecisionAtK(t *testing.T) {
|
||||
fx := Fixture{
|
||||
Query: "q",
|
||||
Expect: []Expected{{MBID: "a"}, {MBID: "b"}},
|
||||
}
|
||||
ranked := []Result{{MBID: "a"}, {MBID: "x"}, {MBID: "b"}, {MBID: "y"}}
|
||||
|
||||
got := scoreQuery(fx, ranked, 4)
|
||||
|
||||
// 2 relevant out of k=4.
|
||||
if !approx(got.PrecisionAtK, 0.5) {
|
||||
t.Errorf("P@4 = %v, want 0.5", got.PrecisionAtK)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNDCGRespectsOrdering(t *testing.T) {
|
||||
expect := []Expected{{MBID: "a", Grade: 3}, {MBID: "b", Grade: 1}}
|
||||
|
||||
// Ideal ordering: high-grade result first.
|
||||
good := scoreQuery(
|
||||
Fixture{Query: "q", Expect: expect},
|
||||
[]Result{{MBID: "a"}, {MBID: "b"}, {MBID: "z"}},
|
||||
5,
|
||||
)
|
||||
|
||||
// Worse ordering: high-grade result buried below an irrelevant one.
|
||||
bad := scoreQuery(
|
||||
Fixture{Query: "q", Expect: expect},
|
||||
[]Result{{MBID: "z"}, {MBID: "b"}, {MBID: "a"}},
|
||||
5,
|
||||
)
|
||||
|
||||
if !approx(good.NDCGAtK, 1.0) {
|
||||
t.Errorf("ideal ordering nDCG = %v, want 1.0", good.NDCGAtK)
|
||||
}
|
||||
|
||||
if bad.NDCGAtK >= good.NDCGAtK {
|
||||
t.Errorf("worse ordering nDCG %v should be < ideal %v", bad.NDCGAtK, good.NDCGAtK)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTypeMustMatchWhenPinned(t *testing.T) {
|
||||
fx := Fixture{
|
||||
Query: "q",
|
||||
Expect: []Expected{{Type: "artist", MBID: "a"}},
|
||||
}
|
||||
|
||||
// Same MBID but wrong entity type — must not count.
|
||||
wrongType := scoreQuery(fx, []Result{{EntityType: "recording", MBID: "a"}}, 5)
|
||||
if wrongType.Hit() {
|
||||
t.Error("result with wrong entity type counted as a hit")
|
||||
}
|
||||
|
||||
rightType := scoreQuery(fx, []Result{{EntityType: "artist", MBID: "a"}}, 5)
|
||||
if !rightType.Hit() {
|
||||
t.Error("result with matching entity type did not count")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluateAggregates(t *testing.T) {
|
||||
fixtures := []Fixture{
|
||||
{Query: "hit-top", Expect: []Expected{{MBID: "a"}}},
|
||||
{Query: "hit-second", Expect: []Expected{{MBID: "a"}}},
|
||||
{Query: "miss", Expect: []Expected{{MBID: "a"}}},
|
||||
}
|
||||
|
||||
r := rankerFromIDs(map[string][]Result{
|
||||
"hit-top": {{MBID: "a"}},
|
||||
"hit-second": {{MBID: "x"}, {MBID: "a"}},
|
||||
"miss": {{MBID: "x"}, {MBID: "y"}},
|
||||
})
|
||||
|
||||
report := Evaluate(r, fixtures, 5)
|
||||
|
||||
// MRR = (1 + 1/2 + 0) / 3.
|
||||
wantMRR := (1.0 + 0.5 + 0.0) / 3.0
|
||||
if !approx(report.MRR, wantMRR) {
|
||||
t.Errorf("MRR = %v, want %v", report.MRR, wantMRR)
|
||||
}
|
||||
|
||||
// 2 of 3 queries surfaced the result somewhere in topK.
|
||||
if !approx(report.HitRate, 2.0/3.0) {
|
||||
t.Errorf("HitRate = %v, want %v", report.HitRate, 2.0/3.0)
|
||||
}
|
||||
|
||||
// Only 1 of 3 had it at rank 1.
|
||||
if !approx(report.Top1Rate, 1.0/3.0) {
|
||||
t.Errorf("Top1Rate = %v, want %v", report.Top1Rate, 1.0/3.0)
|
||||
}
|
||||
|
||||
if len(report.Misses()) != 2 {
|
||||
t.Errorf("Misses = %d, want 2", len(report.Misses()))
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFixtures(t *testing.T) {
|
||||
const doc = `[
|
||||
{"query": "radiohead", "expect": [{"type": "artist", "mbid": "abc"}]},
|
||||
{"query": "ok computer", "note": "album not band", "expect": [{"mbid": "def", "grade": 2}]}
|
||||
]`
|
||||
|
||||
fixtures, err := ParseFixtures(strings.NewReader(doc))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFixtures: %v", err)
|
||||
}
|
||||
|
||||
if len(fixtures) != 2 {
|
||||
t.Fatalf("got %d fixtures, want 2", len(fixtures))
|
||||
}
|
||||
|
||||
if fixtures[0].Expect[0].MBID != "abc" {
|
||||
t.Errorf("MBID = %q, want abc", fixtures[0].Expect[0].MBID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFixturesEmpty(t *testing.T) {
|
||||
_, err := ParseFixtures(strings.NewReader(`[]`))
|
||||
if err == nil {
|
||||
t.Fatal("expected ErrNoFixtures, got nil")
|
||||
}
|
||||
}
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
[
|
||||
{
|
||||
"query": "radiohead",
|
||||
"note": "single-word artist name — should resolve to the artist, not an album titled similarly",
|
||||
"expect": [{ "type": "artist", "mbid": "a74b1b7f-71a5-4011-9441-d0b5e4122711" }]
|
||||
},
|
||||
{
|
||||
"query": "the beatles",
|
||||
"note": "common-word prefix must not let the article dominate",
|
||||
"expect": [{ "type": "artist", "mbid": "b10bbbfc-cf9e-42e0-be17-e2c3e1d2600d" }]
|
||||
},
|
||||
{
|
||||
"query": "abbey road",
|
||||
"note": "album title — should rank the release group above any track of the same name",
|
||||
"expect": [{ "type": "release_group", "mbid": "" }]
|
||||
},
|
||||
{
|
||||
"query": "calling you blue october",
|
||||
"note": "composite title+artist query — recording should win even if the artist is unindexed",
|
||||
"expect": [{ "type": "recording", "mbid": "" }]
|
||||
},
|
||||
{
|
||||
"query": "the teenagers",
|
||||
"note": "regression: must rank The Teenagers above The Beatles despite far lower popularity",
|
||||
"expect": [{ "type": "artist", "mbid": "" }]
|
||||
},
|
||||
{
|
||||
"query": "beyonce",
|
||||
"note": "diacritic folding (migration 37): unaccented query must find the accented artist Beyoncé",
|
||||
"expect": [{ "type": "artist", "mbid": "859d0860-d480-4efd-970c-c05d5f1776b8" }]
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,150 @@
|
||||
package explore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"testing"
|
||||
|
||||
"yellowjacket/backend/database"
|
||||
"yellowjacket/backend/explore/eval"
|
||||
)
|
||||
|
||||
// seedIndexRow inserts one explore_index row. The FTS triggers keep
|
||||
// explore_index_fts in sync automatically.
|
||||
func seedIndexRow(
|
||||
t *testing.T,
|
||||
db *database.DB,
|
||||
entityType, mbid, title, artist string,
|
||||
popularity int,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
_, err := db.ExecContext(`
|
||||
INSERT INTO explore_index
|
||||
(entity_type, mbid, title, artist_name, artist_mbid, popularity, listener_count)
|
||||
VALUES (?, ?, ?, ?, '', ?, ?)
|
||||
`, entityType, mbid, title, artist, popularity, popularity/10)
|
||||
if err != nil {
|
||||
t.Fatalf("seed %s/%s: %v", entityType, mbid, err)
|
||||
}
|
||||
}
|
||||
|
||||
// newTestIndex builds a SearchIndex over a seeded in-memory DB. lb and
|
||||
// artistImg are nil because Search touches neither.
|
||||
func newTestIndex(t *testing.T, db *database.DB) *SearchIndex {
|
||||
t.Helper()
|
||||
|
||||
idx := NewSearchIndex(db, nil, nil, slog.Default())
|
||||
idx.MarkReadyIfPopulated()
|
||||
|
||||
return idx
|
||||
}
|
||||
|
||||
// TestEvalHarnessIndexRanking is the end-to-end wiring of the eval
|
||||
// harness against the real FTS index Search. It seeds a controlled
|
||||
// corpus where the correct answer is known, then asserts the harness
|
||||
// reports a perfect score — proving both the index ranking and the
|
||||
// harness plumbing on a case we fully control.
|
||||
func TestEvalHarnessIndexRanking(t *testing.T) {
|
||||
db := database.NewTestDB(t)
|
||||
|
||||
// Popular exact-match artist should beat a more obscure namesake
|
||||
// and an unrelated album.
|
||||
seedIndexRow(t, db, "artist", "rh", "Radiohead", "Radiohead", 5_000_000)
|
||||
seedIndexRow(t, db, "artist", "radio-obscure", "Radio Birdman", "Radio Birdman", 40_000)
|
||||
seedIndexRow(t, db, "release_group", "okc", "OK Computer", "Radiohead", 2_000_000)
|
||||
seedIndexRow(t, db, "artist", "beatles", "The Beatles", "The Beatles", 9_000_000)
|
||||
seedIndexRow(t, db, "artist", "teenagers", "The Teenagers", "The Teenagers", 60_000)
|
||||
|
||||
idx := newTestIndex(t, db)
|
||||
|
||||
ranker := eval.RankerFunc(func(query string, limit int) []eval.Result {
|
||||
hits := idx.Search(context.Background(), query, limit)
|
||||
out := make([]eval.Result, 0, len(hits))
|
||||
|
||||
for _, h := range hits {
|
||||
out = append(out, eval.Result{EntityType: h.EntityType, MBID: h.MBID})
|
||||
}
|
||||
|
||||
return out
|
||||
})
|
||||
|
||||
fixtures := []eval.Fixture{
|
||||
{
|
||||
Query: "radiohead",
|
||||
Note: "popular exact artist match",
|
||||
Expect: []eval.Expected{{Type: "artist", MBID: "rh"}},
|
||||
},
|
||||
{
|
||||
Query: "the teenagers",
|
||||
Note: "low-popularity exact match must beat high-popularity article match",
|
||||
Expect: []eval.Expected{{Type: "artist", MBID: "teenagers"}},
|
||||
},
|
||||
}
|
||||
|
||||
report := eval.Evaluate(ranker, fixtures, 5)
|
||||
t.Log("\n" + report.Format())
|
||||
|
||||
if report.HitRate < 1.0 {
|
||||
t.Errorf("expected every query to surface its result, got hit rate %.3f", report.HitRate)
|
||||
}
|
||||
|
||||
if report.Top1Rate < 1.0 {
|
||||
t.Errorf("expected every result at rank 1, got top-1 rate %.3f:\n%s",
|
||||
report.Top1Rate, report.Format())
|
||||
}
|
||||
}
|
||||
|
||||
// TestExploreFTSDiacriticFolding proves migration 37: an unaccented
|
||||
// query must find an accented title (and vice versa) now that
|
||||
// explore_index_fts folds diacritics.
|
||||
func TestExploreFTSDiacriticFolding(t *testing.T) {
|
||||
db := database.NewTestDB(t)
|
||||
|
||||
seedIndexRow(t, db, "artist", "bey", "Beyoncé", "Beyoncé", 8_000_000)
|
||||
seedIndexRow(t, db, "artist", "bjork", "Björk", "Björk", 3_000_000)
|
||||
|
||||
idx := newTestIndex(t, db)
|
||||
|
||||
cases := []struct {
|
||||
query string
|
||||
wantMBID string
|
||||
}{
|
||||
{"beyonce", "bey"}, // unaccented query → accented title
|
||||
{"beyoncé", "bey"}, // accented query still works
|
||||
{"bjork", "bjork"}, // ö → o folding
|
||||
{"björk", "bjork"}, // accented query still works
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.query, func(t *testing.T) {
|
||||
hits := idx.Search(context.Background(), tc.query, 5)
|
||||
if len(hits) == 0 {
|
||||
t.Fatalf("query %q returned no hits", tc.query)
|
||||
}
|
||||
|
||||
if hits[0].MBID != tc.wantMBID {
|
||||
t.Errorf("query %q: top hit = %q, want %q", tc.query, hits[0].MBID, tc.wantMBID)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestEvalFixtureFileParses guards the checked-in fixture file so a
|
||||
// malformed edit fails fast rather than silently skipping queries.
|
||||
func TestEvalFixtureFileParses(t *testing.T) {
|
||||
fixtures, err := eval.LoadFixtures("eval/testdata/eval_queries.json")
|
||||
if err != nil {
|
||||
t.Fatalf("load fixtures: %v", err)
|
||||
}
|
||||
|
||||
for i, fx := range fixtures {
|
||||
if fx.Query == "" {
|
||||
t.Errorf("fixture %d has empty query", i)
|
||||
}
|
||||
|
||||
if len(fx.Expect) == 0 {
|
||||
t.Errorf("fixture %q has no expectations", fx.Query)
|
||||
}
|
||||
}
|
||||
}
|
||||
+726
-1416
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,58 @@
|
||||
package explore
|
||||
|
||||
import "strings"
|
||||
|
||||
// Character-bigram similarity, used by the search index's typo-tolerant
|
||||
// rescue pass (see fuzzyRescue). A name is represented as the set of its
|
||||
// sliding 2-rune windows; a single-character typo alters only the one or
|
||||
// two bigrams that span it, so most of the set survives. Overlap between
|
||||
// two such sets (Dice coefficient) therefore stays high across a
|
||||
// misspelling, where prefix matching collapses to nothing.
|
||||
|
||||
// fuzzyNormalize lowercases and collapses runs of whitespace to a single
|
||||
// space so bigram sets are stable across casing and spacing noise.
|
||||
func fuzzyNormalize(s string) string {
|
||||
return strings.Join(strings.Fields(strings.ToLower(s)), " ")
|
||||
}
|
||||
|
||||
// fuzzyBigrams returns the set of character bigrams of s after
|
||||
// normalization. Returns nil for inputs shorter than two runes, which
|
||||
// have no bigram and can't be scored.
|
||||
func fuzzyBigrams(s string) map[string]struct{} {
|
||||
runes := []rune(fuzzyNormalize(s))
|
||||
if len(runes) < 2 {
|
||||
return nil
|
||||
}
|
||||
|
||||
set := make(map[string]struct{}, len(runes))
|
||||
|
||||
for i := 0; i+1 < len(runes); i++ {
|
||||
set[string(runes[i:i+2])] = struct{}{}
|
||||
}
|
||||
|
||||
return set
|
||||
}
|
||||
|
||||
// diceCoefficient is 2·|A∩B| / (|A|+|B|), a similarity in [0, 1] where 1
|
||||
// is an identical bigram set and 0 is disjoint. Iterating the smaller
|
||||
// set keeps the intersection count cheap.
|
||||
func diceCoefficient(a, b map[string]struct{}) float64 {
|
||||
if len(a) == 0 || len(b) == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
small, large := a, b
|
||||
if len(large) < len(small) {
|
||||
small, large = large, small
|
||||
}
|
||||
|
||||
intersection := 0
|
||||
|
||||
for bg := range small {
|
||||
if _, ok := large[bg]; ok {
|
||||
intersection++
|
||||
}
|
||||
}
|
||||
|
||||
return 2 * float64(intersection) / float64(len(a)+len(b))
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package explore
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestDiceCoefficientTypoTolerance(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
a, b string
|
||||
wantMin float64 // score must be at least this
|
||||
wantMax float64 // ...and at most this
|
||||
}{
|
||||
{name: "identical", a: "beatles", b: "beatles", wantMin: 1.0, wantMax: 1.0},
|
||||
{name: "single typo", a: "beetles", b: "beatles", wantMin: 0.5, wantMax: 0.9},
|
||||
{name: "missing char", a: "nirvna", b: "nirvana", wantMin: 0.5, wantMax: 0.95},
|
||||
// Longer-name typo: a single substitution stays comfortably above
|
||||
// the rescue threshold, which is the realistic case (artist and
|
||||
// album names are rarely as short as five characters).
|
||||
{name: "longer name typo", a: "metalica", b: "metallica", wantMin: 0.5, wantMax: 0.95},
|
||||
// Adjacent transposition in a short word is bigrams' known weak
|
||||
// spot — it breaks most windows, so it scores below threshold.
|
||||
// Documented, not a bug: bigram overlap targets substitution,
|
||||
// insertion, and deletion typos.
|
||||
{name: "short transposition", a: "raido", b: "radio", wantMin: 0.0, wantMax: 0.34},
|
||||
{name: "case and space", a: "The Beatles", b: "the beatles", wantMin: 1.0, wantMax: 1.0},
|
||||
{name: "unrelated", a: "beatles", b: "metallica", wantMin: 0.0, wantMax: 0.34},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := diceCoefficient(fuzzyBigrams(tc.a), fuzzyBigrams(tc.b))
|
||||
if got < tc.wantMin || got > tc.wantMax {
|
||||
t.Errorf("diceCoefficient(%q, %q) = %.3f, want in [%.2f, %.2f]",
|
||||
tc.a, tc.b, got, tc.wantMin, tc.wantMax)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFuzzyBigramsShortInput(t *testing.T) {
|
||||
if got := fuzzyBigrams("a"); got != nil {
|
||||
t.Errorf("fuzzyBigrams(%q) = %v, want nil", "a", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
package explore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
// lrclibBaseURL is the LRCLIB lyrics API. LRCLIB is a free,
|
||||
// community-maintained lyrics database (plain + synced) with no
|
||||
// auth required.
|
||||
lrclibBaseURL = "https://lrclib.net"
|
||||
|
||||
// lrclibUserAgent identifies the app per LRCLIB's guidelines.
|
||||
lrclibUserAgent = "YellowJacket (https://github.com/yellowjacket)"
|
||||
|
||||
// lrclibRate is the requests-per-second budget for LRCLIB. The
|
||||
// service has no hard published limit but asks clients to be
|
||||
// gentle; this keeps the library backfill polite.
|
||||
lrclibRate = 3
|
||||
)
|
||||
|
||||
// ErrLyricsNotFound is returned when LRCLIB has no match for the
|
||||
// requested track.
|
||||
var ErrLyricsNotFound = errors.New("lyrics not found")
|
||||
|
||||
// Lyrics holds the plain and (optional) time-synced lyrics for a
|
||||
// track, plus whether the track is marked instrumental.
|
||||
type Lyrics struct {
|
||||
Plain string `json:"plain"`
|
||||
Synced string `json:"synced"`
|
||||
Instrumental bool `json:"instrumental"`
|
||||
}
|
||||
|
||||
// LRCLibClient is a thin, rate-limited, cached HTTP client for the
|
||||
// LRCLIB lyrics API.
|
||||
type LRCLibClient struct {
|
||||
http *http.Client
|
||||
limiter *RateLimiter
|
||||
cache *Cache
|
||||
logger *slog.Logger
|
||||
baseURL string // overridable in tests
|
||||
}
|
||||
|
||||
// NewLRCLibClient creates an LRCLIB client sharing the given cache.
|
||||
func NewLRCLibClient(cache *Cache, logger *slog.Logger) *LRCLibClient {
|
||||
return &LRCLibClient{
|
||||
http: &http.Client{Timeout: 20 * time.Second},
|
||||
limiter: NewRateLimiterN(lrclibRate),
|
||||
cache: cache,
|
||||
logger: logger,
|
||||
baseURL: lrclibBaseURL,
|
||||
}
|
||||
}
|
||||
|
||||
// lrclibResponse is the wire shape of LRCLIB's /api/get response.
|
||||
type lrclibResponse struct {
|
||||
ID int64 `json:"id"`
|
||||
TrackName string `json:"trackName"`
|
||||
ArtistName string `json:"artistName"`
|
||||
AlbumName string `json:"albumName"`
|
||||
Duration float64 `json:"duration"`
|
||||
Instrumental bool `json:"instrumental"`
|
||||
PlainLyrics string `json:"plainLyrics"`
|
||||
SyncedLyrics string `json:"syncedLyrics"`
|
||||
}
|
||||
|
||||
// GetLyrics fetches lyrics for a track by artist, title, album, and
|
||||
// duration (seconds; pass 0 if unknown). LRCLIB matches on the
|
||||
// metadata with a small duration tolerance. Returns ErrLyricsNotFound
|
||||
// when no match exists. Successful and negative results are both
|
||||
// cached so a repeated backfill doesn't re-hit the network.
|
||||
func (c *LRCLibClient) GetLyrics(
|
||||
ctx context.Context, artist, title, album string, durationSec int,
|
||||
) (*Lyrics, error) {
|
||||
if artist == "" || title == "" {
|
||||
return nil, ErrLyricsNotFound
|
||||
}
|
||||
|
||||
q := url.Values{}
|
||||
q.Set("artist_name", artist)
|
||||
q.Set("track_name", title)
|
||||
|
||||
if album != "" {
|
||||
q.Set("album_name", album)
|
||||
}
|
||||
|
||||
if durationSec > 0 {
|
||||
q.Set("duration", strconv.Itoa(durationSec))
|
||||
}
|
||||
|
||||
reqURL := c.baseURL + "/api/get?" + q.Encode()
|
||||
cacheKey := "lrclib:get:" + q.Encode()
|
||||
|
||||
if data, ok := c.cache.Get(cacheKey); ok {
|
||||
return decodeCachedLyrics(data)
|
||||
}
|
||||
|
||||
body, status, err := c.doGet(ctx, reqURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("lrclib get: %w", err)
|
||||
}
|
||||
|
||||
if status == http.StatusNotFound {
|
||||
// Cache the miss as a sentinel so we don't re-request it.
|
||||
c.cache.Set(cacheKey, []byte(lyricsMissSentinel), cacheTTLSearch, "", "lyrics")
|
||||
|
||||
return nil, ErrLyricsNotFound
|
||||
}
|
||||
|
||||
if status < 200 || status >= 300 {
|
||||
return nil, fmt.Errorf("lrclib get: %w: status %d", ErrListenBrainzHTTP, status)
|
||||
}
|
||||
|
||||
var wire lrclibResponse
|
||||
if err := json.Unmarshal(body, &wire); err != nil {
|
||||
return nil, fmt.Errorf("lrclib get unmarshal: %w", err)
|
||||
}
|
||||
|
||||
lyrics := &Lyrics{
|
||||
Plain: wire.PlainLyrics,
|
||||
Synced: wire.SyncedLyrics,
|
||||
Instrumental: wire.Instrumental,
|
||||
}
|
||||
|
||||
// Persist the normalized result (not the raw wire body) so the
|
||||
// cached shape matches what callers expect.
|
||||
if encoded, err := json.Marshal(lyrics); err == nil {
|
||||
c.cache.Set(cacheKey, encoded, cacheTTLSearch, "", "lyrics")
|
||||
}
|
||||
|
||||
return lyrics, nil
|
||||
}
|
||||
|
||||
// lyricsMissSentinel marks a cached "no lyrics found" result.
|
||||
const lyricsMissSentinel = "\x00miss"
|
||||
|
||||
// decodeCachedLyrics interprets a cached LRCLIB payload, mapping the
|
||||
// miss sentinel back to ErrLyricsNotFound.
|
||||
func decodeCachedLyrics(data []byte) (*Lyrics, error) {
|
||||
if string(data) == lyricsMissSentinel {
|
||||
return nil, ErrLyricsNotFound
|
||||
}
|
||||
|
||||
var lyrics Lyrics
|
||||
if err := json.Unmarshal(data, &lyrics); err != nil {
|
||||
return nil, fmt.Errorf("lrclib cache decode: %w", err)
|
||||
}
|
||||
|
||||
return &lyrics, nil
|
||||
}
|
||||
|
||||
// doGet performs a rate-limited GET and returns the body and status.
|
||||
// Unlike the ListenBrainz client, a 404 is a normal "no lyrics"
|
||||
// outcome, so the status is returned rather than folded into an error.
|
||||
func (c *LRCLibClient) doGet(ctx context.Context, reqURL string) ([]byte, int, error) {
|
||||
if err := c.limiter.Wait(ctx); err != nil {
|
||||
return nil, 0, fmt.Errorf("rate limiter: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL, nil)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
req.Header.Set("User-Agent", lrclibUserAgent)
|
||||
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, resp.StatusCode, err
|
||||
}
|
||||
|
||||
return body, resp.StatusCode, nil
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
package explore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"yellowjacket/backend/database"
|
||||
)
|
||||
|
||||
// newTestLRCLib builds an LRCLIB client whose HTTP calls hit the given
|
||||
// test server, sharing a real (in-memory) cache so caching behaviour is
|
||||
// exercised.
|
||||
func newTestLRCLib(t *testing.T, handler http.HandlerFunc) (*LRCLibClient, *httptest.Server) {
|
||||
t.Helper()
|
||||
|
||||
srv := httptest.NewServer(handler)
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
db := database.NewTestDB(t)
|
||||
cache := NewCache(db, testLogger())
|
||||
|
||||
c := NewLRCLibClient(cache, testLogger())
|
||||
// Point the client at the test server instead of the real API by
|
||||
// overriding its transport to rewrite the host.
|
||||
c.http = srv.Client()
|
||||
c.baseURL = srv.URL
|
||||
|
||||
return c, srv
|
||||
}
|
||||
|
||||
func TestLRCLibGetLyrics(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var calls int
|
||||
|
||||
c, _ := newTestLRCLib(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
calls++
|
||||
|
||||
if r.URL.Path != "/api/get" {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if r.URL.Query().Get("track_name") == "Missing" {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"id": 42,
|
||||
"trackName": "Yesterday",
|
||||
"artistName": "The Beatles",
|
||||
"albumName": "Help!",
|
||||
"duration": 125,
|
||||
"instrumental": false,
|
||||
"plainLyrics": "Yesterday, all my troubles seemed so far away",
|
||||
"syncedLyrics": "[00:00.00] Yesterday"
|
||||
}`))
|
||||
})
|
||||
|
||||
t.Run("hit returns plain + synced lyrics", func(t *testing.T) {
|
||||
lyrics, err := c.GetLyrics(context.Background(), "The Beatles", "Yesterday", "Help!", 125)
|
||||
if err != nil {
|
||||
t.Fatalf("GetLyrics: %v", err)
|
||||
}
|
||||
|
||||
if lyrics.Plain == "" || lyrics.Synced == "" {
|
||||
t.Errorf("expected plain and synced lyrics, got %+v", lyrics)
|
||||
}
|
||||
|
||||
if lyrics.Instrumental {
|
||||
t.Error("expected non-instrumental")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("second identical request is served from cache", func(t *testing.T) {
|
||||
before := calls
|
||||
|
||||
if _, err := c.GetLyrics(
|
||||
context.Background(),
|
||||
"The Beatles",
|
||||
"Yesterday",
|
||||
"Help!",
|
||||
125,
|
||||
); err != nil {
|
||||
t.Fatalf("GetLyrics: %v", err)
|
||||
}
|
||||
|
||||
if calls != before {
|
||||
t.Errorf("expected cache hit (no new HTTP call), calls went %d → %d", before, calls)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("404 maps to ErrLyricsNotFound", func(t *testing.T) {
|
||||
_, err := c.GetLyrics(context.Background(), "Nobody", "Missing", "", 0)
|
||||
if !errors.Is(err, ErrLyricsNotFound) {
|
||||
t.Errorf("expected ErrLyricsNotFound, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("missing artist/title short-circuits without a request", func(t *testing.T) {
|
||||
before := calls
|
||||
|
||||
if _, err := c.GetLyrics(
|
||||
context.Background(),
|
||||
"",
|
||||
"Yesterday",
|
||||
"",
|
||||
0,
|
||||
); !errors.Is(
|
||||
err,
|
||||
ErrLyricsNotFound,
|
||||
) {
|
||||
t.Errorf("expected ErrLyricsNotFound, got %v", err)
|
||||
}
|
||||
|
||||
if calls != before {
|
||||
t.Error("expected no HTTP call for empty artist")
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
package explore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"yellowjacket/backend/database"
|
||||
)
|
||||
|
||||
// LyricsResult is a single lyric-search hit, mapped from the DB layer
|
||||
// into the camelCase shape the frontend consumes.
|
||||
type LyricsResult struct {
|
||||
RecordingID int64 `json:"recordingId"`
|
||||
FilePath string `json:"filePath"`
|
||||
LengthMs int64 `json:"lengthMs"`
|
||||
Title string `json:"title"`
|
||||
Artist string `json:"artist"`
|
||||
Album string `json:"album"`
|
||||
}
|
||||
|
||||
// TrackLyrics is the stored or freshly-fetched lyrics for one track.
|
||||
// Source is "embedded" (from the file's tags / library DB), "lrclib"
|
||||
// (fetched on demand), or "" when none are available.
|
||||
type TrackLyrics struct {
|
||||
Plain string `json:"plain"`
|
||||
Synced string `json:"synced"`
|
||||
Instrumental bool `json:"instrumental"`
|
||||
Source string `json:"source"`
|
||||
}
|
||||
|
||||
const (
|
||||
// lyricsSearchLimit caps lyric-search hits returned to the UI.
|
||||
lyricsSearchLimit = 30
|
||||
|
||||
// lyricsBackfillBatch is how many missing-lyrics recordings the
|
||||
// background backfill processes per pass.
|
||||
lyricsBackfillBatch = 200
|
||||
|
||||
// lyricsBackfillMaxPasses bounds a single backfill run so it can't
|
||||
// loop forever on a huge library; the next launch resumes where
|
||||
// this one left off (candidates with lyrics now filled are skipped).
|
||||
lyricsBackfillMaxPasses = 25
|
||||
)
|
||||
|
||||
// SearchLyrics finds library tracks whose lyrics contain the given
|
||||
// fragment, ranked by relevance. Pure local FTS — no network.
|
||||
func (e *Service) SearchLyrics(query string) []LyricsResult {
|
||||
hits, err := e.db.SearchLyrics(query, lyricsSearchLimit)
|
||||
if err != nil {
|
||||
e.logger.Warn("lyrics search failed", "query", query, "err", err)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
out := make([]LyricsResult, 0, len(hits))
|
||||
for _, h := range hits {
|
||||
out = append(out, LyricsResult{
|
||||
RecordingID: h.RecordingID,
|
||||
FilePath: h.FilePath,
|
||||
LengthMs: h.LengthMilliseconds,
|
||||
Title: h.Title,
|
||||
Artist: h.Artist,
|
||||
Album: h.Album,
|
||||
})
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// GetTrackLyrics returns lyrics for a recording. If the library
|
||||
// already has them (from embedded tags) they're returned as-is;
|
||||
// otherwise it fetches from LRCLIB, persists them (updating the FTS
|
||||
// index), and returns them. Never returns an error to the frontend —
|
||||
// a miss just yields an empty result.
|
||||
func (e *Service) GetTrackLyrics(recordingID int64) TrackLyrics {
|
||||
stored, err := e.db.GetRecordingLyrics(recordingID)
|
||||
if err == nil && stored != "" {
|
||||
return TrackLyrics{Plain: stored, Source: "embedded"}
|
||||
}
|
||||
|
||||
lookup, err := e.db.RecordingLyricLookup(recordingID)
|
||||
if err != nil || lookup == nil {
|
||||
return TrackLyrics{}
|
||||
}
|
||||
|
||||
fetched := e.fetchAndStoreLyrics(e.ctx, *lookup)
|
||||
if fetched == nil {
|
||||
return TrackLyrics{}
|
||||
}
|
||||
|
||||
return TrackLyrics{
|
||||
Plain: fetched.Plain,
|
||||
Synced: fetched.Synced,
|
||||
Instrumental: fetched.Instrumental,
|
||||
Source: "lrclib",
|
||||
}
|
||||
}
|
||||
|
||||
// RebuildLyricsIndex rebuilds the FTS lyrics index from the current
|
||||
// library. Cheap; safe to call after every scan.
|
||||
func (e *Service) RebuildLyricsIndex() {
|
||||
if err := e.db.RebuildLyricsIndex(); err != nil {
|
||||
e.logger.Warn("lyrics index rebuild failed", "err", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
e.index.setMeta(lyricsIndexReadyKey, "1")
|
||||
e.logger.Info("lyrics index rebuilt")
|
||||
}
|
||||
|
||||
// RebuildLyricsIndexIfNeeded rebuilds the lyrics FTS only when it has not
|
||||
// been built since the last library change. The backfill keeps the index
|
||||
// in sync incrementally thereafter, so on an unchanged library the full
|
||||
// rebuild is redundant; the scan-completion path calls the unconditional
|
||||
// form.
|
||||
func (e *Service) RebuildLyricsIndexIfNeeded() {
|
||||
if e.index.hasMeta(lyricsIndexReadyKey) {
|
||||
return
|
||||
}
|
||||
|
||||
e.RebuildLyricsIndex()
|
||||
}
|
||||
|
||||
// BackfillLibraryLyrics fetches lyrics from LRCLIB for library tracks
|
||||
// that don't have them, in the background. Idempotent and bounded —
|
||||
// each recording is tried once (a miss is cached), and a run stops
|
||||
// after a fixed number of passes, resuming on the next launch.
|
||||
func (e *Service) BackfillLibraryLyrics() {
|
||||
go e.backfillLibraryLyrics(e.ctx)
|
||||
}
|
||||
|
||||
func (e *Service) backfillLibraryLyrics(ctx context.Context) {
|
||||
total := 0
|
||||
|
||||
for range lyricsBackfillMaxPasses {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
|
||||
candidates, err := e.db.RecordingsMissingLyrics(lyricsBackfillBatch)
|
||||
if err != nil {
|
||||
e.logger.Warn("lyrics backfill: query failed", "err", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if len(candidates) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
filled := 0
|
||||
|
||||
for _, c := range candidates {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if e.fetchAndStoreLyrics(ctx, c) != nil {
|
||||
filled++
|
||||
total++
|
||||
}
|
||||
}
|
||||
|
||||
// If a whole batch produced no stored lyrics, every remaining
|
||||
// candidate is a cached miss with no new data — stop early
|
||||
// rather than spinning through identical misses.
|
||||
if filled == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if total > 0 {
|
||||
e.logger.Info("lyrics backfill complete", "filled", total)
|
||||
}
|
||||
}
|
||||
|
||||
// fetchAndStoreLyrics looks up a single candidate on LRCLIB and, on a
|
||||
// non-instrumental hit with plain lyrics, persists it to the recording
|
||||
// (which also updates the FTS index). Returns the fetched lyrics, or
|
||||
// nil on any miss/error. Instrumental hits are recorded as an empty
|
||||
// lyrics string so they still count as "resolved" and aren't retried.
|
||||
func (e *Service) fetchAndStoreLyrics(
|
||||
ctx context.Context, c database.LyricsCandidate,
|
||||
) *Lyrics {
|
||||
durationSec := int(c.LengthMilliseconds / 1000) //nolint:mnd
|
||||
|
||||
lyrics, err := e.lrclib.GetLyrics(ctx, c.Artist, c.Title, c.Album, durationSec)
|
||||
if err != nil {
|
||||
if !errors.Is(err, ErrLyricsNotFound) {
|
||||
e.logger.Debug("lyrics fetch failed",
|
||||
"artist", c.Artist, "title", c.Title, "err", err,
|
||||
)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
if lyrics.Instrumental || lyrics.Plain == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := e.db.SetRecordingLyrics(c.RecordingID, lyrics.Plain); err != nil {
|
||||
e.logger.Warn("lyrics store failed", "recordingId", c.RecordingID, "err", err)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
return lyrics
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package explore
|
||||
|
||||
import "testing"
|
||||
|
||||
// TestMergeIndexHitsBlendedScale is the regression test for the
|
||||
// merge→filter scale mismatch. An exact-match, in-library album that
|
||||
// exists only in the local index (not returned by MB) with very low
|
||||
// popularity must survive filterAndCap and outrank a weakly-matching MB
|
||||
// result — because index hits are now scored on the same blended scale.
|
||||
//
|
||||
// Under the previous behaviour the index hit was scored as
|
||||
// scalePopularity(50)*0.5 ≈ 12, below minBlendedScore (15), so
|
||||
// filterAndCap dropped it entirely.
|
||||
func TestMergeIndexHitsBlendedScale(t *testing.T) {
|
||||
// A weakly-matching MB release group, already reranked: ~0.35
|
||||
// relevance, no popularity → blended Score ≈ 35.
|
||||
result := MBSearchResult{
|
||||
ReleaseGroups: []MBReleaseGroup{
|
||||
{MBID: "mb-weak", Title: "Live At Wembley", Score: 35, Popularity: 0},
|
||||
},
|
||||
}
|
||||
|
||||
// An exact-match, in-library album from the local index with only
|
||||
// 50 listens.
|
||||
hits := []SearchIndexResult{
|
||||
{
|
||||
EntityType: "release_group",
|
||||
MBID: "idx-exact",
|
||||
Title: "Abbey Road",
|
||||
ArtistName: "The Beatles",
|
||||
Popularity: 50,
|
||||
InLibrary: true,
|
||||
},
|
||||
}
|
||||
|
||||
mergeIndexHits("abbey road", &result, hits)
|
||||
|
||||
if got := result.ReleaseGroups[0].MBID; got != "idx-exact" {
|
||||
t.Fatalf("expected exact in-library index hit to rank first, got %q (score %d)",
|
||||
got, result.ReleaseGroups[0].Score)
|
||||
}
|
||||
|
||||
idxScore := result.ReleaseGroups[0].Score
|
||||
if idxScore < minBlendedScore {
|
||||
t.Errorf("index hit score %d below minBlendedScore %d — would be filtered out",
|
||||
idxScore, minBlendedScore)
|
||||
}
|
||||
|
||||
// It must survive the filter that previously dropped it.
|
||||
filterAndCap(&result)
|
||||
|
||||
if len(result.ReleaseGroups) == 0 || result.ReleaseGroups[0].MBID != "idx-exact" {
|
||||
t.Fatalf("index hit did not survive filterAndCap: %+v", result.ReleaseGroups)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMergeIndexHitsSortsRecordings guards the recordings path, which
|
||||
// previously had no post-merge sort: a high-scoring index recording was
|
||||
// merged but its position depended on prepend order, not score.
|
||||
func TestMergeIndexHitsSortsRecordings(t *testing.T) {
|
||||
result := MBSearchResult{
|
||||
Recordings: []MBRecording{
|
||||
{MBID: "mb-a", Title: "Filler", Score: 60, Popularity: 100},
|
||||
{MBID: "mb-b", Title: "Filler Two", Score: 20, Popularity: 50},
|
||||
},
|
||||
}
|
||||
|
||||
hits := []SearchIndexResult{
|
||||
{
|
||||
EntityType: "recording",
|
||||
MBID: "idx-exact",
|
||||
Title: "Calling You",
|
||||
Popularity: 100_000,
|
||||
InLibrary: true,
|
||||
},
|
||||
}
|
||||
|
||||
mergeIndexHits("calling you", &result, hits)
|
||||
|
||||
// Exact + in-library + decent popularity should land on top, and
|
||||
// the whole list must be in descending Score order.
|
||||
if result.Recordings[0].MBID != "idx-exact" {
|
||||
t.Errorf("exact in-library recording should rank first, got %q", result.Recordings[0].MBID)
|
||||
}
|
||||
|
||||
for i := 1; i < len(result.Recordings); i++ {
|
||||
if result.Recordings[i-1].Score < result.Recordings[i].Score {
|
||||
t.Errorf("recordings not sorted by score descending: %+v", result.Recordings)
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIndexHitRelevanceTiers(t *testing.T) {
|
||||
const q = "abbey road"
|
||||
|
||||
exact := indexHitRelevance(q, "Abbey Road", "")
|
||||
prefix := indexHitRelevance(q, "Abbey Road Sessions", "")
|
||||
word := indexHitRelevance(q, "Live: Abbey Road Medley", "")
|
||||
none := indexHitRelevance(q, "Something Unrelated", "")
|
||||
|
||||
if !(exact > prefix && prefix > word && word >= indexRelevanceFloor) {
|
||||
t.Errorf("relevance tiers not ordered: exact=%v prefix=%v word=%v", exact, prefix, word)
|
||||
}
|
||||
|
||||
if exact != indexRelExact {
|
||||
t.Errorf("exact relevance = %v, want %v", exact, indexRelExact)
|
||||
}
|
||||
|
||||
// A non-matching title still gets the floor (FTS matched something).
|
||||
if none != indexRelevanceFloor {
|
||||
t.Errorf("non-match relevance = %v, want floor %v", none, indexRelevanceFloor)
|
||||
}
|
||||
|
||||
// Artist-credit match should count when the title doesn't.
|
||||
artistMatch := indexHitRelevance("the beatles", "Abbey Road", "The Beatles")
|
||||
if artistMatch != indexRelExact {
|
||||
t.Errorf("artist-credit exact match = %v, want %v", artistMatch, indexRelExact)
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
@@ -18,6 +19,12 @@ const (
|
||||
// cacheTTLEntity is the TTL for lookup/browse results (entity data
|
||||
// changes rarely).
|
||||
cacheTTLEntity = 7 * 24 * time.Hour
|
||||
// cacheTTLReleases is the TTL for a release group's releases +
|
||||
// tracklists. This data is effectively immutable once published, so
|
||||
// it's cached far longer than other entities: it's the local store
|
||||
// that keeps an album page's cold fetch a once-per-quarter event
|
||||
// rather than a weekly one.
|
||||
cacheTTLReleases = 90 * 24 * time.Hour
|
||||
)
|
||||
|
||||
// MusicBrainzClient wraps the musicbrainzws2 library with a local
|
||||
@@ -360,15 +367,27 @@ func (c *MusicBrainzClient) LookupRelease(
|
||||
return &out, nil
|
||||
}
|
||||
|
||||
// BrowseReleases fetches the releases for a given release group
|
||||
// MBID, including media/track information. Cached for 7 days.
|
||||
func (c *MusicBrainzClient) BrowseReleases(
|
||||
ctx context.Context, releaseGroupMBID string,
|
||||
) ([]MBRelease, error) {
|
||||
cacheKey := "mb:browse:releases:" + releaseGroupMBID
|
||||
// MBRecordingRelease is a slim reference to one release a recording
|
||||
// appears on — enough for the autotagger to pick a representative
|
||||
// release and then LookupRelease it in full.
|
||||
type MBRecordingRelease struct {
|
||||
MBID string `json:"mbid"`
|
||||
Title string `json:"title"`
|
||||
Status string `json:"status"`
|
||||
Date string `json:"date"`
|
||||
}
|
||||
|
||||
// LookupRecordingReleases fetches the releases a recording appears on
|
||||
// (id / title / status / date only). Used by the autotag recording-
|
||||
// search path to resolve a picked recording to a concrete release.
|
||||
// Cached for 7 days.
|
||||
func (c *MusicBrainzClient) LookupRecordingReleases(
|
||||
ctx context.Context, recordingMBID string,
|
||||
) ([]MBRecordingRelease, error) {
|
||||
cacheKey := "mb:lookup:recording-releases:" + recordingMBID
|
||||
|
||||
if data, ok := c.cache.Get(cacheKey); ok {
|
||||
var out []MBRelease
|
||||
var out []MBRecordingRelease
|
||||
if err := json.Unmarshal(data, &out); err == nil {
|
||||
return out, nil
|
||||
}
|
||||
@@ -378,6 +397,77 @@ func (c *MusicBrainzClient) BrowseReleases(
|
||||
return nil, err
|
||||
}
|
||||
|
||||
c.logger.Info("musicbrainz lookup recording releases", "mbid", recordingMBID)
|
||||
|
||||
rec, err := c.mb.LookupRecording(
|
||||
ctx,
|
||||
mbtypes.MBID(recordingMBID),
|
||||
musicbrainzws2.IncludesFilter{Includes: []string{"releases"}},
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
out := make([]MBRecordingRelease, 0, len(rec.Releases))
|
||||
for _, rel := range rec.Releases {
|
||||
out = append(out, MBRecordingRelease{
|
||||
MBID: string(rel.ID),
|
||||
Title: rel.Title,
|
||||
Status: rel.Status,
|
||||
Date: rel.Date.String(),
|
||||
})
|
||||
}
|
||||
|
||||
c.cacheJSON(cacheKey, out, cacheTTLEntity, recordingMBID, "recording")
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// BrowseReleases fetches the releases for a given release group
|
||||
// MBID, including media/track information. Cached for 7 days.
|
||||
// browseReleasesCacheKey returns the response-cache key for a release
|
||||
// group's releases.
|
||||
func browseReleasesCacheKey(releaseGroupMBID string) string {
|
||||
return "mb:browse:releases:" + releaseGroupMBID
|
||||
}
|
||||
|
||||
// BrowseReleasesCached returns a release group's releases from the local
|
||||
// response cache only, never hitting the network. The bool reports
|
||||
// whether a fresh (unexpired) cache entry was found. Used by the album
|
||||
// page's local-first path so a cold fetch can be deferred to the
|
||||
// background instead of blocking the request.
|
||||
func (c *MusicBrainzClient) BrowseReleasesCached(
|
||||
releaseGroupMBID string,
|
||||
) ([]MBRelease, bool) {
|
||||
data, ok := c.cache.Get(browseReleasesCacheKey(releaseGroupMBID))
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
var out []MBRelease
|
||||
if err := json.Unmarshal(data, &out); err != nil {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
return out, true
|
||||
}
|
||||
|
||||
// BrowseReleases fetches all releases (with recordings + media) for a
|
||||
// release group, serving from the local response cache when warm and
|
||||
// otherwise hitting MusicBrainz and caching the result.
|
||||
func (c *MusicBrainzClient) BrowseReleases(
|
||||
ctx context.Context, releaseGroupMBID string,
|
||||
) ([]MBRelease, error) {
|
||||
cacheKey := browseReleasesCacheKey(releaseGroupMBID)
|
||||
|
||||
if out, ok := c.BrowseReleasesCached(releaseGroupMBID); ok {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
if err := c.limiter.Wait(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
c.logger.Info("musicbrainz browse releases",
|
||||
"releaseGroupMBID", releaseGroupMBID,
|
||||
)
|
||||
@@ -395,7 +485,7 @@ func (c *MusicBrainzClient) BrowseReleases(
|
||||
|
||||
out := convertReleases(result.Releases)
|
||||
|
||||
c.cacheJSON(cacheKey, out, cacheTTLEntity, releaseGroupMBID, "release-group")
|
||||
c.cacheJSON(cacheKey, out, cacheTTLReleases, releaseGroupMBID, "release-group")
|
||||
|
||||
return out, nil
|
||||
}
|
||||
@@ -534,7 +624,19 @@ func convertRelease(r musicbrainzws2.Release) MBRelease {
|
||||
}
|
||||
|
||||
for _, m := range r.Media {
|
||||
// Skip video media outright — DVD/Blu-ray bonus discs
|
||||
// inflate track counts and wreck track-count-based scoring
|
||||
// (beets ignores video/data tracks for the same reason).
|
||||
if isVideoFormat(m.Format) {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, t := range m.Tracks {
|
||||
// Same for individual video recordings on audio media.
|
||||
if t.Recording.IsVideo {
|
||||
continue
|
||||
}
|
||||
|
||||
// Use the recording MBID, not the track MBID. Tracks
|
||||
// and recordings have distinct MBIDs in MusicBrainz:
|
||||
// a track is the placement of a recording on a specific
|
||||
@@ -565,6 +667,25 @@ func convertRelease(r musicbrainzws2.Release) MBRelease {
|
||||
return rel
|
||||
}
|
||||
|
||||
// isVideoFormat reports whether a medium's format string names a
|
||||
// video carrier. "DVD-Audio" stays audio; bare "DVD", "DVD-Video",
|
||||
// "Blu-ray", "HD-DVD", "VHS", "VCD"/"SVCD" are video.
|
||||
func isVideoFormat(format string) bool {
|
||||
f := strings.ToLower(format)
|
||||
|
||||
if strings.Contains(f, "dvd-audio") {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, v := range []string{"dvd", "blu-ray", "bluray", "hd-dvd", "vhs", "vcd"} {
|
||||
if strings.Contains(f, v) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func convertReleases(releases []musicbrainzws2.Release) []MBRelease {
|
||||
out := make([]MBRelease, len(releases))
|
||||
for i, r := range releases {
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
package explore
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
)
|
||||
|
||||
// This file sketches a learned ranking model that replaces the
|
||||
// hand-tuned scoring constants (the fw* feature weights, tierBonus,
|
||||
// rgTierBonus, the intent-prior multipliers) with weights that can be
|
||||
// trained from the user's own click history.
|
||||
//
|
||||
// Status: NOT yet wired into Search(). The integration point is the
|
||||
// candidate scorers in the top-results pipeline
|
||||
// (scoreArtistCandidate / scoreReleaseGroupCandidate /
|
||||
// scoreRecordingCandidate) and the main rerank. Swap those additive
|
||||
// constant sums for RankFeatures + LinearModel.Score once the eval
|
||||
// harness has real fixtures to prove the change is a win.
|
||||
//
|
||||
// Why a linear model and not something fancier: it needs no scale, no
|
||||
// GPU, trains in microseconds on a desktop's worth of clicks, and —
|
||||
// crucially — personalises to ONE user. That is the lever a
|
||||
// multi-million-user system gets from aggregate logs; we get the same
|
||||
// shape of signal from a single user's repeated intent.
|
||||
//
|
||||
// Training prerequisite (important): logistic training needs negatives,
|
||||
// i.e. results that were SHOWN but NOT clicked. search_clicks records
|
||||
// only clicks today. To train properly, log impressions too (the
|
||||
// MBIDs shown for a query); clicked rows are positive labels, the rest
|
||||
// of the shown set are negatives. Until impressions are logged, use
|
||||
// DefaultModel(), whose weights reproduce the current behaviour.
|
||||
|
||||
// RankFeatures is the feature vector for a single candidate. Every
|
||||
// field is normalised to roughly [0,1] so weights are comparable.
|
||||
type RankFeatures struct {
|
||||
// Textual match strength against the query (mutually exclusive
|
||||
// tiers collapsed to a single 0..1 magnitude: exact=1.0,
|
||||
// prefix=0.6, whole-word=0.4, substring=0.2, none=0).
|
||||
NameMatch float64
|
||||
|
||||
// Artist-credit match strength, same scale. Lets "abbey road
|
||||
// beatles" reward the album credited to The Beatles.
|
||||
ArtistMatch float64
|
||||
|
||||
// Log-scaled popularity and listener count, both via normLog so
|
||||
// they share the fixed reference scale.
|
||||
LogPopularity float64
|
||||
LogListeners float64
|
||||
|
||||
// Personalisation signals.
|
||||
InLibrary float64 // 1.0 when owned, else 0
|
||||
IsSimilar float64 // 0..1 similarity to an owned artist
|
||||
|
||||
// Recency-decayed per-query click signal for this candidate.
|
||||
ClickRate float64
|
||||
}
|
||||
|
||||
// rankFeatureCount is the number of features (excluding bias). Used by
|
||||
// the gradient step to iterate fields generically.
|
||||
const rankFeatureCount = 7
|
||||
|
||||
// asSlice returns the features in a stable order so Score and Update
|
||||
// agree on indexing.
|
||||
func (f RankFeatures) asSlice() [rankFeatureCount]float64 {
|
||||
return [rankFeatureCount]float64{
|
||||
f.NameMatch,
|
||||
f.ArtistMatch,
|
||||
f.LogPopularity,
|
||||
f.LogListeners,
|
||||
f.InLibrary,
|
||||
f.IsSimilar,
|
||||
f.ClickRate,
|
||||
}
|
||||
}
|
||||
|
||||
// LinearModel scores a candidate as bias + Σ wᵢ·featureᵢ. For ranking
|
||||
// the raw score is what matters; the logistic squashing is used only
|
||||
// during training to produce a probability for the gradient.
|
||||
type LinearModel struct {
|
||||
Bias float64 `json:"bias"`
|
||||
Weights [rankFeatureCount]float64 `json:"weights"`
|
||||
}
|
||||
|
||||
// DefaultModel returns weights that reproduce the current hand-tuned
|
||||
// scorer, so swapping the model in with no training leaves behaviour
|
||||
// unchanged. The values mirror the fw* constants in explore.go.
|
||||
func DefaultModel() LinearModel {
|
||||
return LinearModel{
|
||||
Bias: 0,
|
||||
Weights: [rankFeatureCount]float64{
|
||||
1.00, // NameMatch ~ fwExactTitle / fwPrefixTitle blend
|
||||
0.90, // ArtistMatch ~ fwExactArtist
|
||||
0.80, // LogPopularity ~ fwListenLog
|
||||
0.60, // LogListeners ~ fwListenerLog
|
||||
0.50, // InLibrary ~ fwInLibrary
|
||||
0.20, // IsSimilar ~ fwSimilar
|
||||
0.30, // ClickRate ~ click feature cap
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Score returns the un-squashed ranking score for a candidate. Higher
|
||||
// is better. This is the value to sort by.
|
||||
func (m LinearModel) Score(f RankFeatures) float64 {
|
||||
score := m.Bias
|
||||
fs := f.asSlice()
|
||||
|
||||
for i := range fs {
|
||||
score += m.Weights[i] * fs[i]
|
||||
}
|
||||
|
||||
return score
|
||||
}
|
||||
|
||||
// Probability squashes the score to (0,1) via the logistic function.
|
||||
// Used during training to compute the gradient.
|
||||
func (m LinearModel) Probability(f RankFeatures) float64 {
|
||||
return 1.0 / (1.0 + math.Exp(-m.Score(f)))
|
||||
}
|
||||
|
||||
// Sample is one training example: a candidate's features and whether
|
||||
// the user clicked it (1.0) or saw-but-skipped it (0.0).
|
||||
type Sample struct {
|
||||
Features RankFeatures
|
||||
Label float64
|
||||
}
|
||||
|
||||
// Update performs one logistic-regression SGD step toward the label.
|
||||
// learningRate is typically ~0.05. Returns the pre-update prediction
|
||||
// so callers can track convergence.
|
||||
func (m *LinearModel) Update(s Sample, learningRate float64) float64 {
|
||||
pred := m.Probability(s.Features)
|
||||
err := s.Label - pred
|
||||
fs := s.Features.asSlice()
|
||||
|
||||
m.Bias += learningRate * err
|
||||
|
||||
for i := range fs {
|
||||
m.Weights[i] += learningRate * err * fs[i]
|
||||
}
|
||||
|
||||
return pred
|
||||
}
|
||||
|
||||
// Train runs SGD over the samples for the given number of epochs. A
|
||||
// few hundred clicks over a handful of epochs converges fine; this is
|
||||
// cheap enough to run on startup or after a search session.
|
||||
func Train(model *LinearModel, samples []Sample, epochs int, learningRate float64) {
|
||||
for range epochs {
|
||||
for _, s := range samples {
|
||||
model.Update(s, learningRate)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// matchStrength collapses the mutually-exclusive textual match tiers
|
||||
// used throughout the current scorers into a single magnitude, so the
|
||||
// model has one weight to learn instead of four overlapping constants.
|
||||
func matchStrength(exact, prefix, wholeWord, substring bool) float64 {
|
||||
switch {
|
||||
case exact:
|
||||
return 1.0
|
||||
case prefix:
|
||||
return 0.6
|
||||
case wholeWord:
|
||||
return 0.4
|
||||
case substring:
|
||||
return 0.2
|
||||
default:
|
||||
return 0.0
|
||||
}
|
||||
}
|
||||
|
||||
// SaveModel writes the model as JSON. Callers persist this to disk or
|
||||
// the index meta table; the model is small (8 floats).
|
||||
func SaveModel(w io.Writer, m LinearModel) error {
|
||||
if err := json.NewEncoder(w).Encode(m); err != nil {
|
||||
return fmt.Errorf("ranker: encode model: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// LoadModel reads a model previously written by SaveModel.
|
||||
func LoadModel(r io.Reader) (LinearModel, error) {
|
||||
var m LinearModel
|
||||
|
||||
if err := json.NewDecoder(r).Decode(&m); err != nil {
|
||||
return LinearModel{}, fmt.Errorf("ranker: decode model: %w", err)
|
||||
}
|
||||
|
||||
return m, nil
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package explore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"math"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDefaultModelScoreOrdering(t *testing.T) {
|
||||
m := DefaultModel()
|
||||
|
||||
// A popular, in-library exact match must outscore an obscure
|
||||
// substring match.
|
||||
strong := RankFeatures{NameMatch: 1.0, LogPopularity: 0.9, InLibrary: 1.0}
|
||||
weak := RankFeatures{NameMatch: 0.2, LogPopularity: 0.1}
|
||||
|
||||
if m.Score(strong) <= m.Score(weak) {
|
||||
t.Errorf("strong candidate %.3f should outscore weak %.3f",
|
||||
m.Score(strong), m.Score(weak))
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateMovesTowardLabel(t *testing.T) {
|
||||
m := DefaultModel()
|
||||
|
||||
// A candidate the user repeatedly clicks should see its predicted
|
||||
// probability rise after training on positive labels.
|
||||
f := RankFeatures{NameMatch: 0.4, LogPopularity: 0.2}
|
||||
|
||||
before := m.Probability(f)
|
||||
|
||||
for range 50 {
|
||||
m.Update(Sample{Features: f, Label: 1.0}, 0.1)
|
||||
}
|
||||
|
||||
after := m.Probability(f)
|
||||
|
||||
if after <= before {
|
||||
t.Errorf("probability should rise toward positive label: before=%.4f after=%.4f",
|
||||
before, after)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateLearnsNegative(t *testing.T) {
|
||||
m := DefaultModel()
|
||||
|
||||
f := RankFeatures{NameMatch: 0.9, LogPopularity: 0.9}
|
||||
|
||||
before := m.Probability(f)
|
||||
|
||||
// Shown repeatedly, never clicked — probability should fall.
|
||||
for range 50 {
|
||||
m.Update(Sample{Features: f, Label: 0.0}, 0.1)
|
||||
}
|
||||
|
||||
after := m.Probability(f)
|
||||
|
||||
if after >= before {
|
||||
t.Errorf("probability should fall toward negative label: before=%.4f after=%.4f",
|
||||
before, after)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchStrengthTiers(t *testing.T) {
|
||||
if matchStrength(true, false, false, false) != 1.0 {
|
||||
t.Error("exact match should be 1.0")
|
||||
}
|
||||
|
||||
if matchStrength(false, false, false, false) != 0.0 {
|
||||
t.Error("no match should be 0.0")
|
||||
}
|
||||
|
||||
// Tiers must be strictly ordered.
|
||||
exact := matchStrength(true, false, false, false)
|
||||
prefix := matchStrength(false, true, false, false)
|
||||
word := matchStrength(false, false, true, false)
|
||||
sub := matchStrength(false, false, false, true)
|
||||
|
||||
if !(exact > prefix && prefix > word && word > sub) {
|
||||
t.Errorf("tiers not strictly ordered: %v %v %v %v", exact, prefix, word, sub)
|
||||
}
|
||||
}
|
||||
|
||||
func TestModelRoundTrip(t *testing.T) {
|
||||
m := DefaultModel()
|
||||
m.Bias = 0.123
|
||||
m.Weights[0] = 0.777
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := SaveModel(&buf, m); err != nil {
|
||||
t.Fatalf("SaveModel: %v", err)
|
||||
}
|
||||
|
||||
got, err := LoadModel(&buf)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadModel: %v", err)
|
||||
}
|
||||
|
||||
if math.Abs(got.Bias-m.Bias) > 1e-9 || math.Abs(got.Weights[0]-m.Weights[0]) > 1e-9 {
|
||||
t.Errorf("round trip mismatch: got %+v want %+v", got, m)
|
||||
}
|
||||
}
|
||||
+927
-1408
File diff suppressed because it is too large
Load Diff
+31
-13
@@ -24,6 +24,7 @@ type TopResult struct {
|
||||
MBID string `json:"mbid"`
|
||||
Name string `json:"name"`
|
||||
ArtistCredit string `json:"artistCredit,omitempty"` // for tracks/albums
|
||||
ArtistMBID string `json:"artistMbid,omitempty"` // for linking the artist subtitle
|
||||
IntentScore float64 `json:"intentScore"`
|
||||
// Artist-specific
|
||||
ArtistType string `json:"artistType,omitempty"` // "Group", "Person"
|
||||
@@ -31,8 +32,14 @@ type TopResult struct {
|
||||
// Album-specific
|
||||
PrimaryType string `json:"primaryType,omitempty"`
|
||||
Year string `json:"year,omitempty"`
|
||||
// Track-specific
|
||||
Length int `json:"length,omitempty"`
|
||||
// Track-specific. ReleaseGroupMBID is resolved (from CAAReleaseMBID)
|
||||
// so a track click can open its album page with the track highlighted,
|
||||
// matching how tracks behave everywhere else. ReleaseName is the album
|
||||
// title used for the album page header.
|
||||
Length int `json:"length,omitempty"`
|
||||
CAAReleaseMBID string `json:"caaReleaseMbid,omitempty"`
|
||||
ReleaseGroupMBID string `json:"releaseGroupMbid,omitempty"`
|
||||
ReleaseName string `json:"releaseName,omitempty"`
|
||||
// Library status — populated from index cross-reference columns.
|
||||
InLibrary bool `json:"inLibrary"`
|
||||
}
|
||||
@@ -64,8 +71,9 @@ type MBReleaseGroup struct {
|
||||
SecondaryTypes []string `json:"secondaryTypes,omitempty"`
|
||||
FirstReleaseDate string `json:"firstReleaseDate"`
|
||||
ArtistCredit string `json:"artistCredit"`
|
||||
Score int `json:"-"` // MB search relevance, used for reranking
|
||||
Popularity int `json:"popularity"` // raw LB listen count (0 if unknown)
|
||||
ArtistMBID string `json:"artistMbid,omitempty"` // for linking the artist to its detail page
|
||||
Score int `json:"-"` // MB search relevance, used for reranking
|
||||
Popularity int `json:"popularity"` // raw LB listen count (0 if unknown)
|
||||
ListenerCount int `json:"listenerCount"`
|
||||
InLibrary bool `json:"inLibrary"` // true if the user owns this album
|
||||
LocalID int64 `json:"localId,omitempty"` // local release_group row ID
|
||||
@@ -85,15 +93,22 @@ type MBRelease struct {
|
||||
// MBRecording is a Wails-friendly projection of a MusicBrainz
|
||||
// recording.
|
||||
type MBRecording struct {
|
||||
MBID string `json:"mbid"`
|
||||
Title string `json:"title"`
|
||||
Length int `json:"length"`
|
||||
ArtistCredit string `json:"artistCredit"`
|
||||
Score int `json:"score"`
|
||||
Popularity int `json:"popularity"` // raw LB listen count (0 if unknown)
|
||||
ListenerCount int `json:"listenerCount"`
|
||||
InLibrary bool `json:"inLibrary"` // true if the user owns this recording
|
||||
LocalID int64 `json:"localId,omitempty"` // local recording row ID
|
||||
MBID string `json:"mbid"`
|
||||
Title string `json:"title"`
|
||||
Length int `json:"length"`
|
||||
ArtistCredit string `json:"artistCredit"`
|
||||
ArtistMBID string `json:"artistMbid,omitempty"` // for linking the artist to its detail page
|
||||
Score int `json:"score"`
|
||||
Popularity int `json:"popularity"` // raw LB listen count (0 if unknown)
|
||||
ListenerCount int `json:"listenerCount"`
|
||||
CAAReleaseMBID string `json:"caaReleaseMbid,omitempty"` // parent release, for album navigation
|
||||
// ReleaseGroupMBID is resolved from CAAReleaseMBID so a track can
|
||||
// link to its album page with the track highlighted, matching how
|
||||
// tracks behave everywhere else.
|
||||
ReleaseGroupMBID string `json:"releaseGroupMbid,omitempty"`
|
||||
ReleaseName string `json:"releaseName,omitempty"` // album title
|
||||
InLibrary bool `json:"inLibrary"` // true if the user owns this recording
|
||||
LocalID int64 `json:"localId,omitempty"` // local recording row ID
|
||||
}
|
||||
|
||||
// MBTrack is a Wails-friendly projection of a MusicBrainz track.
|
||||
@@ -119,6 +134,9 @@ type LBTopRecording struct {
|
||||
TrackName string `json:"trackName"`
|
||||
TotalListenCount int `json:"totalListenCount"`
|
||||
CAAReleaseMBID string `json:"caaReleaseMbid"`
|
||||
// ReleaseGroupMBID is resolved from CAAReleaseMBID so a top-track
|
||||
// row can link to its album page with the track highlighted.
|
||||
ReleaseGroupMBID string `json:"releaseGroupMbid,omitempty"`
|
||||
ReleaseName string `json:"releaseName"`
|
||||
Length int `json:"length"` // milliseconds (from LB API)
|
||||
InLibrary bool `json:"inLibrary"`
|
||||
|
||||
Reference in New Issue
Block a user