From 65048401e85f717ffe7e964525c218f52f731382 Mon Sep 17 00:00:00 2001 From: Caleb Allen Date: Fri, 24 Jul 2026 12:14:20 -0400 Subject: [PATCH] feat: autotag scoring overhaul, dump-based explore index, and lyrics search MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Consolidates in-progress work across autotag, explore, and library: - autotag: beets/Picard-informed scoring engine — ID-first matching, VA handling, recommendation tiers, and a merged distance/rank cascade, with an eval harness for regression tracking. - explore: offline MusicBrainz dump import/incremental refresh replaces the legacy tier crawl; index-first local search with fuzzy matching and a dedicated ranker; disk-free guards for dump downloads. - library: artist-credit extraction and matching. - lyrics: owned-library lyric search (FTS) with LRCLIB backfill. Also: rewrite README to be user-focused, and migrate upstream to git.ljones.me/yonlu/yellowjacket. Co-Authored-By: Claude Opus 4.8 --- .planning/NOTES.md | 70 - .planning/ROADMAP.md | 61 - .planning/autotag.md | 162 ++ .../plans/active/010-autotag-review-ui.md | 30 - .../plans/completed/001-v1.0-consolidation.md | 22 - .../plans/completed/002-v1.1-multi-library.md | 23 - .../plans/completed/003-v1.2-tag-editing.md | 24 - .../completed/004-v1.2.1-format-parity.md | 18 - .../plans/completed/005-smart-playlists.md | 25 - .planning/plans/completed/006-play-history.md | 19 - .../plans/completed/007-explore-browser.md | 24 - .../completed/008-autotag-schema-grouping.md | 20 - .../completed/009-autotag-scoring-engine.md | 32 - .../plans/pending/011-autotag-auto-accept.md | 26 - .../pending/012-autotag-settings-polish.md | 31 - README.md | 158 +- backend/app.go | 64 +- backend/autotag/align.go | 176 +- backend/autotag/distance.go | 191 +- backend/autotag/distance_test.go | 121 +- backend/autotag/eval/eval.go | 108 + backend/autotag/eval/evaluate.go | 123 + .../autotag/eval/testdata/scoring_cases.json | 225 ++ backend/autotag/eval_harness_test.go | 102 + backend/autotag/mb.go | 378 ++- backend/autotag/mb_test.go | 326 ++- backend/autotag/normalize.go | 128 +- backend/autotag/normalize_test.go | 6 +- backend/autotag/rank.go | 352 ++- backend/autotag/rank_test.go | 307 ++- backend/autotag/recommend.go | 133 + backend/autotag/recommend_test.go | 130 + backend/autotag/scorer.go | 241 +- backend/autotag/scorer_test.go | 198 +- backend/autotag/types.go | 32 +- backend/autotagservice/service.go | 441 +++- backend/config/config.go | 41 +- backend/config/window.go | 18 +- backend/database/database.go | 612 ++++- backend/database/lyrics_search.go | 295 +++ backend/database/lyrics_search_test.go | 252 ++ .../database/sql/queries/release_groups.sql | 44 + .../sql/queries/tagging_candidates.sql | 15 + .../database/sql/queries/tagging_items.sql | 4 +- backend/database/sql/schemas/lyrics_index.sql | 15 + backend/database/sql/schemas/playlists.sql | 1 + .../sql/schemas/tagging_candidates.sql | 18 + backend/database/sql/sqlcgen/models.go | 23 +- backend/database/sql/sqlcgen/playlists.sql.go | 9 +- .../sql/sqlcgen/release_groups.sql.go | 52 + .../sql/sqlcgen/tagging_candidates.sql.go | 51 + .../database/sql/sqlcgen/tagging_items.sql.go | 4 +- backend/database/tagging_items_test.go | 50 + backend/database/testhelper.go | 11 +- backend/events/events.go | 18 + backend/explore/autotagclient.go | 52 +- backend/explore/diskfree_unix.go | 19 + backend/explore/diskfree_windows.go | 22 + backend/explore/dumpcatalog.go | 1079 ++++++++ backend/explore/dumpcounts.go | 584 +++++ backend/explore/dumpimport.go | 574 ++++ backend/explore/dumpimport_test.go | 925 +++++++ backend/explore/dumpincremental.go | 522 ++++ backend/explore/dumpincremental_test.go | 209 ++ backend/explore/dumppatch.go | 279 ++ backend/explore/dumpstream.go | 329 +++ backend/explore/eval/eval.go | 115 + backend/explore/eval/metrics.go | 228 ++ backend/explore/eval/metrics_test.go | 193 ++ .../explore/eval/testdata/eval_queries.json | 32 + backend/explore/eval_harness_test.go | 150 ++ backend/explore/explore.go | 2142 +++++---------- backend/explore/fuzzy.go | 58 + backend/explore/fuzzy_test.go | 43 + backend/explore/lrclib.go | 189 ++ backend/explore/lrclib_test.go | 127 + backend/explore/lyrics.go | 210 ++ backend/explore/mergeindexhits_test.go | 121 + backend/explore/musicbrainz.go | 137 +- backend/explore/ranker.go | 195 ++ backend/explore/ranker_test.go | 102 + backend/explore/searchindex.go | 2335 +++++++---------- backend/explore/types.go | 44 +- backend/library/artistcredit.go | 72 + backend/library/artistcredit_test.go | 126 + backend/library/crud.go | 9 + backend/library/library.go | 31 +- backend/library/query.go | 37 +- backend/player/buffered_streamer.go | 27 +- backend/player/player.go | 17 +- backend/playlist/playlist.go | 165 +- backend/playlist/smart_test.go | 140 +- backend/queue/persistence.go | 6 +- backend/smartplaylist/smartplaylist.go | 90 +- backend/smartplaylist/smartplaylist_test.go | 117 + frontend/pnpm-workspace.yaml | 2 + .../components/autotag-view/autotag-view.ts | 1660 +++++++++--- .../src/components/config-page/config-page.ts | 2 +- .../src/components/cover-grid/cover-grid.ts | 5 +- .../explore-album-details.ts | 77 +- .../explore-artist-details.ts | 133 +- .../components/explore-view/explore-view.ts | 776 +++--- .../smart-playlist-details.ts | 894 ++++++- .../smart-playlist-editor.ts | 35 +- .../top-results-row/top-results-row.ts | 25 +- frontend/src/events.ts | 3 + frontend/src/utils/text-diff.ts | 41 + .../wailsjs/go/autotagservice/Service.d.ts | 6 + frontend/wailsjs/go/autotagservice/Service.js | 12 + frontend/wailsjs/go/explore/Service.d.ts | 26 +- frontend/wailsjs/go/explore/Service.js | 52 +- frontend/wailsjs/go/models.ts | 92 + frontend/wailsjs/go/playlist/Service.d.ts | 4 + frontend/wailsjs/go/playlist/Service.js | 8 + go.mod | 10 +- go.sum | 20 +- main.go | 5 +- 117 files changed, 17033 insertions(+), 4767 deletions(-) delete mode 100644 .planning/NOTES.md delete mode 100644 .planning/ROADMAP.md create mode 100644 .planning/autotag.md delete mode 100644 .planning/plans/active/010-autotag-review-ui.md delete mode 100644 .planning/plans/completed/001-v1.0-consolidation.md delete mode 100644 .planning/plans/completed/002-v1.1-multi-library.md delete mode 100644 .planning/plans/completed/003-v1.2-tag-editing.md delete mode 100644 .planning/plans/completed/004-v1.2.1-format-parity.md delete mode 100644 .planning/plans/completed/005-smart-playlists.md delete mode 100644 .planning/plans/completed/006-play-history.md delete mode 100644 .planning/plans/completed/007-explore-browser.md delete mode 100644 .planning/plans/completed/008-autotag-schema-grouping.md delete mode 100644 .planning/plans/completed/009-autotag-scoring-engine.md delete mode 100644 .planning/plans/pending/011-autotag-auto-accept.md delete mode 100644 .planning/plans/pending/012-autotag-settings-polish.md create mode 100644 backend/autotag/eval/eval.go create mode 100644 backend/autotag/eval/evaluate.go create mode 100644 backend/autotag/eval/testdata/scoring_cases.json create mode 100644 backend/autotag/eval_harness_test.go create mode 100644 backend/autotag/recommend.go create mode 100644 backend/autotag/recommend_test.go create mode 100644 backend/database/lyrics_search.go create mode 100644 backend/database/lyrics_search_test.go create mode 100644 backend/database/sql/queries/tagging_candidates.sql create mode 100644 backend/database/sql/schemas/lyrics_index.sql create mode 100644 backend/database/sql/schemas/tagging_candidates.sql create mode 100644 backend/database/sql/sqlcgen/tagging_candidates.sql.go create mode 100644 backend/explore/diskfree_unix.go create mode 100644 backend/explore/diskfree_windows.go create mode 100644 backend/explore/dumpcatalog.go create mode 100644 backend/explore/dumpcounts.go create mode 100644 backend/explore/dumpimport.go create mode 100644 backend/explore/dumpimport_test.go create mode 100644 backend/explore/dumpincremental.go create mode 100644 backend/explore/dumpincremental_test.go create mode 100644 backend/explore/dumppatch.go create mode 100644 backend/explore/dumpstream.go create mode 100644 backend/explore/eval/eval.go create mode 100644 backend/explore/eval/metrics.go create mode 100644 backend/explore/eval/metrics_test.go create mode 100644 backend/explore/eval/testdata/eval_queries.json create mode 100644 backend/explore/eval_harness_test.go create mode 100644 backend/explore/fuzzy.go create mode 100644 backend/explore/fuzzy_test.go create mode 100644 backend/explore/lrclib.go create mode 100644 backend/explore/lrclib_test.go create mode 100644 backend/explore/lyrics.go create mode 100644 backend/explore/mergeindexhits_test.go create mode 100644 backend/explore/ranker.go create mode 100644 backend/explore/ranker_test.go create mode 100644 backend/library/artistcredit.go create mode 100644 backend/library/artistcredit_test.go create mode 100644 frontend/pnpm-workspace.yaml diff --git a/.planning/NOTES.md b/.planning/NOTES.md deleted file mode 100644 index f6b92c3..0000000 --- a/.planning/NOTES.md +++ /dev/null @@ -1,70 +0,0 @@ -# Notes - -Miscellaneous things worth knowing that aren't documented elsewhere. CLAUDE.md owns the architecture overview, the build commands, and the per-package responsibilities — this file is for the gotchas, the deferred-but-tracked items, the "we already considered and rejected" decisions, and the things that bite if you forget them. - -## Hard SQLite-driver constraints (forget at your peril) - -- `MaxOpenConns(1)` is set on the SQLite connection. **Holding `*sql.Rows` open while calling another function that queries the same `*database.DB` will deadlock.** Close rows explicitly before downstream calls; don't rely on `defer` when the deferred close has to happen *after* a downstream query. This bit smart-playlist S01/T03 and the smart-playlist auto-edit feature; it bites the play-history hook so `OnPlaybackFinished` unlocks the player mutex *before* recording a play. -- No CGo: the driver is `modernc.org/sqlite`. Cannot switch to a CGo-based driver — design around it. -- Connection pooling, dynamic ORDER BY in sqlc, and `FILTER (WHERE ...)` in older modernc versions all require care. (`FILTER` works in current `modernc.org/sqlite`, but if it ever fails, fall back to `SUM(CASE WHEN ... THEN 1 ELSE 0 END)`.) - -## Explore + Autotagger API-call-minimization playbook - -Every MusicBrainz interaction follows this resolution order — check before sending: - -1. **Local DB** — does a matching `release_groups` row already exist? (zero network cost) -2. **Existing partial MBIDs** on the album's tracks — use as Lucene filters (`arid:`, `rgid:`) to narrow search and produce deterministic cache keys. -3. **`http_cache`** (already wrapped transparently by `MusicBrainzClient`) — serves hits for any previously-fetched entity. -4. **Live MB fetch** — last resort. - -Plus: share fan-out (one `LookupArtist` per album, not per track); persist decisions to `tagging_items` so reopening the review UI fires zero MB calls; prefetch album N+1 while user reviews album N; cover art is pull-on-apply only; auto-accept never fetches incrementally. - -Cache TTLs: 24 h for searches (data updates often), 7 d for entity lookups (artist details, discography are stable). - -## Tag-editing pipeline invariants - -- **`AtomicWrite`** writes to `.yj-tmp` in the same directory, then renames. Same-directory rename avoids cross-device issues; deterministic suffix enables orphan cleanup on startup. -- **Upsert-and-relink for entity sync.** Never mutate shared `artists` / `albums` / `genres` rows in place — create new ones or relink to existing rows. Safe under concurrent reads. -- **Currently-playing file gets stopped before its tag write.** `PlayerStopper` interface (implemented by `playerAdapter` in `app.go`) breaks the import cycle. -- **Scan and write are mutually exclusive** via `pipelineMu` in the library package. -- **Batch writes coalesce events** with a `suppressEvents` flag — one `TrackMetadataChanged` per batch instead of N. - -## Frontend gotchas - -- **Wails TS bindings for smart-playlist methods are manually maintained** in `frontend/wailsjs/go/playlist/Service.{d.ts,js}`. The Wails build does *not* re-generate them in worktrees, and no build-time check catches drift if Go signatures change. Same for any explore method added outside a clean Wails build. -- **`pnpm build` runs from `frontend/`, not project root.** Root `package.json` is empty. -- **Combobox blur-vs-click race** — the `mousedown` + `preventDefault()` + `requestAnimationFrame` fallback in `combobox.ts` is fragile if option rendering moves into a separate shadow DOM. Re-verify click-to-select after any combobox refactor. -- **`go build ./...` fails in git worktrees** because `main.go:28` embeds `frontend/dist`, which doesn't exist in a fresh worktree. Use `go build ./backend/...` for backend-only verification. -- Explore detail components duplicate `CoverArtGroupURL`, `nameToHue`, `extractYear`, `formatDuration` from `explore-view`. Three consumers as of v1.3 planning. If a fourth emerges, extract to `explore-utils.ts`. - -## Lint baseline - -`make lint` may report a small number of pre-existing warnings (3 wsl_v5 in `database_test.go`, 1 gci + 1 revive in `smartplaylist.go` last time anyone counted). Don't chase these in unrelated PRs. Note them as pre-existing in verification evidence and move on. Anything new must be clean. - -## Out-of-scope decisions worth remembering the *why* for - -- **Separate databases per library** — defeats unified presentation; rejected. -- **Auto-dedup across libraries** — complex matching logic, not table stakes. -- **Parallel library scanning** — SQLite single-writer; pointless. -- **ORM / query builder** — fights existing sqlc architecture. -- **Connection pooling for SQLite** — meaningless under `MaxOpenConns(1)`. -- **Database health checking / reconnection** — desktop app context, low priority. -- **Cosmetic file splitting** — large files are only a problem if they cause real issues. Extract for reuse or correctness, not aesthetics. -- **Parenthesized boolean logic in smart playlists** — UI complexity not worth the use case; AND-only with multi-value `is_any_of` covers the vast majority. -- **"Is favorited" as a smart-playlist filter** — favoriting is a special-case relationship, not a queryable field. -- **Playing audio remotely from MB** — MB is a metadata catalog, not a streaming service. -- **Fuzzy auto-accept threshold slider** — strict all-match is the trustworthy default; a slider is a power-user foot-gun. -- **Manual MB search UI in autotag** — Paste-URL covers the escape hatch; full search is surface area we don't need yet. -- **Configurable field whitelist for autotag writes** — hardcoded list in v1; add config only if users actually ask. -- **Folder-level cover art (`folder.jpg`/`cover.png`)** — separate feature area from embedded art. -- **Per-library autotagger on/off** — single global setting; per-library adds UI for no clear benefit. - -## Known gap from milestone 007 - -R032 (offline visual indicator for cached Explore data) was scoped into M004 but not implemented. The cache layer works correctly — entries are served when offline until TTL expires — but `Cache.Get()` doesn't propagate a "from cache" flag, and no frontend component renders a "Cached" badge. If picked up: backend modifies `Cache.Get()` to return a `fromCache` bool, frontend adds a subtle badge. ~30 min of work. - -## Open architecture questions - -- **VA compilation detection threshold for the autotagger.** Per-track artist credits differing from album-artist is the easy heuristic. What's the threshold on number of differing tracks before we relax the artist-match rule? Will surface during scoring tuning in plan 009. -- **Cover Art Archive minimum-dimension check.** REVIEW-05 (plan 010) sets 500 px on the shortest side. Coarse but cheap. Expect tuning once we see real CAA quality variance across genres. -- **Rate-limit priority queue design.** CFG-02 (plan 012) wants user-initiated MB calls to jump ahead of background auto-accept. Implementation strategy is open — priority channel? Two limiters with a yield mechanism? Solve when 012 starts. diff --git a/.planning/ROADMAP.md b/.planning/ROADMAP.md deleted file mode 100644 index 31b76fd..0000000 --- a/.planning/ROADMAP.md +++ /dev/null @@ -1,61 +0,0 @@ -# YellowJacket — Roadmap - -A cross-platform desktop music player. Plays local MP3 / FLAC / OGG / WAV; manages multiple library directories via SQLite; queue, playlists, smart playlists, play history, full-text search, cover art, MPRIS controls on Linux, full single + batch tag editing across all four formats, and a read-only MusicBrainz / ListenBrainz catalog browser. - -The guiding rule for everything below is "the music player works reliably and feels solid" — every interaction should be correct, responsive, and trustworthy. New surface area only goes in once the foundation under it is stable. - -## Capability set - -- **Playback** — play / pause / seek / volume on MP3, FLAC, OGG, WAV. Queue with shuffle (Fisher-Yates), repeat (off / one / all), auto-advance, persistent across restarts. -- **Library** — multiple library directories, concurrent metadata extraction, adaptive scan concurrency by disk type, cancellable / pausable scans, cross-library playlists with phantom-track preservation when a library is removed. -- **Search & browse** — FTS5 across tracks/artists/albums/paths; browse by albums / artists / genres; library filter respected everywhere; virtual scrolling. -- **Playlists** — CRUD, M3U8 import/export, favorites, smart playlists (rule-based saved queries with combobox editor + live preview), default playlist. -- **Tag editing** — single + batch edit of 8 fields across all four formats; cover art embed/replace/remove; crash-safe atomic writes; instant DB + FTS5 sync, no rescan needed. -- **Play history** — natural-finish play counting, `last_played` timestamp, play-history log, integration into smart playlist rule fields. -- **Explore** — read-only MusicBrainz / ListenBrainz browser: search → artist page → album detail with release-version selection, with rate-limited APIs and SQLite-cached responses. -- **Configuration & system** — TOML config with live reload, configurable keyboard shortcuts (record-style capture, scope-aware dispatch), theme tokens (accent / background shade), MPRIS2 on Linux. - -## Milestone sequence - -| # | Milestone | Status | File | -|---|-----------|--------|------| -| 001 | v1.0 Consolidation | shipped 2026-03-05 | [completed/001-v1.0-consolidation.md](plans/completed/001-v1.0-consolidation.md) | -| 002 | v1.1 Multi-Library Support | shipped 2026-03-16 | [completed/002-v1.1-multi-library.md](plans/completed/002-v1.1-multi-library.md) | -| 003 | v1.2 Tag Editing (MP3 + FLAC) | shipped 2026-03-18 | [completed/003-v1.2-tag-editing.md](plans/completed/003-v1.2-tag-editing.md) | -| 004 | v1.2.1 Format Parity (OGG + WAV) | shipped 2026-03-21 | [completed/004-v1.2.1-format-parity.md](plans/completed/004-v1.2.1-format-parity.md) | -| 005 | Smart Playlists | shipped 2026-03-22 | [completed/005-smart-playlists.md](plans/completed/005-smart-playlists.md) | -| 006 | Play History & Play Count | shipped 2026-03-22 | [completed/006-play-history.md](plans/completed/006-play-history.md) | -| 007 | MusicBrainz/ListenBrainz Explore Browser | shipped 2026-03-24 | [completed/007-explore-browser.md](plans/completed/007-explore-browser.md) | -| 008 | Autotag — Schema & Grouping Foundation | shipped 2026-04-20 | [completed/008-autotag-schema-grouping.md](plans/completed/008-autotag-schema-grouping.md) | -| 009 | Autotag — Scoring Engine & MB Orchestration | shipped 2026-04-21 | [completed/009-autotag-scoring-engine.md](plans/completed/009-autotag-scoring-engine.md) | -| 010 | Autotag — Review UI & Apply Pipeline | **active** | [active/010-autotag-review-ui.md](plans/active/010-autotag-review-ui.md) | -| 011 | Autotag — Auto-Accept & Entry Points | pending | [pending/011-autotag-auto-accept.md](plans/pending/011-autotag-auto-accept.md) | -| 012 | Autotag — Settings & Polish | pending | [pending/012-autotag-settings-polish.md](plans/pending/012-autotag-settings-polish.md) | - -The active phase is the **MusicBrainz autotagger** (collectively v1.3). It builds on the explore-browser API client + cache foundation from milestone 007. Plans 008-012 are sequential — each depends on the prior one. See `NOTES.md` for the API-call-minimization playbook and the design principles that constrain every Autotag plan. - -## Beyond v1.3 (not yet planned) - -Captured here so they don't get lost; not yet promoted to plan files. - -- **ListenBrainz scrobbling.** Submit play data when a track crosses the scrobble threshold (`min(duration / 2, 4 minutes)`). Three submission types — `playing_now`, `single`, `import`. Hook slots in next to the existing play-history pipeline; MBIDs already flow through the stack thanks to v1.3. -- **Gapless playback + crossfade.** Seamless transitions with optional crossfade. -- **Layout customization system.** Section-based UI customization; components declare size constraints, users configure per-section. -- **Plugin system.** Full-access API for UI components and backend hooks. -- **AcoustID fingerprinting** via fpcalc — slots in behind the v1.3 `Identifier` interface seam. - -## Out of scope - -Things deliberately **not** going in. See `NOTES.md` for the reasoning behind each. - -- Separate databases per library, auto-dedup across libraries, user access control per library. -- Parallel library scanning (SQLite single-writer). -- Cross-platform media controls beyond MPRIS on Linux. -- Database health checking / reconnection. -- ORM or query builder, connection pooling for SQLite. -- Parenthesized boolean logic in smart playlists; "is favorited" as a filter; queue that re-evaluates rules during playback. -- Playing audio from MusicBrainz remotely (it's a metadata catalog, not a streaming service). -- Integrating any service beyond MusicBrainz / ListenBrainz / Cover Art Archive without explicit user approval. -- Fuzzy auto-accept threshold sliders (strict all-match is the trustworthy default). -- Manual MB search UI (Paste-URL covers the escape-hatch case). -- Cover art replacement during auto-accept (highest-regret op — never automatic in v1). diff --git a/.planning/autotag.md b/.planning/autotag.md new file mode 100644 index 0000000..268ab99 --- /dev/null +++ b/.planning/autotag.md @@ -0,0 +1,162 @@ +# Autotag (v1.3) — MusicBrainz Autotagger + +The MusicBrainz autotagger, collectively **v1.3**. Builds on the explore-browser API client + cache foundation. Five sequential phases (008–012), each depending on the prior one. + +| Phase | Title | Status | +|-------|-------|--------| +| 008 | Schema & Grouping Foundation | shipped 2026-04-20 | +| 009 | Scoring Engine & MB Orchestration | shipped 2026-04-21 | +| 010 | Review UI & Apply Pipeline | **active** | +| 011 | Auto-Accept & Entry Points | pending | +| 012 | Settings & Polish | pending | + +--- + +## 008 — Schema & Grouping Foundation · shipped 2026-04-20 + +> First v1.3 phase. Lays down the schema + bookkeeping — no scoring, no UI, no MB calls yet. + +### What landed + +- **008.1 — Migration 31: `audio_files.tag_status`.** Text column with inline `CHECK(tag_status IN (...))` constraint (SQLite `ALTER TABLE ADD COLUMN` supports column constraints, so both fresh and upgraded DBs enforce it). Partial index `idx_audio_files_tag_status_untagged ON audio_files(library_id) WHERE tag_status = 'untagged'` powers the pending badge. Backfill sets `user_confirmed` where the joined recording already has an MBID; everything else defaults to `untagged`. +- **008.2 — Migration 32: `tagging_items` + `group_key`.** New `tagging_items` table (PK `group_key`, `status CHECK IN ('pending','matched','confirmed','skipped')`, two indexes including the partial `WHERE status = 'pending'` for the badge). `audio_files.group_key TEXT NOT NULL DEFAULT ''` with partial index `WHERE group_key != ''`. Column-referencing indexes live in the migration, not the schema file, because `CREATE TABLE IF NOT EXISTS` is a no-op on existing tables and the partial-index predicate would reference a column that hasn't been added yet. +- **008.3 — `backend/autotag.GroupKey` + scan-path integration.** Lower-case hex SHA-1 over `libraryID || 0 || parentDirLower || 0 || albumTrimmed || 0 || discNumber` — intentionally shallow, deeper normalization is scoring territory (009). `saveAudioFile` switched to `CreateAudioFileWithGroupKey` + `UpsertTaggingItemOnTrackAdd`. `updateAudioFileMetadata` rebinds on key change: decrement old group → delete if empty → upsert new group → write new key onto the audio_files row. Migration 32 streams existing rows in batches of 500 and aggregates `tagging_items` at the end, defaulting status to `confirmed` when every track in the group is `user_confirmed`. +- **008.4 — Pending-queue sqlc queries.** `CountPendingTaggingItems`, three `ListPendingTaggingItems...` variants (alphabetical, by-score nulls-last, by-recent) — sqlc has no dynamic ORDER BY so each sort has its own query. `CAST(@param AS INTEGER|TEXT)` hints give typed params (otherwise sqlc emits `interface{}`). `GetTaggingItem` and `ListAudioFilesInTaggingGroup` round out the queue API. `EXPLAIN QUERY PLAN` test asserts the badge query uses `idx_tagging_items_status_pending` so a future schema change that breaks the partial index fails loudly. + +### Key decisions retained + +- **Hash algorithm in Go, not SQL.** `autotag.GroupKey` is the single source of truth for the key format so we can evolve it without coupling to SQLite functions. +- **SHA-1 over alternatives.** Matches the codebase's existing non-crypto deterministic-key convention. Collision risk at album-group cardinality (millions) is irrelevant. +- **Null-byte separators between hash inputs.** Prevents `a|b` vs `ab|` ambiguity. +- **Per-track integration inside `commitBatch`'s transaction**, not a post-scan callback — keeps `tagging_items` coherent after partial scans. +- **`best_match_release_mbid`, `score`, `last_checked_at` shapes fixed now** even though they stay NULL until 009-010. Avoids schema churn later. + +--- + +## 009 — Scoring Engine & MB Orchestration · shipped 2026-04-21 + +> Second v1.3 phase. Given an album-group, produce a ranked list of candidate releases with per-track alignment data, using as few MusicBrainz calls as the API-minimization playbook allows. No UI yet — 010 surfaces this to the user. + +### What landed + +- **`backend/autotag/` domain types** — `Candidate`, `TrackAlignment`, `GroupScore`, `LocalTrack`, `CandidateSource` (`local` / `musicbrainz`), `AlignmentStatus` (`matched` / `missing` / `extra` / `mismatched`). +- **`Normalize(s)`** — Unicode NFC → qualifier-suffix strip (`(Remastered 2009)`, `[Bonus Track]`, `(feat. X)`, etc.) → case fold → punctuation drop → whitespace collapse. Comparison-only, not human-readable. +- **Per-track distance** — weighted 60% title similarity (1.0 − Levenshtein/max-len), 30% length delta (linear: ≤1 s = 1.0, ≥30 s = 0.0, neutral 0.5 when either side unknown), 10% track-number match. +- **Greedy alignment** — `AlignTracks` picks the highest-scoring (local, cand) pair iteratively; not Hungarian-optimal but fine at album cardinalities. Emits `matched` / `missing` / `extra` / `mismatched` rows so the review UI can render the diff. +- **Local-first resolver** — `ListLocalReleaseGroupCandidates` sqlc query pre-filters on `rg.mbid != '' AND r.mbid != '' AND rg.name = ? COLLATE NOCASE`, then Go applies full normalization. Candidate track lists only include recordings that themselves have MBIDs (avoids untagged dupes polluting the "canonical" view when the same `(name, artist)` release group is shared across libraries). +- **MB orchestration** — `MBClient` interface (`SearchReleaseGroups`, `BrowseReleases`, `LookupArtist`) hides `explore.MusicBrainzClient`; `backend/explore/autotagclient.go` adapts one to the other. `buildMBQuery` assembles `release:"X" AND arid: AND tracks:N` — `arid:` makes the search cache key deterministic, `tracks:N` filters out box-set-style releases. One search per album + one `BrowseReleases` per candidate RG. +- **Release-level ranker** — aggregate track score (70%) + track-count match (15%, zero at ≥50% delta) + meta (15%, avg of year/Official/country bonuses). `Scorer.ScoreGroup` hits local first, runs MB only when no local candidate scores ≥ 0.90. +- **Persistence** — `SetTaggingItemBestMatch` writes `best_match_release_mbid`, `score`, `last_checked_at = CURRENT_TIMESTAMP`, `status = 'matched'`. + +### Key decisions retained + +- **SHA-1-grade normalization vs. full MB-equivalent.** Qualifier regex handles the common cases (remaster, deluxe, explicit, feat., bonus, etc.) without dragging in a full MB title-parsing library. Edges that bite in real libraries will show up in 010 review UX and can be patched then. +- **`*sqlcgen.Queries` as the DB boundary for autotag**, not `*database.DB`. The `database` package already depends on `autotag.GroupKey` (from 008.3), so reversing the dependency via `database` would create a cycle. Using `sqlcgen` directly is acyclic and keeps `autotag` swappable. +- **Scorer constructor takes `MBClient` as interface, not `*explore.MusicBrainzClient`.** Lets tests inject a stub without spinning up the HTTP + cache layer. The concrete adapter lives in `explore/autotagclient.go`. +- **`localSufficient = 0.90` threshold for skipping MB.** Empirical guess — will get retuned in 011 auto-accept phase when we observe real corpus scores. +- **Weights `60/30/10` for title/length/track-number.** Cribbed from beets' broad intuition; tuned to emphasize title-matching since length data can be unreliable from Vorbis Comments. Tests document the expected floors (e.g. "exact match should score ≥ 0.99") so nudging weights won't silently regress. +- **`release_groups` and `recordings` must both carry MBIDs** for a local candidate. Otherwise untagged dupes of the same album (across libraries) falsely expand the "canonical" track list. +- **UTF-8 em-dashes in SQL comments broke sqlc's string-literal emitter**, truncating generated query strings mid-word. All autotag SQL comments use ASCII punctuation. + +### Known follow-ups into 010+ + +- **`guessArtistMBID` is a stub** returning `""` because `LocalTrack` doesn't currently carry artist MBIDs. 010 should thread artist MBIDs through `ListAudioFilesInTaggingGroup` so the MB resolver can use `arid:` filters. +- **`yearBonus` uses `time.Now().Year()`** as a placeholder target. Should become the earliest release-date hint from the group's tracks once 010 provides it. +- **VA compilation detection threshold** is still open. The scorer doesn't special-case per-track artist credits differing from album-artist. + +--- + +## 010 — Review UI & Apply Pipeline · ACTIVE + +> Third v1.3 phase. User reviews one album at a time, sees the diff clearly, and applies or skips — file tags get written, DB gets synced, cover art follows the never-replace-existing rule. + +**Requirements:** REVIEW-01..07 · **Depends on:** 009 (needs candidates + scores) + +### Success criteria + +1. `/autotag` shows the next pending album with its top candidate as a field-by-field diff. Missing-from-local and extra-in-local tracks are shown explicitly. +2. Keyboard shortcuts work without the mouse: `A` apply, `M` more candidates, `S` skip, `L` leave-as-is, `U` paste URL, `→`/`←` navigate. +3. Apply writes tags to every track in the group via the existing format-specific writers + atomic write + DB sync + FTS5 sync. Whitelisted fields only: title, artist, album, album-artist, year, track#, track-total, disc#, disc-total, all MBIDs. +4. Cover art rule: embed only when the file has no existing art **and** CAA returns ≥500 px on the shortest side. Never replace existing embedded art (auto or manual). +5. First-ever apply per library shows an irreversibility warning. "Don't show again" sets a flag on the `libraries` row; never shows again for that library. +6. While the user reviews album N, candidates for album N+1 are prefetched into `http_cache` so advancing feels instant. +7. "Paste MB URL" dialog accepts a release URL, extracts the MBID, runs one `LookupRelease`, renders the diff against the current album. + +### Sub-plans + +- Wails bindings — `StartAutotagQueue`, `GetCurrentCandidate`, `GetCandidates(groupKey)`, `Apply(groupKey, releaseMBID)`, `Skip`, `LeaveAsIs(groupKey)`, `RetagGroup(groupKey)`. +- `/autotag` page layout — focused album header, diff table, candidate sidebar, missing/extra panel. +- Keyboard shortcut wiring through the existing scope-aware dispatch. +- Apply pipeline integration with existing tag writers + DB sync. +- Cover art apply rule + CAA fetch + 500 px minimum check. +- File-write warning dialog with per-library persistence. +- Prefetch-next-album goroutine, rate-limiter aware. +- Paste-MB-URL escape hatch. + +### Risk callout + +Every apply rewrites a file. The `AtomicWrite` pipeline mitigates corruption risk; the per-library warning mitigates surprise. Dry-run mode (from 009) lets developers validate scoring changes without file writes. + +--- + +## 011 — Auto-Accept & Entry Points · pending + +> Fourth v1.3 phase. The strict all-match auto-accept path runs as a background job; the tool is reachable from every place a user expects. + +**Requirements:** AUTO-01..06 · **Depends on:** 010 (needs the apply pipeline) + +### Success criteria + +1. An album-group qualifies for auto-accept iff: exact track-count match, every track's normalized title matches, every track's length within ±2s, no cover-art replacement required, no existing-MBID conflicts. Decision uses already-cached candidate data — **no additional MB calls**. +2. Auto-accept job processes all qualifying groups in the queue, emits progress events, is cancellable at any point, honors the shared rate limiter. +3. Right-click on a track / album / artist exposes "Autotag this album" (queues + jumps to review) and "Retag" (flips status to `untagged` and requeues). +4. After a library scan finishes with N new untagged albums, a non-blocking toast appears linking to `/autotag`. +5. Sidebar has an "Autotag" nav entry with a pending-count badge, updates reactively. +6. Pasting a MB release URL into the Paste-URL dialog renders a full diff against the current album with one `LookupRelease` call. + +### Sub-plans + +- Strict all-match rule + unit tests. +- Auto-accept background job — progress events, cancellation, queue traversal. +- Context menu integrations on track/album/artist views. +- Post-scan toast wiring via the existing scan-complete event. +- Sidebar nav entry + pending-count badge store integration. + +### Risk callout + +Release selection can pick the wrong edition. The exact-track-count gate prevents most silent misbehavior, but bonus-track editions and remaster reissues with matching track counts are genuine ambiguity. Manual review handles the edge cases — that's why auto-accept is strict by design, and the slider for fuzzy auto-accept is explicitly out of scope. + +--- + +## 012 — Settings & Polish · pending + +> Fifth and final v1.3 phase. Configuration surfaces in the existing settings system; the known sharp edges (rate-limit contention, VA compilations, singleton files) get sanded; the fingerprinting seam is in place for the future. + +**Requirements:** CFG-01..06 · **Depends on:** 011 + +### Success criteria + +1. Autotag settings panel accessible via the existing templ/HTMX settings UI. Exposes: enable/disable auto-accept, per-library file-write warning reset, default review filter, default sort order. +2. Shared rate limiter distinguishes interactive from background requests. User-initiated MB calls (paste-URL, opening a review, explore browsing) are never blocked behind a running auto-accept job. +3. VA compilation albums (per-track artist credits differ from album-artist) are detected. The auto-accept artist-match rule relaxes for them; the ranker prefers MB releases credited to "Various Artists". +4. Singleton files (`track_count = 1`, no sibling context) use a recording-level match path (`SearchRecordings` with title + artist + length filters). Lower confidence ceiling — never eligible for auto-accept regardless of confidence. +5. `type Identifier interface { Identify(path) ([]Candidate, error) }` exists with `MetadataIdentifier` as the v1 implementation. No fpcalc integration, but the seam is in place for a future `AcoustIDIdentifier`. +6. User-facing quickstart docs exist; CLAUDE.md gets a `backend/autotag/` package description; scoring-function dev notes are committed. + +### Sub-plans + +- Autotag settings panel (templ + HTMX). +- Rate-limiter priority support. +- VA compilation detection and scoring adjustments. +- Singleton-file match path. +- `Identifier` interface seam with `MetadataIdentifier`. +- Docs — user quickstart + dev notes + CLAUDE.md update. + +--- + +## Ship criteria for v1.3 overall + +- All 29 SCHEMA/MATCH/REVIEW/AUTO/CFG requirements complete. +- All five phases' success criteria verified end-to-end on a real library (10k+ tracks, mixed match quality). +- Auto-accept run against a well-tagged subset produces zero incorrect matches. +- Manual review workflow can process 100 albums in under 30 minutes without mouse use. diff --git a/.planning/plans/active/010-autotag-review-ui.md b/.planning/plans/active/010-autotag-review-ui.md deleted file mode 100644 index ef571bd..0000000 --- a/.planning/plans/active/010-autotag-review-ui.md +++ /dev/null @@ -1,30 +0,0 @@ -# 010 — Autotag: Review UI & Apply Pipeline - -> Third v1.3 phase. User reviews one album at a time, sees the diff clearly, and applies or skips — file tags get written, DB gets synced, cover art follows the never-replace-existing rule. - -**Status:** pending · **Requirements:** REVIEW-01..07 · **Depends on:** 009 (needs candidates + scores) - -## Success criteria - -1. `/autotag` shows the next pending album with its top candidate as a field-by-field diff. Missing-from-local and extra-in-local tracks are shown explicitly. -2. Keyboard shortcuts work without the mouse: `A` apply, `M` more candidates, `S` skip, `L` leave-as-is, `U` paste URL, `→`/`←` navigate. -3. Apply writes tags to every track in the group via the existing format-specific writers + atomic write + DB sync + FTS5 sync. Whitelisted fields only: title, artist, album, album-artist, year, track#, track-total, disc#, disc-total, all MBIDs. -4. Cover art rule: embed only when the file has no existing art **and** CAA returns ≥500 px on the shortest side. Never replace existing embedded art (auto or manual). -5. First-ever apply per library shows an irreversibility warning. "Don't show again" sets a flag on the `libraries` row; never shows again for that library. -6. While the user reviews album N, candidates for album N+1 are prefetched into `http_cache` so advancing feels instant. -7. "Paste MB URL" dialog accepts a release URL, extracts the MBID, runs one `LookupRelease`, renders the diff against the current album. - -## Sub-plans - -- Wails bindings — `StartAutotagQueue`, `GetCurrentCandidate`, `GetCandidates(groupKey)`, `Apply(groupKey, releaseMBID)`, `Skip`, `LeaveAsIs(groupKey)`, `RetagGroup(groupKey)`. -- `/autotag` page layout — focused album header, diff table, candidate sidebar, missing/extra panel. -- Keyboard shortcut wiring through the existing scope-aware dispatch. -- Apply pipeline integration with existing tag writers + DB sync. -- Cover art apply rule + CAA fetch + 500 px minimum check. -- File-write warning dialog with per-library persistence. -- Prefetch-next-album goroutine, rate-limiter aware. -- Paste-MB-URL escape hatch. - -## Risk callout - -Every apply rewrites a file. The `AtomicWrite` pipeline mitigates corruption risk; the per-library warning mitigates surprise. Dry-run mode (from 009) lets developers validate scoring changes without file writes. diff --git a/.planning/plans/completed/001-v1.0-consolidation.md b/.planning/plans/completed/001-v1.0-consolidation.md deleted file mode 100644 index e19cf8f..0000000 --- a/.planning/plans/completed/001-v1.0-consolidation.md +++ /dev/null @@ -1,22 +0,0 @@ -# 001 — v1.0 Consolidation - -> Foundation milestone: fixed all SetContext data races, established the test infrastructure, consolidated SQL, and tuned backend + frontend performance so feature work could safely follow. - -**Shipped:** 2026-03-05 · **Phases:** 1-8 - -## What landed - -- **Phase 1 — Concurrency race fixes.** Mutex-protected `SetContext` across queue, library, playlist, player; collapsed Player double-lock. App runs `-race`-clean. -- **Phase 2 — Backend correctness.** `startupErr` moved to struct field; config files written 0o644; MPRIS callback errors logged; `Library.Scan()` separates warnings (`ScanMetrics`) from fatal errors (return value). -- **Phase 3 — Test infrastructure.** `database.NewTestDB(t)` returns an in-memory SQLite with full migrations + production PRAGMAs (`synchronous=NORMAL`, `cache_size=-8000`, `mmap_size=67108864`). -- **Phase 4 — Queue/Config/Player tests.** ~30 tests covering SetQueue, navigation, shuffle, repeat modes, persistence, config load/save roundtrip, volume conversion, format detection. -- **Phase 5 — Database/Library tests.** ~25 tests covering FTS5 search, search index rebuild, migrations, scan logic, entity cache. -- **Phase 6 — SQL consolidation.** 5-table FTS5 JOIN consolidated into the `track_metadata` VIEW; AST-based event codegen (Go → TypeScript) with pre-commit hook; `sqlc.slice()` for queue batch lookups; every hand-crafted SQL statement carries a `// SAFETY:` comment explaining why. -- **Phase 7 — Backend performance.** Incremental queue persistence (O(1) add/remove), SetQueue Phase-2 dedup, library store deferred loading. -- **Phase 8 — Frontend performance.** Lit `repeat()` with stable keys; `queueMicrotask` notification batching; design token system; visual consistency audit across 15 components. - -## Key decisions retained - -- **No cosmetic file splitting** — extract only when reuse or correctness demands it. -- **Tests as a means to safe refactoring**, not a coverage target. -- **Fix races → tests → refactoring** order is mandatory: can't run `-race`-clean tests with active races; can't safely refactor without tests. diff --git a/.planning/plans/completed/002-v1.1-multi-library.md b/.planning/plans/completed/002-v1.1-multi-library.md deleted file mode 100644 index cfa3753..0000000 --- a/.planning/plans/completed/002-v1.1-multi-library.md +++ /dev/null @@ -1,23 +0,0 @@ -# 002 — v1.1 Multi-Library Support - -> Lets users register and manage multiple music library directories with per-library scanning, filtered views, cross-library playlists, and phantom-track preservation when a library is removed. - -**Shipped:** 2026-03-16 · **Phases:** 9-14 - -## What landed - -- **Phase 9 — Scan cancellation & keyboard shortcuts.** Cancellable/pausable library scans via per-scan context; configurable shortcuts with record-style capture, scope-aware dispatch, conflict detection. -- **Phase 10 — Schema & migration.** New `libraries` table, `library_id` FK on `audio_files`, phantom columns on `playlist_tracks`, seamless single-directory → named-library migration on first launch. -- **Phase 11 — Per-library scan pipeline.** Sequential scan-queue coordinator (single writer is the SQLite reality); per-library progress UI with library name; cancel scope respects the active library. -- **Phase 12 — Library CRUD & data integrity.** Add/rename/remove library through UI; atomic orphan cleanup (shared artists/albums/genres survive); FTS5 entries removed; queue tracks from a removed library cascade-deleted with playback continuing. -- **Phase 13 — Library views & phantom tracks.** Library filter dropdown across all views; tracks/albums/artists/genres/search respect the active filter; cross-library playlists work naturally; removed-library tracks become greyed-out phantoms with preserved metadata, auto-resolved on rescan via `ScanHooks` + M3U8 matching. -- **Phase 14 — Performance optimization.** CSS containment + GPU promotion on scroll containers; view caching (display toggle, no DOM destruction); event delegation on virtualizer; `queueMicrotask` notification batching; pprof profiling guide. - -## Key decisions retained - -- **Hybrid model — `library_id` only on `audio_files`.** Physical files belong to a library; logical entities (artist/album/genre) are shared. -- **Libraries live in DB, not TOML.** UI-driven CRUD shouldn't require TOML manipulation. -- **`SET NULL` on `playlist_tracks` FK.** Phantom rows preserve playlist structure when a library is removed. -- **Backend filters, frontend doesn't.** `ByLibrary` SQL variants keep the UI responsive at 150k tracks. -- **Sequential scanning.** SQLite single-writer makes parallel scans pointless. -- **`ScanHooks` callback pattern.** Breaks the library → playlist circular dep needed for phantom resolution. diff --git a/.planning/plans/completed/003-v1.2-tag-editing.md b/.planning/plans/completed/003-v1.2-tag-editing.md deleted file mode 100644 index 91c7df0..0000000 --- a/.planning/plans/completed/003-v1.2-tag-editing.md +++ /dev/null @@ -1,24 +0,0 @@ -# 003 — v1.2 Tag Editing (MP3 + FLAC) - -> First write path into user files: edit any track's metadata or cover art (single or batch) with crash-safe writes, then synchronize the database, FTS5 index, and entity graph in the same operation. MP3 and FLAC only — OGG and WAV ship in v1.2.1. - -**Shipped:** 2026-03-18 · **Phases:** 15-18 - -## What landed - -- **Phase 15 — Schema migration & write safety.** FTS5 migrated to `contentless_delete=1` so per-row deletes/updates work without ghost search hits. New `backend/fileutil` package provides `AtomicWrite` (write to `.yj-tmp` sibling, then rename) with crash-safe semantics and orphan cleanup on startup. -- **Phase 16 — Tag writers + DB sync.** - - `backend/tagwriter/` writes ID3v2 (MP3) and Vorbis Comments (FLAC), including embedded cover art. - - `WriteTrackTags` pipeline: file write → transactional DB sync (entity upsert-and-relink + FTS5 update + orphan cleanup) → event emission. - - `PlayerStopper` interface (implemented by `playerAdapter` in `app.go`) lets tagwriter request playback stop without an import cycle. - - Scan/write mutual exclusion via `pipelineMu`. -- **Phase 17 — Single-track editor.** Right-click any track anywhere; 8 editable fields (title/artist/album/genre/year/track#/disc#/composer); cover art pick/replace/remove; diff-only saves; all visible views refresh without rescan. -- **Phase 18 — Batch editor.** Multi-select; three-state field model (keep / set / clear) driven implicitly by `editValues` presence; confirmation guard; progress bar with cancellation; partial-failure reporting; batch cover art set/clear. - -## Key decisions retained - -- **Upsert-and-relink for entity sync.** Never mutate shared entities — create new rows or relink to existing ones. Safe under concurrent reads. -- **`AtomicWrite` with deterministic `.yj-tmp` suffix.** Same-directory rename avoids cross-device issues; deterministic naming enables orphan cleanup. -- **`suppressEvents` flag for batch coalescing.** One `TrackMetadataChanged` event per batch, not N individual ones. -- **Three-state field model via dirty tracking.** `editValues` presence is the signal — no explicit state enum needed. -- **go-flac ecosystem.** Small library (44★) but the only pure-Go FLAC writer. dhowden/tag reads back what go-flac writes; 7 round-trip tests validate. diff --git a/.planning/plans/completed/004-v1.2.1-format-parity.md b/.planning/plans/completed/004-v1.2.1-format-parity.md deleted file mode 100644 index 1aa7efa..0000000 --- a/.planning/plans/completed/004-v1.2.1-format-parity.md +++ /dev/null @@ -1,18 +0,0 @@ -# 004 — v1.2.1 Format Parity (OGG + WAV) - -> Closes the tag-writing format gap: every file YellowJacket plays can also be edited. OGG Vorbis and WAV writers, plus a wsl_v5 lint cleanup. (This was tracked as M001 in the GSD system; S01–S18 in that milestone were just retroactive imports of v1.0–v1.2.) - -**Shipped:** 2026-03-21 · **Phases:** 19-21 - -## What landed - -- **Phase 19 — WAV tag writer.** Custom RIFF parser with lenient-read / strict-write behavior. ID3v2 chunk placed at end of file after preserved chunks. Case-insensitive `id3`/`ID3` chunk detection. Existing ID3v2 tags merged so unknown frames written by other tools survive. Read-back uses `bogem/id3v2.ParseReader` (handles cleared tags correctly; dhowden/tag does not). -- **Phase 20 — OGG Vorbis tag writer.** Custom OGG page parser/writer with MSB-first CRC32 (Go's `hash/crc32` uses incompatible reflected bit ordering, so a precomputed 256-entry lookup table lives in-package). Vorbis Comment serializer with `METADATA_BLOCK_PICTURE` cover art. Raw byte preservation for non-edited entries — avoids corrupting non-UTF-8 tags. Tests use a programmatic fixture builder (no embedded binaries). CRC32 validated against independently computed known vectors (`OggS` → `0x5fb0a94f`, `{1..8}` → `0x7d0f3681`). -- **Phase 21 — Cleanup.** wsl → wsl_v5 upgrade, all 19 pre-existing warnings fixed. - -## Test coverage - -- 7 FLAC round-trip tests (pre-existing). -- WAV round-trip tests covering all 8 text fields + cover art. -- OGG round-trip tests with CRC32 validation against known vectors. -- All pass with `-race`. diff --git a/.planning/plans/completed/005-smart-playlists.md b/.planning/plans/completed/005-smart-playlists.md deleted file mode 100644 index 83b1ef0..0000000 --- a/.planning/plans/completed/005-smart-playlists.md +++ /dev/null @@ -1,25 +0,0 @@ -# 005 — Smart Playlists - -> Rule-based dynamic playlists: saved queries that evaluate against the library on demand, with a typeable combobox rule editor, live preview, and queue-snapshot-on-play semantics. AND-only logic with multi-value "is any of" — no nested boolean groups. - -**Shipped:** 2026-03-22 · **Slices:** S01-S04 (M002 in GSD) - -## What landed - -- **S01 — Schema & rule engine.** Migration 9 adds `is_smart` + `smart_rules` (JSON) columns to the existing `playlists` table — extends, doesn't duplicate. `backend/smartplaylist/` builds parameterized WHERE clauses via a hardcoded `fieldMap` whitelist (16 columns from `track_metadata`). 7 text operators (`is`, `is_not`, `contains`, `does_not_contain`, `starts_with`, `ends_with`, `is_any_of`) and 5 numeric (`is`, `is_not`, `greater_than`, `less_than`, `between`). 49 unit tests cover all operators, the genre dual-path, and SQL-injection rejection. Optional result limit + sort (whitelisted fields, `ORDER BY RANDOM()` allowed). -- **S02 — CRUD & sidebar.** sqlc regenerated for the new columns; `IsSmart bool` propagated through `playlist.Summary` to TypeScript models. Smart playlists render with a filter icon and "Smart" badge. New `smart-playlist-details` Lit element follows the `genre-details` pattern; refresh/play/shuffle wired. Drag-drop onto smart playlists blocked. -- **S03 — Rule editor UI.** Reusable `yj-combobox` LitElement (typeable input, ARIA, keyboard nav). `smart-playlist-editor` builds rules row-by-row with field combobox, operator switch (text/numeric auto), value input. Live preview at 300 ms debounce calling `PreviewSmartPlaylist`. -- **S04 — Integration.** Create → navigate → `auto-edit` flow — detail view awaits initial data load (avoids `MaxOpenConns(1)` deadlock) before opening the editor. - -## Key decisions retained - -- **One playlists table** — `is_smart` flag, not a parallel table. One code path for listing. -- **Saved queries, not saved track lists** — no `playlist_tracks` rows for smart playlists; results evaluated on demand. -- **Snapshot to queue on play.** Queue must be stable during playback, not dynamically re-evaluating. -- **Genre dual-path.** Exact ops (`is`, `is_not`, `is_any_of`) use a subquery against `recording_genres JOIN genres` because the `GROUP_CONCAT(g.name, '||')` column in `track_metadata` can't match individual names. LIKE-based ops use the concatenated column. -- **Wails TS bindings for the 5 smart-playlist methods are manually maintained** (`Service.d.ts` / `Service.js`). No build-time check catches drift if Go signatures change. - -## Fragility notes - -- **Combobox blur-vs-click race.** The `mousedown` + `preventDefault()` + `requestAnimationFrame` fallback in `combobox.ts` is battle-tested but brittle if anyone restructures option rendering into a separate shadow DOM. Re-verify click-to-select in any combobox refactor. -- **`MaxOpenConns(1)` deadlock.** Holding `*sql.Rows` open while calling functions that query the same `*database.DB` will deadlock. Close rows explicitly before downstream calls; never rely on `defer` here. diff --git a/.planning/plans/completed/006-play-history.md b/.planning/plans/completed/006-play-history.md deleted file mode 100644 index 19c7455..0000000 --- a/.planning/plans/completed/006-play-history.md +++ /dev/null @@ -1,19 +0,0 @@ -# 006 — Play History & Play Count - -> Tracks every natural track completion. Powers smart-playlist rules like "most played", "never played", "not played in 30 days", and exposes play count as an optional track-list column. - -**Shipped:** 2026-03-22 · **Slices:** S01-S03 (M003 in GSD) - -## What landed - -- **S01 — Schema, migration, recording.** Migration 10 adds `play_history` table and denormalized `play_count` + `last_played` columns on `audio_files`; `track_metadata` VIEW recreated to include them. `recordPlay()` pipeline: insert history row → increment count → update timestamp → emit event. `OnPlaybackFinished` hook unlocks the player mutex *before* the DB write (avoids SQLite deadlock). -- **S02 — Smart playlist integration.** `play_count` (numeric) and `days_since_played` (computed) added to the rule-engine field whitelist. Explicit NULL handling for never-played tracks in `days_since_played` conditions. -- **S03 — UI.** Optional play-count column in track lists; `PlayCount` / `LastPlayed` added to TypeScript models; editor dropdown entries for the new fields. - -## Key decisions retained - -- **Natural finish only** — no percentage threshold, no skip counting. (If you scrub past the end, that doesn't count.) -- **Denormalized `play_count`/`last_played` on `audio_files`** — the alternative was per-query aggregation against `play_history`; the cost wasn't worth it. -- **`days_since_played` is a computed field**, not raw timestamp comparison in the rule. Lets the rule engine speak the user's vocabulary. -- **"Recently played" is a smart playlist**, not a dedicated view. Reuses the existing rule engine instead of adding a parallel query path. -- **Mutex unlock before `recordPlay()`** is mandatory — locking around the DB write under `MaxOpenConns(1)` deadlocks. diff --git a/.planning/plans/completed/007-explore-browser.md b/.planning/plans/completed/007-explore-browser.md deleted file mode 100644 index d715bf5..0000000 --- a/.planning/plans/completed/007-explore-browser.md +++ /dev/null @@ -1,24 +0,0 @@ -# 007 — MusicBrainz/ListenBrainz Explore Browser - -> Read-only remote catalog browser: search MB/LB, browse artist pages with ListenBrainz-ranked top tracks and similar artists, view album detail with release-version selection. Rate-limited (1 req/sec), SQLite-cached, with cover art from CAA. Lays the API client + cache foundation that v1.3 Autotagger builds on. - -**Shipped:** 2026-03-24 · **Slices:** S01-S04 (M004 in GSD) - -## What landed - -- **S01 — API clients, cache, explore shell.** `backend/explore/` — MusicBrainz client (wraps `musicbrainzws2`, which provides its own internal rate limiter), ListenBrainz client (custom `RateLimiter` via `golang.org/x/time/rate`, 1 req/sec, burst=1), Cover Art Archive client (URL builders only). All clients follow cache-first pattern: check cache → call API → wrap → cache → return. Migration 11 adds `explore_cache` table (URL key, JSON body, datetime expiry, nullable `mbid` + `entity_type` columns for future autotagger correlation). `idx_explore_cache_mbid` index. TTLs: 24 h for search, 7 d for entity lookups. Sidebar "Explore" entry with globe icon. -- **S02 — Smart search.** Unified `Search()` runs concurrent sub-searches with `WaitGroup` + `Mutex`; frontend renders Top Results / Artists / Albums / Tracks with 300 ms debounce and stale-response discarding via version counter. CAA thumbnails for all result types. -- **S03 — Remote artist page.** `explore-artist-details` (904 LOC). Four independent sections via `Promise.allSettled`: header, top tracks (ListenBrainz `TopRecordingsForArtist` with formatted listen counts), discography (grouped Album/EP/Single/Compilation/Other, newest-first), similar artists (recursive navigation). ListenBrainz Labs `SimilarArtists` failures degrade silently to console.error (it's an experimental endpoint). -- **S04 — Album detail & release versions.** `explore-album-details` (828 LOC). Release fingerprinting (sort tracks by `(discNumber, position)`, join MBIDs with `|`) collapses identical editions into one cluster; track-count min/max labelling shows differences across editions; default selection is the earliest-dated release. Disc separators when any track has `discNumber > 1`. - -## Known gap (carried into v1.3 follow-up scope) - -- **R032 — offline visual indicator** was in the milestone scope but never implemented. Cache infrastructure works (cached entries served when offline until TTL expires), but `Cache.Get()` doesn't propagate a "from cache" flag to the frontend, and no component renders a "Cached" badge. Functional offline mode works; the UI cue is missing. - -## Key decisions retained - -- **Wrapper types in `backend/explore/types.go` for Wails bindings**, not raw `musicbrainzws2` types. Avoids ugly TS namespaces and serialization issues with unexported fields. -- **Dual rate limiters** — accept it. The `musicbrainzws2` library has its own that can't easily be disabled or shared; our limiter governs LB + CAA. Both enforce 1 req/sec independently. -- **MBID + entity_type columns in `explore_cache`** — scaffolding for the v1.3 autotagger. No autotagging logic in M004, just storage shape. -- **`CoverArtGroupURL` is a synchronous inline string builder.** Pure deterministic template; making it async via Wails would add per-render round-trips for nothing. -- **Detail components share design tokens via `tokens.css.ts`** but are independent Lit components — no inheritance, no shared base. Data sources differ entirely (local SQLite vs remote API). diff --git a/.planning/plans/completed/008-autotag-schema-grouping.md b/.planning/plans/completed/008-autotag-schema-grouping.md deleted file mode 100644 index 5c85924..0000000 --- a/.planning/plans/completed/008-autotag-schema-grouping.md +++ /dev/null @@ -1,20 +0,0 @@ -# 008 — Autotag: Schema & Grouping Foundation - -> First v1.3 phase. Lays down the schema + bookkeeping for the MusicBrainz autotagger — no scoring, no UI, no MB calls yet. Subsequent autotag phases (009-012) build on this. - -**Shipped:** 2026-04-20 · **Sub-plans:** 008.1-008.4 - -## What landed - -- **008.1 — Migration 31: `audio_files.tag_status`.** Text column with inline `CHECK(tag_status IN (...))` constraint (SQLite `ALTER TABLE ADD COLUMN` supports column constraints, so both fresh and upgraded DBs enforce it). Partial index `idx_audio_files_tag_status_untagged ON audio_files(library_id) WHERE tag_status = 'untagged'` powers the pending badge. Backfill sets `user_confirmed` where the joined recording already has an MBID; everything else defaults to `untagged`. -- **008.2 — Migration 32: `tagging_items` + `group_key`.** New `tagging_items` table (PK `group_key`, `status CHECK IN ('pending','matched','confirmed','skipped')`, two indexes including the partial `WHERE status = 'pending'` for the badge). `audio_files.group_key TEXT NOT NULL DEFAULT ''` with partial index `WHERE group_key != ''`. Column-referencing indexes live in the migration, not the schema file, because `CREATE TABLE IF NOT EXISTS` is a no-op on existing tables and the partial-index predicate would reference a column that hasn't been added yet. -- **008.3 — `backend/autotag.GroupKey` + scan-path integration.** Lower-case hex SHA-1 over `libraryID || 0 || parentDirLower || 0 || albumTrimmed || 0 || discNumber` — intentionally shallow, deeper normalization is scoring territory (009). `saveAudioFile` switched to `CreateAudioFileWithGroupKey` + `UpsertTaggingItemOnTrackAdd`. `updateAudioFileMetadata` rebinds on key change: decrement old group → delete if empty → upsert new group → write new key onto the audio_files row. Migration 32 streams existing rows in batches of 500 and aggregates `tagging_items` at the end, defaulting status to `confirmed` when every track in the group is `user_confirmed`. -- **008.4 — Pending-queue sqlc queries.** `CountPendingTaggingItems`, three `ListPendingTaggingItems...` variants (alphabetical, by-score nulls-last, by-recent) — sqlc has no dynamic ORDER BY so each sort has its own query. `CAST(@param AS INTEGER|TEXT)` hints give typed params (otherwise sqlc emits `interface{}`). `GetTaggingItem` and `ListAudioFilesInTaggingGroup` round out the queue API. `EXPLAIN QUERY PLAN` test asserts the badge query uses `idx_tagging_items_status_pending` so a future schema change that breaks the partial index fails loudly. - -## Key decisions retained - -- **Hash algorithm in Go, not SQL.** `autotag.GroupKey` is the single source of truth for the key format so we can evolve it without coupling to SQLite functions. -- **SHA-1 over alternatives.** Matches the codebase's existing non-crypto deterministic-key convention. Collision risk at album-group cardinality (millions) is irrelevant. -- **Null-byte separators between hash inputs.** Prevents `a|b` vs `ab|` ambiguity. -- **Per-track integration inside `commitBatch`'s transaction**, not a post-scan callback — keeps `tagging_items` coherent after partial scans. -- **`best_match_release_mbid`, `score`, `last_checked_at` shapes fixed now** even though they stay NULL until 009-010. Avoids schema churn later. diff --git a/.planning/plans/completed/009-autotag-scoring-engine.md b/.planning/plans/completed/009-autotag-scoring-engine.md deleted file mode 100644 index ad394ca..0000000 --- a/.planning/plans/completed/009-autotag-scoring-engine.md +++ /dev/null @@ -1,32 +0,0 @@ -# 009 — Autotag: Scoring Engine & MB Orchestration - -> Second v1.3 phase. Given an album-group, produce a ranked list of candidate releases with per-track alignment data, using as few MusicBrainz calls as the API-minimization playbook allows. No UI yet — 010 surfaces this to the user. - -**Shipped:** 2026-04-21 · **Sub-plans:** types + normalization + distance, local resolver, MB orchestration, ranker + persistence, test corpus - -## What landed - -- **`backend/autotag/` domain types** — `Candidate`, `TrackAlignment`, `GroupScore`, `LocalTrack`, `CandidateSource` (`local` / `musicbrainz`), `AlignmentStatus` (`matched` / `missing` / `extra` / `mismatched`). -- **`Normalize(s)`** — Unicode NFC → qualifier-suffix strip (`(Remastered 2009)`, `[Bonus Track]`, `(feat. X)`, etc.) → case fold → punctuation drop → whitespace collapse. Comparison-only, not human-readable. -- **Per-track distance** — weighted 60% title similarity (1.0 − Levenshtein/max-len), 30% length delta (linear: ≤1 s = 1.0, ≥30 s = 0.0, neutral 0.5 when either side unknown), 10% track-number match. -- **Greedy alignment** — `AlignTracks` picks the highest-scoring (local, cand) pair iteratively; not Hungarian-optimal but fine at album cardinalities. Emits `matched` / `missing` / `extra` / `mismatched` rows so the review UI can render the diff. -- **Local-first resolver** — `ListLocalReleaseGroupCandidates` sqlc query pre-filters on `rg.mbid != '' AND r.mbid != '' AND rg.name = ? COLLATE NOCASE`, then Go applies full normalization. Candidate track lists only include recordings that themselves have MBIDs (avoids untagged dupes polluting the "canonical" view when the same `(name, artist)` release group is shared across libraries). -- **MB orchestration** — `MBClient` interface (`SearchReleaseGroups`, `BrowseReleases`, `LookupArtist`) hides `explore.MusicBrainzClient`; `backend/explore/autotagclient.go` adapts one to the other. `buildMBQuery` assembles `release:"X" AND arid: AND tracks:N` — `arid:` makes the search cache key deterministic, `tracks:N` filters out box-set-style releases. One search per album + one `BrowseReleases` per candidate RG. -- **Release-level ranker** — aggregate track score (70%) + track-count match (15%, zero at ≥50% delta) + meta (15%, avg of year/Official/country bonuses). `Scorer.ScoreGroup` hits local first, runs MB only when no local candidate scores ≥ 0.90. -- **Persistence** — `SetTaggingItemBestMatch` writes `best_match_release_mbid`, `score`, `last_checked_at = CURRENT_TIMESTAMP`, `status = 'matched'`. - -## Key decisions retained - -- **SHA-1-grade normalization vs. full MB-equivalent.** Qualifier regex handles the common cases (remaster, deluxe, explicit, feat., bonus, etc.) without dragging in a full MB title-parsing library. Edges that bite in real libraries will show up in 010 review UX and can be patched then. -- **`*sqlcgen.Queries` as the DB boundary for autotag**, not `*database.DB`. The `database` package already depends on `autotag.GroupKey` (from 008.3), so reversing the dependency via `database` would create a cycle. Using `sqlcgen` directly is acyclic and keeps `autotag` swappable. -- **Scorer constructor takes `MBClient` as interface, not `*explore.MusicBrainzClient`.** Lets tests inject a stub without spinning up the HTTP + cache layer. The concrete adapter lives in `explore/autotagclient.go`. -- **`localSufficient = 0.90` threshold for skipping MB.** Empirical guess — will get retuned in 011 auto-accept phase when we observe real corpus scores. -- **Weights `60/30/10` for title/length/track-number.** Cribbed from beets' broad intuition; tuned to emphasize title-matching since length data can be unreliable from Vorbis Comments. Tests document the expected floors (e.g. "exact match should score ≥ 0.99") so nudging weights won't silently regress. -- **`release_groups` and `recordings` must both carry MBIDs** for a local candidate. Otherwise untagged dupes of the same album (across libraries) falsely expand the "canonical" track list. -- **UTF-8 em-dashes in SQL comments broke sqlc's string-literal emitter**, truncating generated query strings mid-word. All autotag SQL comments use ASCII punctuation. - -## Known follow-ups into 010+ - -- **`guessArtistMBID` is a stub** returning `""` because `LocalTrack` doesn't currently carry artist MBIDs. 010 should thread artist MBIDs through `ListAudioFilesInTaggingGroup` so the MB resolver can use `arid:` filters. -- **`yearBonus` uses `time.Now().Year()`** as a placeholder target. Should become the earliest release-date hint from the group's tracks once 010 provides it. -- **VA compilation detection threshold** is still open (noted in `.planning/NOTES.md`). The scorer doesn't special-case per-track artist credits differing from album-artist. diff --git a/.planning/plans/pending/011-autotag-auto-accept.md b/.planning/plans/pending/011-autotag-auto-accept.md deleted file mode 100644 index d3926dc..0000000 --- a/.planning/plans/pending/011-autotag-auto-accept.md +++ /dev/null @@ -1,26 +0,0 @@ -# 011 — Autotag: Auto-Accept & Entry Points - -> Fourth v1.3 phase. The strict all-match auto-accept path runs as a background job; the tool is reachable from every place a user expects. - -**Status:** pending · **Requirements:** AUTO-01..06 · **Depends on:** 010 (needs the apply pipeline) - -## Success criteria - -1. An album-group qualifies for auto-accept iff: exact track-count match, every track's normalized title matches, every track's length within ±2s, no cover-art replacement required, no existing-MBID conflicts. Decision uses already-cached candidate data — **no additional MB calls**. -2. Auto-accept job processes all qualifying groups in the queue, emits progress events, is cancellable at any point, honors the shared rate limiter. -3. Right-click on a track / album / artist exposes "Autotag this album" (queues + jumps to review) and "Retag" (flips status to `untagged` and requeues). -4. After a library scan finishes with N new untagged albums, a non-blocking toast appears linking to `/autotag`. -5. Sidebar has an "Autotag" nav entry with a pending-count badge, updates reactively. -6. Pasting a MB release URL into the Paste-URL dialog renders a full diff against the current album with one `LookupRelease` call. - -## Sub-plans - -- Strict all-match rule + unit tests. -- Auto-accept background job — progress events, cancellation, queue traversal. -- Context menu integrations on track/album/artist views. -- Post-scan toast wiring via the existing scan-complete event. -- Sidebar nav entry + pending-count badge store integration. - -## Risk callout - -Release selection can pick the wrong edition. The exact-track-count gate prevents most silent misbehavior, but bonus-track editions and remaster reissues with matching track counts are genuine ambiguity. Manual review handles the edge cases — that's why auto-accept is strict by design, and the slider for fuzzy auto-accept is explicitly out of scope. diff --git a/.planning/plans/pending/012-autotag-settings-polish.md b/.planning/plans/pending/012-autotag-settings-polish.md deleted file mode 100644 index 6326acd..0000000 --- a/.planning/plans/pending/012-autotag-settings-polish.md +++ /dev/null @@ -1,31 +0,0 @@ -# 012 — Autotag: Settings & Polish - -> Fifth and final v1.3 phase. Configuration surfaces in the existing settings system; the known sharp edges (rate-limit contention, VA compilations, singleton files) get sanded; the fingerprinting seam is in place for the future. - -**Status:** pending · **Requirements:** CFG-01..06 · **Depends on:** 011 - -## Success criteria - -1. Autotag settings panel accessible via the existing templ/HTMX settings UI. Exposes: enable/disable auto-accept, per-library file-write warning reset, default review filter, default sort order. -2. Shared rate limiter distinguishes interactive from background requests. User-initiated MB calls (paste-URL, opening a review, explore browsing) are never blocked behind a running auto-accept job. -3. VA compilation albums (per-track artist credits differ from album-artist) are detected. The auto-accept artist-match rule relaxes for them; the ranker prefers MB releases credited to "Various Artists". -4. Singleton files (`track_count = 1`, no sibling context) use a recording-level match path (`SearchRecordings` with title + artist + length filters). Lower confidence ceiling — never eligible for auto-accept regardless of confidence. -5. `type Identifier interface { Identify(path) ([]Candidate, error) }` exists with `MetadataIdentifier` as the v1 implementation. No fpcalc integration, but the seam is in place for a future `AcoustIDIdentifier`. -6. User-facing quickstart docs exist; CLAUDE.md gets a `backend/autotag/` package description; scoring-function dev notes are committed. - -## Sub-plans - -- Autotag settings panel (templ + HTMX). -- Rate-limiter priority support. -- VA compilation detection and scoring adjustments. -- Singleton-file match path. -- `Identifier` interface seam with `MetadataIdentifier`. -- Docs — user quickstart + dev notes + CLAUDE.md update. - -## Ship criteria for v1.3 overall - -- All 29 SCHEMA/MATCH/REVIEW/AUTO/CFG requirements complete. -- All five phases' success criteria verified end-to-end on a real library (10k+ tracks, mixed match quality). -- Auto-accept run against a well-tagged subset produces zero incorrect matches. -- Manual review workflow can process 100 albums in under 30 minutes without mouse use. -- Recap files moved to `.planning/plans/completed/`. diff --git a/README.md b/README.md index 35143ff..394987b 100644 --- a/README.md +++ b/README.md @@ -1,61 +1,77 @@ # YellowJacket -[![CI](https://github.com/onion-4-dinner/yellowjacket/actions/workflows/ci.yml/badge.svg)](https://github.com/onion-4-dinner/yellowjacket/actions/workflows/ci.yml) -[![Release](https://github.com/onion-4-dinner/yellowjacket/actions/workflows/release.yml/badge.svg)](https://github.com/onion-4-dinner/yellowjacket/actions/workflows/release.yml) +*Music how it was meant to bee.* -Music how it was meant to bee. +YellowJacket is a fast, cross-platform desktop music player for your local +collection. It plays your files, keeps your library tidy, and helps you discover +and organize your music — all in a clean, responsive interface. No accounts, no +streaming, no telemetry: just your music on your machine. -YellowJacket is a cross-platform desktop music player built with Go and web technologies. It focuses on local music library management with a clean, responsive interface. +Runs on **Linux**, **macOS**, and **Windows**. + +## Features + +### Play your music +- Plays **MP3, FLAC, OGG Vorbis, and WAV** +- Play, pause, seek, and volume control with a mute toggle +- Gapless, glitch-free seeking backed by a read-ahead buffer +- A queue you can add to, reorder, and shuffle, with play-next support +- Shuffle and repeat (off / all / one) +- Picks up right where you left off — remembers your track, position, and volume between sessions +- Media-key and MPRIS support on Linux, so your desktop's playback controls just work + +### Keep your library organized +- Point it at your music folders and it scans them automatically +- Reads tags and embedded cover art, and de-duplicates artwork so it isn't stored twice +- Incremental sync — only new or changed files get reprocessed, and deleted files are cleaned up +- Browse by **album**, **artist**, or **genre**, or search across everything +- Mark favorites and see what you've been listening to with play history +- Edit track tags directly when something's off + +### Playlists +- Create playlists, drag tracks in, and reorder them +- **Smart playlists** that build themselves from rules (by genre, rating, play count, and more) +- Pin a default playlist and spot duplicate tracks at a glance + +### Discover and clean up (powered by MusicBrainz) +- **Explore** — browse artists, releases, and genres from the MusicBrainz catalog, not just what's already in your library +- **Auto-tag** — match your files against MusicBrainz to fill in correct artist, album, and track metadata, with a review step before anything is written +- **Lyrics search** — find a track by a line you remember ## Install -Grab the latest release for your platform: +Download the latest build for your platform from the +[releases page](https://git.ljones.me/yonlu/yellowjacket/releases). -**[Download Latest Release](https://github.com/onion-4-dinner/yellowjacket/releases/latest)** - -| Platform | Binary | -|----------|--------| +| Platform | Download | +|----------|----------| | Linux | `yellowjacket-linux-amd64` | | macOS | `yellowjacket-darwin-universal.app.zip` (Apple Silicon + Intel) | | Windows | `yellowjacket-windows-amd64.exe` | -## Features +Prefer to build it yourself? See [Building from source](#building-from-source). -**Playback** -- Play, pause, seek, and volume control with mute toggle -- Support for MP3, FLAC, OGG Vorbis, and WAV -- Queue management with add, remove, reorder, and play-next -- Shuffle mode (Fisher-Yates) and repeat modes (off, all, one) -- Session persistence -- resumes volume, track, and seek position on restart +## Getting started -**Library** -- Concurrent library scanning with automatic metadata extraction -- ID3v2, Vorbis Comments, and other tag format support -- Embedded cover art extraction with content-hash deduplication -- Incremental sync -- only processes new or changed files -- Orphan cleanup for deleted files +1. Launch YellowJacket. +2. Open **Settings** and add the folder(s) where your music lives. +3. Let the initial scan finish — you'll see progress as it works. +4. Browse by album, artist, or genre, queue something up, and press play. -**Interface** -- Album cover grid view and track list view -- Now playing display with cover art -- Resizable sidebar navigation -- Slide-out queue panel -- Context menus for tracks and albums (play, add to queue, play next) -- Settings page for library directory configuration +Your library and settings are stored locally: -## Architecture +| | Linux / macOS | Windows | +|---|---|---| +| Config | `~/.config/yellowjacket/` | `%LOCALAPPDATA%\yellowjacket\config` | +| Library data | `~/.local/share/yellowjacket/` | `%LOCALAPPDATA%\yellowjacket\data` | -YellowJacket uses the [Wails v2](https://wails.io/) framework to bridge a Go backend with a TypeScript/[Lit](https://lit.dev/) frontend running in a native webview. +## Building from source -- **Go backend** -- audio decoding and playback ([beep](https://github.com/gopxl/beep)), library scanning, SQLite database, cover art serving, TOML configuration -- **TypeScript frontend** -- Lit web components, singleton stores with reactive controllers, [Web Awesome](https://www.webawesome.com/) UI components -- **Communication** -- bidirectional event system via Wails runtime; backend is the source of truth -- **Database** -- SQLite (pure-Go driver) with type-safe queries generated by [sqlc](https://sqlc.dev/); MusicBrainz-style data model (artists, artist credits, recordings, release groups) -- **Config page** -- HTMX-based, loads HTML fragments rendered by Go [templ](https://templ.guide/) templates +YellowJacket is built with [Go](https://go.dev/) and a +[Lit](https://lit.dev/)/TypeScript frontend, bridged by the +[Wails](https://wails.io/) framework. -## Development - -### Prerequisites +**Prerequisites** | Tool | Version | |------|---------| @@ -64,66 +80,22 @@ YellowJacket uses the [Wails v2](https://wails.io/) framework to bridge a Go bac | pnpm | 10+ | | Wails CLI | v2 (`go install github.com/wailsapp/wails/v2/cmd/wails@latest`) | -**Linux system dependencies:** +On Linux, install the system libraries Wails needs: + ```bash sudo apt-get install libasound2-dev libgtk-3-dev libwebkit2gtk-4.1-dev ``` -On macOS and Windows, no additional system dependencies are needed. Run `wails doctor` to verify your environment. +macOS and Windows need no extra system packages. Run `wails doctor` to check your +environment. -### Build & Run +**Build** ```bash -make setup # Install git hooks (lefthook) -make dev # Development with hot-reload -make build-dev # Debug build -make build-prod # Production build (obfuscated + UPX compressed) -make generate # Run code generators (sqlc, templ) +make setup # install tooling and git hooks +make dev # run with hot-reload +make build-prod # produce a release binary ``` -### Testing - -```bash -make test # All tests (race detector, no cache, 2min timeout) - -# Run tests manually (build tag required): -go test -tags webkit2_41 ./backend/player/ # Single package -go test -tags webkit2_41 -run TestFunctionName ./backend/player/ # Single test -go test -tags webkit2_41 -v -run TestFunctionName ./backend/player/ # Verbose -``` - -### Linting - -```bash -make lint # golangci-lint (v2 config, strict rules) -``` - -### Data Locations - -| | Linux | macOS | Windows | -|---|---|---|---| -| Config | `~/.config/yellowjacket/` | `~/.config/yellowjacket/` | `%LOCALAPPDATA%\yellowjacket\config` | -| Data/DB | `~/.local/share/yellowjacket/` | `~/.local/share/yellowjacket/` | `%LOCALAPPDATA%\yellowjacket\data` | - -### Project Structure - -``` -backend/ Go backend - player/ Audio playback (beep) - queue/ Queue management, shuffle, repeat - library/ Library scanning, metadata extraction, cover art - metadata/ Audio decoding and tag extraction - database/ SQLite connection, sqlc-generated queries - config/ TOML config, HTTP handler for settings page - events/ Event name constants (mirrored in frontend) - models/ Shared data types (Album, Track, Artist) - system/ OS-specific paths -frontend/src/ TypeScript/Lit frontend - components/ UI components (player, sidebar, track list, cover grid, queue) - store/ Singleton stores and reactive controllers - pages/ Config page (HTMX entry point) -internal/dev/ Build-tag dev/prod detection -test_data/ Audio test fixtures -``` - -Further development documentation is available in [`docs/dev/`](./docs/dev/overview.md). +More detail for contributors lives in +[`docs/dev/overview.md`](./docs/dev/overview.md) and [`CLAUDE.md`](./CLAUDE.md). diff --git a/backend/app.go b/backend/app.go index c5beece..da78050 100644 --- a/backend/app.go +++ b/backend/app.go @@ -247,20 +247,28 @@ func (yj *YellowJacketApp) OnStartup(ctx context.Context) { RepopulatePlaylists: yj.playlist.RepopulateFromM3U, ResolvePhantoms: yj.playlist.ResolvePhantomTracksAfterScan, OnAllScansComplete: func() { - // Index new library artists (blocks until done). - yj.explore.IndexNewArtists() - yj.explore.WaitForIndexIdle() - - // Populate local_*_id cross-reference columns on - // explore_index so "is this in my library?" is O(1). + // Fold the local library into the search index: every + // MB-verified owned artist/album/track is upserted and + // flagged in_library, straight from the library tables + // with no API calls. Deep discographies stay lazy. yj.explore.PopulateLocalCrossReferences() - // Always start the full build — it's incremental and - // will skip tiers that are already fresh. This ensures - // sitewide + similar artist tiers run even if the index - // already has library data. + // Start (or resume) the dump-based index build. Skips + // itself once the one-time import has completed, so this + // is cheap on every startup. yj.explore.StartIndexBuild() + // Fold in any new incremental listen dumps to keep + // popularity fresh (weekly-gated, background, no API). + // No-op while the full import above is still running. + yj.explore.RefreshListenCounts() + + // Refresh the lyric-search FTS index from the just-scanned + // library, then backfill any missing lyrics from LRCLIB in + // the background (bounded, resumable, idempotent). + yj.explore.RebuildLyricsIndex() + yj.explore.BackfillLibraryLyrics() + // Sweep the autotag queue for newly-discovered pending // items so the user sees match scores ready when they // next open the review page. The worker is idempotent @@ -276,6 +284,10 @@ func (yj *YellowJacketApp) OnStartup(ctx context.Context) { yj.library.SetRemovalHooks(library.RemovalHooks{ StopPlayback: func() { yj.player.UnloadTrack() }, CompactQueue: yj.queue.CompactAfterLibraryRemoval, + // Removal deletes owned content outside a scan, so force the + // gated library-sync steps to re-run on the next launch and + // clear stale in_library flags / orphaned lyric-index rows. + PostRemove: yj.explore.InvalidateLibrarySync, }) // Register playback finished handler to drive queue auto-advance. @@ -333,6 +345,21 @@ func (yj *YellowJacketApp) OnStartup(ctx context.Context) { func (yj *YellowJacketApp) OnBeforeClose(ctx context.Context) bool { w, h := wailsruntime.WindowGetSize(ctx) + // Guard against a bogus size clobbering a good saved one. During + // teardown / hot-reload the runtime can report a zero or below- + // minimum size; persisting that would shrink the window to the + // minimum on next launch. Keep the previously-saved size instead. + if w < config.MinWidth || h < config.MinHeight { + yj.logger.Warn("OnBeforeClose: ignoring bogus window size", + "width", w, + "height", h, + "kept_width", yj.appConfig.Window.Width, + "kept_height", yj.appConfig.Window.Height, + ) + + return false + } + yj.logger.Info("OnBeforeClose: saving window state", "width", w, "height", h, @@ -393,7 +420,24 @@ func (yj *YellowJacketApp) OnDomReady(ctx context.Context) { // index build directly. If scans WERE queued, the // OnAllScansComplete hook starts it after they finish. if yj.library.GetScanQueueLength() == 0 && !yj.library.IsScanActive() { + // Keep the search index's in_library flags and owned-entity + // rows in sync. Gated: the library is unchanged here, so + // this only does work on the first launch after an upgrade + // or index wipe — steady-state launches skip the write burst. + yj.explore.PopulateLocalCrossReferencesIfNeeded() + yj.explore.StartIndexBuild() + + // Weekly-gated incremental popularity refresh (background, + // no API). No-op while the full import is running. + yj.explore.RefreshListenCounts() + + // Same for the lyric-search index. The backfill keeps it in + // sync incrementally, so on an unchanged library the full + // rebuild is redundant and gated out; the backfill still runs + // to fill any remaining gaps. + yj.explore.RebuildLyricsIndexIfNeeded() + yj.explore.BackfillLibraryLyrics() } // Kick off the autotag prefetch worker so any unscored diff --git a/backend/autotag/align.go b/backend/autotag/align.go index ce3137b..ada52db 100644 --- a/backend/autotag/align.go +++ b/backend/autotag/align.go @@ -1,5 +1,7 @@ package autotag +import "slices" + // alignTitleFloor is the minimum title similarity required to // pair a local file with a candidate track at all. Below this, // even the best available pair is treated as no pair: the local @@ -10,14 +12,28 @@ package autotag // happens to share a track number stays in its own group. const alignTitleFloor = 0.30 -// AlignTracks pairs local tracks with candidate tracks using a -// greedy best-score algorithm: repeatedly pick the (local, cand) -// pair with the highest trackDistance that hasn't already been -// claimed. Not optimal (Hungarian would be), but good enough for -// the small cardinalities we see (album tracks, ~10-50) and much -// simpler. +// alignPair carries the per-combination scores AlignTracks computes +// once per (local, candidate) pair. +type alignPair struct { + li int + ci int + score float64 + title float64 + length float64 +} + +// AlignTracks pairs local tracks with candidate tracks in two +// passes: // -// Pairs whose title similarity is below alignTitleFloor are +// 1. Recording-MBID locks: a local track whose RecordingMBID equals +// a candidate track's MBID is the same recording by definition — +// it pairs unconditionally, regardless of how the titles compare. +// 2. Greedy best-score: repeatedly pick the remaining (local, cand) +// pair with the highest track score. Not optimal (Hungarian +// would be), but good enough for the small cardinalities we see +// (album tracks, ~10-50) and much simpler. +// +// Greedy pairs whose title similarity is below alignTitleFloor are // rejected: the local stays "unmatched" and the candidate slot // surfaces as "missing". This lets a wrong-track-number-but- // matching-title file pair correctly while keeping a totally @@ -25,44 +41,72 @@ const alignTitleFloor = 0.30 // // Returns one TrackAlignment per local track (status = matched, // mismatched, or unmatched) plus additional missing alignments -// for candidate tracks with no local file. The caller sums Score -// fields to get a release-level score. +// for candidate tracks with no local file. func AlignTracks(locals []LocalTrack, cands []CandidateTrack) []TrackAlignment { - type pair struct { - li int - ci int - score float64 - title float64 + localUsed := make([]bool, len(locals)) + candUsed := make([]bool, len(cands)) + alignments := make([]TrackAlignment, len(locals)) + localMatched := 0 + + // Pass 1: recording-MBID locks. + for li, local := range locals { + if local.RecordingMBID == "" { + continue + } + + for ci, cand := range cands { + if candUsed[ci] || cand.MBID == "" || cand.MBID != local.RecordingMBID { + continue + } + + localUsed[li] = true + candUsed[ci] = true + localMatched++ + alignments[li] = mkAlignment(li, local, cand, alignPair{ + title: titleSimilarity(local.Title, cand.Title), + length: lengthScore(local.LengthMillis, cand.LengthMillis), + }, true) + + break + } } - // Score every (local, cand) combination. - pairs := make([]pair, 0, len(locals)*len(cands)) + // Pass 2: greedy best-score over the remaining combinations. + pairs := make([]alignPair, 0, len(locals)*len(cands)) for li, local := range locals { + if localUsed[li] { + continue + } + for ci, cand := range cands { - pairs = append(pairs, pair{ - li: li, - ci: ci, - score: trackDistance(local, cand), - title: titleSimilarity(local.Title, cand.Title), + if candUsed[ci] { + continue + } + + title := titleSimilarity(local.Title, cand.Title) + length := lengthScore(local.LengthMillis, cand.LengthMillis) + pairs = append(pairs, alignPair{ + li: li, + ci: ci, + score: combineTrackScore(title, length, trackNumberOK(local, cand)), + title: title, + length: length, }) } } - // Sort descending by score — pick best pairs first. Insertion - // sort keeps the dependency surface zero; N^2 here is fine for - // album-sized inputs. - for i := 1; i < len(pairs); i++ { - for j := i; j > 0 && pairs[j].score > pairs[j-1].score; j-- { - pairs[j], pairs[j-1] = pairs[j-1], pairs[j] + // Sort descending by score — pick best pairs first. + slices.SortStableFunc(pairs, func(a, b alignPair) int { + switch { + case a.score > b.score: + return -1 + case a.score < b.score: + return 1 + default: + return 0 } - } - - localUsed := make([]bool, len(locals)) - candUsed := make([]bool, len(cands)) - alignments := make([]TrackAlignment, len(locals)) - - localMatched := 0 + }) for _, p := range pairs { if localMatched == len(locals) { @@ -84,34 +128,7 @@ func AlignTracks(locals []LocalTrack, cands []CandidateTrack) []TrackAlignment { localUsed[p.li] = true candUsed[p.ci] = true localMatched++ - - l := locals[p.li] - c := cands[p.ci] - - status := AlignmentMatched - if p.title < titleReject { - status = AlignmentMismatched - } - - delta := l.LengthMillis - c.LengthMillis - if delta < 0 { - delta = -delta - } - - alignments[p.li] = TrackAlignment{ - LocalIndex: p.li, - LocalTitle: l.Title, - LocalLengthMillis: l.LengthMillis, - CandidatePosition: c.Position, - CandidateDiscNumber: c.DiscNumber, - CandidateTitle: c.Title, - CandidateMBID: c.MBID, - CandidateLength: c.LengthMillis, - TitleScore: p.title, - LengthDeltaMs: delta, - TrackNumberOK: l.TrackNumber > 0 && l.TrackNumber == c.Position, - Status: status, - } + alignments[p.li] = mkAlignment(p.li, locals[p.li], cands[p.ci], p, false) } // Local tracks left unclaimed → folder has them, candidate @@ -151,3 +168,38 @@ func AlignTracks(locals []LocalTrack, cands []CandidateTrack) []TrackAlignment { return alignments } + +// mkAlignment builds the matched/mismatched alignment for one +// claimed (local, candidate) pair. idMatch pairs are always +// "matched" — same recording MBID means same recording, however +// the titles are spelled. +func mkAlignment( + li int, l LocalTrack, c CandidateTrack, p alignPair, idMatch bool, +) TrackAlignment { + status := AlignmentMatched + if !idMatch && p.title < titleReject { + status = AlignmentMismatched + } + + delta := l.LengthMillis - c.LengthMillis + if delta < 0 { + delta = -delta + } + + return TrackAlignment{ + LocalIndex: li, + LocalTitle: l.Title, + LocalLengthMillis: l.LengthMillis, + CandidatePosition: c.Position, + CandidateDiscNumber: c.DiscNumber, + CandidateTitle: c.Title, + CandidateMBID: c.MBID, + CandidateLength: c.LengthMillis, + TitleScore: p.title, + LengthScore: p.length, + LengthDeltaMs: delta, + TrackNumberOK: trackNumberOK(l, c), + IDMatch: idMatch, + Status: status, + } +} diff --git a/backend/autotag/distance.go b/backend/autotag/distance.go index f8e91f4..16adb23 100644 --- a/backend/autotag/distance.go +++ b/backend/autotag/distance.go @@ -1,5 +1,11 @@ package autotag +import ( + "regexp" + "strings" + "unicode" +) + // levenshtein returns the Levenshtein edit distance between a and // b, operating on runes so multi-byte characters count as one edit. // Allocates a single O(min(len)) scratch slice. @@ -63,29 +69,142 @@ func min3(a, b, c int) int { return c } -// titleSimilarity returns a score in [0, 1] from normalized edit -// distance. 1.0 means identical after normalization, 0.0 means -// fully dissimilar. Both sides are normalized inside. -func titleSimilarity(a, b string) float64 { - na := Normalize(a) - nb := Normalize(b) +// sdPattern couples a regexp with the weight its removal carries in +// stringDist: when deleting the matched portion from both strings +// shrinks their distance, the recovered distance is re-added at +// `weight` instead of counting fully. Weight 0 makes a difference +// in that portion free (known-cosmetic); higher weights make it +// cheap but not free. Modeled on beets' SD_PATTERNS. +type sdPattern struct { + re *regexp.Regexp + weight float64 +} - if na == "" && nb == "" { - return 1.0 - } +// sdPatterns are applied in order; earlier patterns claim their +// portion of the string first. Known qualifiers (whitelist) are +// free; generic parenthetical / bracketed content, featured-artist +// credits, leading articles, and part suffixes are de-weighted. +var sdPatterns = []sdPattern{ + {qualifierPattern, 0.0}, + {dashQualifierPattern, 0.0}, + {regexp.MustCompile(`^the `), 0.1}, + {regexp.MustCompile(`\b(featuring|feat\.?|ft\.?)[ :].*$`), 0.1}, + {regexp.MustCompile(`\(.*?\)`), 0.3}, + {regexp.MustCompile(`\[.*?\]`), 0.3}, + {regexp.MustCompile(`(, )?\b(pt\.|part) .+$`), 0.2}, +} - longest := len(na) - if len(nb) > longest { - longest = len(nb) - } +// sdEndWords are articles that user tags sometimes rotate to the +// end with a comma: "Beatles, The" ≡ "The Beatles". +var sdEndWords = []string{"the", "a", "an"} - if longest == 0 { +// stringDistBasic is the normalized edit distance between the two +// strings reduced to lowercase alphanumerics, in [0, 1]. Inputs +// are assumed to be lowercased and ASCII-folded already. +func stringDistBasic(a, b string) float64 { + a = alnumOnly(a) + b = alnumOnly(b) + + if a == "" && b == "" { return 0.0 } - dist := levenshtein(na, nb) + longest := max(len(a), len(b)) - return 1.0 - float64(dist)/float64(longest) + return float64(levenshtein(a, b)) / float64(longest) +} + +// alnumOnly strips everything but letters and digits. +func alnumOnly(s string) string { + var b strings.Builder + + b.Grow(len(s)) + + for _, r := range s { + if unicode.IsLetter(r) || unicode.IsDigit(r) { + b.WriteRune(r) + } + } + + return b.String() +} + +// stringDist returns an "intuitive" distance between two titles or +// artist credits, in [0, 1]. It is a normalized edit distance with +// tweaks reflecting how music metadata actually differs (ported +// from beets' string_dist): +// +// - accents transliterated, case ignored +// - "X, The" rotated back to "The X" (same for "A"/"An") +// - "&" ≡ "and" +// - known qualifier suffixes ("(Remastered 2009)", "- Radio Edit") +// are free; unknown parenthesized/bracketed content, featured- +// artist credits, leading articles, and "Part N" suffixes are +// de-weighted rather than counting as full edits +func stringDist(a, b string) float64 { + a = strings.ToLower(asciiFold(a)) + b = strings.ToLower(asciiFold(b)) + + a = rotateEndWord(a) + b = rotateEndWord(b) + + a = strings.ReplaceAll(a, "&", " and ") + b = strings.ReplaceAll(b, "&", " and ") + + base := stringDistBasic(a, b) + penalty := 0.0 + + for _, p := range sdPatterns { + ca := p.re.ReplaceAllString(a, "") + cb := p.re.ReplaceAllString(b, "") + + if ca == a && cb == b { + continue + } + + // The pattern was present: measure how much of the distance + // it accounted for and re-add that share at reduced weight. + caseDist := stringDistBasic(ca, cb) + + delta := base - caseDist + if delta <= 0 { + continue + } + + a, b = ca, cb + base = caseDist + penalty += p.weight * delta + } + + return base + penalty +} + +// rotateEndWord undoes sort-style article rotation: "beatles, the" +// → "the beatles". Input must be lowercased. +func rotateEndWord(s string) string { + for _, w := range sdEndWords { + suffix := ", " + w + if strings.HasSuffix(s, suffix) { + return w + " " + s[:len(s)-len(suffix)] + } + } + + return s +} + +// titleSimilarity returns a score in [0, 1] from stringDist. 1.0 +// means identical after normalization, 0.0 means fully dissimilar. +func titleSimilarity(a, b string) float64 { + if strings.TrimSpace(a) == "" && strings.TrimSpace(b) == "" { + return 1.0 + } + + sim := 1.0 - stringDist(a, b) + if sim < 0 { + return 0.0 + } + + return sim } // Scoring weights for the per-track distance function. Local reads @@ -98,13 +217,17 @@ const ( // Length deltas at or below lengthExactMs score 1.0 — matches // the frontend's "subtle drift" threshold so anything the UI - // hides also doesn't count against the score. Past the - // threshold the penalty scales with delta / candidateMs (i.e. - // percentage of candidate-track length): a 5 s delta on a + // hides also doesn't count against the score. MB recording + // lengths routinely differ from file durations by a few seconds + // (encoder padding, different masters), so the grace band is + // deliberately wider than perceptual accuracy; Picard tolerates + // up to 30 s linearly and beets grants a flat 10 s grace. Past + // the threshold the penalty scales with delta / candidateMs + // (i.e. percentage of candidate-track length): a 5 s delta on a // 4 min track is small, the same delta on a 30 s interlude is // huge. At lengthFullyWrongPct of candidate length the score // hits zero; beyond that it stays clamped to zero. - lengthExactMs int64 = 2000 + lengthExactMs int64 = 5000 lengthFullyWrongPct float64 = 0.20 // A title below titleReject has too little signal for this @@ -139,17 +262,31 @@ func lengthScore(localMs, candidateMs int64) float64 { return 1.0 - pct/lengthFullyWrongPct } -// trackDistance scores how well one local track aligns with one -// candidate track. Higher is better. Caller decides what to do -// with the result — this function has no threshold. -func trackDistance(local LocalTrack, cand CandidateTrack) float64 { - title := titleSimilarity(local.Title, cand.Title) - length := lengthScore(local.LengthMillis, cand.LengthMillis) +// trackNumberOK reports whether the local track number agrees with +// the candidate position (0 = unknown, never a match). +func trackNumberOK(local LocalTrack, cand CandidateTrack) bool { + return local.TrackNumber > 0 && local.TrackNumber == cand.Position +} +// combineTrackScore folds the per-track components into one score. +// Split out so AlignTracks can compute the components once per pair +// and still share the exact formula with trackDistance. +func combineTrackScore(title, length float64, numberOK bool) float64 { var trackOK float64 - if local.TrackNumber > 0 && local.TrackNumber == cand.Position { + if numberOK { trackOK = 1.0 } return title*weightTitle + length*weightLength + trackOK*weightTrackNumber } + +// trackDistance scores how well one local track aligns with one +// candidate track. Higher is better. Caller decides what to do +// with the result — this function has no threshold. +func trackDistance(local LocalTrack, cand CandidateTrack) float64 { + return combineTrackScore( + titleSimilarity(local.Title, cand.Title), + lengthScore(local.LengthMillis, cand.LengthMillis), + trackNumberOK(local, cand), + ) +} diff --git a/backend/autotag/distance_test.go b/backend/autotag/distance_test.go index 76a5a3d..e67d720 100644 --- a/backend/autotag/distance_test.go +++ b/backend/autotag/distance_test.go @@ -33,10 +33,18 @@ func TestTitleSimilarity(t *testing.T) { minScore float64 }{ {"Hey Jude", "Hey Jude", 1.00}, - {"Hey Jude", "Hey Jude (Remastered 2009)", 1.00}, // qualifier stripped + {"Hey Jude", "Hey Jude (Remastered 2009)", 1.00}, // whitelisted qualifier: free + {"Hey Jude", "Hey Jude - 2015 Remaster", 1.00}, // dash-suffix qualifier: free {"Hey Jude", "HEY JUDE!", 1.00}, // case + punct - {"Hey Jude", "Hay Jude", 0.85}, // one char off - {"Hey Jude", "Let It Be", 0.00}, // different + {"Beyoncé", "Beyonce", 1.00}, // transliteration + {"Simon & Garfunkel", "Simon and Garfunkel", 1.00}, + {"Beatles, The", "The Beatles", 1.00}, // article rotation + {"Hey Jude", "Hay Jude", 0.85}, // one char off + // Unknown parenthetical content is de-weighted, not free — + // still similar, but detectably not identical. + {"Song Title (Special Whatever)", "Song Title", 0.80}, + {"Yellow (feat. Somebody)", "Yellow", 0.90}, // feat credit nearly free + {"Hey Jude", "Let It Be", 0.00}, // different } for _, tc := range cases { @@ -66,9 +74,9 @@ func TestLengthScore(t *testing.T) { want float64 }{ {200000, 200000, 1.0}, // exact - {200000, 200500, 1.0}, // 0.5s — under 2s threshold - {200000, 201900, 1.0}, // 1.9s — under 2s threshold - {200000, 202000, 1.0}, // exactly 2s — still full credit + {200000, 200500, 1.0}, // 0.5s — under the grace band + {200000, 204900, 1.0}, // 4.9s — under the grace band + {200000, 205000, 1.0}, // exactly 5s — still full credit {0, 200000, 0.5}, // unknown local → neutral {200000, 0, 0.5}, // unknown candidate → neutral @@ -130,3 +138,104 @@ func TestTrackDistance(t *testing.T) { t.Errorf("wrong match = %.2f, want < 0.2", got) } } + +func TestDominantArtist(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + local []LocalTrack + want string + }{ + {"empty", nil, ""}, + {"all blank", []LocalTrack{{Artist: ""}, {Artist: ""}}, ""}, + { + "unanimous", + []LocalTrack{{Artist: "Radiohead"}, {Artist: "Radiohead"}}, + "Radiohead", + }, + { + "majority wins over a stray", + []LocalTrack{{Artist: "Radiohead"}, {Artist: "Radiohead"}, {Artist: "Guest"}}, + "Radiohead", + }, + { + "blanks ignored, one real value wins", + []LocalTrack{{Artist: ""}, {Artist: "Bjork"}, {Artist: ""}}, + "Bjork", + }, + } + + for _, tc := range cases { + if got := dominantArtist(tc.local); got != tc.want { + t.Errorf("%s: dominantArtist = %q, want %q", tc.name, got, tc.want) + } + } +} + +func TestArtistCreditFit(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + local, cnd string + wantMin float64 // when > 0, require >= ; when 0, require <= 0.5 (mismatch) + }{ + {"unknown local is neutral", "", "Whoever", 1.0}, + {"unknown candidate is neutral", "Whoever", "", 1.0}, + {"exact", "The Beatles", "The Beatles", 1.0}, + {"case/punct only", "The Beatles", "THE BEATLES!", 1.0}, + {"almost-right stays high", "Beyonce", "Beyoncé", 0.85}, + {"completely different artist", "The Beatles", "Metallica", 0.0}, + } + + for _, tc := range cases { + got := artistCreditFit(tc.local, tc.cnd) + if tc.wantMin == 0 { + if got > 0.5 { //nolint:mnd + t.Errorf( + "%s: artistCreditFit(%q,%q) = %.2f, want low", + tc.name, + tc.local, + tc.cnd, + got, + ) + } + + continue + } + + if got < tc.wantMin { + t.Errorf( + "%s: artistCreditFit(%q,%q) = %.2f, want >= %.2f", + tc.name, + tc.local, + tc.cnd, + got, + tc.wantMin, + ) + } + } +} + +func TestEvidenceFactor(t *testing.T) { + t.Parallel() + + cases := []struct { + tracks int + want float64 + }{ + {0, evidenceFloor}, // no tracks — treated as minimum evidence + {1, evidenceFloor}, // singleton — the harshest case + {2, 0.925}, // halfway between floor and full + {3, 1.0}, // full evidence + {10, 1.0}, // large album — unscaled + } + + for _, tc := range cases { + got := evidenceFactor(tc.tracks) + if diff := got - tc.want; diff < -0.001 || diff > 0.001 { + t.Errorf("evidenceFactor(%d) = %.4f, want %.4f", tc.tracks, got, tc.want) + } + } +} diff --git a/backend/autotag/eval/eval.go b/backend/autotag/eval/eval.go new file mode 100644 index 0000000..eb8c43d --- /dev/null +++ b/backend/autotag/eval/eval.go @@ -0,0 +1,108 @@ +// Package eval is the autotag candidate-scoring evaluation harness. +// It turns "this match feels wrong" into a number that goes up or +// down, so a scoring change can be validated against a frozen set of +// labelled cases instead of tuned by anecdote. +// +// A case describes a local album-group (the files on disk) plus a set +// of candidate releases, and pins expectations: which candidate must +// rank first, and per-candidate score floors/ceilings. The harness +// is decoupled from the scorer: a caller adapts whatever ranking +// function it wants to measure to the Ranker interface (the autotag +// package wires autotag.RankCandidates to it in eval_harness_test.go). +// +// The point of the ceiling assertions is negative testing: a known +// wrong candidate (same title, different artist) must stay BELOW a +// confidence bar, which is exactly the false-positive class the +// artist term + evidence scaling exist to suppress. +package eval + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" +) + +// ErrNoCases is returned when a fixture file contains zero cases. +var ErrNoCases = errors.New("eval: fixture set is empty") + +// LocalTrackFixture is one on-disk track, in the shape a case author +// hand-writes. Millisecond durations match the scorer's units. +type LocalTrackFixture struct { + Title string `json:"title"` + Artist string `json:"artist,omitempty"` + Track int `json:"track,omitempty"` + Disc int `json:"disc,omitempty"` + LengthMs int64 `json:"lengthMs,omitempty"` +} + +// CandidateTrackFixture is one track inside a candidate release. +type CandidateTrackFixture struct { + Pos int `json:"pos"` + Disc int `json:"disc,omitempty"` + Title string `json:"title"` + LengthMs int64 `json:"lengthMs,omitempty"` +} + +// CandidateFixture is one candidate release the scorer must rank. +// Source is "local" or "musicbrainz" (default) — it drives +// evidence scaling, so it matters for singleton cases. +type CandidateFixture struct { + MBID string `json:"mbid"` + Title string `json:"title,omitempty"` + ArtistCredit string `json:"artistCredit,omitempty"` + Status string `json:"status,omitempty"` + Country string `json:"country,omitempty"` + PrimaryType string `json:"primaryType,omitempty"` + Source string `json:"source,omitempty"` + Tracks []CandidateTrackFixture `json:"tracks"` +} + +// Case is a single labelled scoring scenario. Every real-world +// mismatch worth guarding against belongs here so it can never +// silently regress. +// +// - AlbumName / AlbumArtist mirror the tagging item's fields — +// leave them empty to keep the album-title / artist terms +// neutral for the case. +// - ExpectTop, when set, is the MBID that must rank first. +// - MaxScore pins per-MBID ceilings (candidate must score <= value). +// - MinScore pins per-MBID floors (candidate must score >= value). +type Case struct { + Note string `json:"note,omitempty"` + AlbumName string `json:"albumName,omitempty"` + AlbumArtist string `json:"albumArtist,omitempty"` + Local []LocalTrackFixture `json:"local"` + Candidates []CandidateFixture `json:"candidates"` + ExpectTop string `json:"expectTop,omitempty"` + MaxScore map[string]float64 `json:"maxScore,omitempty"` + MinScore map[string]float64 `json:"minScore,omitempty"` +} + +// LoadCases reads a JSON fixture file from disk. +func LoadCases(path string) ([]Case, error) { + f, err := os.Open(path) //nolint:gosec // path is a test fixture, not user input + if err != nil { + return nil, fmt.Errorf("eval: open cases: %w", err) + } + + defer func() { _ = f.Close() }() + + return ParseCases(f) +} + +// ParseCases decodes a JSON case set from a reader. +func ParseCases(r io.Reader) ([]Case, error) { + var cases []Case + + if err := json.NewDecoder(r).Decode(&cases); err != nil { + return nil, fmt.Errorf("eval: decode cases: %w", err) + } + + if len(cases) == 0 { + return nil, ErrNoCases + } + + return cases, nil +} diff --git a/backend/autotag/eval/evaluate.go b/backend/autotag/eval/evaluate.go new file mode 100644 index 0000000..dab1697 --- /dev/null +++ b/backend/autotag/eval/evaluate.go @@ -0,0 +1,123 @@ +package eval + +import "fmt" + +// ScoredCandidate is one ranked candidate reduced to what the harness +// checks: its MBID and final score. The slice a Ranker returns must +// be ordered best-first. +type ScoredCandidate struct { + MBID string + Score float64 +} + +// Ranker scores and orders the candidates of a single case. Best +// candidate first. Implemented by adapting the real scorer (see +// eval_harness_test.go), which is why the eval package itself never +// imports autotag. +type Ranker interface { + Rank(c Case) []ScoredCandidate +} + +// RankerFunc adapts a plain function to the Ranker interface. +type RankerFunc func(c Case) []ScoredCandidate + +// Rank calls the underlying function. +func (f RankerFunc) Rank(c Case) []ScoredCandidate { + return f(c) +} + +// CaseResult records how one case fared. Failures is empty when the +// case passed every pinned expectation. +type CaseResult struct { + Case Case + Failures []string +} + +// Passed reports whether the case met every expectation. +func (r CaseResult) Passed() bool { + return len(r.Failures) == 0 +} + +// Report aggregates results across a case set. +type Report struct { + Results []CaseResult +} + +// Passed counts cases that met every expectation. +func (r Report) Passed() int { + n := 0 + + for _, c := range r.Results { + if c.Passed() { + n++ + } + } + + return n +} + +// Accuracy is the fraction of cases that passed, in [0, 1]. +func (r Report) Accuracy() float64 { + if len(r.Results) == 0 { + return 0 + } + + return float64(r.Passed()) / float64(len(r.Results)) +} + +// Evaluate runs every case through the ranker and checks its pinned +// expectations (ExpectTop, MaxScore ceilings, MinScore floors), +// returning a Report the caller can assert on and print. +func Evaluate(cases []Case, ranker Ranker) Report { + rep := Report{Results: make([]CaseResult, 0, len(cases))} + + for _, c := range cases { + rep.Results = append(rep.Results, evaluateCase(c, ranker)) + } + + return rep +} + +func evaluateCase(c Case, ranker Ranker) CaseResult { + ranked := ranker.Rank(c) + + res := CaseResult{Case: c} + + byMBID := make(map[string]float64, len(ranked)) + for _, r := range ranked { + byMBID[r.MBID] = r.Score + } + + if c.ExpectTop != "" { + switch { + case len(ranked) == 0: + res.Failures = append( + res.Failures, + "expected top "+c.ExpectTop+" but ranking was empty", + ) + case ranked[0].MBID != c.ExpectTop: + res.Failures = append(res.Failures, fmt.Sprintf( + "top = %q (%.3f), want %q (%.3f)", + ranked[0].MBID, ranked[0].Score, c.ExpectTop, byMBID[c.ExpectTop], + )) + } + } + + for mbid, ceil := range c.MaxScore { + if got, ok := byMBID[mbid]; ok && got > ceil { + res.Failures = append(res.Failures, fmt.Sprintf( + "%s scored %.3f, want <= %.3f", mbid, got, ceil, + )) + } + } + + for mbid, floor := range c.MinScore { + if got, ok := byMBID[mbid]; ok && got < floor { + res.Failures = append(res.Failures, fmt.Sprintf( + "%s scored %.3f, want >= %.3f", mbid, got, floor, + )) + } + } + + return res +} diff --git a/backend/autotag/eval/testdata/scoring_cases.json b/backend/autotag/eval/testdata/scoring_cases.json new file mode 100644 index 0000000..3d6538e --- /dev/null +++ b/backend/autotag/eval/testdata/scoring_cases.json @@ -0,0 +1,225 @@ +[ + { + "note": "seed: single with a generic title must reject a same-title, different-artist, wrong-length MB hit and prefer the correct artist. This is the reported 92%-false-positive class.", + "local": [ + { "title": "Intro", "artist": "Real Artist", "track": 1, "lengthMs": 90000 } + ], + "candidates": [ + { + "mbid": "right", + "artistCredit": "Real Artist", + "status": "Official", + "source": "musicbrainz", + "tracks": [{ "pos": 1, "title": "Intro", "lengthMs": 90000 }] + }, + { + "mbid": "wrong-artist", + "artistCredit": "Some Other Band", + "status": "Official", + "source": "musicbrainz", + "tracks": [{ "pos": 1, "title": "Intro", "lengthMs": 240000 }] + } + ], + "expectTop": "right", + "maxScore": { "wrong-artist": 0.75 } + }, + { + "note": "seed: full-album exact match should still read as a confident, near-perfect match (evidence scaling must not punish real albums).", + "local": [ + { "title": "Song A", "artist": "The Band", "track": 1, "lengthMs": 200000 }, + { "title": "Song B", "artist": "The Band", "track": 2, "lengthMs": 210000 }, + { "title": "Song C", "artist": "The Band", "track": 3, "lengthMs": 195000 }, + { "title": "Song D", "artist": "The Band", "track": 4, "lengthMs": 220000 } + ], + "candidates": [ + { + "mbid": "album", + "artistCredit": "The Band", + "status": "Official", + "source": "musicbrainz", + "tracks": [ + { "pos": 1, "title": "Song A", "lengthMs": 200000 }, + { "pos": 2, "title": "Song B", "lengthMs": 210000 }, + { "pos": 3, "title": "Song C", "lengthMs": 195000 }, + { "pos": 4, "title": "Song D", "lengthMs": 220000 } + ] + } + ], + "expectTop": "album", + "minScore": { "album": 0.9 } + }, + { + "note": "seed: near-right artist (accent/spelling) must stay a strong match; soft artist term, not a gate.", + "local": [ + { "title": "Jolene", "artist": "Beyonce", "track": 1, "lengthMs": 200000 }, + { "title": "Halo", "artist": "Beyonce", "track": 2, "lengthMs": 220000 } + ], + "candidates": [ + { + "mbid": "accented", + "artistCredit": "Beyoncé", + "status": "Official", + "source": "musicbrainz", + "tracks": [ + { "pos": 1, "title": "Jolene", "lengthMs": 200000 }, + { "pos": 2, "title": "Halo", "lengthMs": 220000 } + ] + } + ], + "expectTop": "accented", + "minScore": { "accented": 0.85 } + }, + { + "note": "VA compilation: per-track artists all differ, album-artist says Various Artists. The VA candidate must not be penalised for its artist credit and must read as a strong match.", + "albumName": "Now That's What I Call Music! 60", + "albumArtist": "Various Artists", + "local": [ + { "title": "Song One", "artist": "Artist A", "track": 1, "lengthMs": 200000 }, + { "title": "Song Two", "artist": "Artist B", "track": 2, "lengthMs": 210000 }, + { "title": "Song Three", "artist": "Artist C", "track": 3, "lengthMs": 195000 }, + { "title": "Song Four", "artist": "Artist D", "track": 4, "lengthMs": 205000 } + ], + "candidates": [ + { + "mbid": "va-comp", + "title": "Now That's What I Call Music! 60", + "artistCredit": "Various Artists", + "status": "Official", + "primaryType": "Compilation", + "source": "musicbrainz", + "tracks": [ + { "pos": 1, "title": "Song One", "lengthMs": 200000 }, + { "pos": 2, "title": "Song Two", "lengthMs": 210000 }, + { "pos": 3, "title": "Song Three", "lengthMs": 195000 }, + { "pos": 4, "title": "Song Four", "lengthMs": 205000 } + ] + } + ], + "expectTop": "va-comp", + "minScore": { "va-comp": 0.85 } + }, + { + "note": "same recordings, two release groups: folder album name must pull the studio album above the greatest-hits comp with an identical tracklist.", + "albumName": "The Studio Album", + "albumArtist": "The Band", + "local": [ + { "title": "Song A", "artist": "The Band", "track": 1, "lengthMs": 200000 }, + { "title": "Song B", "artist": "The Band", "track": 2, "lengthMs": 210000 }, + { "title": "Song C", "artist": "The Band", "track": 3, "lengthMs": 195000 } + ], + "candidates": [ + { + "mbid": "studio", + "title": "The Studio Album", + "artistCredit": "The Band", + "status": "Official", + "primaryType": "Album", + "source": "musicbrainz", + "tracks": [ + { "pos": 1, "title": "Song A", "lengthMs": 200000 }, + { "pos": 2, "title": "Song B", "lengthMs": 210000 }, + { "pos": 3, "title": "Song C", "lengthMs": 195000 } + ] + }, + { + "mbid": "hits-comp", + "title": "Greatest Hits", + "artistCredit": "The Band", + "status": "Official", + "primaryType": "Compilation", + "source": "musicbrainz", + "tracks": [ + { "pos": 1, "title": "Song A", "lengthMs": 200000 }, + { "pos": 2, "title": "Song B", "lengthMs": 210000 }, + { "pos": 3, "title": "Song C", "lengthMs": 195000 } + ] + } + ], + "expectTop": "studio", + "minScore": { "studio": 0.9 } + }, + { + "note": "disc 1 of a 2-disc release: per-disc group must not be punished for 'missing' disc 2, and must not lose to a random single-disc release with fewer matching tracks.", + "albumName": "The Double Album", + "albumArtist": "The Band", + "local": [ + { "title": "D1 Track 1", "artist": "The Band", "track": 1, "disc": 1, "lengthMs": 200000 }, + { "title": "D1 Track 2", "artist": "The Band", "track": 2, "disc": 1, "lengthMs": 210000 }, + { "title": "D1 Track 3", "artist": "The Band", "track": 3, "disc": 1, "lengthMs": 195000 } + ], + "candidates": [ + { + "mbid": "double", + "title": "The Double Album", + "artistCredit": "The Band", + "status": "Official", + "primaryType": "Album", + "source": "musicbrainz", + "tracks": [ + { "pos": 1, "disc": 1, "title": "D1 Track 1", "lengthMs": 200000 }, + { "pos": 2, "disc": 1, "title": "D1 Track 2", "lengthMs": 210000 }, + { "pos": 3, "disc": 1, "title": "D1 Track 3", "lengthMs": 195000 }, + { "pos": 1, "disc": 2, "title": "D2 Track 1", "lengthMs": 220000 }, + { "pos": 2, "disc": 2, "title": "D2 Track 2", "lengthMs": 230000 }, + { "pos": 3, "disc": 2, "title": "D2 Track 3", "lengthMs": 240000 } + ] + } + ], + "expectTop": "double", + "minScore": { "double": 0.9 } + }, + { + "note": "streaming-style dash qualifiers: '- 2011 Remaster' suffixes on every local title must not cost title score against the clean MB tracklist.", + "albumName": "Classic Album", + "albumArtist": "The Band", + "local": [ + { "title": "Opener - 2011 Remaster", "artist": "The Band", "track": 1, "lengthMs": 200000 }, + { "title": "Middle Cut - 2011 Remaster", "artist": "The Band", "track": 2, "lengthMs": 210000 }, + { "title": "Closer - 2011 Remaster", "artist": "The Band", "track": 3, "lengthMs": 195000 } + ], + "candidates": [ + { + "mbid": "clean-titles", + "title": "Classic Album", + "artistCredit": "The Band", + "status": "Official", + "primaryType": "Album", + "source": "musicbrainz", + "tracks": [ + { "pos": 1, "title": "Opener", "lengthMs": 200000 }, + { "pos": 2, "title": "Middle Cut", "lengthMs": 210000 }, + { "pos": 3, "title": "Closer", "lengthMs": 195000 } + ] + } + ], + "expectTop": "clean-titles", + "minScore": { "clean-titles": 0.9 } + }, + { + "note": "sorted-artist tag: 'Beatles, The' must match 'The Beatles' releases at full strength (article rotation).", + "albumName": "Abbey Road", + "albumArtist": "Beatles, The", + "local": [ + { "title": "Come Together", "artist": "Beatles, The", "track": 1, "lengthMs": 259000 }, + { "title": "Something", "artist": "Beatles, The", "track": 2, "lengthMs": 182000 }, + { "title": "Octopus's Garden", "artist": "Beatles, The", "track": 3, "lengthMs": 170000 } + ], + "candidates": [ + { + "mbid": "abbey", + "title": "Abbey Road", + "artistCredit": "The Beatles", + "status": "Official", + "primaryType": "Album", + "source": "musicbrainz", + "tracks": [ + { "pos": 1, "title": "Come Together", "lengthMs": 259000 }, + { "pos": 2, "title": "Something", "lengthMs": 182000 }, + { "pos": 3, "title": "Octopus's Garden", "lengthMs": 170000 } + ] + } + ], + "expectTop": "abbey", + "minScore": { "abbey": 0.9 } + } +] diff --git a/backend/autotag/eval_harness_test.go b/backend/autotag/eval_harness_test.go new file mode 100644 index 0000000..1cc1486 --- /dev/null +++ b/backend/autotag/eval_harness_test.go @@ -0,0 +1,102 @@ +package autotag_test + +import ( + "testing" + + "yellowjacket/backend/autotag" + "yellowjacket/backend/autotag/eval" +) + +// adaptRanker converts an eval.Case (hand-written fixture shapes) into +// the autotag domain types, runs the real RankCandidates, and maps +// the result back to []eval.ScoredCandidate. This is the one place +// the eval package's decoupled fixtures meet the concrete scorer. +func adaptRanker(c eval.Case) []eval.ScoredCandidate { + locals := make([]autotag.LocalTrack, len(c.Local)) + for i, l := range c.Local { + locals[i] = autotag.LocalTrack{ + Title: l.Title, + Artist: l.Artist, + TrackNumber: l.Track, + DiscNumber: l.Disc, + LengthMillis: l.LengthMs, + } + } + + cands := make([]autotag.Candidate, len(c.Candidates)) + for i, cf := range c.Candidates { + tracks := make([]autotag.CandidateTrack, len(cf.Tracks)) + for j, tf := range cf.Tracks { + tracks[j] = autotag.CandidateTrack{ + Position: tf.Pos, + DiscNumber: tf.Disc, + Title: tf.Title, + LengthMillis: tf.LengthMs, + } + } + + cands[i] = autotag.Candidate{ + ReleaseMBID: cf.MBID, + Title: cf.Title, + ArtistCredit: cf.ArtistCredit, + Status: cf.Status, + Country: cf.Country, + PrimaryType: cf.PrimaryType, + Source: candidateSource(cf.Source), + Tracks: tracks, + } + } + + ranked := autotag.RankCandidates(autotag.Group{ + AlbumName: c.AlbumName, + AlbumArtist: c.AlbumArtist, + Tracks: locals, + }, cands) + + out := make([]eval.ScoredCandidate, len(ranked)) + for i, r := range ranked { + out[i] = eval.ScoredCandidate{MBID: r.ReleaseMBID, Score: r.Score} + } + + return out +} + +// candidateSource maps the fixture string to the domain type, +// defaulting to MusicBrainz (the interesting, evidence-scaled path). +func candidateSource(s string) autotag.CandidateSource { + if s == string(autotag.SourceLocal) { + return autotag.SourceLocal + } + + return autotag.SourceMusicBrainz +} + +// TestScoringCorpus runs the frozen labelled corpus through the real +// ranker. Add real-world mismatches to testdata/scoring_cases.json — +// every case that fails here is a scoring regression, and the harness +// prints exactly which expectation broke. +func TestScoringCorpus(t *testing.T) { + t.Parallel() + + cases, err := eval.LoadCases("eval/testdata/scoring_cases.json") + if err != nil { + t.Fatalf("load cases: %v", err) + } + + report := eval.Evaluate(cases, eval.RankerFunc(adaptRanker)) + + for _, r := range report.Results { + if r.Passed() { + continue + } + + for _, f := range r.Failures { + t.Errorf("case %q: %s", r.Case.Note, f) + } + } + + t.Logf( + "scoring corpus: %d/%d cases passed (%.0f%% accuracy)", + report.Passed(), len(report.Results), report.Accuracy()*100, + ) +} diff --git a/backend/autotag/mb.go b/backend/autotag/mb.go index ede81aa..80ba3f9 100644 --- a/backend/autotag/mb.go +++ b/backend/autotag/mb.go @@ -32,6 +32,25 @@ type MBRelease struct { Tracks []CandidateTrack } +// MBRecordingHit is the minimal projection of a MusicBrainz recording +// search result — used by the in-app search path (singletons). +type MBRecordingHit struct { + MBID string + Title string + ArtistCredit string + LengthMillis int64 +} + +// MBReleaseRef is a slim reference to one release a recording appears +// on. The resolver ranks these to pick a representative release and +// then resolves it in full via ResolveOneReleaseMBID. +type MBReleaseRef struct { + MBID string + Title string + Status string + Date string +} + // MBClient is the subset of the explore.MusicBrainzClient surface // the autotagger depends on. Implementations must be cache-first // — repeated calls with the same inputs must not repeat network @@ -42,16 +61,42 @@ type MBClient interface { query string, limit int, ) ([]MBReleaseGroupHit, int, error) + SearchRecordings( + ctx context.Context, + query string, + limit int, + ) ([]MBRecordingHit, int, error) + LookupRecordingReleases(ctx context.Context, recordingMBID string) ([]MBReleaseRef, error) BrowseReleases(ctx context.Context, releaseGroupMBID string) ([]MBRelease, error) LookupRelease(ctx context.Context, releaseMBID string) (MBRelease, error) LookupReleaseGroup(ctx context.Context, releaseGroupMBID string) (MBReleaseGroupHit, error) - LookupArtist(ctx context.Context, mbid string) (string, error) // returns sort name or name } +// mbidVariousArtists is the MusicBrainz artist MBID for the special +// "Various Artists" entity — used as an arid filter when a group +// looks like a compilation. +const mbidVariousArtists = "89ad4ac3-39f7-470e-963a-56509c546377" + +// cascadeSufficient is the merged-best score at which the cascade +// stops issuing looser queries. "First step with any hits" is the +// wrong stop condition — a strict query can return plausible-but- +// wrong release groups and starve the looser steps of the chance to +// surface the right one. Scoring is free; searches and browses are +// rate-limited network calls, so the cascade pays for another step +// only while the best candidate so far is still mediocre. +const cascadeSufficient = 0.70 + +// hitBrowseFloor is the minimum title-or-artist similarity a search +// hit needs before the resolver pays a rate-limited BrowseReleases +// call for it. Hits failing both checks are junk from MB's fuzzy +// tokenizer. +const hitBrowseFloor = 0.30 + // MBResolver orchestrates MusicBrainz lookups for a tagging group. -// Strategy: normalize the user-provided album/artist first, then -// issue a cascade of progressively looser Lucene queries, stopping -// at the first one that yields enough candidates. +// Strategy: use recording MBIDs already present in local tags when +// possible (exact, cheap); otherwise issue a cascade of +// progressively looser Lucene queries, merging results until a +// candidate scores well enough to stop. type MBResolver struct { client MBClient logger *slog.Logger @@ -73,27 +118,21 @@ type mbQueryStep struct { } // ResolveMB returns MB-sourced candidates for a tagging group. -// Runs a cascade of Lucene queries, returning at the first step -// that produces results. Each search hit fans out to -// BrowseReleases (one per release-group) for track-level data. -func (r *MBResolver) ResolveMB( - ctx context.Context, - albumName, albumArtist string, - trackCount int, - knownArtistMBID string, -) ([]Candidate, error) { - if albumName == "" { - return nil, nil - } - - nAlbum := Normalize(albumName) - nArtist := Normalize(albumArtist) - +// Runs the Lucene query cascade, accumulating deduplicated +// candidates across steps and stopping once the best merged +// candidate scores at least cascadeSufficient against the group. +func (r *MBResolver) ResolveMB(ctx context.Context, g Group) ([]Candidate, error) { + nAlbum := Normalize(g.AlbumName) if nAlbum == "" { return nil, nil } - steps := buildMBQueryCascade(nAlbum, nArtist, trackCount, knownArtistMBID) + nArtist := Normalize(groupArtist(g)) + steps := buildMBQueryCascade(nAlbum, nArtist, len(g.Tracks), vaLikely(g)) + + seen := make(map[string]bool) + + var merged []Candidate for _, step := range steps { hits, _, err := r.client.SearchReleaseGroups(ctx, step.query, r.limit) @@ -106,35 +145,65 @@ func (r *MBResolver) ResolveMB( continue } - if len(hits) == 0 { + added := r.fanOutBrowse(ctx, g, hits, step.label, seen, &merged) + + r.logger.Debug( + "MB search step done", + "step", step.label, "hits", len(hits), "new_candidates", added, + ) + + if added == 0 { + continue + } + + // Score what we have so far; good enough means the looser + // (noisier, costlier) steps aren't needed. + ranked := RankCandidates(g, merged) + if len(ranked) > 0 && ranked[0].Score >= cascadeSufficient { + r.logger.Info( + "MB cascade stopped — sufficient candidate", + "step", step.label, "score", ranked[0].Score, + ) + + break + } + } + + return merged, nil +} + +// fanOutBrowse iterates search hits, fetches each plausible +// release-group's releases, and appends previously-unseen ones to +// merged as Candidates. Returns how many candidates were added. +// Errors on individual browses are logged and skipped. +func (r *MBResolver) fanOutBrowse( + ctx context.Context, + g Group, + hits []MBReleaseGroupHit, + step string, + seen map[string]bool, + merged *[]Candidate, +) int { + added := 0 + + for _, h := range hits { + if seen["rg:"+h.MBID] { + continue + } + + seen["rg:"+h.MBID] = true + + // Don't pay a rate-limited browse for a hit that resembles + // neither the folder's album name nor its artist. + if !hitPlausible(g, h) { r.logger.Debug( - "MB search step empty — trying next", - "step", step.label, "query", step.query, + "skipping implausible search hit", + "title", h.Title, "artist", h.ArtistCredit, ) continue } - r.logger.Info( - "MB search step succeeded", - "step", step.label, "hits", len(hits), - ) - - return r.fanOutBrowse(ctx, hits, step.label), nil - } - - return nil, nil -} - -// fanOutBrowse iterates search hits, fetches each release-group's -// releases, and returns them as Candidates. Errors on individual -// browses are logged and skipped. -func (r *MBResolver) fanOutBrowse( - ctx context.Context, hits []MBReleaseGroupHit, step string, -) []Candidate { - var out []Candidate - - for _, h := range hits { releases, err := r.client.BrowseReleases(ctx, h.MBID) if err != nil { r.logger.Warn( @@ -146,11 +215,105 @@ func (r *MBResolver) fanOutBrowse( } for _, rel := range releases { - out = append(out, mkCandidate(h, rel, step)) + if rel.MBID != "" && seen[rel.MBID] { + continue + } + + seen[rel.MBID] = true + + *merged = append(*merged, mkCandidate(h, rel, step)) + added++ } } - return out + return added +} + +// hitPlausible reports whether a release-group search hit is worth +// a BrowseReleases round-trip: its title or artist must bear at +// least a loose resemblance to the group's. Unknown local fields +// never disqualify a hit. +func hitPlausible(g Group, h MBReleaseGroupHit) bool { + if g.AlbumName != "" && h.Title != "" && + titleSimilarity(g.AlbumName, h.Title) >= hitBrowseFloor { + return true + } + + artist := groupArtist(g) + if artist != "" && h.ArtistCredit != "" && + titleSimilarity(artist, h.ArtistCredit) >= hitBrowseFloor { + return true + } + + // Nothing to compare against (or a VA credit): stay permissive. + return g.AlbumName == "" || h.Title == "" || isVAName(h.ArtistCredit) +} + +// ResolveByRecordingMBIDs resolves candidates from recording MBIDs +// already present in the local tags — the highest-precision signal +// available, and the reason previously-tagged files should never +// need a fuzzy search. Each recording is looked up, releases are +// counted as votes, and the best-voted release (Official and +// earliest among ties) is resolved in full with provenance "id". +// Returns nil when no recording resolves to any release. +func (r *MBResolver) ResolveByRecordingMBIDs( + ctx context.Context, recordingMBIDs []string, +) ([]Candidate, error) { + votes := make(map[string]int) + refs := make(map[string]MBReleaseRef) + + for _, id := range recordingMBIDs { + rels, err := r.client.LookupRecordingReleases(ctx, id) + if err != nil { + r.logger.Warn( + "recording lookup failed — skipping", + "recording_mbid", id, "err", err, + ) + + continue + } + + counted := make(map[string]bool, len(rels)) + + for _, ref := range rels { + if ref.MBID == "" || counted[ref.MBID] { + continue + } + + counted[ref.MBID] = true + votes[ref.MBID]++ + refs[ref.MBID] = ref + } + } + + if len(votes) == 0 { + return nil, nil + } + + // Highest vote count wins; betterRelease breaks ties so the + // pick is deterministic and favours Official + earliest. + var ( + bestMBID string + bestVotes int + ) + + for mbid, n := range votes { + switch { + case n > bestVotes: + bestMBID, bestVotes = mbid, n + case n == bestVotes && betterRelease(refs[mbid], refs[bestMBID]): + bestMBID = mbid + } + } + + cand, err := r.ResolveOneReleaseMBID(ctx, bestMBID) + if err != nil { + return nil, fmt.Errorf("resolve voted release %s: %w", bestMBID, err) + } + + cand.Provenance = "id" + + return []Candidate{cand}, nil } // ResolveOneReleaseMBID fetches a single release by MBID and @@ -208,6 +371,7 @@ func mkCandidate(h MBReleaseGroupHit, rel MBRelease, step string) Candidate { OriginalDate: h.FirstDate, Country: rel.Country, Status: rel.Status, + PrimaryType: h.PrimaryType, TrackCount: len(rel.Tracks), Tracks: rel.Tracks, Source: SourceMusicBrainz, @@ -215,19 +379,131 @@ func mkCandidate(h MBReleaseGroupHit, rel MBRelease, step string) Candidate { } } +// SearchReleaseGroupHits runs a single release-group search from a +// user-supplied album + artist (the in-app "suggest a candidate" +// path). Both fields are normalized and phrase-quoted; artist is +// dropped from the query when empty. +func (r *MBResolver) SearchReleaseGroupHits( + ctx context.Context, album, artist string, +) ([]MBReleaseGroupHit, error) { + query := "release:" + luceneQuote(Normalize(album)) + if a := Normalize(artist); a != "" { + query += " AND artist:" + luceneQuote(a) + } + + hits, _, err := r.client.SearchReleaseGroups(ctx, query, r.limit) + if err != nil { + return nil, fmt.Errorf("search release groups: %w", err) + } + + return hits, nil +} + +// SearchRecordingHits runs a single recording search from a user- +// supplied title + artist — the singleton path, where the folder has +// one track and release-group search is too coarse. +func (r *MBResolver) SearchRecordingHits( + ctx context.Context, title, artist string, +) ([]MBRecordingHit, error) { + query := "recording:" + luceneQuote(Normalize(title)) + if a := Normalize(artist); a != "" { + query += " AND artist:" + luceneQuote(a) + } + + hits, _, err := r.client.SearchRecordings(ctx, query, r.limit) + if err != nil { + return nil, fmt.Errorf("search recordings: %w", err) + } + + return hits, nil +} + +// ResolveOneRecordingMBID turns a picked recording into a fully-scored +// Candidate by resolving it to a representative release (so the +// existing release-based diff + Apply pipeline works unchanged). +// Picks the release the same way a human would default: prefer an +// Official status, then the earliest date. Provenance is +// "search-recording" so the UI can label where it came from. +func (r *MBResolver) ResolveOneRecordingMBID( + ctx context.Context, recordingMBID string, +) (Candidate, error) { + refs, err := r.client.LookupRecordingReleases(ctx, recordingMBID) + if err != nil { + return Candidate{}, fmt.Errorf("lookup recording releases: %w", err) + } + + best := pickRepresentativeRelease(refs) + if best.MBID == "" { + return Candidate{}, fmt.Errorf("%w: %s", errNoReleasesForRecording, recordingMBID) + } + + cand, err := r.ResolveOneReleaseMBID(ctx, best.MBID) + if err != nil { + return Candidate{}, err + } + + cand.Provenance = "search-recording" + + return cand, nil +} + +// pickRepresentativeRelease chooses the release most likely to be the +// one the user means: an Official release beats a non-Official one, +// and among equals the earliest date wins (favouring the original +// over later reissues). Returns the zero value for an empty slice. +func pickRepresentativeRelease(refs []MBReleaseRef) MBReleaseRef { + var best MBReleaseRef + + for _, ref := range refs { + if best.MBID == "" || betterRelease(ref, best) { + best = ref + } + } + + return best +} + +// betterRelease reports whether a should be preferred over b. +func betterRelease(a, b MBReleaseRef) bool { + aOfficial := strings.EqualFold(a.Status, "Official") + bOfficial := strings.EqualFold(b.Status, "Official") + + if aOfficial != bOfficial { + return aOfficial + } + + // Same official-ness: earlier date wins. Empty dates sort last + // so a dated release beats an undated one. + switch { + case a.Date == "": + return false + case b.Date == "": + return true + default: + return a.Date < b.Date + } +} + +// errNoReleasesForRecording signals a recording that resolved to zero +// releases — nothing to diff or apply against. +var errNoReleasesForRecording = errors.New("autotag: recording has no releases") + // buildMBQueryCascade returns the Lucene queries to try in order. // Cascade: // -// 1. Full: release + arid/artist + tracks:N +// 1. Full: release + artist (or VA arid) + tracks:N // 2. Drop tracks:N (bonus tracks, live editions, etc.) // 3. Drop artist entirely (wrong artist tag is common) // 4. Fuzzy title (unquoted; Lucene does token/prefix match) // -// Each step is only added when it would differ from the previous. +// VA-likely groups filter on the Various Artists arid instead of an +// artist name — per-track artists on a compilation say nothing +// about the release's artist credit. Each step is only added when +// it would differ from the previous. func buildMBQueryCascade( normAlbum, normArtist string, trackCount int, - artistMBID string, + va bool, ) []mbQueryStep { var steps []mbQueryStep @@ -235,8 +511,8 @@ func buildMBQueryCascade( artistClause := "" switch { - case artistMBID != "": - artistClause = "arid:" + artistMBID + case va: + artistClause = "arid:" + mbidVariousArtists case normArtist != "": artistClause = "artist:" + luceneQuote(normArtist) } diff --git a/backend/autotag/mb_test.go b/backend/autotag/mb_test.go index 6cd7207..89e500a 100644 --- a/backend/autotag/mb_test.go +++ b/backend/autotag/mb_test.go @@ -15,11 +15,14 @@ var errFakeNotFound = errors.New("fake: not found") // cascade tests orchestrate `searchByStep` so step N returns hits // only when the resolver has already tried steps < N. type fakeMBClient struct { - queries []string - searchByStep map[int][]MBReleaseGroupHit - browseByMBID map[string][]MBRelease - lookupRels map[string]MBRelease - lookupRGs map[string]MBReleaseGroupHit + queries []string + searchByStep map[int][]MBReleaseGroupHit + browseByMBID map[string][]MBRelease + browseCalls map[string]int + lookupRels map[string]MBRelease + lookupRGs map[string]MBReleaseGroupHit + searchRecs []MBRecordingHit + recRelsByMBID map[string][]MBReleaseRef } func (f *fakeMBClient) SearchReleaseGroups( @@ -36,6 +39,12 @@ func (f *fakeMBClient) SearchReleaseGroups( func (f *fakeMBClient) BrowseReleases( _ context.Context, mbid string, ) ([]MBRelease, error) { + if f.browseCalls == nil { + f.browseCalls = make(map[string]int) + } + + f.browseCalls[mbid]++ + return f.browseByMBID[mbid], nil } @@ -61,8 +70,18 @@ func (f *fakeMBClient) LookupReleaseGroup( return rg, nil } -func (f *fakeMBClient) LookupArtist(_ context.Context, _ string) (string, error) { - return "", nil +func (f *fakeMBClient) SearchRecordings( + _ context.Context, query string, _ int, +) ([]MBRecordingHit, int, error) { + f.queries = append(f.queries, query) + + return f.searchRecs, len(f.searchRecs), nil +} + +func (f *fakeMBClient) LookupRecordingReleases( + _ context.Context, mbid string, +) ([]MBReleaseRef, error) { + return f.recRelsByMBID[mbid], nil } func TestBuildMBQueryCascade_StepsOrder(t *testing.T) { @@ -70,7 +89,7 @@ func TestBuildMBQueryCascade_StepsOrder(t *testing.T) { // Normalize() is a no-op for these inputs — ASCII, no // qualifier suffix, no punctuation — so cascade builds directly. - steps := buildMBQueryCascade("abbey road", "the beatles", 17, "") + steps := buildMBQueryCascade("abbey road", "the beatles", 17, false) if len(steps) < 3 { t.Fatalf("expected ≥3 cascade steps, got %d", len(steps)) @@ -94,7 +113,7 @@ func TestBuildMBQueryCascade_NormalizesInputs(t *testing.T) { // Qualifier suffix "(Remastered 2009)" must be stripped by // the *caller*; verify the query emitter doesn't reintroduce it. - steps := buildMBQueryCascade("abbey road", "", 0, "") + steps := buildMBQueryCascade("abbey road", "", 0, false) for _, step := range steps { if strings.Contains(strings.ToLower(step.query), "remastered") { t.Errorf("step %q should not contain qualifier: %q", step.label, step.query) @@ -102,26 +121,56 @@ func TestBuildMBQueryCascade_NormalizesInputs(t *testing.T) { } } -func TestMBResolver_CascadeStopsOnFirstHit(t *testing.T) { +func TestBuildMBQueryCascade_VariousArtists(t *testing.T) { + t.Parallel() + + // VA-likely groups must filter on the Various Artists arid, not + // on whatever plurality artist the compilation's tracks have. + steps := buildMBQueryCascade("now that's music", "artist one", 12, true) + + if !strings.Contains(steps[0].query, "arid:"+mbidVariousArtists) { + t.Errorf("VA step 1 should carry the VA arid, got %q", steps[0].query) + } + + if strings.Contains(steps[0].query, "artist:") { + t.Errorf("VA step 1 must not carry an artist: clause, got %q", steps[0].query) + } +} + +// abbeyRoadGroup is a group whose single track matches the rg1 +// fixture release well enough to clear cascadeSufficient. +func abbeyRoadGroup() Group { + return Group{ + AlbumName: "Abbey Road", + AlbumArtist: "The Beatles", + Tracks: []LocalTrack{{ + Title: "Come Together", TrackNumber: 1, LengthMillis: 259000, + }}, + } +} + +func TestMBResolver_CascadeStopsWhenSufficient(t *testing.T) { t.Parallel() fake := &fakeMBClient{ searchByStep: map[int][]MBReleaseGroupHit{ - // step 0 (strict) returns nothing; step 1 (no-track-count) hits. + // step 0 (strict) returns nothing; step 1 (no-track-count) + // hits with a release that scores well against the group. 1: {{MBID: "rg1", Title: "Abbey Road"}}, }, browseByMBID: map[string][]MBRelease{ - "rg1": {{MBID: "rel1", Title: "Abbey Road", Tracks: []CandidateTrack{ - {Position: 1, Title: "Come Together"}, - }}}, + "rg1": {{ + MBID: "rel1", Title: "Abbey Road", Status: "Official", + Tracks: []CandidateTrack{ + {Position: 1, Title: "Come Together", LengthMillis: 259000}, + }, + }}, }, } r := NewMBResolver(fake, slog.New(slog.DiscardHandler)) - cands, err := r.ResolveMB( - context.Background(), "Abbey Road", "The Beatles", 17, "", - ) + cands, err := r.ResolveMB(context.Background(), abbeyRoadGroup()) if err != nil { t.Fatalf("ResolveMB: %v", err) } @@ -139,13 +188,123 @@ func TestMBResolver_CascadeStopsOnFirstHit(t *testing.T) { } } +func TestMBResolver_CascadeContinuesPastMediocreHits(t *testing.T) { + t.Parallel() + + // Step 0 returns a same-title release whose track list doesn't + // match the folder at all — plausible enough to browse, but it + // must NOT stop the cascade ("first non-empty step wins" was the + // old, wrong behavior). Step 1 surfaces the real album; both + // candidates come back merged. + fake := &fakeMBClient{ + searchByStep: map[int][]MBReleaseGroupHit{ + 0: {{MBID: "rg-decoy", Title: "Abbey Road"}}, + 1: {{MBID: "rg-real", Title: "Abbey Road"}}, + }, + browseByMBID: map[string][]MBRelease{ + "rg-decoy": {{ + MBID: "rel-decoy", Title: "Abbey Road", + Tracks: []CandidateTrack{ + {Position: 1, Title: "Something Else Entirely", LengthMillis: 111000}, + {Position: 2, Title: "Not It Either", LengthMillis: 122000}, + }, + }}, + "rg-real": {{ + MBID: "rel-real", Title: "Abbey Road", Status: "Official", + Tracks: []CandidateTrack{ + {Position: 1, Title: "Come Together", LengthMillis: 259000}, + }, + }}, + }, + } + + r := NewMBResolver(fake, slog.New(slog.DiscardHandler)) + + cands, err := r.ResolveMB(context.Background(), abbeyRoadGroup()) + if err != nil { + t.Fatalf("ResolveMB: %v", err) + } + + if len(cands) != 2 { //nolint:mnd + t.Fatalf("expected merged candidates from both steps, got %d", len(cands)) + } + + if len(fake.queries) < 2 { //nolint:mnd + t.Errorf( + "cascade should have continued past the decoy step, got %d queries", + len(fake.queries), + ) + } +} + +func TestMBResolver_CascadeBrowsesEachReleaseGroupOnce(t *testing.T) { + t.Parallel() + + // The same release group surfacing at multiple cascade steps must + // only be browsed (and returned) once. + fake := &fakeMBClient{ + searchByStep: map[int][]MBReleaseGroupHit{ + 0: {{MBID: "rg-dup", Title: "Abbey Road"}}, + 1: {{MBID: "rg-dup", Title: "Abbey Road"}}, + 2: {{MBID: "rg-dup", Title: "Abbey Road"}}, + 3: {{MBID: "rg-dup", Title: "Abbey Road"}}, + }, + browseByMBID: map[string][]MBRelease{ + // Poor track match so the cascade keeps going. + "rg-dup": {{ + MBID: "rel-dup", Title: "Abbey Road", + Tracks: []CandidateTrack{ + {Position: 1, Title: "Unrelated", LengthMillis: 100000}, + }, + }}, + }, + } + + r := NewMBResolver(fake, slog.New(slog.DiscardHandler)) + + cands, err := r.ResolveMB(context.Background(), abbeyRoadGroup()) + if err != nil { + t.Fatalf("ResolveMB: %v", err) + } + + if len(cands) != 1 { + t.Fatalf("expected 1 deduplicated candidate, got %d", len(cands)) + } + + if fake.browseCalls["rg-dup"] != 1 { + t.Errorf("browse calls for rg-dup = %d, want 1", fake.browseCalls["rg-dup"]) + } +} + +func TestMBResolver_SkipsImplausibleHits(t *testing.T) { + t.Parallel() + + // A hit resembling neither the album name nor the artist must + // not cost a browse round-trip. + fake := &fakeMBClient{ + searchByStep: map[int][]MBReleaseGroupHit{ + 0: {{MBID: "rg-junk", Title: "Polka Party Hits", ArtistCredit: "Zzyzx Ensemble"}}, + }, + } + + r := NewMBResolver(fake, slog.New(slog.DiscardHandler)) + + if _, err := r.ResolveMB(context.Background(), abbeyRoadGroup()); err != nil { + t.Fatalf("ResolveMB: %v", err) + } + + if fake.browseCalls["rg-junk"] != 0 { + t.Errorf("junk hit was browsed %d times, want 0", fake.browseCalls["rg-junk"]) + } +} + func TestMBResolver_AbortsOnEmptyAlbumName(t *testing.T) { t.Parallel() fake := &fakeMBClient{} r := NewMBResolver(fake, slog.New(slog.DiscardHandler)) - cands, err := r.ResolveMB(context.Background(), "", "", 0, "") + cands, err := r.ResolveMB(context.Background(), Group{}) if err != nil { t.Fatalf("ResolveMB: %v", err) } @@ -191,3 +350,134 @@ func TestMBResolver_ResolveOneReleaseMBID(t *testing.T) { t.Errorf("expected 1 track, got %d", len(cand.Tracks)) } } + +func TestResolveByRecordingMBIDs_VotesAcrossRecordings(t *testing.T) { + t.Parallel() + + // rec-1 and rec-2 both appear on rel-shared; rec-1 also appears + // on rel-solo. The shared release gets 2 votes and wins. + fake := &fakeMBClient{ + recRelsByMBID: map[string][]MBReleaseRef{ + "rec-1": { + {MBID: "rel-shared", Status: "Official", Date: "1969"}, + {MBID: "rel-solo", Status: "Official", Date: "1968"}, + }, + "rec-2": { + {MBID: "rel-shared", Status: "Official", Date: "1969"}, + }, + }, + lookupRels: map[string]MBRelease{ + "rel-shared": { + MBID: "rel-shared", Title: "Abbey Road", + Tracks: []CandidateTrack{ + {Position: 1, Title: "Come Together", MBID: "rec-1"}, + {Position: 2, Title: "Something", MBID: "rec-2"}, + }, + }, + }, + } + + r := NewMBResolver(fake, slog.New(slog.DiscardHandler)) + + cands, err := r.ResolveByRecordingMBIDs( + context.Background(), []string{"rec-1", "rec-2"}, + ) + if err != nil { + t.Fatalf("ResolveByRecordingMBIDs: %v", err) + } + + if len(cands) != 1 { + t.Fatalf("expected 1 candidate, got %d", len(cands)) + } + + if cands[0].ReleaseMBID != "rel-shared" { + t.Errorf("release = %q, want 'rel-shared' (2 votes beats 1)", cands[0].ReleaseMBID) + } + + if cands[0].Provenance != "id" { + t.Errorf("provenance = %q, want 'id'", cands[0].Provenance) + } +} + +func TestResolveByRecordingMBIDs_NoResults(t *testing.T) { + t.Parallel() + + fake := &fakeMBClient{recRelsByMBID: map[string][]MBReleaseRef{}} + r := NewMBResolver(fake, slog.New(slog.DiscardHandler)) + + cands, err := r.ResolveByRecordingMBIDs(context.Background(), []string{"rec-x"}) + if err != nil { + t.Fatalf("ResolveByRecordingMBIDs: %v", err) + } + + if cands != nil { + t.Errorf("expected nil candidates for unknown recordings, got %d", len(cands)) + } +} + +func TestPickRepresentativeRelease(t *testing.T) { + t.Parallel() + + refs := []MBReleaseRef{ + {MBID: "promo-1970", Status: "Promotion", Date: "1970"}, + {MBID: "official-1975", Status: "Official", Date: "1975"}, + {MBID: "official-1969", Status: "Official", Date: "1969"}, + {MBID: "undated-official", Status: "Official", Date: ""}, + } + + // Official beats Promotion; among Official the earliest date wins. + best := pickRepresentativeRelease(refs) + if best.MBID != "official-1969" { + t.Errorf("best = %q, want 'official-1969'", best.MBID) + } + + if got := pickRepresentativeRelease(nil); got.MBID != "" { + t.Errorf("empty input = %+v, want zero", got) + } +} + +func TestResolveOneRecordingMBID_ResolvesToRepresentativeRelease(t *testing.T) { + t.Parallel() + + fake := &fakeMBClient{ + recRelsByMBID: map[string][]MBReleaseRef{ + "rec-1": { + {MBID: "rel-reissue", Status: "Official", Date: "2011"}, + {MBID: "rel-original", Status: "Official", Date: "1979"}, + }, + }, + lookupRels: map[string]MBRelease{ + "rel-original": { + MBID: "rel-original", Title: "The Wall", + Tracks: []CandidateTrack{{Position: 1, Title: "Hey You"}}, + }, + }, + } + + r := NewMBResolver(fake, slog.New(slog.DiscardHandler)) + + cand, err := r.ResolveOneRecordingMBID(context.Background(), "rec-1") + if err != nil { + t.Fatalf("ResolveOneRecordingMBID: %v", err) + } + + // Earliest official release chosen and resolved in full. + if cand.ReleaseMBID != "rel-original" { + t.Errorf("release = %q, want 'rel-original'", cand.ReleaseMBID) + } + + if cand.Provenance != "search-recording" { + t.Errorf("provenance = %q, want 'search-recording'", cand.Provenance) + } +} + +func TestResolveOneRecordingMBID_NoReleases(t *testing.T) { + t.Parallel() + + fake := &fakeMBClient{recRelsByMBID: map[string][]MBReleaseRef{}} + r := NewMBResolver(fake, slog.New(slog.DiscardHandler)) + + if _, err := r.ResolveOneRecordingMBID(context.Background(), "rec-x"); err == nil { + t.Fatal("expected error for recording with no releases") + } +} diff --git a/backend/autotag/normalize.go b/backend/autotag/normalize.go index 25dcd64..f033e6e 100644 --- a/backend/autotag/normalize.go +++ b/backend/autotag/normalize.go @@ -5,60 +5,128 @@ import ( "strings" "unicode" + "golang.org/x/text/runes" + "golang.org/x/text/transform" "golang.org/x/text/unicode/norm" ) +// qualifierAlternation is the regex alternation of parenthesized / +// dash-suffixed qualifiers that MB sometimes adds to titles but user +// tags often omit — e.g. `Remastered 2009`, `Bonus Track`, `feat. X`. +// Shared between the parenthesized and dash-suffix qualifier patterns. +const qualifierAlternation = `remaster(ed)?(\s+\d{4})?|` + + `re-?master(ed)?(\s+\d{4})?|` + + `\d{4}\s+remaster(ed)?|` + + `deluxe(\s+(edition|version))?|` + + `expanded(\s+(edition|version))?|` + + `anniversary(\s+(edition|version))?|` + + `explicit|` + + `clean|` + + `bonus\s+track|` + + `live(\s+at\s+[^\)\]]*)?|` + + `acoustic|` + + `radio\s+edit|` + + `single\s+version|` + + `album\s+version|` + + `original\s+mix|` + + `instrumental|` + + `demo|` + + `mono|` + + `stereo|` + + `feat\.?\s+[^\)\]]*|` + + `featuring\s+[^\)\]]*|` + + `ft\.?\s+[^\)\]]*` + // qualifierPattern matches common parenthesized / bracketed -// qualifiers that MB sometimes adds to titles but user tags often -// omit — e.g. `(Remastered 2009)`, `[Bonus Track]`, `(feat. X)`. -// Case-insensitive. Only strips when the qualifier is at a -// word-boundary to avoid mangling titles like "Untitled (1)". +// qualifiers. Case-insensitive. Only strips when the qualifier is +// at a word-boundary to avoid mangling titles like "Untitled (1)". var qualifierPattern = regexp.MustCompile( - `(?i)\s*[\(\[]\s*(` + - `remaster(ed)?(\s+\d{4})?|` + - `re-?master(ed)?(\s+\d{4})?|` + - `\d{4}\s+remaster(ed)?|` + - `deluxe(\s+edition)?|` + - `expanded(\s+edition)?|` + - `explicit|` + - `clean|` + - `bonus\s+track|` + - `live(\s+at\s+[^\)\]]*)?|` + - `acoustic|` + - `radio\s+edit|` + - `single\s+version|` + - `album\s+version|` + - `instrumental|` + - `demo|` + - `mono|` + - `stereo|` + - `feat\.?\s+[^\)\]]*|` + - `featuring\s+[^\)\]]*|` + - `ft\.?\s+[^\)\]]*` + - `)\s*[\)\]]`, + `(?i)\s*[\(\[]\s*(` + qualifierAlternation + `)\s*[\)\]]`, +) + +// dashQualifierPattern matches the dash-suffix form of the same +// qualifiers — `Song - 2009 Remaster`, `Song – Radio Edit` — which +// streaming-service-derived tags use instead of parentheses. +var dashQualifierPattern = regexp.MustCompile( + `(?i)\s+[-–—]\s+(` + qualifierAlternation + `)\s*$`, ) // whitespaceCollapse replaces runs of whitespace with a single space. var whitespaceCollapse = regexp.MustCompile(`\s+`) +// asciiSpecials maps letters that unicode decomposition alone can't +// reduce to ASCII (they aren't combining-mark compositions). +var asciiSpecials = strings.NewReplacer( + "ß", "ss", "ẞ", "SS", + "æ", "ae", "Æ", "AE", + "œ", "oe", "Œ", "OE", + "ø", "o", "Ø", "O", + "đ", "d", "Đ", "D", + "ð", "d", "Ð", "D", + "þ", "th", "Þ", "Th", + "ł", "l", "Ł", "L", + "ı", "i", +) + +// asciiFold transliterates accented characters to their closest +// ASCII equivalent ("Beyoncé" → "Beyonce", "Björk" → "Bjork") so +// diacritic differences between user tags and MB data don't count +// as edits. Non-Latin scripts pass through unchanged. +func asciiFold(s string) string { + // Fast path: nothing to fold in pure-ASCII strings. + ascii := true + + for i := range len(s) { + if s[i] >= 0x80 { + ascii = false + + break + } + } + + if ascii { + return s + } + + s = asciiSpecials.Replace(s) + + // NFKD splits accented letters into base + combining marks; + // dropping the marks (unicode.Mn) leaves the base letter. The + // chain is stateful, so build it per call — it's cheap and this + // keeps concurrent scorers safe. + t := transform.Chain(norm.NFKD, runes.Remove(runes.In(unicode.Mn)), norm.NFC) + + folded, _, err := transform.String(t, s) + if err != nil { + return s + } + + return folded +} + // Normalize returns a comparison-friendly form of a title or // artist-credit string: // -// - NFC unicode composition +// - ASCII transliteration (accents folded) // - qualifier suffixes stripped (see qualifierPattern) -// - all punctuation dropped +// - "&" replaced with "and" +// - all remaining punctuation dropped // - case folded (lowercased) // - whitespace collapsed and trimmed // // The result is not intended to be human-readable — only used for -// equality and edit-distance comparisons inside the scorer. +// equality comparisons (local candidate matching) and search query +// building. Fuzzy comparisons go through titleSimilarity, which +// keeps more structure. func Normalize(s string) string { if s == "" { return "" } - s = norm.NFC.String(s) + s = asciiFold(s) s = qualifierPattern.ReplaceAllString(s, "") + s = dashQualifierPattern.ReplaceAllString(s, "") + s = strings.ReplaceAll(s, "&", " and ") var b strings.Builder diff --git a/backend/autotag/normalize_test.go b/backend/autotag/normalize_test.go index b7e6895..d8ca33b 100644 --- a/backend/autotag/normalize_test.go +++ b/backend/autotag/normalize_test.go @@ -20,12 +20,16 @@ func TestNormalize(t *testing.T) { "Sgt. Pepper's Lonely Hearts Club Band!", "sgt peppers lonely hearts club band", }, - "NFC unicode": {"Beyoncé", "beyoncé"}, + "accents fold to ascii": {"Beyoncé", "beyonce"}, + "eszett folds": {"Motörhead & Björk", "motorhead and bjork"}, + "ampersand becomes and": {"Simon & Garfunkel", "simon and garfunkel"}, "remastered qualifier": {"Abbey Road (Remastered 2009)", "abbey road"}, "remaster no year": {"Abbey Road (Remaster)", "abbey road"}, "explicit qualifier": {"Lemonade [Explicit]", "lemonade"}, "feat qualifier": {"Yellow (feat. Coldplay)", "yellow"}, "bonus track qualifier": {"Hey Jude [Bonus Track]", "hey jude"}, + "dash suffix qualifier": {"Hey Jude - 2015 Remaster", "hey jude"}, + "dash radio edit": {"One More Time - Radio Edit", "one more time"}, "collapse whitespace": {" Abbey Road ", "abbey road"}, "non-ascii digits kept": {"Track 7", "track 7"}, "numbered title not mangled": {"Untitled (1)", "untitled 1"}, diff --git a/backend/autotag/rank.go b/backend/autotag/rank.go index a498e55..ad3c358 100644 --- a/backend/autotag/rank.go +++ b/backend/autotag/rank.go @@ -8,23 +8,90 @@ import ( // Release-level scoring weights. Aggregate track score is the // dominant signal — the release-level signals are tie-breakers -// when the track alignment is roughly comparable. +// when the track alignment is roughly comparable. Weights sum to +// 1.0 so a perfect candidate scores 1.0 before the evidence scale. const ( - weightTrackAggregate = 0.70 - weightTrackCountMatch = 0.15 - weightReleaseMeta = 0.15 // Official + country averaged + weightTrackAggregate = 0.55 + weightArtist = 0.12 // album-artist vs candidate artist-credit + weightAlbumTitle = 0.10 // folder album name vs candidate release title + weightTrackCountMatch = 0.13 + weightReleaseMeta = 0.10 // official + country + RG type, averaged // Country preference: a very mild nudge toward releases from // the user's locale. Will become a config option in 012. preferredCountry = "US" + + // Evidence scaling: a folder with very few tracks offers little + // corroborating signal, so even a perfect title+length match on + // a single track is inherently less trustworthy than the same + // match across a full album. The final release score is scaled + // by evidenceFactor(localTrackCount): folders at or above + // evidenceFullTracks are unscaled; smaller folders are pulled + // toward evidenceFloor. This is the "harsher on singletons" + // lever — a single can never present as a near-certain match on + // its own, which is also why 012 keeps singletons out of + // auto-accept entirely. + evidenceFloor = 0.85 + evidenceFullTracks = 3 ) +// vaNames are artist strings that signal "various artists" — used +// both to detect VA-likely folders and to recognize VA candidate +// credits. Mirrors beets' VA_ARTISTS. +var vaNames = map[string]bool{ + "various artists": true, + "various": true, + "va": true, + "v a": true, // "V.A." after Normalize + "unknown": true, +} + +// isVAName reports whether an artist string reads as "various +// artists". Empty strings are NOT VA — they're unknown, which the +// artist term already treats as neutral. +func isVAName(s string) bool { + return vaNames[Normalize(s)] +} + +// vaLikely reports whether a group is probably a various-artists +// compilation: the album-artist tag says so outright, or the +// per-track artists have no consensus (≥2 distinct values, or none +// at all). Mirrors beets' va_likely heuristic. +func vaLikely(g Group) bool { + if isVAName(g.AlbumArtist) { + return true + } + + if g.AlbumArtist != "" { + return false + } + + distinct := make(map[string]bool, 2) //nolint:mnd + + for _, t := range g.Tracks { + if t.Artist == "" { + continue + } + + distinct[Normalize(t.Artist)] = true + } + + return len(distinct) != 1 +} + // ScoreCandidate fills in c.Alignments, c.Score, c.Breakdown, and -// c.TrackCount for a single candidate against the given local -// tracks. The returned Candidate is safe to copy — no shared -// state with the caller's slice. -func ScoreCandidate(local []LocalTrack, c Candidate, localTrackCount int) Candidate { - c.Alignments = AlignTracks(local, c.Tracks) +// c.TrackCount for a single candidate against the given group. +// The returned Candidate is safe to copy — no shared state with +// the caller's slice. +func ScoreCandidate(g Group, c Candidate) Candidate { + local := g.Tracks + + // When the group is one disc of a multi-disc candidate, align + // and count against that disc only — a "disc 1 of 2" folder is + // complete for its disc, not half an album. + targets := alignmentTargets(local, c.Tracks) + + c.Alignments = AlignTracks(local, targets) var ( titleSum float64 @@ -38,10 +105,19 @@ func ScoreCandidate(local []LocalTrack, c Candidate, localTrackCount int) Candid } counted++ - titleSum += a.TitleScore - l := local[a.LocalIndex] - lengthSum += lengthScore(l.LengthMillis, a.CandidateLength) + // A recording-MBID lock is identity, not similarity: the + // title may be garbled in the local tag, but the track IS + // the candidate's track. Count it as a perfect title so a + // confirmed match isn't dragged down by its own typos (the + // UI still shows the textual diff). + if a.IDMatch { + titleSum++ + } else { + titleSum += a.TitleScore + } + + lengthSum += a.LengthScore } titleAvg, lengthAvg := 0.0, 0.0 @@ -63,31 +139,213 @@ func ScoreCandidate(local []LocalTrack, c Candidate, localTrackCount int) Candid trackAgg := ((titleAvg*weightTitle + lengthAvg*weightLength) / trackWeightSum) * coverage - trackCountScore := trackCountMatch(len(c.Tracks), localTrackCount) - // Release-meta is just official-status + country preference, - // averaged. We used to mix in a year bonus too, but that - // compared candidate years against time.Now() — penalising - // every album that wasn't from this year, regardless of how - // well it matched the local files. See git history. - const metaTerms = 2.0 + trackCountScore := trackCountMatch(len(targets), len(local)) - meta := (officialBonus(c.Status) + countryBonus(c.Country)) / metaTerms + // Artist fit: compare the folder's artist against the + // candidate's release artist-credit. This is a SOFT signal, not + // a gate — user artist tags are often slightly wrong (misspelled, + // "&" vs "and", missing "feat."), so an almost-right artist still + // matches well while a completely different artist is penalised. + // Critically, artist is otherwise only a *search* filter (see + // buildMBQueryCascade), and that cascade drops the artist clause + // on its looser steps — so without this term a same-title, + // different-artist release scores as if the artist matched. + artistFit := artistCreditFit(groupArtist(g), c.ArtistCredit) - c.Score = trackAgg*weightTrackAggregate + + // Album-title fit: the same soft-signal contract for the release + // title. Without it, a compilation containing the same + // recordings scores as if it WERE the album ("Greatest Hits" vs + // the studio album with an identical tracklist). + albumFit := albumTitleFit(g.AlbumName, c.Title) + + // Release-meta: official-status + country preference + release- + // group type, averaged. All mild tie-breakers. (We used to mix + // in a year bonus too, but that compared candidate years against + // time.Now() — see git history.) + const metaTerms = 3.0 + + meta := (officialBonus(c.Status) + countryBonus(c.Country) + rgTypeBonus(c.PrimaryType)) / + metaTerms + + // Evidence scaling applies only to MusicBrainz candidates. A + // local candidate is the *same* release-group already tagged with + // MBIDs in another library — its confidence comes from that + // confirmed tagging, not from thin per-track heuristics, so a + // small-folder local match stays fully trusted (and keeps + // clearing the localSufficient MB-skip short-circuit). MB + // matches, by contrast, are fuzzy search results where a + // single-track folder genuinely offers little corroboration. + evidence := 1.0 + if c.Source == SourceMusicBrainz { + evidence = evidenceFactor(len(local)) + } + + c.Score = (trackAgg*weightTrackAggregate + + artistFit*weightArtist + + albumFit*weightAlbumTitle + trackCountScore*weightTrackCountMatch + - meta*weightReleaseMeta + meta*weightReleaseMeta) * evidence c.Breakdown = ScoreBreakdown{ TitleAvg: titleAvg, LengthAvg: lengthAvg, + ArtistFit: artistFit, + AlbumFit: albumFit, TrackCountFit: trackCountScore, ReleaseMeta: meta, + Evidence: evidence, } c.TrackCount = len(c.Tracks) return c } +// alignmentTargets returns the candidate tracks the local group +// should be aligned against. When every local track sits on the +// same disc D and the candidate spans multiple discs including D, +// only disc D's tracks are targets — the group key is per-disc, so +// a single-disc folder must not be penalised for "missing" the +// candidate's other discs. +func alignmentTargets(local []LocalTrack, cands []CandidateTrack) []CandidateTrack { + disc := uniformDisc(local) + if disc == 0 { + return cands + } + + var ( + onDisc int + multiDiscs bool + ) + + for _, c := range cands { + if c.DiscNumber == disc { + onDisc++ + } else if c.DiscNumber > 0 { + multiDiscs = true + } + } + + if !multiDiscs || onDisc == 0 { + return cands + } + + out := make([]CandidateTrack, 0, onDisc) + + for _, c := range cands { + if c.DiscNumber == disc { + out = append(out, c) + } + } + + return out +} + +// uniformDisc returns the disc number shared by every local track, +// or 0 when discs are mixed or unknown. +func uniformDisc(local []LocalTrack) int { + disc := 0 + + for _, t := range local { + switch { + case t.DiscNumber <= 0: + return 0 + case disc == 0: + disc = t.DiscNumber + case t.DiscNumber != disc: + return 0 + } + } + + return disc +} + +// groupArtist returns the artist string to compare candidates +// against: the tagging item's album-artist when it's a real name, +// otherwise the most common per-track artist. Returns "" when +// nothing is known (neutral, no penalty). +func groupArtist(g Group) string { + if g.AlbumArtist != "" && !isVAName(g.AlbumArtist) { + return g.AlbumArtist + } + + return dominantArtist(g.Tracks) +} + +// dominantArtist returns the most common non-empty per-track artist +// in a local group. Ties resolve to the first-seen value so the +// result is deterministic. Returns "" when no track has an artist, +// which artistCreditFit treats as "unknown, no penalty". +func dominantArtist(local []LocalTrack) string { + counts := make(map[string]int, len(local)) + + var ( + best string + bestCount int + ) + + for _, t := range local { + if t.Artist == "" { + continue + } + + counts[t.Artist]++ + if counts[t.Artist] > bestCount { + best = t.Artist + bestCount = counts[t.Artist] + } + } + + return best +} + +// artistCreditFit scores how well a folder's artist matches a +// candidate's release artist-credit, in [0, 1]. Returns 1.0 (no +// penalty) when either side is unknown or reads as "various +// artists": absence of artist data must not push a candidate down, +// and VA credits are placeholders, not disagreements. Reuses the +// edit-distance similarity so near-right artists stay high. +func artistCreditFit(localArtist, candidateArtist string) float64 { + if localArtist == "" || candidateArtist == "" { + return 1.0 + } + + if isVAName(localArtist) || isVAName(candidateArtist) { + return 1.0 + } + + return titleSimilarity(localArtist, candidateArtist) +} + +// albumTitleFit scores how well the folder's album name matches the +// candidate's release title, in [0, 1]. Neutral (1.0) when either +// side is unknown — same soft-signal contract as artistCreditFit. +func albumTitleFit(albumName, candidateTitle string) float64 { + if albumName == "" || candidateTitle == "" { + return 1.0 + } + + return titleSimilarity(albumName, candidateTitle) +} + +// evidenceFactor scales the release score down when a folder has too +// few tracks to corroborate the match. Folders at or above +// evidenceFullTracks are unscaled (1.0); a single-track folder is +// pulled to evidenceFloor; two tracks land halfway. See the +// evidence-scaling note on the weight constants. +func evidenceFactor(localTrackCount int) float64 { + if localTrackCount >= evidenceFullTracks { + return 1.0 + } + + if localTrackCount <= 1 { + return evidenceFloor + } + + span := float64(localTrackCount-1) / float64(evidenceFullTracks-1) + + return evidenceFloor + (1.0-evidenceFloor)*span +} + // trackCountMatch returns 1.0 when equal, 0.0 when off by >= 50%, // linear between. func trackCountMatch(a, b int) float64 { @@ -104,10 +362,7 @@ func trackCountMatch(a, b int) float64 { diff = -diff } - larger := a - if b > larger { - larger = b - } + larger := max(a, b) frac := float64(diff) / float64(larger) @@ -124,14 +379,11 @@ func trackCountMatch(a, b int) float64 { func officialBonus(status string) float64 { const partial = 0.5 - switch strings.ToLower(status) { - case "official": + if strings.EqualFold(status, "official") { return 1.0 - case "": - return partial - default: - return partial } + + return partial } // countryBonus gives a mild nudge toward releases from the @@ -153,6 +405,32 @@ func countryBonus(country string) float64 { return neutral } +// rgTypeBonus nudges toward studio albums over compilations and +// live releases when the track evidence is otherwise comparable — +// Picard weights release type heavily for the same reason. The +// nudge is mild: a genuine single folder still matches its Single +// release because track count and alignment dominate. Unknown +// types (including all local candidates) sit near the top so the +// term only separates candidates we positively know differ. +func rgTypeBonus(primaryType string) float64 { + switch strings.ToLower(primaryType) { + case "album": + return 1.0 + case "ep": + return 0.9 + case "single": + return 0.85 + case "": + return 0.85 + case "soundtrack": + return 0.7 + case "compilation", "live": + return 0.6 + default: + return 0.7 + } +} + // parseYear pulls the first 4-digit year out of date strings like // "2009", "2009-05-18", "". func parseYear(date string) int { @@ -168,13 +446,13 @@ func parseYear(date string) int { return y } -// RankCandidates scores each candidate against the local tracks -// and returns a new slice sorted descending by score. Input slice -// is not modified. -func RankCandidates(local []LocalTrack, candidates []Candidate) []Candidate { +// RankCandidates scores each candidate against the group and +// returns a new slice sorted descending by score. Input slice is +// not modified. +func RankCandidates(g Group, candidates []Candidate) []Candidate { scored := make([]Candidate, 0, len(candidates)) for _, c := range candidates { - scored = append(scored, ScoreCandidate(local, c, len(local))) + scored = append(scored, ScoreCandidate(g, c)) } sort.SliceStable(scored, func(i, j int) bool { diff --git a/backend/autotag/rank_test.go b/backend/autotag/rank_test.go index c390593..53f59b2 100644 --- a/backend/autotag/rank_test.go +++ b/backend/autotag/rank_test.go @@ -37,7 +37,10 @@ func TestRankCandidates_PrefersExactTrackCountMatch(t *testing.T) { }, } - ranked := autotag.RankCandidates(local, []autotag.Candidate{longer, matching}) + ranked := autotag.RankCandidates( + autotag.Group{Tracks: local}, + []autotag.Candidate{longer, matching}, + ) if ranked[0].ReleaseMBID != "exact" { t.Errorf( "top = %q (score %.2f vs %.2f), want 'exact'", @@ -69,7 +72,10 @@ func TestRankCandidates_PrefersOfficial(t *testing.T) { Tracks: tracks, } - ranked := autotag.RankCandidates(local, []autotag.Candidate{promo, official}) + ranked := autotag.RankCandidates( + autotag.Group{Tracks: local}, + []autotag.Candidate{promo, official}, + ) if ranked[0].ReleaseMBID != "official" { t.Errorf("top = %q, want 'official'", ranked[0].ReleaseMBID) } @@ -98,7 +104,7 @@ func TestRankCandidates_MultiDisc(t *testing.T) { }, } - ranked := autotag.RankCandidates(local, []autotag.Candidate{cand}) + ranked := autotag.RankCandidates(autotag.Group{Tracks: local}, []autotag.Candidate{cand}) if ranked[0].Score < 0.75 { //nolint:mnd t.Errorf("multi-disc exact match = %.2f, want >= 0.75", ranked[0].Score) } @@ -127,12 +133,123 @@ func TestRankCandidates_VariousArtists(t *testing.T) { }, } - ranked := autotag.RankCandidates(local, []autotag.Candidate{cand}) + ranked := autotag.RankCandidates(autotag.Group{Tracks: local}, []autotag.Candidate{cand}) if ranked[0].Score < 0.75 { //nolint:mnd t.Errorf("VA-compilation exact track match = %.2f, want >= 0.75", ranked[0].Score) } } +func TestRankCandidates_PenalizesWrongArtistSingleton(t *testing.T) { + t.Parallel() + + // The reported false positive: a single with a generic title + // matched a same-titled song by a DIFFERENT artist with a very + // different length, at high confidence. Artist was only a search + // filter, never a scoring signal, so nothing pulled the wrong + // candidate down. Now the artist term + singleton evidence + // scaling must keep it well below a confident match. + local := []autotag.LocalTrack{ + {Title: "Intro", Artist: "Real Artist", TrackNumber: 1, LengthMillis: 90000}, + } + + // Same title, wrong artist, and a wildly different length — an MB + // search hit that happens to share a common title. + wrongArtist := autotag.Candidate{ + ReleaseMBID: "wrong", + Title: "Intro", + ArtistCredit: "Some Other Band", + Status: "Official", + Source: autotag.SourceMusicBrainz, + Tracks: []autotag.CandidateTrack{ + {Position: 1, Title: "Intro", LengthMillis: 240000}, + }, + } + + // The correct release: same title, right artist, right length. + rightArtist := autotag.Candidate{ + ReleaseMBID: "right", + Title: "Intro", + ArtistCredit: "Real Artist", + Status: "Official", + Source: autotag.SourceMusicBrainz, + Tracks: []autotag.CandidateTrack{ + {Position: 1, Title: "Intro", LengthMillis: 90000}, + }, + } + + ranked := autotag.RankCandidates( + autotag.Group{Tracks: local}, + []autotag.Candidate{wrongArtist, rightArtist}, + ) + + if ranked[0].ReleaseMBID != "right" { + t.Fatalf( + "top = %q (%.2f vs %.2f), want 'right'", + ranked[0].ReleaseMBID, ranked[0].Score, ranked[1].Score, + ) + } + + // The wrong-artist candidate must not read as a confident match. + var wrongScore float64 + + for _, c := range ranked { + if c.ReleaseMBID == "wrong" { + wrongScore = c.Score + } + } + + if wrongScore >= 0.75 { //nolint:mnd + t.Errorf("wrong-artist singleton scored %.2f, want < 0.75", wrongScore) + } +} + +func TestRankCandidates_EvidenceScalingIsSourceAware(t *testing.T) { + t.Parallel() + + // A perfect single-track match: identical title, artist, length. + // From MusicBrainz it should be evidence-scaled (thin corroboration + // on one track); from a local library it should NOT be — a local + // candidate is the same release-group already tagged with MBIDs + // elsewhere, so its confidence is not heuristic. + local := []autotag.LocalTrack{ + {Title: "Solo", Artist: "Someone", TrackNumber: 1, LengthMillis: 200000}, + } + + tracks := []autotag.CandidateTrack{ + {Position: 1, Title: "Solo", LengthMillis: 200000}, + } + + mbCand := autotag.Candidate{ + ReleaseMBID: "mb", ArtistCredit: "Someone", Status: "Official", + Source: autotag.SourceMusicBrainz, Tracks: tracks, + } + localCand := autotag.Candidate{ + ReleaseMBID: "local", ArtistCredit: "Someone", Status: "Official", + Source: autotag.SourceLocal, Tracks: tracks, + } + + scoredMB := autotag.RankCandidates(autotag.Group{Tracks: local}, []autotag.Candidate{mbCand})[0] + scoredLocal := autotag.RankCandidates(autotag.Group{Tracks: local}, []autotag.Candidate{localCand})[0] + + if scoredMB.Breakdown.Evidence >= 1.0 { + t.Errorf("MB singleton evidence = %.3f, want < 1.0", scoredMB.Breakdown.Evidence) + } + + if scoredLocal.Breakdown.Evidence < 1.0 { + t.Errorf( + "local singleton evidence = %.3f, want 1.0 (not scaled)", + scoredLocal.Breakdown.Evidence, + ) + } + + if scoredLocal.Score <= scoredMB.Score { + t.Errorf( + "local perfect single (%.3f) should outscore the evidence-scaled MB single (%.3f)", + scoredLocal.Score, scoredMB.Score, + ) + } +} + func TestRankCandidates_AmbiguousAlbumNames(t *testing.T) { t.Parallel() @@ -164,7 +281,10 @@ func TestRankCandidates_AmbiguousAlbumNames(t *testing.T) { }, } - ranked := autotag.RankCandidates(local, []autotag.Candidate{queen, eagles}) + ranked := autotag.RankCandidates( + autotag.Group{Tracks: local}, + []autotag.Candidate{queen, eagles}, + ) if ranked[0].ReleaseMBID != "eagles-gh" { t.Errorf( "top = %q (%.2f vs %.2f), want 'eagles-gh'", @@ -172,3 +292,180 @@ func TestRankCandidates_AmbiguousAlbumNames(t *testing.T) { ) } } + +func TestRankCandidates_AlbumTitleSeparatesCompilation(t *testing.T) { + t.Parallel() + + // Identical tracklists: the studio album and a greatest-hits comp + // that contains the same recordings. Track alignment can't + // separate them — the folder's album name must. + local := []autotag.LocalTrack{ + {Title: "Song A", Artist: "The Band", TrackNumber: 1, LengthMillis: 200000}, + {Title: "Song B", Artist: "The Band", TrackNumber: 2, LengthMillis: 210000}, + {Title: "Song C", Artist: "The Band", TrackNumber: 3, LengthMillis: 195000}, + } + + tracks := []autotag.CandidateTrack{ + {Position: 1, Title: "Song A", LengthMillis: 200000}, + {Position: 2, Title: "Song B", LengthMillis: 210000}, + {Position: 3, Title: "Song C", LengthMillis: 195000}, + } + + album := autotag.Candidate{ + ReleaseMBID: "studio", Title: "The Studio Album", + ArtistCredit: "The Band", Status: "Official", Tracks: tracks, + } + comp := autotag.Candidate{ + ReleaseMBID: "comp", Title: "Greatest Hits", + ArtistCredit: "The Band", Status: "Official", Tracks: tracks, + } + + g := autotag.Group{ + AlbumName: "The Studio Album", AlbumArtist: "The Band", Tracks: local, + } + + ranked := autotag.RankCandidates(g, []autotag.Candidate{comp, album}) + if ranked[0].ReleaseMBID != "studio" { + t.Errorf( + "top = %q (%.3f vs %.3f), want 'studio' (album-title term)", + ranked[0].ReleaseMBID, ranked[0].Score, ranked[1].Score, + ) + } +} + +func TestRankCandidates_ReleaseGroupTypeBreaksTies(t *testing.T) { + t.Parallel() + + // Same tracks, same title — one RG is an Album, the other a + // Compilation. Type preference should break the tie toward the + // studio album (Picard weights release type for the same reason). + local := []autotag.LocalTrack{ + {Title: "Song A", TrackNumber: 1, LengthMillis: 200000}, + {Title: "Song B", TrackNumber: 2, LengthMillis: 210000}, + {Title: "Song C", TrackNumber: 3, LengthMillis: 195000}, + } + + tracks := []autotag.CandidateTrack{ + {Position: 1, Title: "Song A", LengthMillis: 200000}, + {Position: 2, Title: "Song B", LengthMillis: 210000}, + {Position: 3, Title: "Song C", LengthMillis: 195000}, + } + + album := autotag.Candidate{ + ReleaseMBID: "album", Title: "X", Status: "Official", + PrimaryType: "Album", Tracks: tracks, + } + comp := autotag.Candidate{ + ReleaseMBID: "comp", Title: "X", Status: "Official", + PrimaryType: "Compilation", Tracks: tracks, + } + + ranked := autotag.RankCandidates( + autotag.Group{Tracks: local}, []autotag.Candidate{comp, album}, + ) + if ranked[0].ReleaseMBID != "album" { + t.Errorf( + "top = %q (%.3f vs %.3f), want 'album' (RG type preference)", + ranked[0].ReleaseMBID, ranked[0].Score, ranked[1].Score, + ) + } +} + +func TestRankCandidates_SingleDiscGroupAgainstMultiDiscRelease(t *testing.T) { + t.Parallel() + + // The group key is per-disc, so "disc 1 of 2" folders score + // against multi-disc releases. Alignment and track count must + // compare against disc 1's tracks only — not get punished for + // "missing" all of disc 2. + local := []autotag.LocalTrack{ + {Title: "D1T1", DiscNumber: 1, TrackNumber: 1, LengthMillis: 200000}, + {Title: "D1T2", DiscNumber: 1, TrackNumber: 2, LengthMillis: 210000}, + {Title: "D1T3", DiscNumber: 1, TrackNumber: 3, LengthMillis: 195000}, + } + + cand := autotag.Candidate{ + ReleaseMBID: "2disc", + Status: "Official", + Tracks: []autotag.CandidateTrack{ + {Position: 1, DiscNumber: 1, Title: "D1T1", LengthMillis: 200000}, + {Position: 2, DiscNumber: 1, Title: "D1T2", LengthMillis: 210000}, + {Position: 3, DiscNumber: 1, Title: "D1T3", LengthMillis: 195000}, + {Position: 1, DiscNumber: 2, Title: "D2T1", LengthMillis: 220000}, + {Position: 2, DiscNumber: 2, Title: "D2T2", LengthMillis: 230000}, + {Position: 3, DiscNumber: 2, Title: "D2T3", LengthMillis: 240000}, + }, + } + + ranked := autotag.RankCandidates( + autotag.Group{Tracks: local}, []autotag.Candidate{cand}, + ) + + top := ranked[0] + if top.Score < 0.85 { //nolint:mnd + t.Errorf("disc-1 folder vs 2-disc release = %.3f, want >= 0.85", top.Score) + } + + if top.Breakdown.TrackCountFit != 1.0 { + t.Errorf( + "track count fit = %.2f, want 1.0 (counted against disc 1 only)", + top.Breakdown.TrackCountFit, + ) + } + + // No "missing" rows for disc 2 — the folder is complete for its + // disc. + for _, a := range top.Alignments { + if a.Status == autotag.AlignmentMissing { + t.Errorf("unexpected missing alignment for %q", a.CandidateTitle) + } + } +} + +func TestRankCandidates_RecordingMBIDLocksAlignment(t *testing.T) { + t.Parallel() + + // The local title is garbled, but its recording MBID matches a + // candidate track — identity beats similarity: the pair must + // align, count as matched, and not drag the title average down. + local := []autotag.LocalTrack{ + {Title: "trck 01", RecordingMBID: "rec-a", TrackNumber: 1, LengthMillis: 200000}, + {Title: "Song B", TrackNumber: 2, LengthMillis: 210000}, + {Title: "Song C", TrackNumber: 3, LengthMillis: 195000}, + } + + cand := autotag.Candidate{ + ReleaseMBID: "rel", + Status: "Official", + Tracks: []autotag.CandidateTrack{ + {Position: 1, Title: "Song A", LengthMillis: 200000, MBID: "rec-a"}, + {Position: 2, Title: "Song B", LengthMillis: 210000}, + {Position: 3, Title: "Song C", LengthMillis: 195000}, + }, + } + + ranked := autotag.RankCandidates( + autotag.Group{Tracks: local}, []autotag.Candidate{cand}, + ) + + top := ranked[0] + + var locked *autotag.TrackAlignment + + for i := range top.Alignments { + if top.Alignments[i].LocalIndex == 0 { + locked = &top.Alignments[i] + } + } + + if locked == nil || locked.Status != autotag.AlignmentMatched || !locked.IDMatch { + t.Fatalf("garbled-title track should be ID-locked matched, got %+v", locked) + } + + if top.Breakdown.TitleAvg < 0.99 { + t.Errorf( + "title avg = %.3f, want ~1.0 (ID-locked pair counts as perfect title)", + top.Breakdown.TitleAvg, + ) + } +} diff --git a/backend/autotag/recommend.go b/backend/autotag/recommend.go new file mode 100644 index 0000000..ab68afa --- /dev/null +++ b/backend/autotag/recommend.go @@ -0,0 +1,133 @@ +package autotag + +// Recommendation is a qualitative confidence tier for a group's +// ranked candidates — the piece a raw score can't express on its +// own. Modeled on beets' Recommendation enum: the tier starts from +// the top candidate's absolute score and is then CAPPED by defects +// (ambiguity with a different release group, missing/unmatched +// tracks, thin evidence). Auto-accept (plan 011) should require +// RecommendationStrong; the review UI can badge the rest. +type Recommendation string + +// Recommendation tiers, weakest to strongest. +const ( + RecommendationNone Recommendation = "none" + RecommendationLow Recommendation = "low" + RecommendationMedium Recommendation = "medium" + RecommendationStrong Recommendation = "strong" +) + +const ( + // Absolute score tiers. + strongScoreThresh = 0.90 + mediumScoreThresh = 0.75 + + // A runner-up from a DIFFERENT release group within this margin + // of the top score makes the match ambiguous — two genuinely + // different albums both fit, so a human should look. Editions + // of the same release group are expected to score nearly + // identically and never count as ambiguity. + ambiguityMargin = 0.05 +) + +// Recommend derives the confidence tier for a ranked candidate +// list. candidates must already be sorted best-first (the shape +// RankCandidates returns). +func Recommend(g Group, candidates []Candidate) Recommendation { + if len(candidates) == 0 { + return RecommendationNone + } + + top := candidates[0] + + var rec Recommendation + + switch { + case top.Score >= strongScoreThresh: + rec = RecommendationStrong + case top.Score >= mediumScoreThresh: + rec = RecommendationMedium + default: + return RecommendationLow + } + + // Cap: a different release group scoring within the ambiguity + // margin means the score alone can't pick between two albums. + if rivalWithinMargin(top, candidates[1:]) { + rec = minRecommendation(rec, RecommendationMedium) + } + + // Cap: missing or unmatched tracks mean the alignment itself is + // incomplete, however good the matched tracks look (beets caps + // these penalties at "medium" the same way). + for _, a := range top.Alignments { + if a.Status == AlignmentMissing || a.Status == AlignmentUnmatched { + rec = minRecommendation(rec, RecommendationMedium) + + break + } + } + + // Cap: tiny folders can't corroborate a match strongly enough + // to act on without review, whatever the arithmetic says. + if len(g.Tracks) < evidenceFullTracks { + rec = minRecommendation(rec, RecommendationMedium) + } + + return rec +} + +// rivalWithinMargin reports whether any candidate from a different +// release group scores within ambiguityMargin of the top candidate. +func rivalWithinMargin(top Candidate, rest []Candidate) bool { + for _, c := range rest { + if top.Score-c.Score > ambiguityMargin { + // Sorted descending: everything further is farther away. + return false + } + + if !sameReleaseGroup(top, c) { + return true + } + } + + return false +} + +// sameReleaseGroup reports whether two candidates belong to the +// same release group — by MBID when both carry one, by normalized +// title + artist-credit otherwise (local candidates may lack RG +// MBIDs). +func sameReleaseGroup(a, b Candidate) bool { + if a.ReleaseGroupMBID != "" && b.ReleaseGroupMBID != "" { + return a.ReleaseGroupMBID == b.ReleaseGroupMBID + } + + return Normalize(a.Title) == Normalize(b.Title) && + Normalize(a.ArtistCredit) == Normalize(b.ArtistCredit) +} + +// recommendationRank orders tiers for min-comparison. +func recommendationRank(r Recommendation) int { + switch r { + case RecommendationNone: + return 0 + case RecommendationLow: + return 1 + case RecommendationMedium: + return 2 + case RecommendationStrong: + return 3 + default: + return 0 + } +} + +// minRecommendation returns the weaker of two tiers. +func minRecommendation(a, b Recommendation) Recommendation { + if recommendationRank(a) <= recommendationRank(b) { + return a + } + + return b +} diff --git a/backend/autotag/recommend_test.go b/backend/autotag/recommend_test.go new file mode 100644 index 0000000..8c38b6e --- /dev/null +++ b/backend/autotag/recommend_test.go @@ -0,0 +1,130 @@ +package autotag + +import "testing" + +// mkScoredCandidate builds a minimal candidate with a preset score +// for Recommend tests — Recommend never re-scores, it only reads. +func mkScoredCandidate(rgMBID string, score float64) Candidate { + return Candidate{ + ReleaseMBID: "rel-" + rgMBID, + ReleaseGroupMBID: rgMBID, + Score: score, + } +} + +func fullGroup() Group { + return Group{Tracks: []LocalTrack{{Title: "A"}, {Title: "B"}, {Title: "C"}}} +} + +func TestRecommend_Tiers(t *testing.T) { + t.Parallel() + + g := fullGroup() + + cases := []struct { + name string + cands []Candidate + want Recommendation + }{ + {"no candidates", nil, RecommendationNone}, + {"strong", []Candidate{mkScoredCandidate("rg1", 0.95)}, RecommendationStrong}, + {"medium", []Candidate{mkScoredCandidate("rg1", 0.80)}, RecommendationMedium}, + {"low", []Candidate{mkScoredCandidate("rg1", 0.50)}, RecommendationLow}, + } + + for _, tc := range cases { + if got := Recommend(g, tc.cands); got != tc.want { + t.Errorf("%s: Recommend = %q, want %q", tc.name, got, tc.want) + } + } +} + +func TestRecommend_AmbiguousRivalCapsAtMedium(t *testing.T) { + t.Parallel() + + // A different release group within the margin → ambiguous, even + // though the top score alone reads strong. + cands := []Candidate{ + mkScoredCandidate("rg1", 0.95), + mkScoredCandidate("rg2", 0.93), + } + + if got := Recommend(fullGroup(), cands); got != RecommendationMedium { + t.Errorf("ambiguous rival: Recommend = %q, want medium", got) + } +} + +func TestRecommend_SameRGEditionsAreNotAmbiguous(t *testing.T) { + t.Parallel() + + // Multiple editions of the SAME release group score nearly + // identically by construction — that's not ambiguity. + cands := []Candidate{ + mkScoredCandidate("rg1", 0.95), + mkScoredCandidate("rg1", 0.94), + mkScoredCandidate("rg1", 0.93), + } + + if got := Recommend(fullGroup(), cands); got != RecommendationStrong { + t.Errorf("same-RG editions: Recommend = %q, want strong", got) + } +} + +func TestRecommend_DistantRivalDoesNotCap(t *testing.T) { + t.Parallel() + + cands := []Candidate{ + mkScoredCandidate("rg1", 0.95), + mkScoredCandidate("rg2", 0.60), + } + + if got := Recommend(fullGroup(), cands); got != RecommendationStrong { + t.Errorf("distant rival: Recommend = %q, want strong", got) + } +} + +func TestRecommend_AlignmentDefectsCapAtMedium(t *testing.T) { + t.Parallel() + + top := mkScoredCandidate("rg1", 0.95) + top.Alignments = []TrackAlignment{ + {Status: AlignmentMatched}, + {Status: AlignmentMissing, LocalIndex: -1}, + } + + if got := Recommend(fullGroup(), []Candidate{top}); got != RecommendationMedium { + t.Errorf("missing track: Recommend = %q, want medium", got) + } +} + +func TestRecommend_ThinEvidenceCapsAtMedium(t *testing.T) { + t.Parallel() + + // A 2-track folder can't be auto-accept confident however well + // it matches. (Local candidates skip evidence *scaling* but not + // this cap — acting without review still needs corroboration.) + g := Group{Tracks: []LocalTrack{{Title: "A"}, {Title: "B"}}} + cands := []Candidate{mkScoredCandidate("rg1", 0.96)} + + if got := Recommend(g, cands); got != RecommendationMedium { + t.Errorf("thin evidence: Recommend = %q, want medium", got) + } +} + +func TestRecommend_LocalCandidatesWithoutRGMBIDCompareByTitle(t *testing.T) { + t.Parallel() + + // Local candidates may lack RG MBIDs; same title+artist means + // same release group for ambiguity purposes. + a := Candidate{Title: "Album", ArtistCredit: "Band", Score: 0.95} + b := Candidate{Title: "Album", ArtistCredit: "Band", Score: 0.94} + + if got := Recommend(fullGroup(), []Candidate{a, b}); got != RecommendationStrong { + t.Errorf("same title/artist locals: Recommend = %q, want strong", got) + } + + c := Candidate{Title: "Different Album", ArtistCredit: "Band", Score: 0.94} + if got := Recommend(fullGroup(), []Candidate{a, c}); got != RecommendationMedium { + t.Errorf("different-title rival: Recommend = %q, want medium", got) + } +} diff --git a/backend/autotag/scorer.go b/backend/autotag/scorer.go index 77c0be3..56c8a1c 100644 --- a/backend/autotag/scorer.go +++ b/backend/autotag/scorer.go @@ -40,13 +40,51 @@ func NewScorer(q *sqlcgen.Queries, mb MBClient, logger *slog.Logger) *Scorer { } } -// ScoreGroup produces the full GroupScore for one tagging item. -// Always runs both the local resolver (free) and the MB resolver -// (cache-first, so repeats cost nothing). Candidates from both -// sources are merged and ranked — the UI displays them with -// provenance badges so the user can compare. +// localSufficient is the local-candidate score above which a +// local-first caller skips the MusicBrainz round-trip entirely: a +// local candidate at or above this is a strong match (the same album +// already tagged correctly in another library), so the MB cascade +// wouldn't change the top pick and isn't worth a rate-limited network +// call. Interactive scoring ignores this and always consults MB so +// the review UI can show both sources side by side. +const localSufficient = 0.90 + +// idSufficient is the score at which an ID-resolved candidate (built +// from recording MBIDs already present in the local tags) makes the +// fuzzy search cascade unnecessary — mirrors beets, where a strong +// mb_albumid match returns immediately without a text search. +const idSufficient = 0.90 + +// maxIDSampleTracks caps how many local recording MBIDs the ID-first +// path looks up — three spread across the folder corroborate a +// release without paying for a lookup per track. +const maxIDSampleTracks = 3 + +// ScoreGroup produces the full GroupScore for one tagging item, +// always consulting MusicBrainz (when a client is configured) so the +// review UI can display local + MB candidates side by side with +// provenance badges. Use this on interactive paths where the user is +// looking at the result. func (s *Scorer) ScoreGroup( ctx context.Context, groupKey string, +) (*GroupScore, error) { + return s.scoreGroup(ctx, groupKey, false) +} + +// ScoreGroupLocalFirst is the cheap variant for background work +// (prefetch): it scores local candidates first and skips the +// MusicBrainz cascade when the best local candidate is already a +// strong match (score >= localSufficient). Falls back to the full +// MB-consulting path otherwise, so albums with no strong local match +// still get a real score for the sidebar pill. +func (s *Scorer) ScoreGroupLocalFirst( + ctx context.Context, groupKey string, +) (*GroupScore, error) { + return s.scoreGroup(ctx, groupKey, true) +} + +func (s *Scorer) scoreGroup( + ctx context.Context, groupKey string, localFirst bool, ) (*GroupScore, error) { item, err := s.q.GetTaggingItem(ctx, groupKey) if err != nil { @@ -62,36 +100,153 @@ func (s *Scorer) ScoreGroup( return nil, err } + g := Group{ + AlbumName: item.AlbumName, + AlbumArtist: item.AlbumArtist, + Tracks: locals, + } + localHits, err := s.local.ResolveLocal(ctx, item.AlbumName) if err != nil { return nil, err } + // Local-first short-circuit: pre-score the free local candidates + // and, if one is already a strong match, skip the MB round-trip. + var localCandidates []Candidate + + skipMB := false + + if localFirst && s.mb != nil { + localCandidates = RankCandidates(g, localHits) + skipMB = len(localCandidates) > 0 && localCandidates[0].Score >= localSufficient + } + var mbHits []Candidate - if s.mb != nil { - mbHits, err = s.mb.ResolveMB( - ctx, - item.AlbumName, - item.AlbumArtist, - len(locals), - guessArtistMBID(locals), - ) + + if s.mb != nil && !skipMB { + mbHits = s.resolveMBCandidates(ctx, g, groupKey) + } + + // Reuse the pre-ranked local list when we skipped MB; otherwise + // rank the merged set. + candidates := localCandidates + if !skipMB { + candidates = RankCandidates(g, append(localHits, mbHits...)) + } + + return &GroupScore{ + GroupKey: groupKey, + AlbumName: item.AlbumName, + AlbumArtist: item.AlbumArtist, + LocalTracks: locals, + Candidates: candidates, + Recommendation: Recommend(g, candidates), + }, nil +} + +// resolveMBCandidates gathers MusicBrainz candidates for a group: +// ID-first (recording MBIDs already in the tags), then the search +// cascade when the ID path didn't produce a strong match. Failures +// on either path degrade to fewer candidates, never to an error — +// local candidates must still surface when MB is unreachable. +func (s *Scorer) resolveMBCandidates( + ctx context.Context, g Group, groupKey string, +) []Candidate { + var out []Candidate + + if ids := sampleRecordingMBIDs(g.Tracks); len(ids) > 0 { + idCands, err := s.mb.ResolveByRecordingMBIDs(ctx, ids) if err != nil { s.log.Warn( - "MB resolve failed — returning local-only candidates", - "group_key", groupKey, - "err", err, + "MB ID-first resolve failed — falling back to search", + "group_key", groupKey, "err", err, ) } + + if len(idCands) > 0 { + ranked := RankCandidates(g, idCands) + if ranked[0].Score >= idSufficient { + s.log.Info( + "MB ID-first match — skipping search cascade", + "group_key", groupKey, "score", ranked[0].Score, + ) + + return idCands + } + + out = idCands + } } - candidates := RankCandidates(locals, append(localHits, mbHits...)) + searchHits, err := s.mb.ResolveMB(ctx, g) + if err != nil { + s.log.Warn( + "MB resolve failed — returning local-only candidates", + "group_key", groupKey, "err", err, + ) - return &GroupScore{ - GroupKey: groupKey, - LocalTracks: locals, - Candidates: candidates, - }, nil + return out + } + + return append(out, dropDuplicateReleases(out, searchHits)...) +} + +// dropDuplicateReleases filters from `extra` any candidate whose +// release MBID already appears in `have`. +func dropDuplicateReleases(have, extra []Candidate) []Candidate { + if len(have) == 0 { + return extra + } + + seen := make(map[string]bool, len(have)) + + for _, c := range have { + if c.ReleaseMBID != "" { + seen[c.ReleaseMBID] = true + } + } + + out := make([]Candidate, 0, len(extra)) + + for _, c := range extra { + if c.ReleaseMBID != "" && seen[c.ReleaseMBID] { + continue + } + + out = append(out, c) + } + + return out +} + +// sampleRecordingMBIDs picks up to maxIDSampleTracks distinct +// recording MBIDs spread across the group (first, middle, last) — +// enough to corroborate a release via voting without a lookup per +// track. +func sampleRecordingMBIDs(tracks []LocalTrack) []string { + distinct := make([]string, 0, len(tracks)) + seen := make(map[string]bool, len(tracks)) + + for _, t := range tracks { + if t.RecordingMBID == "" || seen[t.RecordingMBID] { + continue + } + + seen[t.RecordingMBID] = true + + distinct = append(distinct, t.RecordingMBID) + } + + if len(distinct) <= maxIDSampleTracks { + return distinct + } + + return []string{ + distinct[0], + distinct[len(distinct)/2], + distinct[len(distinct)-1], + } } // LocalTracksForGroup exposes the local resolver so callers that @@ -104,32 +259,6 @@ func (s *Scorer) LocalTracksForGroup( return s.local.LocalTracksForGroup(ctx, groupKey) } -// PersistBest writes the top candidate's release MBID and score -// onto the tagging_items row, bumping status to 'matched' when a -// candidate exists. No-op when candidates is empty (keeps the -// current status — likely 'pending'). -func (s *Scorer) PersistBest( - ctx context.Context, score *GroupScore, -) error { - if score == nil || len(score.Candidates) == 0 { - return nil - } - - top := score.Candidates[0] - - mbid := top.ReleaseMBID - if mbid == "" { - mbid = top.ReleaseGroupMBID - } - - return s.q.SetTaggingItemBestMatch(ctx, sqlcgen.SetTaggingItemBestMatchParams{ - BestMatchReleaseMbid: sql.NullString{String: mbid, Valid: mbid != ""}, - Score: sql.NullFloat64{Float64: top.Score, Valid: true}, - Status: "matched", - GroupKey: score.GroupKey, - }) -} - // PersistScore writes the top candidate's release MBID and score // onto the tagging_items row WITHOUT touching its status. Use // this from paths that want the sidebar pill / sort to reflect a @@ -156,19 +285,3 @@ func (s *Scorer) PersistScore( GroupKey: score.GroupKey, }) } - -// guessArtistMBID returns the first non-empty recording MBID- -// derived artist hint we can find. Tracks carry recording MBIDs, -// not artist MBIDs, but existing partial tags are often consistent -// enough that any non-empty MBID signals "this album already has -// some MB lineage". A follow-up in 010 can resolve actual artist -// MBIDs via the artists table. -func guessArtistMBID(tracks []LocalTrack) string { - // Phase 009 doesn't wire up per-track artist MBIDs through - // the LocalTrack struct yet — keeping the hook so the MB - // resolver still compiles with the empty hint. Real artist - // MBIDs flow in once 010 wires them into LocalTrack. - _ = tracks - - return "" -} diff --git a/backend/autotag/scorer_test.go b/backend/autotag/scorer_test.go index 2494ffa..58307f9 100644 --- a/backend/autotag/scorer_test.go +++ b/backend/autotag/scorer_test.go @@ -166,7 +166,179 @@ func TestScorer_LocalHitSurfacesFirst(t *testing.T) { } } -func TestScorer_PersistBestWritesMatched(t *testing.T) { +// TestScorer_LocalFirstSkipsMB asserts the background (prefetch) +// scoring path makes zero MusicBrainz calls when a local candidate is +// already a strong match — the whole point of ScoreGroupLocalFirst. +func TestScorer_LocalFirstSkipsMB(t *testing.T) { + t.Parallel() + + db := database.NewTestDB(t) + + // A canonical local album (MBIDs present) that the pending group + // matches track-for-track — the local candidate scores >= 0.90. + seed(t, db, seededAlbum{ + groupKey: "g-canonical", + albumName: "Good Album", + releaseMBID: "rg-abcd", + tracks: []seededTrack{ + { + filePath: "/lib/a.mp3", title: "Song A", + trackNumber: 1, lengthMillis: 200000, recordingMBID: "rec-a", + }, + { + filePath: "/lib/b.mp3", title: "Song B", + trackNumber: 2, lengthMillis: 180000, recordingMBID: "rec-b", + }, + }, + }) + seed(t, db, seededAlbum{ + groupKey: "g-pending", + albumName: "Good Album", + tracks: []seededTrack{ + {filePath: "/other/a.mp3", title: "Song A", trackNumber: 1, lengthMillis: 200000}, + {filePath: "/other/b.mp3", title: "Song B", trackNumber: 2, lengthMillis: 180000}, + }, + }) + + mbCalls := 0 + fakeMB := &countingMBClient{onSearch: func() { mbCalls++ }} + + scorer := autotag.NewScorer(db.Queries, fakeMB, slog.New(slog.DiscardHandler)) + + result, err := scorer.ScoreGroupLocalFirst(context.Background(), "g-pending") + if err != nil { + t.Fatalf("ScoreGroupLocalFirst: %v", err) + } + + if mbCalls != 0 { + t.Errorf("made %d MB calls, want 0 (strong local match should skip MB)", mbCalls) + } + + if len(result.Candidates) == 0 || result.Candidates[0].ReleaseGroupMBID != "rg-abcd" { + t.Errorf("top candidate = %+v, want local rg-abcd", result.Candidates) + } +} + +// TestScorer_IDFirstSkipsSearch asserts that a group whose tracks +// already carry recording MBIDs resolves through the ID-first path +// — the release the recordings vote for is looked up directly and +// the fuzzy search cascade never runs. +func TestScorer_IDFirstSkipsSearch(t *testing.T) { + t.Parallel() + + db := database.NewTestDB(t) + + seed(t, db, seededAlbum{ + groupKey: "g-tagged", + albumName: "Good Album", + tracks: []seededTrack{ + { + filePath: "/lib/a.mp3", title: "Song A", + trackNumber: 1, lengthMillis: 200000, recordingMBID: "rec-a", + }, + { + filePath: "/lib/b.mp3", title: "Song B", + trackNumber: 2, lengthMillis: 180000, recordingMBID: "rec-b", + }, + }, + }) + + fake := &idFakeClient{ + recRels: map[string][]autotag.MBReleaseRef{ + "rec-a": {{MBID: "rel-full", Status: "Official", Date: "1999"}}, + "rec-b": {{MBID: "rel-full", Status: "Official", Date: "1999"}}, + }, + releases: map[string]autotag.MBRelease{ + "rel-full": { + MBID: "rel-full", Title: "Good Album", + ArtistCredit: "Test Artist", Status: "Official", Country: "US", + Tracks: []autotag.CandidateTrack{ + {Position: 1, Title: "Song A", LengthMillis: 200000, MBID: "rec-a"}, + {Position: 2, Title: "Song B", LengthMillis: 180000, MBID: "rec-b"}, + }, + }, + }, + } + + scorer := autotag.NewScorer(db.Queries, fake, slog.New(slog.DiscardHandler)) + + result, err := scorer.ScoreGroup(context.Background(), "g-tagged") + if err != nil { + t.Fatalf("ScoreGroup: %v", err) + } + + if fake.searches != 0 { + t.Errorf("made %d search calls, want 0 (ID-first should skip the cascade)", fake.searches) + } + + if len(result.Candidates) == 0 { + t.Fatal("no candidates") + } + + top := result.Candidates[0] + if top.ReleaseMBID != "rel-full" || top.Provenance != "id" { + t.Errorf( + "top = %q via %q (%.3f), want 'rel-full' via 'id'", + top.ReleaseMBID, top.Provenance, top.Score, + ) + } +} + +// idFakeClient serves canned recording→release lookups and counts +// search calls so the ID-first test can assert the cascade stayed +// cold. +type idFakeClient struct { + searches int + recRels map[string][]autotag.MBReleaseRef + releases map[string]autotag.MBRelease +} + +func (c *idFakeClient) SearchReleaseGroups( + _ context.Context, _ string, _ int, +) ([]autotag.MBReleaseGroupHit, int, error) { + c.searches++ + + return nil, 0, nil +} + +func (c *idFakeClient) SearchRecordings( + _ context.Context, _ string, _ int, +) ([]autotag.MBRecordingHit, int, error) { + c.searches++ + + return nil, 0, nil +} + +func (c *idFakeClient) LookupRecordingReleases( + _ context.Context, mbid string, +) ([]autotag.MBReleaseRef, error) { + return c.recRels[mbid], nil +} + +func (c *idFakeClient) BrowseReleases( + _ context.Context, _ string, +) ([]autotag.MBRelease, error) { + return nil, nil +} + +func (c *idFakeClient) LookupRelease( + _ context.Context, mbid string, +) (autotag.MBRelease, error) { + rel, ok := c.releases[mbid] + if !ok { + return autotag.MBRelease{}, autotag.ErrGroupNotFound + } + + return rel, nil +} + +func (c *idFakeClient) LookupReleaseGroup( + _ context.Context, _ string, +) (autotag.MBReleaseGroupHit, error) { + return autotag.MBReleaseGroupHit{}, nil +} + +func TestScorer_PersistScoreWritesTopMatch(t *testing.T) { t.Parallel() db := database.NewTestDB(t) @@ -200,8 +372,8 @@ func TestScorer_PersistBestWritesMatched(t *testing.T) { t.Fatalf("ScoreGroup: %v", err) } - if err := scorer.PersistBest(context.Background(), result); err != nil { - t.Fatalf("PersistBest: %v", err) + if err := scorer.PersistScore(context.Background(), result); err != nil { + t.Fatalf("PersistScore: %v", err) } got, err := db.Queries.GetTaggingItem(context.Background(), "g-pending") @@ -209,8 +381,10 @@ func TestScorer_PersistBestWritesMatched(t *testing.T) { t.Fatalf("reload: %v", err) } - if got.Status != "matched" { - t.Errorf("status = %q, want 'matched'", got.Status) + // PersistScore records the pill score + best match but must leave + // the review status untouched so the folder stays in the queue. + if got.Status != "pending" { + t.Errorf("status = %q, want 'pending' (PersistScore must not flip status)", got.Status) } if !got.BestMatchReleaseMbid.Valid || got.BestMatchReleaseMbid.String != "rg-abcd" { @@ -258,10 +432,20 @@ func (c *countingMBClient) BrowseReleases( return nil, nil } -func (c *countingMBClient) LookupArtist(_ context.Context, _ string) (string, error) { +func (c *countingMBClient) SearchRecordings( + _ context.Context, _ string, _ int, +) ([]autotag.MBRecordingHit, int, error) { c.onSearch() - return "", nil + return nil, 0, nil +} + +func (c *countingMBClient) LookupRecordingReleases( + _ context.Context, _ string, +) ([]autotag.MBReleaseRef, error) { + c.onSearch() + + return nil, nil } func (c *countingMBClient) LookupRelease(_ context.Context, _ string) (autotag.MBRelease, error) { diff --git a/backend/autotag/types.go b/backend/autotag/types.go index 7166cfb..fff8aec 100644 --- a/backend/autotag/types.go +++ b/backend/autotag/types.go @@ -14,6 +14,15 @@ type LocalTrack struct { RecordingMBID string } +// Group is the folder-level context candidates are ranked against: +// the tagging item's album name/artist plus its local tracks. The +// album fields are soft signals — empty values never penalize. +type Group struct { + AlbumName string + AlbumArtist string + Tracks []LocalTrack +} + // CandidateSource distinguishes candidates served from the local // release_groups cache (zero network cost) from those fetched live. type CandidateSource string @@ -42,22 +51,26 @@ type Candidate struct { OriginalDate string // "YYYY" or "YYYY-MM-DD" — release group's first release Country string Status string // "Official", "Promotion", ... + PrimaryType string // release group's primary type: "Album", "Single", ... TrackCount int Tracks []CandidateTrack Alignments []TrackAlignment Score float64 // 0..1, higher is better Breakdown ScoreBreakdown Source CandidateSource - Provenance string // cascade step that produced this ("strict", "fuzzy-title", "paste", "local") + Provenance string // cascade step that produced this ("strict", "fuzzy-title", "id", "paste", "local") } -// ScoreBreakdown exposes the four inputs that go into Candidate.Score -// so the review UI can explain the ranking to the user. +// ScoreBreakdown exposes the inputs that go into Candidate.Score so +// the review UI can explain the ranking to the user. type ScoreBreakdown struct { TitleAvg float64 // average per-track title similarity (0..1) LengthAvg float64 // average per-track length similarity (0..1) + ArtistFit float64 // album-artist vs candidate artist-credit similarity (0..1) + AlbumFit float64 // folder album name vs candidate release title similarity (0..1) TrackCountFit float64 // 1.0 when local and candidate track counts match - ReleaseMeta float64 // year + official + country, averaged + ReleaseMeta float64 // official + country + release-group type, averaged + Evidence float64 // confidence scale from corroborating track count (0..1) } // CandidateTrack is one track inside a candidate release. @@ -97,14 +110,19 @@ type TrackAlignment struct { CandidateLength int64 TitleScore float64 // 0..1 from normalized-title edit distance + LengthScore float64 // 0..1 from lengthScore; 0.5 (neutral) if either side unknown LengthDeltaMs int64 // abs(local - candidate); 0 if either side missing TrackNumberOK bool // local track number matches candidate position + IDMatch bool // local recording MBID equals candidate track MBID Status AlignmentStatus } // GroupScore is the scorer's full output for one tagging group. type GroupScore struct { - GroupKey string - LocalTracks []LocalTrack - Candidates []Candidate // sorted by Score, descending + GroupKey string + AlbumName string + AlbumArtist string + LocalTracks []LocalTrack + Candidates []Candidate // sorted by Score, descending + Recommendation Recommendation } diff --git a/backend/autotagservice/service.go b/backend/autotagservice/service.go index c7d6b12..b68eb32 100644 --- a/backend/autotagservice/service.go +++ b/backend/autotagservice/service.go @@ -8,9 +8,12 @@ package autotagservice import ( "context" "database/sql" + "encoding/base64" + "encoding/json" "errors" "fmt" "log/slog" + "os" "path/filepath" "strings" "sync" @@ -23,6 +26,7 @@ import ( "yellowjacket/backend/database/sql/sqlcgen" "yellowjacket/backend/events" "yellowjacket/backend/explore" + "yellowjacket/backend/metadata" "yellowjacket/backend/tagwriter" ) @@ -81,6 +85,13 @@ type Service struct { exp *explore.Service logger *slog.Logger ctx context.Context + // ctxReady reports whether ctx is the Wails lifecycle context set + // via SetContext (rather than the context.Background() default). It + // gates event emission: calling wailsruntime.EventsEmit with a + // non-runtime context triggers log.Fatalf (os.Exit) inside Wails, so + // a background worker that fires before OnStartup wires the context + // would otherwise take the whole app down on launch. + ctxReady bool // Queue cursor — the group_key of the last item returned. // GetNextPending uses it to advance. Reset by StartAutotagQueue. @@ -88,12 +99,6 @@ type Service struct { cursor string libraryID int64 - // Candidate cache: freezes the candidate list the user saw so - // Apply operates on the exact release they selected even if a - // racing rescore would have changed the ranking. Keyed by - // group_key; replaced by GetCandidates / GetCandidatesForPasteURL. - candidateCache map[string][]autotag.Candidate - // In-flight set for ApplyAsync. Protects against the user // firing two Apply jobs on the same group while the first is // still running. Keyed by group_key; entries removed when the @@ -194,7 +199,6 @@ func NewService( exp: exp, logger: logger, ctx: context.Background(), - candidateCache: make(map[string][]autotag.Candidate), runningApplies: make(map[string]struct{}), } } @@ -202,7 +206,36 @@ func NewService( // SetContext stores the Wails runtime context (called from // OnStartup). func (s *Service) SetContext(ctx context.Context) { + s.mu.Lock() + defer s.mu.Unlock() + s.ctx = ctx + s.ctxReady = ctx != nil +} + +// emitEvent emits a Wails runtime event, but only when the stored +// context actually carries the Wails runtime. Wails' EventsEmit calls +// log.Fatalf — which os.Exit()s the process and cannot be recovered — +// whenever the context lacks its internal "events" value (e.g. the +// context.Background() default, or any non-lifecycle context). A +// background worker (the prefetch/apply sweeps) that emits before, or +// independently of, OnStartup wiring the real context would otherwise +// take the whole app down on launch. We replicate Wails' own +// precondition here so a not-yet-ready context degrades to a no-op +// instead of a crash. +func (s *Service) emitEvent(eventName string, data any) { + s.mu.Lock() + ready := s.ctxReady + ctx := s.ctx + s.mu.Unlock() + + // hasWailsRuntime mirrors the check in wails/pkg/runtime.getEvents: + // the runtime is present only when ctx.Value("events") is non-nil. + if !ready || ctx == nil || ctx.Value("events") == nil { + return + } + + wailsruntime.EventsEmit(ctx, eventName, data) } // StartBackgroundPrefetch kicks off (or restarts) the prefetch @@ -335,7 +368,7 @@ func (s *Service) startPrefetch(libraryID int64) { // folder, which populates the cache via the foreground // GetCandidates path. Skip the redundant re-score. if cached := s.lookupCachedCandidates(key); len(cached) > 0 { - wailsruntime.EventsEmit(s.ctx, events.AutotagPrefetchProgress, map[string]any{ + s.emitEvent(events.AutotagPrefetchProgress, map[string]any{ "processed": i + 1, "total": total, }) @@ -343,7 +376,10 @@ func (s *Service) startPrefetch(libraryID int64) { continue } - score, err := s.scorer.ScoreGroup(ctx, key) + // Local-first: the background sweep skips the MusicBrainz + // cascade when a local candidate already scores well, so a + // library with cross-library duplicates costs no network here. + score, err := s.scorer.ScoreGroupLocalFirst(ctx, key) if err != nil { s.logger.Debug( "prefetch: score failed — skipping", @@ -360,13 +396,13 @@ func (s *Service) startPrefetch(libraryID int64) { } } - wailsruntime.EventsEmit(s.ctx, events.AutotagPrefetchProgress, map[string]any{ + s.emitEvent(events.AutotagPrefetchProgress, map[string]any{ "processed": i + 1, "total": total, }) } - wailsruntime.EventsEmit(s.ctx, events.AutotagPrefetchFinished, map[string]any{ + s.emitEvent(events.AutotagPrefetchFinished, map[string]any{ "processed": total, "total": total, }) @@ -548,11 +584,164 @@ func (s *Service) GetCandidateCoverArt( return s.exp.GetCandidateThumbnail(releaseMBID, releaseGroupMBID) } +// GetLocalCoverArt returns a data-URI for the artwork associated +// with the group's local files, so the review UI can show "what the +// folder already looks like" beside the fetched candidate art. +// It first checks each file for embedded ID3/Vorbis pictures, then +// falls back to a sidecar cover image (cover.jpg, folder.png, …) +// sitting in the album directory. Untagged folders carry one or the +// other far more often than they have a DB release-group cover, so we +// read the files directly. Returns "" when nothing is found. +func (s *Service) GetLocalCoverArt(groupKey string) string { + locals, err := s.scorer.LocalTracksForGroup(s.ctx, groupKey) + if err != nil { + s.logger.Warn( + "local cover art: list tracks failed", + "group_key", groupKey, "err", err, + ) + + return "" + } + + // 1. Embedded artwork — guaranteed to travel with the track. + for _, t := range locals { + if t.FilePath == "" { + continue + } + + meta, err := metadata.ExtractTags(t.FilePath) + if err != nil || meta == nil || meta.Picture == nil { + continue + } + + if len(meta.Picture.Data) == 0 { + continue + } + + mime := meta.Picture.MIMEType + if mime == "" { + mime = "image/jpeg" + } + + return "data:" + mime + ";base64," + + base64.StdEncoding.EncodeToString(meta.Picture.Data) + } + + // 2. Sidecar cover image file in the album directory. Scan the + // distinct directories the tracks live in (usually just one). + seen := make(map[string]struct{}, 1) + + for _, t := range locals { + if t.FilePath == "" { + continue + } + + dir := filepath.Dir(t.FilePath) + if _, ok := seen[dir]; ok { + continue + } + + seen[dir] = struct{}{} + + if uri := folderCoverDataURI(dir); uri != "" { + return uri + } + } + + return "" +} + +// coverImageExts maps recognised sidecar cover-image extensions to +// their MIME type for the data-URI. +var coverImageExts = map[string]string{ + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".png": "image/png", + ".gif": "image/gif", + ".webp": "image/webp", + ".bmp": "image/bmp", +} + +// preferredCoverStems lists the base filenames (lower-cased, minus +// extension) commonly used for album artwork, in priority order. +var preferredCoverStems = []string{ + "cover", "folder", "front", "album", "albumart", "albumartsmall", "thumb", +} + +// folderCoverDataURI looks for a conventional cover-image file in dir +// and returns it as a base64 data-URI, or "" when none is found. +// Preferred filenames (cover.*, folder.*, …) win; any other image +// file in the directory is used as a last resort. Matching is +// case-insensitive so COVER.JPG and Folder.Png both hit. +func folderCoverDataURI(dir string) string { + entries, err := os.ReadDir(dir) + if err != nil { + return "" + } + + byStem := make(map[string]string) // lower stem -> filename + + var fallback string + + for _, e := range entries { + if e.IsDir() { + continue + } + + name := e.Name() + ext := strings.ToLower(filepath.Ext(name)) + + if _, ok := coverImageExts[ext]; !ok { + continue + } + + stem := strings.ToLower(strings.TrimSuffix(name, filepath.Ext(name))) + if _, dup := byStem[stem]; !dup { + byStem[stem] = name + } + + if fallback == "" { + fallback = name + } + } + + name := fallback + + for _, stem := range preferredCoverStems { + if match, ok := byStem[stem]; ok { + name = match + + break + } + } + + if name == "" { + return "" + } + + data, err := os.ReadFile(filepath.Join(dir, name)) + if err != nil || len(data) == 0 { + return "" + } + + mime := coverImageExts[strings.ToLower(filepath.Ext(name))] + if mime == "" { + mime = "image/jpeg" + } + + return "data:" + mime + ";base64," + base64.StdEncoding.EncodeToString(data) +} + // ScoreView is the Wails-friendly projection of autotag.GroupScore. type ScoreView struct { GroupKey string `json:"groupKey"` LocalTracks []LocalTrackView `json:"localTracks"` Candidates []CandidateView `json:"candidates"` + // Recommendation is the qualitative confidence tier for the + // ranked list: "none", "low", "medium", or "strong". Unlike the + // raw score it accounts for ambiguity (a rival release group + // scoring nearly as high) and alignment defects. + Recommendation string `json:"recommendation"` } // LocalTrackView mirrors autotag.LocalTrack. @@ -583,6 +772,7 @@ type CandidateView struct { OriginalDate string `json:"originalDate"` Country string `json:"country"` Status string `json:"status"` + PrimaryType string `json:"primaryType"` TrackCount int `json:"trackCount"` Score float64 `json:"score"` Breakdown ScoreBreakdownView `json:"breakdown"` @@ -597,8 +787,11 @@ type CandidateView struct { type ScoreBreakdownView struct { TitleAvg float64 `json:"titleAvg"` LengthAvg float64 `json:"lengthAvg"` + ArtistFit float64 `json:"artistFit"` + AlbumFit float64 `json:"albumFit"` TrackCountFit float64 `json:"trackCountFit"` ReleaseMeta float64 `json:"releaseMeta"` + Evidence float64 `json:"evidence"` } // AlignmentView mirrors autotag.TrackAlignment. LocalIndex of -1 @@ -676,21 +869,50 @@ func (s *Service) GetCandidates(groupKey string) (*ScoreView, error) { return scoreToView(score, s.exp), nil } -// cacheCandidates replaces the candidate list for the given group. -// Thread-safe. +// cacheCandidates durably persists the scored candidate list for a +// group as a JSON blob in tagging_candidates. This freezes the exact +// list the user saw (so Apply operates on the release they selected +// even if a racing rescore would reorder it) AND survives process +// restarts, so a later session reuses the result instead of re-hitting +// MusicBrainz. Errors are logged, not returned — a failed persist +// only costs a recompute, never correctness. func (s *Service) cacheCandidates(groupKey string, cands []autotag.Candidate) { - s.mu.Lock() - defer s.mu.Unlock() + blob, err := json.Marshal(cands) + if err != nil { + s.logger.Warn("cache candidates: marshal failed", "group_key", groupKey, "err", err) - s.candidateCache[groupKey] = cands + return + } + + if err := s.db.Queries.UpsertTaggingCandidates(s.ctx, sqlcgen.UpsertTaggingCandidatesParams{ + GroupKey: groupKey, + Candidates: string(blob), + }); err != nil { + s.logger.Warn("cache candidates: persist failed", "group_key", groupKey, "err", err) + } } -// lookupCachedCandidates returns the cached list, or empty if none. +// lookupCachedCandidates returns the durably-stored candidate list for +// a group, or nil when none has been computed yet (or the blob can't +// be decoded — treated as a miss so the caller recomputes). func (s *Service) lookupCachedCandidates(groupKey string) []autotag.Candidate { - s.mu.Lock() - defer s.mu.Unlock() + blob, err := s.db.Queries.GetTaggingCandidates(s.ctx, groupKey) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) && !isNoRows(err) { + s.logger.Warn("lookup candidates: query failed", "group_key", groupKey, "err", err) + } - return s.candidateCache[groupKey] + return nil + } + + var cands []autotag.Candidate + if err := json.Unmarshal([]byte(blob), &cands); err != nil { + s.logger.Warn("lookup candidates: unmarshal failed", "group_key", groupKey, "err", err) + + return nil + } + + return cands } // ApplyResultView mirrors autotag.ApplyResult for the frontend. @@ -713,33 +935,9 @@ type FailureView struct { // always applies the exact release they saw in the UI. // Empty releaseMBID picks the top-scored candidate. func (s *Service) Apply(groupKey, releaseMBID string) (*ApplyResultView, error) { - cands := s.lookupCachedCandidates(groupKey) - if len(cands) == 0 { - // Cache miss (user reloaded the page?): re-score and carry - // on — worst case they get the current top-ranked release. - score, err := s.scorer.ScoreGroup(s.ctx, groupKey) - if err != nil { - return nil, fmt.Errorf("score before apply: %w", err) - } - - cands = score.Candidates - s.cacheCandidates(groupKey, cands) - } - - locals, err := s.scorer.LocalTracksForGroup(s.ctx, groupKey) + plan, err := s.prepareApplyPlan(groupKey, releaseMBID) if err != nil { - return nil, fmt.Errorf("load locals: %w", err) - } - - groupScore := &autotag.GroupScore{ - GroupKey: groupKey, - LocalTracks: locals, - Candidates: cands, - } - - plan, err := s.applier.BuildPlan(groupScore, releaseMBID) - if err != nil { - return nil, fmt.Errorf("build plan: %w", err) + return nil, err } result, err := s.applier.Apply(s.ctx, plan, nil) @@ -799,7 +997,7 @@ func (s *Service) ApplyAsync(groupKey, releaseMBID string) error { } } - wailsruntime.EventsEmit(s.ctx, events.AutotagApplyStarted, map[string]any{ + s.emitEvent(events.AutotagApplyStarted, map[string]any{ "groupKey": groupKey, "total": total, }) @@ -852,7 +1050,7 @@ func (s *Service) runApply(groupKey string, plan *autotag.ApplyPlan, total int) defer s.endApply(groupKey) onProgress := func(current, total, succeeded, failed int) { - wailsruntime.EventsEmit(s.ctx, events.AutotagApplyProgress, map[string]any{ + s.emitEvent(events.AutotagApplyProgress, map[string]any{ "groupKey": groupKey, "current": current, "total": total, @@ -879,7 +1077,7 @@ func (s *Service) runApply(groupKey string, plan *autotag.ApplyPlan, total int) finished["error"] = err.Error() } - wailsruntime.EventsEmit(s.ctx, events.AutotagApplyFinished, finished) + s.emitEvent(events.AutotagApplyFinished, finished) } // tryStartApply records that an Apply for the given group is @@ -939,8 +1137,14 @@ func (s *Service) LeaveAsIs(groupKey string) error { } // RetagGroup flips a group back to 'pending' so the user can -// re-review after an apply or skip. +// re-review after an apply or skip. Drops the durably-cached +// candidate list too, so the next open recomputes against fresh +// MusicBrainz data rather than reusing the stale stored result. func (s *Service) RetagGroup(groupKey string) error { + if err := s.db.Queries.DeleteTaggingCandidates(s.ctx, groupKey); err != nil { + s.logger.Warn("retag: clear cached candidates failed", "group_key", groupKey, "err", err) + } + return s.db.Queries.SetTaggingItemStatus( s.ctx, sqlcgen.SetTaggingItemStatusParams{Status: "pending", GroupKey: groupKey}, @@ -980,7 +1184,11 @@ func (s *Service) GetCandidatesForPasteURL( // splice it to the top of the list. RankCandidates would // re-sort by score; we explicitly keep the pasted candidate // first so the user's manual pick is visually anchored. - scored := autotag.ScoreCandidate(score.LocalTracks, pasted, len(score.LocalTracks)) + scored := autotag.ScoreCandidate(autotag.Group{ + AlbumName: score.AlbumName, + AlbumArtist: score.AlbumArtist, + Tracks: score.LocalTracks, + }, pasted) merged := append([]autotag.Candidate{scored}, score.Candidates...) score.Candidates = merged @@ -989,6 +1197,122 @@ func (s *Service) GetCandidatesForPasteURL( return scoreToView(score, s.exp), nil } +// SearchHitView is one in-app MusicBrainz search result surfaced to +// the review UI. Kind is "releasegroup" or "recording" so the +// frontend knows which resolve path SelectSearchCandidate must take. +type SearchHitView struct { + MBID string `json:"mbid"` + Kind string `json:"kind"` + Title string `json:"title"` + Artist string `json:"artist"` + Detail string `json:"detail"` +} + +// SearchCandidates runs an in-app MusicBrainz search — the "suggest a +// new candidate" escape hatch when the automatic cascade misses. +// kind is "recording" (title + artist, for singletons) or anything +// else (album + artist release-group search). Returns lightweight +// hits; SelectSearchCandidate resolves the picked one into a scored +// candidate. +func (s *Service) SearchCandidates( + kind, query, artist string, +) ([]SearchHitView, error) { + if kind == "recording" { + hits, err := s.mbr.SearchRecordingHits(s.ctx, query, artist) + if err != nil { + return nil, fmt.Errorf("search recordings: %w", err) + } + + out := make([]SearchHitView, 0, len(hits)) + for _, h := range hits { + out = append(out, SearchHitView{ + MBID: h.MBID, + Kind: "recording", + Title: h.Title, + Artist: h.ArtistCredit, + Detail: formatMillis(h.LengthMillis), + }) + } + + return out, nil + } + + hits, err := s.mbr.SearchReleaseGroupHits(s.ctx, query, artist) + if err != nil { + return nil, fmt.Errorf("search release groups: %w", err) + } + + out := make([]SearchHitView, 0, len(hits)) + for _, h := range hits { + detail := h.PrimaryType + if y := h.FirstDate; len(y) >= 4 { //nolint:mnd + detail = strings.TrimSpace(y[:4] + " " + detail) + } + + out = append(out, SearchHitView{ + MBID: h.MBID, + Kind: "releasegroup", + Title: h.Title, + Artist: h.ArtistCredit, + Detail: detail, + }) + } + + return out, nil +} + +// SelectSearchCandidate resolves a picked search hit into a fully- +// scored candidate and splices it to the top of the group's candidate +// list — the same shape as the paste-URL path, so Apply works +// unchanged. A "recording" hit is resolved to a representative +// release first. +func (s *Service) SelectSearchCandidate( + groupKey, kind, mbid string, +) (*ScoreView, error) { + score, err := s.scorer.ScoreGroup(s.ctx, groupKey) + if err != nil { + return nil, fmt.Errorf("score group: %w", err) + } + + var picked autotag.Candidate + + if kind == "recording" { + picked, err = s.mbr.ResolveOneRecordingMBID(s.ctx, mbid) + } else { + picked, err = s.mbr.ResolveOneReleaseMBID(s.ctx, mbid) + } + + if err != nil { + return nil, fmt.Errorf("resolve %s %s: %w", kind, mbid, err) + } + + scored := autotag.ScoreCandidate(autotag.Group{ + AlbumName: score.AlbumName, + AlbumArtist: score.AlbumArtist, + Tracks: score.LocalTracks, + }, picked) + merged := append([]autotag.Candidate{scored}, score.Candidates...) + score.Candidates = merged + + s.cacheCandidates(groupKey, merged) + + return scoreToView(score, s.exp), nil +} + +// formatMillis renders a millisecond duration as m:ss for the search +// result detail line; empty for unknown lengths. +func formatMillis(ms int64) string { + if ms <= 0 { + return "" + } + + sec := ms / 1000 + minutes := sec / 60 + seconds := sec % 60 + + return fmt.Sprintf("%d:%02d", minutes, seconds) +} + var errInvalidMBURL = errors.New("autotag: not a valid MusicBrainz release URL") // extractReleaseMBID pulls the UUID out of a MusicBrainz release @@ -1033,7 +1357,16 @@ func extractReleaseMBID(url string) string { // top-ranked candidate; pass nil to skip cover art entirely (used // only by paths that don't need art). func scoreToView(s *autotag.GroupScore, exp *explore.Service) *ScoreView { - out := &ScoreView{GroupKey: s.GroupKey} + rec := s.Recommendation + if rec == "" { + // Paths that rebuild a GroupScore from cached candidates + // don't run the scorer; derive the tier here. + rec = autotag.Recommend( + autotag.Group{Tracks: s.LocalTracks}, s.Candidates, + ) + } + + out := &ScoreView{GroupKey: s.GroupKey, Recommendation: string(rec)} for _, l := range s.LocalTracks { out.LocalTracks = append(out.LocalTracks, LocalTrackView{ @@ -1058,6 +1391,7 @@ func scoreToView(s *autotag.GroupScore, exp *explore.Service) *ScoreView { OriginalDate: c.OriginalDate, Country: c.Country, Status: c.Status, + PrimaryType: c.PrimaryType, TrackCount: c.TrackCount, Score: c.Score, Source: string(c.Source), @@ -1065,8 +1399,11 @@ func scoreToView(s *autotag.GroupScore, exp *explore.Service) *ScoreView { Breakdown: ScoreBreakdownView{ TitleAvg: c.Breakdown.TitleAvg, LengthAvg: c.Breakdown.LengthAvg, + ArtistFit: c.Breakdown.ArtistFit, + AlbumFit: c.Breakdown.AlbumFit, TrackCountFit: c.Breakdown.TrackCountFit, ReleaseMeta: c.Breakdown.ReleaseMeta, + Evidence: c.Breakdown.Evidence, }, } diff --git a/backend/config/config.go b/backend/config/config.go index 72b2e5e..267ce0c 100644 --- a/backend/config/config.go +++ b/backend/config/config.go @@ -174,9 +174,46 @@ func (c *Config) Save() error { return fmt.Errorf("could not marshal config struct: %w", err) } - err = os.WriteFile(c.filePath, confFileData, 0o644) + // Write atomically: marshal into a temp file in the same directory, + // then rename it over the target. os.WriteFile truncates the file + // in place before writing, so a crash or kill mid-write (common + // during dev restarts) can leave a truncated — often empty — config. + // An empty TOML file loads "successfully" as all-defaults and then + // gets re-saved as defaults, silently wiping the user's settings. + // A temp-file + rename makes the replacement atomic: a reader always + // sees either the previous file or the complete new one. + tmp, err := os.CreateTemp(path.Dir(c.filePath), "config-*.toml.tmp") if err != nil { - return fmt.Errorf("could not write config file (%s): %w", c.filePath, err) + return fmt.Errorf("could not create temp config file: %w", err) + } + + tmpName := tmp.Name() + + // Best-effort cleanup if we bail before the rename succeeds. + defer func() { _ = os.Remove(tmpName) }() + + if _, err := tmp.Write(confFileData); err != nil { + _ = tmp.Close() + + return fmt.Errorf("could not write temp config file: %w", err) + } + + if err := tmp.Sync(); err != nil { + _ = tmp.Close() + + return fmt.Errorf("could not sync temp config file: %w", err) + } + + if err := tmp.Close(); err != nil { + return fmt.Errorf("could not close temp config file: %w", err) + } + + if err := os.Chmod(tmpName, 0o644); err != nil { + return fmt.Errorf("could not set config file permissions: %w", err) + } + + if err := os.Rename(tmpName, c.filePath); err != nil { + return fmt.Errorf("could not replace config file (%s): %w", c.filePath, err) } c.logger.Debug("saved config to file", "file", c.filePath) diff --git a/backend/config/window.go b/backend/config/window.go index f34faf8..96e6e40 100644 --- a/backend/config/window.go +++ b/backend/config/window.go @@ -1,10 +1,20 @@ package config const ( - // DefaultWidth is the default window width in pixels. - DefaultWidth = 512 - // DefaultHeight is the default window height in pixels. - DefaultHeight = 384 + // DefaultWidth is the default window width in pixels for a fresh + // config. Kept comfortably above the minimum so a first launch + // (or a config with no saved size) opens at a usable size rather + // than the cramped minimum. + DefaultWidth = 1100 + // DefaultHeight is the default window height in pixels for a fresh config. + DefaultHeight = 720 + + // MinWidth is the smallest allowed window width in pixels. Wails + // enforces this at runtime; it is also the floor below which a + // reported size is treated as bogus and not persisted. + MinWidth = 512 + // MinHeight is the smallest allowed window height in pixels. + MinHeight = 384 ) // WindowConfig holds window size preferences. diff --git a/backend/database/database.go b/backend/database/database.go index db1f501..a63becb 100644 --- a/backend/database/database.go +++ b/backend/database/database.go @@ -30,13 +30,45 @@ import ( var schemas embed.FS // DB wraps the SQLite database connection and queries. +// +// Two handles back a single database file. db is the single-writer +// connection (MaxOpenConns 1) used for every write and every +// transaction. readDB is a small multi-connection, query-only pool +// used for standalone reads. Under WAL, readers run concurrently +// with the writer, so a long background write (index build, dump +// patch) no longer blocks interactive searches — the reason searches +// stalled for seconds was that the file was in rollback-journal mode +// with a single shared connection, so any writer locked out readers. type DB struct { - db *sql.DB - Ctx context.Context + db *sql.DB + readDB *sql.DB + Ctx context.Context + // Queries runs on the single-writer connection. Use it for every + // write and for any read that must observe an uncommitted write made + // earlier in the same logical operation. Queries *sqlcgen.Queries - logger *slog.Logger + // ReadQueries runs on the query-only WAL read pool, so standalone + // reads proceed concurrently with a long background write instead of + // queueing behind it on the single writer. It observes only + // committed data. In tests (no read pool) it aliases Queries. + ReadQueries *sqlcgen.Queries + logger *slog.Logger } +// Data-source names. modernc.org/sqlite only honours PRAGMAs passed +// as `_pragma=name(value)` — the mattn-style `_journal_mode=WAL` +// form is silently ignored, which is why WAL was never actually on. +const ( + writeDSNParams = "?_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)" + readDSNParams = "?_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)" + + "&_pragma=query_only(true)&_pragma=synchronous(NORMAL)" + + "&_pragma=cache_size(-8000)&_pragma=mmap_size(67108864)" + // readPoolConns bounds concurrent read connections. A handful is + // plenty for interactive search + art/lookup fan-out and keeps WAL + // reader overhead small. + readPoolConns = 4 +) + // NewDB opens the database and applies schema migrations. func NewDB(logger *slog.Logger) (*DB, error) { defer profiling.TimeOp(logger, "database.NewDB")() @@ -52,7 +84,7 @@ func NewDB(logger *slog.Logger) (*DB, error) { logger.Debug("opening sqlite database", "filepath", sqliteDBFilePath) - db, err := sql.Open("sqlite", sqliteDBFilePath+"?_busy_timeout=5000&_journal_mode=WAL") + db, err := sql.Open("sqlite", sqliteDBFilePath+writeDSNParams) if err != nil { return nil, fmt.Errorf("could not connect to sqlite database: %w", err) } @@ -126,14 +158,37 @@ func NewDB(logger *slog.Logger) (*DB, error) { // Get generated queries queries := sqlcgen.New(db) + // Open a separate query-only read pool. The write handle above + // has already converted the file to WAL, so these connections read + // a consistent snapshot concurrently with in-flight writes. + readDB, err := sql.Open("sqlite", sqliteDBFilePath+readDSNParams) + if err != nil { + return nil, fmt.Errorf("could not open read pool: %w", err) + } + + readDB.SetMaxOpenConns(readPoolConns) + return &DB{ - db: db, - Ctx: dbCtx, - Queries: queries, - logger: logger, + db: db, + readDB: readDB, + Ctx: dbCtx, + Queries: queries, + ReadQueries: sqlcgen.New(readDB), + logger: logger, }, err } +// reader returns the handle standalone reads should use: the +// query-only read pool when present, else the write handle (tests +// share one in-memory connection, which cannot be reopened). +func (d *DB) reader() *sql.DB { + if d.readDB != nil { + return d.readDB + } + + return d.db +} + // BeginTx starts a new database transaction. func (d *DB) BeginTx() (*sql.Tx, error) { return d.db.BeginTx(d.Ctx, nil) @@ -144,9 +199,20 @@ func (d *DB) ExecContext(query string, args ...any) (sql.Result, error) { return d.db.ExecContext(d.Ctx, query, args...) } -// QueryContext executes a query that returns rows. +// QueryContext executes a query that returns rows. Reads run on the +// query-only read pool so they proceed concurrently with writes under +// WAL instead of queueing behind the single writer connection. func (d *DB) QueryContext(query string, args ...any) (*sql.Rows, error) { - return d.db.QueryContext(d.Ctx, query, args...) + return d.reader().QueryContext(d.Ctx, query, args...) +} + +// QueryContextWith executes a query that returns rows using a +// caller-supplied context instead of the DB's lifecycle context. +// This lets an individual query (e.g. a superseded search) be +// cancelled independently. Like QueryContext it runs on the read +// pool. +func (d *DB) QueryContextWith(ctx context.Context, query string, args ...any) (*sql.Rows, error) { + return d.reader().QueryContext(ctx, query, args...) } // Logger returns the structured logger bound to this DB. Callers can @@ -976,6 +1042,468 @@ func runMigrations( } } + if version < 37 { //nolint:mnd + if err := migration37ExploreFTSDiacritics(ctx, db, logger); err != nil { + return err + } + } + + if version < 38 { //nolint:mnd + if err := migration38TaggingCandidates(ctx, db, logger); err != nil { + return err + } + } + + if version < 39 { //nolint:mnd + if err := migration39LyricsIndex(ctx, db, logger); err != nil { + return err + } + } + + if version < 40 { //nolint:mnd + if err := migration40ExploreExactMatchIndexes(ctx, db, logger); err != nil { + return err + } + } + + if version < 41 { //nolint:mnd + if err := migration41ExploreChampionFTS(ctx, db, logger); err != nil { + return err + } + } + + if version < 42 { //nolint:mnd + if err := migration42ReleaseToRG(ctx, db, logger); err != nil { + return err + } + } + + if version < 43 { //nolint:mnd + if err := migration43MergeArtistCredits(ctx, db, logger); err != nil { + return err + } + } + + if version < 44 { //nolint:mnd + if err := migration44ExploreCAAReleaseIndex(ctx, db, logger); err != nil { + return err + } + } + + if version < 45 { //nolint:mnd + if err := migration45Analyze(ctx, db, logger); err != nil { + return err + } + } + + if version < 46 { //nolint:mnd + if err := migration46SmartSnapshot(ctx, db, logger); err != nil { + return err + } + } + + return nil +} + +// migration46SmartSnapshot adds the smart_snapshot_at column to the +// playlists table. Smart playlists now materialize their evaluated +// membership into playlist_tracks and only re-evaluate on demand; the +// timestamp records when that snapshot was last taken (NULL means the +// playlist has never been materialized, so it is backfilled on first +// open). +func migration46SmartSnapshot( + ctx context.Context, + db *sql.DB, + logger *slog.Logger, +) error { + logger.Info("applying migration 46: smart playlist snapshot column") + + if _, err := db.ExecContext(ctx, + `ALTER TABLE playlists + ADD COLUMN smart_snapshot_at DATETIME`, + ); err != nil { + if !isDuplicateColumnErr(err) { + return fmt.Errorf( + "migration 46: could not add smart_snapshot_at column: %w", + err, + ) + } + } + + if _, err := db.ExecContext( + ctx, "PRAGMA user_version = 46", + ); err != nil { + return fmt.Errorf( + "migration 46: set user_version: %w", err, + ) + } + + logger.Info("migration 46 complete") + + return nil +} + +// migration45Analyze runs ANALYZE so SQLite's query planner has real +// table/index statistics. Without stats the planner guesses from row +// counts alone and mis-chose indexes on the ~2M-row explore_index — e.g. +// the top-result parent-release lookup scanned all 400k release_group +// rows via idx_explore_index_entity_pop instead of seeking the new +// idx_explore_caa_release, costing seconds per search. ANALYZE populates +// sqlite_stat1 (a one-time ~1.5s scan) and the planner then picks the +// right index for that query and every other query on these large tables. +func migration45Analyze( + ctx context.Context, + db *sql.DB, + logger *slog.Logger, +) error { + logger.Info("applying migration 45: ANALYZE for query planner statistics") + + if _, err := db.ExecContext(ctx, "ANALYZE"); err != nil { + return fmt.Errorf("migration 45: analyze: %w", err) + } + + if _, err := db.ExecContext(ctx, "PRAGMA user_version = 45"); err != nil { + return fmt.Errorf("migration 45: set user_version: %w", err) + } + + logger.Info("migration 45 complete") + + return nil +} + +// migration44ExploreCAAReleaseIndex adds a partial index on +// caa_release_mbid so the top-result resolver's parent-release-group +// lookup (SearchIndex.ReleaseGroupMBIDsForCAAReleaseMBIDs) seeks the +// index instead of scanning every release_group row in explore_index +// (~150k) on the hot search path. The index is partial, mirroring the +// query's own filter (entity_type = 'release_group' AND +// caa_release_mbid is non-empty), so it stays small and covers exactly the +// rows that lookup can match. Without it, a generic query whose top +// results include recordings with cover art (e.g. "big") spends +// seconds in this scan. +func migration44ExploreCAAReleaseIndex( + ctx context.Context, + db *sql.DB, + logger *slog.Logger, +) error { + logger.Info("applying migration 44: explore caa_release_mbid index") + + if _, err := db.ExecContext(ctx, ` + CREATE INDEX IF NOT EXISTS idx_explore_caa_release + ON explore_index(caa_release_mbid) + WHERE entity_type = 'release_group' AND caa_release_mbid != '' + `); err != nil { + return fmt.Errorf("migration 44: create caa_release index: %w", err) + } + + if _, err := db.ExecContext(ctx, "PRAGMA user_version = 44"); err != nil { + return fmt.Errorf("migration 44: set user_version: %w", err) + } + + logger.Info("migration 44 complete") + + return nil +} + +// migration43MergeArtistCredits repairs artist rows that were created +// from full credit strings. Before the scanner resolved a track's +// primary artist, a credit like "Lana Del Rey ft. Sean Lennon" was +// stored as its own artists row and stamped with the primary artist's +// single MBID — so one MusicBrainz artist fanned out into many rows that +// shared an MBID, and the explore index (last-write-wins per MBID) then +// displayed a featured-credit string as the artist's name. +// +// This collapses every set of artists rows that share an MBID into the +// one "clean" member (a name with no featuring clause), repoints the +// artist_credit_artist links, deletes the redundant rows, and refreshes +// the explore index's artist titles from the survivors. Clusters with +// no clean member (all names carry a marker) are left untouched. +func migration43MergeArtistCredits( + ctx context.Context, + db *sql.DB, + logger *slog.Logger, +) error { + logger.Info("applying migration 43: merge credit-string artists") + + // Map each redundant artist row to the clean canonical row for its + // MBID. "Clean" = a name carrying no featuring marker; the lowest + // id among those is the canonical survivor. + if _, err := db.ExecContext(ctx, ` + CREATE TEMP TABLE artist_merge_map AS + SELECT a.id AS dirty_id, canon.canon_id AS canon_id + FROM artists a + JOIN ( + SELECT mbid, MIN(id) AS canon_id + FROM artists + WHERE mbid IS NOT NULL AND mbid != '' + AND lower(name) NOT LIKE '% feat %' + AND lower(name) NOT LIKE '% feat. %' + AND lower(name) NOT LIKE '% featuring %' + AND lower(name) NOT LIKE '% ft %' + AND lower(name) NOT LIKE '% ft. %' + GROUP BY mbid + ) canon ON canon.mbid = a.mbid + WHERE a.id != canon.canon_id + `); err != nil { + return fmt.Errorf("migration 43: build merge map: %w", err) + } + + // Drop links that would collide with an existing (canonical, credit) + // link after repointing — the unique index would otherwise reject + // the UPDATE. + if _, err := db.ExecContext(ctx, ` + DELETE FROM artist_credit_artist + WHERE id IN ( + SELECT aca.id + FROM artist_credit_artist aca + JOIN artist_merge_map m ON m.dirty_id = aca.artist_id + WHERE EXISTS ( + SELECT 1 FROM artist_credit_artist keep + WHERE keep.artist_id = m.canon_id + AND keep.credit_id = aca.credit_id + ) + ) + `); err != nil { + return fmt.Errorf("migration 43: prune colliding links: %w", err) + } + + // Repoint surviving links to the canonical artist. + if _, err := db.ExecContext(ctx, ` + UPDATE artist_credit_artist + SET artist_id = ( + SELECT canon_id FROM artist_merge_map + WHERE dirty_id = artist_credit_artist.artist_id + ) + WHERE artist_id IN (SELECT dirty_id FROM artist_merge_map) + `); err != nil { + return fmt.Errorf("migration 43: repoint links: %w", err) + } + + // Remove the now-orphaned credit-string artist rows. + if _, err := db.ExecContext(ctx, ` + DELETE FROM artists WHERE id IN (SELECT dirty_id FROM artist_merge_map) + `); err != nil { + return fmt.Errorf("migration 43: delete merged artists: %w", err) + } + + // Refresh explore-index artist rows from the surviving library + // artists so their (previously clobbered) titles show the clean + // name. The AFTER UPDATE trigger keeps explore_index_fts in sync. + // Only rows backed by a library artist are touched; dump-only rows + // are left alone. + if _, err := db.ExecContext(ctx, ` + UPDATE explore_index + SET title = ( + SELECT name FROM artists + WHERE artists.mbid = explore_index.mbid ORDER BY id LIMIT 1), + artist_name = ( + SELECT name FROM artists + WHERE artists.mbid = explore_index.mbid ORDER BY id LIMIT 1), + local_artist_id = ( + SELECT id FROM artists + WHERE artists.mbid = explore_index.mbid ORDER BY id LIMIT 1) + WHERE entity_type = 'artist' + AND EXISTS (SELECT 1 FROM artists WHERE artists.mbid = explore_index.mbid) + `); err != nil { + return fmt.Errorf("migration 43: refresh explore titles: %w", err) + } + + if _, err := db.ExecContext(ctx, "DROP TABLE IF EXISTS artist_merge_map"); err != nil { + return fmt.Errorf("migration 43: drop temp table: %w", err) + } + + if _, err := db.ExecContext(ctx, "PRAGMA user_version = 43"); err != nil { + return fmt.Errorf("migration 43: set user_version: %w", err) + } + + logger.Info("migration 43 complete") + + return nil +} + +// migration42ReleaseToRG creates the release_to_rg mapping table: for +// every release under an indexed release group, which release-group it +// belongs to. It is populated from the canonical dump during a full +// import (the mapping is otherwise in-memory only and discarded). The +// incremental-dump popularity refresh uses it to roll per-release listen +// deltas up to their release group, so album popularity stays fresh +// without any API call. +func migration42ReleaseToRG( + ctx context.Context, + db *sql.DB, + logger *slog.Logger, +) error { + logger.Info("applying migration 42: release_to_rg table") + + if _, err := db.ExecContext(ctx, ` + CREATE TABLE IF NOT EXISTS release_to_rg ( + release_mbid TEXT PRIMARY KEY, + rg_mbid TEXT NOT NULL + ) WITHOUT ROWID + `); err != nil { + return fmt.Errorf("migration 42: create release_to_rg: %w", err) + } + + if _, err := db.ExecContext(ctx, + "PRAGMA user_version = 42", + ); err != nil { + return fmt.Errorf("migration 42: set user_version: %w", err) + } + + logger.Info("migration 42 complete") + + return nil +} + +// migration41ExploreChampionFTS creates the "champion" full-text index: +// a second external-content FTS5 over explore_index that holds only the +// high-popularity / owned rows. Short generic prefixes ("the", "a") +// match hundreds of thousands of rows in the full index, and the +// popularity-blended ORDER BY must score every one of them — seconds of +// work. Routing those queries at the champion index instead scores only +// the ~90k rows that could plausibly win, cutting the query from seconds +// to tens of milliseconds. The table is created empty here; the search +// index populates it at runtime (see SearchIndex.RebuildChampionIndex) +// because the row set derives from popularity, which changes as the +// index is (re)built. +func migration41ExploreChampionFTS( + ctx context.Context, + db *sql.DB, + logger *slog.Logger, +) error { + logger.Info("applying migration 41: explore champion FTS") + + if _, err := db.ExecContext(ctx, ` + CREATE VIRTUAL TABLE IF NOT EXISTS explore_champion_fts USING fts5( + title, artist_name, aliases, + content='explore_index', + content_rowid='id', + tokenize='unicode61 remove_diacritics 2' + ) + `); err != nil { + return fmt.Errorf("migration 41: create champion fts: %w", err) + } + + if _, err := db.ExecContext(ctx, "PRAGMA user_version = 41"); err != nil { + return fmt.Errorf("migration 41: set user_version: %w", err) + } + + logger.Info("migration 41 complete") + + return nil +} + +// migration39LyricsIndex creates the contentless FTS5 lyrics_index +// (see lyrics_index.sql) and back-populates it from any recordings +// that already have embedded lyrics, so lyric search works on +// existing libraries without waiting for a rescan. +func migration39LyricsIndex( + ctx context.Context, + db *sql.DB, + logger *slog.Logger, +) error { + logger.Info("applying migration 39: lyrics_index FTS") + + if _, err := db.ExecContext(ctx, ` + CREATE VIRTUAL TABLE IF NOT EXISTS lyrics_index USING fts5( + lyrics, + content='', + contentless_delete=1, + tokenize='unicode61 remove_diacritics 2' + ) + `); err != nil { + return fmt.Errorf("migration 39: create lyrics_index: %w", err) + } + + // Back-populate from recordings that already carry lyrics. The + // rowid is the recording id so it stays stable across rebuilds. + if _, err := db.ExecContext(ctx, ` + INSERT INTO lyrics_index(rowid, lyrics) + SELECT id, lyrics + FROM recordings + WHERE lyrics IS NOT NULL AND lyrics != '' + `); err != nil { + return fmt.Errorf("migration 39: populate lyrics_index: %w", err) + } + + if _, err := db.ExecContext(ctx, "PRAGMA user_version = 39"); err != nil { + return fmt.Errorf("migration 39: set user_version: %w", err) + } + + logger.Info("migration 39 complete") + + return nil +} + +// migration40ExploreExactMatchIndexes adds partial expression indexes +// on LOWER(title) and LOWER(artist_name) so the interactive top-result +// resolver's exact-match lookup (SearchIndex.ExactMatches) seeks the +// index instead of scanning all ~240k explore_index rows on every +// keystroke. The indexes are partial (WHERE popularity > 0) because +// that lookup always filters on popularity, keeping them small; the +// UNION-of-equalities query shape in ExactMatches is what lets SQLite +// use them (an OR across the two columns forces a scan instead). +func migration40ExploreExactMatchIndexes( + ctx context.Context, + db *sql.DB, + logger *slog.Logger, +) error { + logger.Info("applying migration 40: explore exact-match indexes") + + if _, err := db.ExecContext(ctx, ` + CREATE INDEX IF NOT EXISTS idx_explore_title_lower + ON explore_index(LOWER(title)) + WHERE popularity > 0 + `); err != nil { + return fmt.Errorf("migration 40: create title index: %w", err) + } + + if _, err := db.ExecContext(ctx, ` + CREATE INDEX IF NOT EXISTS idx_explore_artist_lower + ON explore_index(LOWER(artist_name)) + WHERE popularity > 0 + `); err != nil { + return fmt.Errorf("migration 40: create artist index: %w", err) + } + + if _, err := db.ExecContext(ctx, "PRAGMA user_version = 40"); err != nil { + return fmt.Errorf("migration 40: set user_version: %w", err) + } + + logger.Info("migration 40 complete") + + return nil +} + +// migration38TaggingCandidates creates the tagging_candidates table — +// a durable per-group store for the scored candidate list so it is +// computed once and reused across restarts instead of re-hitting +// MusicBrainz every session (see tagging_candidates.sql). A plain +// CREATE TABLE IF NOT EXISTS is safe on both fresh and existing DBs. +func migration38TaggingCandidates( + ctx context.Context, + db *sql.DB, + logger *slog.Logger, +) error { + logger.Info("applying migration 38: tagging_candidates") + + if _, err := db.ExecContext(ctx, ` + CREATE TABLE IF NOT EXISTS tagging_candidates ( + group_key TEXT PRIMARY KEY, + candidates TEXT NOT NULL, + computed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY(group_key) REFERENCES tagging_items(group_key) ON DELETE CASCADE + ) + `); err != nil { + return fmt.Errorf("migration 38: create tagging_candidates: %w", err) + } + + if _, err := db.ExecContext(ctx, "PRAGMA user_version = 38"); err != nil { + return fmt.Errorf("migration 38: set user_version: %w", err) + } + + logger.Info("migration 38 complete") + return nil } @@ -1010,6 +1538,70 @@ func migration36ClearedAt( return nil } +// migration37ExploreFTSDiacritics rebuilds explore_index_fts with the +// "unicode61 remove_diacritics 2" tokeniser so accented queries match +// their unaccented forms (e.g. "beyonce" finds "Beyoncé"), matching the +// library search_index tokeniser. The original table (migration 26) +// was created with the default tokeniser, which does not fold +// diacritics. +// +// Because explore_index_fts is an external-content table over +// explore_index, the rebuild repopulates from the existing content +// rows — no data loss and no need to re-run the expensive tiered index +// build. +func migration37ExploreFTSDiacritics( + ctx context.Context, + db *sql.DB, + logger *slog.Logger, +) error { + logger.Info("applying migration 37: explore_index_fts diacritic folding") + + // Drop the sync triggers and the FTS table, then recreate both. + // The triggers must go first — they reference the FTS table. + stmts := []string{ + `DROP TRIGGER IF EXISTS explore_index_ai`, + `DROP TRIGGER IF EXISTS explore_index_ad`, + `DROP TRIGGER IF EXISTS explore_index_au`, + `DROP TABLE IF EXISTS explore_index_fts`, + `CREATE VIRTUAL TABLE explore_index_fts USING fts5( + title, artist_name, aliases, + content='explore_index', + content_rowid='id', + tokenize='unicode61 remove_diacritics 2' + )`, + `CREATE TRIGGER explore_index_ai AFTER INSERT ON explore_index BEGIN + INSERT INTO explore_index_fts(rowid, title, artist_name, aliases) + VALUES (new.id, new.title, new.artist_name, new.aliases); + END`, + `CREATE TRIGGER explore_index_ad AFTER DELETE ON explore_index BEGIN + INSERT INTO explore_index_fts(explore_index_fts, rowid, title, artist_name, aliases) + VALUES ('delete', old.id, old.title, old.artist_name, old.aliases); + END`, + `CREATE TRIGGER explore_index_au AFTER UPDATE ON explore_index BEGIN + INSERT INTO explore_index_fts(explore_index_fts, rowid, title, artist_name, aliases) + VALUES ('delete', old.id, old.title, old.artist_name, old.aliases); + INSERT INTO explore_index_fts(rowid, title, artist_name, aliases) + VALUES (new.id, new.title, new.artist_name, new.aliases); + END`, + // Repopulate the FTS index from the content table. + `INSERT INTO explore_index_fts(explore_index_fts) VALUES('rebuild')`, + } + + for _, stmt := range stmts { + if _, err := db.ExecContext(ctx, stmt); err != nil { + return fmt.Errorf("migration 37: %w", err) + } + } + + if _, err := db.ExecContext(ctx, "PRAGMA user_version = 37"); err != nil { + return fmt.Errorf("migration 37: set user_version: %w", err) + } + + logger.Info("migration 37 complete") + + return nil +} + // backfills it from file_path, creates the basename index, and // populates the FTS5 search_index table. func migration2BasenameAndFTS( diff --git a/backend/database/lyrics_search.go b/backend/database/lyrics_search.go new file mode 100644 index 0000000..6eddcb0 --- /dev/null +++ b/backend/database/lyrics_search.go @@ -0,0 +1,295 @@ +package database + +import ( + "fmt" + "strings" + "unicode" +) + +// LyricsHit is a single result from a lyric-fragment search: the +// matched recording plus enough metadata to render and play it. +type LyricsHit struct { + RecordingID int64 + FilePath string + LengthMilliseconds int64 + Title string + Artist string + Album string +} + +// SearchLyrics finds recordings whose lyrics match the given query, +// ranked by FTS5 relevance. The query is treated as a phrase so a +// fragment like "hello darkness my old friend" matches consecutive +// words rather than each word independently. Returns nil for an +// empty query. +func (d *DB) SearchLyrics(query string, limit int) ([]LyricsHit, error) { + query = strings.TrimSpace(query) + if query == "" { + return nil, nil + } + + if limit <= 0 { + limit = 25 + } + + ftsQuery := buildLyricsPhraseQuery(query) + if ftsQuery == "" { + return nil, nil + } + + // Map the matched recording (lyrics_index.rowid == recordings.id) + // to a representative playable file via the lowest audio_files id, + // then to the track_metadata VIEW for display fields. + // + // SAFETY: FTS5 MATCH syntax unsupported by sqlc. Query is parameterized; no string interpolation. + rows, err := d.db.QueryContext(d.Ctx, ` + SELECT + r.id, + tm.file_path, + tm.length_milliseconds, + tm.title, + tm.artist_name, + tm.album + FROM lyrics_index li + JOIN recordings r ON r.id = li.rowid + JOIN ( + SELECT recording_id, MIN(id) AS af_id + FROM audio_files + GROUP BY recording_id + ) af ON af.recording_id = r.id + JOIN track_metadata tm ON tm.id = af.af_id + WHERE lyrics_index MATCH ? + ORDER BY rank + LIMIT ? + `, ftsQuery, limit) + if err != nil { + return nil, fmt.Errorf("lyrics search failed: %w", err) + } + + defer func() { _ = rows.Close() }() + + var results []LyricsHit + + for rows.Next() { + var h LyricsHit + if err := rows.Scan( + &h.RecordingID, + &h.FilePath, + &h.LengthMilliseconds, + &h.Title, + &h.Artist, + &h.Album, + ); err != nil { + return nil, fmt.Errorf("could not scan lyrics hit: %w", err) + } + + results = append(results, h) + } + + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("lyrics hit iteration error: %w", err) + } + + return results, nil +} + +// GetRecordingLyrics returns the stored lyrics for a recording, or +// an empty string if none are stored. +func (d *DB) GetRecordingLyrics(recordingID int64) (string, error) { + var lyrics string + + err := d.db.QueryRowContext(d.Ctx, + "SELECT COALESCE(lyrics, '') FROM recordings WHERE id = ?", + recordingID, + ).Scan(&lyrics) + if err != nil { + return "", fmt.Errorf("could not read recording lyrics: %w", err) + } + + return lyrics, nil +} + +// SetRecordingLyrics writes lyrics onto a recording and keeps the FTS +// lyrics_index in sync (delete + reinsert the single row). Used by +// the LRCLIB backfill to persist fetched lyrics. Passing an empty +// string clears both the column and the index entry. +func (d *DB) SetRecordingLyrics(recordingID int64, lyrics string) error { + if _, err := d.db.ExecContext(d.Ctx, + "UPDATE recordings SET lyrics = ? WHERE id = ?", + lyrics, recordingID, + ); err != nil { + return fmt.Errorf("could not update recording lyrics: %w", err) + } + + return d.upsertLyricsIndex(recordingID, lyrics) +} + +// upsertLyricsIndex refreshes a single recording's entry in the +// contentless lyrics_index. contentless_delete=1 makes the DELETE +// valid; an empty lyrics string leaves the row deleted. +func (d *DB) upsertLyricsIndex(recordingID int64, lyrics string) error { + if _, err := d.db.ExecContext(d.Ctx, + "DELETE FROM lyrics_index WHERE rowid = ?", recordingID, + ); err != nil { + return fmt.Errorf("could not delete lyrics_index row: %w", err) + } + + if strings.TrimSpace(lyrics) == "" { + return nil + } + + // SAFETY: FTS5 virtual table INSERT unsupported by sqlc. All values parameterized. + if _, err := d.db.ExecContext(d.Ctx, + "INSERT INTO lyrics_index(rowid, lyrics) VALUES (?, ?)", + recordingID, lyrics, + ); err != nil { + return fmt.Errorf("could not insert lyrics_index row: %w", err) + } + + return nil +} + +// RebuildLyricsIndex repopulates lyrics_index from scratch using the +// current recordings table. Cheap for a personal library and safe to +// run after every scan. +func (d *DB) RebuildLyricsIndex() error { + if _, err := d.db.ExecContext(d.Ctx, + "DELETE FROM lyrics_index", + ); err != nil { + return fmt.Errorf("could not clear lyrics_index: %w", err) + } + + // SAFETY: FTS5 virtual table INSERT unsupported by sqlc. Values sourced from recordings; no user input. + if _, err := d.db.ExecContext(d.Ctx, ` + INSERT INTO lyrics_index(rowid, lyrics) + SELECT id, lyrics + FROM recordings + WHERE lyrics IS NOT NULL AND lyrics != '' + `); err != nil { + return fmt.Errorf("could not rebuild lyrics_index: %w", err) + } + + return nil +} + +// RecordingsMissingLyrics returns recordings that have no stored +// lyrics but do carry the artist/title/duration needed to look them +// up from an external provider. Used by the LRCLIB backfill. The +// limit bounds each batch so the backfill can be run incrementally. +func (d *DB) RecordingsMissingLyrics(limit int) ([]LyricsCandidate, error) { + if limit <= 0 { + limit = 200 + } + + rows, err := d.db.QueryContext(d.Ctx, ` + SELECT + r.id, + COALESCE(r.name, ''), + COALESCE(ac.text, ''), + COALESCE(rg.name, ''), + MIN(af.length_milliseconds) + FROM recordings r + JOIN audio_files af ON af.recording_id = r.id + LEFT JOIN artist_credit ac ON r.artist_credit_id = ac.id + LEFT JOIN ( + SELECT recording_id, MIN(release_group_id) AS release_group_id + FROM release_group_recordings + GROUP BY recording_id + ) rgr ON rgr.recording_id = r.id + LEFT JOIN release_groups rg ON rg.id = rgr.release_group_id + WHERE (r.lyrics IS NULL OR r.lyrics = '') + AND r.name IS NOT NULL AND r.name != '' + AND ac.text IS NOT NULL AND ac.text != '' + GROUP BY r.id + LIMIT ? + `, limit) + if err != nil { + return nil, fmt.Errorf("could not query recordings missing lyrics: %w", err) + } + + defer func() { _ = rows.Close() }() + + var out []LyricsCandidate + + for rows.Next() { + var c LyricsCandidate + if err := rows.Scan( + &c.RecordingID, &c.Title, &c.Artist, &c.Album, &c.LengthMilliseconds, + ); err != nil { + return nil, fmt.Errorf("could not scan lyrics candidate: %w", err) + } + + out = append(out, c) + } + + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("lyrics candidate iteration error: %w", err) + } + + return out, nil +} + +// LyricsCandidate identifies a recording that needs its lyrics fetched +// and carries the fields an external provider matches on. +type LyricsCandidate struct { + RecordingID int64 + Title string + Artist string + Album string + LengthMilliseconds int64 +} + +// RecordingLyricLookup returns the provider-match fields (artist, +// title, album, duration) for a single recording, so lyrics can be +// fetched on demand. Returns nil if the recording has no audio file +// or no artist/title to match on. +func (d *DB) RecordingLyricLookup(recordingID int64) (*LyricsCandidate, error) { + var c LyricsCandidate + + err := d.db.QueryRowContext(d.Ctx, ` + SELECT + r.id, + COALESCE(r.name, ''), + COALESCE(ac.text, ''), + COALESCE(rg.name, ''), + COALESCE(MIN(af.length_milliseconds), 0) + FROM recordings r + JOIN audio_files af ON af.recording_id = r.id + LEFT JOIN artist_credit ac ON r.artist_credit_id = ac.id + LEFT JOIN ( + SELECT recording_id, MIN(release_group_id) AS release_group_id + FROM release_group_recordings + GROUP BY recording_id + ) rgr ON rgr.recording_id = r.id + LEFT JOIN release_groups rg ON rg.id = rgr.release_group_id + WHERE r.id = ? + GROUP BY r.id + `, recordingID).Scan(&c.RecordingID, &c.Title, &c.Artist, &c.Album, &c.LengthMilliseconds) + if err != nil { + return nil, fmt.Errorf("could not look up recording for lyrics: %w", err) + } + + if c.Title == "" || c.Artist == "" { + return nil, nil + } + + return &c, nil +} + +// buildLyricsPhraseQuery turns a user's lyric fragment into an FTS5 +// phrase query — a single double-quoted string of tokens — so the +// words must appear adjacently ("hello darkness my old friend"), +// which is what a lyric search means. All non-alphanumeric runes are +// treated as separators (matching the unicode61 tokeniser), so no +// user character can break out of the quoted phrase. Returns "" when +// the fragment has no searchable tokens. +func buildLyricsPhraseQuery(query string) string { + fields := strings.FieldsFunc(query, func(r rune) bool { + return !unicode.IsLetter(r) && !unicode.IsNumber(r) + }) + if len(fields) == 0 { + return "" + } + + return `"` + strings.Join(fields, " ") + `"` +} diff --git a/backend/database/lyrics_search_test.go b/backend/database/lyrics_search_test.go new file mode 100644 index 0000000..f0a6e15 --- /dev/null +++ b/backend/database/lyrics_search_test.go @@ -0,0 +1,252 @@ +package database + +import ( + "testing" +) + +// seedLyricsTrack inserts the minimal FK chain (artist_credit → +// recording → audio_file → release_group link) for one track with the +// given lyrics, so lyric-search tests have realistic joins. +func seedLyricsTrack( + t *testing.T, + db *DB, + id int64, + title, artist, album, lyrics string, + lenMs int64, +) { + t.Helper() + + if _, err := db.ExecContext( + "INSERT OR IGNORE INTO artist_credit (id, text) VALUES (?, ?)", id, artist, + ); err != nil { + t.Fatalf("insert artist_credit: %v", err) + } + + if _, err := db.ExecContext( + "INSERT OR IGNORE INTO release_groups (id, name) VALUES (?, ?)", id, album, + ); err != nil { + t.Fatalf("insert release_group: %v", err) + } + + if _, err := db.ExecContext( + "INSERT INTO recordings (id, name, artist_credit_id, lyrics) VALUES (?, ?, ?, ?)", + id, title, id, nullableLyrics(lyrics), + ); err != nil { + t.Fatalf("insert recording: %v", err) + } + + if _, err := db.ExecContext( + "INSERT INTO audio_files (id, file_path, length_milliseconds, file_type_id, recording_id) "+ + "VALUES (?, ?, ?, ?, ?)", + id, "/music/track"+itoa(id)+".mp3", lenMs, 0, id, + ); err != nil { + t.Fatalf("insert audio_file: %v", err) + } + + if _, err := db.ExecContext( + "INSERT INTO release_group_recordings (release_group_id, recording_id) VALUES (?, ?)", + id, id, + ); err != nil { + t.Fatalf("insert release_group_recordings: %v", err) + } +} + +func nullableLyrics(l string) any { + if l == "" { + return nil + } + + return l +} + +func itoa(v int64) string { + if v == 0 { + return "0" + } + + var b []byte + + for v > 0 { + b = append([]byte{byte('0' + v%10)}, b...) + v /= 10 + } + + return string(b) +} + +func TestSearchLyrics(t *testing.T) { + t.Parallel() + + db := NewTestDB(t) + + seedLyricsTrack( + t, + db, + 1, + "The Sound of Silence", + "Simon & Garfunkel", + "Sounds of Silence", + "Hello darkness my old friend\nI've come to talk with you again", + 180000, + ) + seedLyricsTrack(t, db, 2, "Bohemian Rhapsody", "Queen", "A Night at the Opera", + "Is this the real life? Is this just fantasy?", 354000) + seedLyricsTrack(t, db, 3, "Instrumental Track", "Some Artist", "Some Album", + "", 200000) // no lyrics — must never appear in results + + if err := db.RebuildLyricsIndex(); err != nil { + t.Fatalf("RebuildLyricsIndex: %v", err) + } + + t.Run("phrase match returns the right track with metadata", func(t *testing.T) { + t.Parallel() + + hits, err := db.SearchLyrics("hello darkness my old friend", 10) + if err != nil { + t.Fatalf("SearchLyrics: %v", err) + } + + if len(hits) != 1 { + t.Fatalf("expected 1 hit, got %d: %+v", len(hits), hits) + } + + h := hits[0] + if h.RecordingID != 1 { + t.Errorf("RecordingID = %d, want 1", h.RecordingID) + } + + if h.Title != "The Sound of Silence" { + t.Errorf("Title = %q, want The Sound of Silence", h.Title) + } + + if h.Artist != "Simon & Garfunkel" { + t.Errorf("Artist = %q, want Simon & Garfunkel", h.Artist) + } + + if h.Album != "Sounds of Silence" { + t.Errorf("Album = %q, want Sounds of Silence", h.Album) + } + + if h.FilePath == "" { + t.Error("FilePath is empty; expected a playable path") + } + }) + + t.Run("adjacency: scrambled words do not match as a phrase", func(t *testing.T) { + t.Parallel() + + hits, err := db.SearchLyrics("friend old darkness", 10) + if err != nil { + t.Fatalf("SearchLyrics: %v", err) + } + + if len(hits) != 0 { + t.Errorf("expected 0 phrase hits for scrambled words, got %d", len(hits)) + } + }) + + t.Run("empty query returns nil", func(t *testing.T) { + t.Parallel() + + hits, err := db.SearchLyrics(" ", 10) + if err != nil { + t.Fatalf("SearchLyrics: %v", err) + } + + if hits != nil { + t.Errorf("expected nil for empty query, got %+v", hits) + } + }) + + t.Run("no match returns no hits", func(t *testing.T) { + t.Parallel() + + hits, err := db.SearchLyrics("this phrase appears in no song", 10) + if err != nil { + t.Fatalf("SearchLyrics: %v", err) + } + + if len(hits) != 0 { + t.Errorf("expected 0 hits, got %d", len(hits)) + } + }) +} + +func TestSetRecordingLyricsUpdatesIndex(t *testing.T) { + t.Parallel() + + db := NewTestDB(t) + + // Track starts with no lyrics. + seedLyricsTrack(t, db, 1, "Yesterday", "The Beatles", "Help!", "", 125000) + + if err := db.RebuildLyricsIndex(); err != nil { + t.Fatalf("RebuildLyricsIndex: %v", err) + } + + // Nothing indexed yet. + if hits, _ := db.SearchLyrics("yesterday all my troubles", 10); len(hits) != 0 { + t.Fatalf("expected 0 hits before backfill, got %d", len(hits)) + } + + // Backfill lyrics — should update both the column and the FTS index. + const lyrics = "Yesterday all my troubles seemed so far away" + if err := db.SetRecordingLyrics(1, lyrics); err != nil { + t.Fatalf("SetRecordingLyrics: %v", err) + } + + stored, err := db.GetRecordingLyrics(1) + if err != nil { + t.Fatalf("GetRecordingLyrics: %v", err) + } + + if stored != lyrics { + t.Errorf("stored lyrics = %q, want %q", stored, lyrics) + } + + hits, err := db.SearchLyrics("all my troubles seemed so far away", 10) + if err != nil { + t.Fatalf("SearchLyrics: %v", err) + } + + if len(hits) != 1 || hits[0].RecordingID != 1 { + t.Fatalf("expected recording 1 after backfill, got %+v", hits) + } +} + +func TestRecordingsMissingLyrics(t *testing.T) { + t.Parallel() + + db := NewTestDB(t) + + seedLyricsTrack(t, db, 1, "Has Lyrics", "Artist A", "Album A", "some words here", 100000) + seedLyricsTrack(t, db, 2, "No Lyrics", "Artist B", "Album B", "", 200000) + + missing, err := db.RecordingsMissingLyrics(50) + if err != nil { + t.Fatalf("RecordingsMissingLyrics: %v", err) + } + + if len(missing) != 1 { + t.Fatalf("expected 1 candidate, got %d: %+v", len(missing), missing) + } + + c := missing[0] + if c.RecordingID != 2 || c.Title != "No Lyrics" || c.Artist != "Artist B" { + t.Errorf("unexpected candidate: %+v", c) + } + + if c.LengthMilliseconds != 200000 { + t.Errorf("LengthMilliseconds = %d, want 200000", c.LengthMilliseconds) + } + + // Single-recording lookup mirrors the batch fields. + one, err := db.RecordingLyricLookup(2) + if err != nil { + t.Fatalf("RecordingLyricLookup: %v", err) + } + + if one == nil || one.Artist != "Artist B" || one.Album != "Album B" { + t.Errorf("unexpected lookup: %+v", one) + } +} diff --git a/backend/database/sql/queries/release_groups.sql b/backend/database/sql/queries/release_groups.sql index 0ad2d9e..ed58984 100644 --- a/backend/database/sql/queries/release_groups.sql +++ b/backend/database/sql/queries/release_groups.sql @@ -64,6 +64,17 @@ SELECT COALESCE(rg.year, 0) AS release_year, rg.mbid, COALESCE(ac.text, fallback_ac.text, '') as artist_name, + -- primary (first-credited) album artist's MBID, for linking the + -- artist name to its detail page. Empty when the album has no + -- MB-tagged album-artist credit. + CAST(COALESCE(( + SELECT a.mbid + FROM artist_credit_artist aca_p + JOIN artists a ON a.id = aca_p.artist_id + WHERE aca_p.credit_id = rg.album_artist_credit_id + ORDER BY aca_p.id + LIMIT 1 + ), '') AS TEXT) as artist_mbid, COALESCE(ca.file_path, '') as cover_art_path FROM release_groups rg LEFT JOIN artist_credit ac ON rg.album_artist_credit_id = ac.id @@ -89,6 +100,17 @@ SELECT COALESCE(rg.year, 0) AS release_year, rg.mbid, COALESCE(ac.text, fallback_ac.text, '') as artist_name, + -- primary (first-credited) album artist's MBID, for linking the + -- artist name to its detail page. Empty when the album has no + -- MB-tagged album-artist credit. + CAST(COALESCE(( + SELECT a.mbid + FROM artist_credit_artist aca_p + JOIN artists a ON a.id = aca_p.artist_id + WHERE aca_p.credit_id = rg.album_artist_credit_id + ORDER BY aca_p.id + LIMIT 1 + ), '') AS TEXT) as artist_mbid, COALESCE(ca.file_path, '') as cover_art_path FROM release_groups rg LEFT JOIN artist_credit ac ON rg.album_artist_credit_id = ac.id @@ -116,6 +138,17 @@ SELECT COALESCE(rg.original_year, rg.year) AS year, COALESCE(rg.year, 0) AS release_year, COALESCE(ac.text, fallback_ac.text, '') as artist_name, + -- primary (first-credited) album artist's MBID, for linking the + -- artist name to its detail page. Empty when the album has no + -- MB-tagged album-artist credit. + CAST(COALESCE(( + SELECT a.mbid + FROM artist_credit_artist aca_p + JOIN artists a ON a.id = aca_p.artist_id + WHERE aca_p.credit_id = rg.album_artist_credit_id + ORDER BY aca_p.id + LIMIT 1 + ), '') AS TEXT) as artist_mbid, COALESCE(ca.file_path, '') as cover_art_path FROM release_groups rg JOIN artist_credit ac ON rg.album_artist_credit_id = ac.id @@ -141,6 +174,17 @@ SELECT COALESCE(rg.original_year, rg.year) AS year, COALESCE(rg.year, 0) AS release_year, COALESCE(ac.text, fallback_ac.text, '') as artist_name, + -- primary (first-credited) album artist's MBID, for linking the + -- artist name to its detail page. Empty when the album has no + -- MB-tagged album-artist credit. + CAST(COALESCE(( + SELECT a.mbid + FROM artist_credit_artist aca_p + JOIN artists a ON a.id = aca_p.artist_id + WHERE aca_p.credit_id = rg.album_artist_credit_id + ORDER BY aca_p.id + LIMIT 1 + ), '') AS TEXT) as artist_mbid, COALESCE(ca.file_path, '') as cover_art_path FROM release_groups rg JOIN artist_credit ac ON rg.album_artist_credit_id = ac.id diff --git a/backend/database/sql/queries/tagging_candidates.sql b/backend/database/sql/queries/tagging_candidates.sql new file mode 100644 index 0000000..7fca135 --- /dev/null +++ b/backend/database/sql/queries/tagging_candidates.sql @@ -0,0 +1,15 @@ +-- name: UpsertTaggingCandidates :exec +INSERT INTO tagging_candidates (group_key, candidates, computed_at) +VALUES (?, ?, CURRENT_TIMESTAMP) +ON CONFLICT(group_key) DO UPDATE SET + candidates = excluded.candidates, + computed_at = excluded.computed_at; + +-- name: GetTaggingCandidates :one +SELECT candidates FROM tagging_candidates +WHERE group_key = ? +LIMIT 1; + +-- name: DeleteTaggingCandidates :exec +DELETE FROM tagging_candidates +WHERE group_key = ?; diff --git a/backend/database/sql/queries/tagging_items.sql b/backend/database/sql/queries/tagging_items.sql index 5002379..8d130fc 100644 --- a/backend/database/sql/queries/tagging_items.sql +++ b/backend/database/sql/queries/tagging_items.sql @@ -62,7 +62,7 @@ SELECT ti.library_id, COALESCE(lb.name, '') AS library_name, COALESCE(lb.path, '') AS library_path, - CAST(COALESCE((SELECT af.file_path FROM audio_files af WHERE af.group_key = ti.group_key LIMIT 1), '') AS TEXT) AS sample_file_path, + CAST(COALESCE((SELECT af.file_path FROM audio_files af WHERE af.group_key = ti.group_key AND af.group_key != '' LIMIT 1), '') AS TEXT) AS sample_file_path, ti.track_count, ti.album_name, ti.album_artist, @@ -93,7 +93,7 @@ SELECT ti.library_id, COALESCE(lb.name, '') AS library_name, COALESCE(lb.path, '') AS library_path, - CAST(COALESCE((SELECT af.file_path FROM audio_files af WHERE af.group_key = ti.group_key LIMIT 1), '') AS TEXT) AS sample_file_path, + CAST(COALESCE((SELECT af.file_path FROM audio_files af WHERE af.group_key = ti.group_key AND af.group_key != '' LIMIT 1), '') AS TEXT) AS sample_file_path, ti.track_count, ti.album_name, ti.album_artist, diff --git a/backend/database/sql/schemas/lyrics_index.sql b/backend/database/sql/schemas/lyrics_index.sql new file mode 100644 index 0000000..04a8816 --- /dev/null +++ b/backend/database/sql/schemas/lyrics_index.sql @@ -0,0 +1,15 @@ +-- Contentless FTS5 index over recording lyrics, enabling +-- "search by a lyric fragment → find the song". The rowid is +-- recordings.id. Only recordings with non-empty lyrics are indexed. +-- +-- content='' means the lyric text itself is NOT stored a second time +-- (it already lives in recordings.lyrics); the index keeps only the +-- tokenised inverted index, so it stays compact even for large +-- libraries. contentless_delete=1 lets us delete/reinsert a single +-- row when a track's lyrics change (scan update or LRCLIB backfill). +CREATE VIRTUAL TABLE IF NOT EXISTS lyrics_index USING fts5( + lyrics, + content='', + contentless_delete=1, + tokenize='unicode61 remove_diacritics 2' +); diff --git a/backend/database/sql/schemas/playlists.sql b/backend/database/sql/schemas/playlists.sql index 337e7e8..7ca1385 100644 --- a/backend/database/sql/schemas/playlists.sql +++ b/backend/database/sql/schemas/playlists.sql @@ -3,6 +3,7 @@ CREATE TABLE IF NOT EXISTS playlists ( name TEXT NOT NULL, is_smart INTEGER NOT NULL DEFAULT 0, smart_rules TEXT, + smart_snapshot_at DATETIME, created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ); diff --git a/backend/database/sql/schemas/tagging_candidates.sql b/backend/database/sql/schemas/tagging_candidates.sql new file mode 100644 index 0000000..afc6b5e --- /dev/null +++ b/backend/database/sql/schemas/tagging_candidates.sql @@ -0,0 +1,18 @@ +-- tagging_candidates durably stores the scored candidate list for a +-- tagging group so it survives process restarts. Without it, only the +-- top score (a single REAL on tagging_items) is persisted; the full +-- candidate list — releases, tracks, alignments — is recomputed every +-- session, re-hitting MusicBrainz whenever the short-TTL http_cache has +-- expired. The blob is written once when a group is first scored and +-- read back on every subsequent open. +-- +-- candidates holds the JSON-encoded []autotag.Candidate. ON DELETE +-- CASCADE ties the blob's lifetime to its tagging_items row: when a +-- group's tracks change, the scan path deletes the old group_key row +-- (and SQLite, with foreign_keys = ON, drops the stale blob with it). +CREATE TABLE IF NOT EXISTS tagging_candidates ( + group_key TEXT PRIMARY KEY, + candidates TEXT NOT NULL, + computed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY(group_key) REFERENCES tagging_items(group_key) ON DELETE CASCADE +); diff --git a/backend/database/sql/sqlcgen/models.go b/backend/database/sql/sqlcgen/models.go index 8453611..9f86982 100644 --- a/backend/database/sql/sqlcgen/models.go +++ b/backend/database/sql/sqlcgen/models.go @@ -85,6 +85,10 @@ type Library struct { AutotagWarningAcked int64 } +type LyricsIndex struct { + Lyrics string +} + type PlayHistory struct { ID int64 AudioFileID int64 @@ -100,12 +104,13 @@ type PlayerState struct { } type Playlist struct { - ID int64 - Name string - IsSmart int64 - SmartRules sql.NullString - CreatedAt time.Time - UpdatedAt time.Time + ID int64 + Name string + IsSmart int64 + SmartRules sql.NullString + SmartSnapshotAt sql.NullTime + CreatedAt time.Time + UpdatedAt time.Time } type PlaylistTrack struct { @@ -184,6 +189,12 @@ type SearchIndex struct { Album string } +type TaggingCandidate struct { + GroupKey string + Candidates string + ComputedAt time.Time +} + type TaggingItem struct { GroupKey string LibraryID int64 diff --git a/backend/database/sql/sqlcgen/playlists.sql.go b/backend/database/sql/sqlcgen/playlists.sql.go index 28dbe35..0ed54f4 100644 --- a/backend/database/sql/sqlcgen/playlists.sql.go +++ b/backend/database/sql/sqlcgen/playlists.sql.go @@ -82,7 +82,7 @@ func (q *Queries) CountPlaylistsByName(ctx context.Context, name string) (int64, const createPlaylist = `-- name: CreatePlaylist :one INSERT INTO playlists (name) VALUES (?) -RETURNING id, name, is_smart, smart_rules, created_at, updated_at +RETURNING id, name, is_smart, smart_rules, smart_snapshot_at, created_at, updated_at ` func (q *Queries) CreatePlaylist(ctx context.Context, name string) (Playlist, error) { @@ -93,6 +93,7 @@ func (q *Queries) CreatePlaylist(ctx context.Context, name string) (Playlist, er &i.Name, &i.IsSmart, &i.SmartRules, + &i.SmartSnapshotAt, &i.CreatedAt, &i.UpdatedAt, ) @@ -205,7 +206,7 @@ func (q *Queries) GetAllPlaylistTracksWithMetadata(ctx context.Context) ([]GetAl } const getAllPlaylists = `-- name: GetAllPlaylists :many -SELECT id, name, is_smart, smart_rules, created_at, updated_at FROM playlists ORDER BY updated_at DESC +SELECT id, name, is_smart, smart_rules, smart_snapshot_at, created_at, updated_at FROM playlists ORDER BY updated_at DESC ` func (q *Queries) GetAllPlaylists(ctx context.Context) ([]Playlist, error) { @@ -222,6 +223,7 @@ func (q *Queries) GetAllPlaylists(ctx context.Context) ([]Playlist, error) { &i.Name, &i.IsSmart, &i.SmartRules, + &i.SmartSnapshotAt, &i.CreatedAt, &i.UpdatedAt, ); err != nil { @@ -251,7 +253,7 @@ func (q *Queries) GetNextPlaylistTrackPosition(ctx context.Context, playlistID i } const getPlaylist = `-- name: GetPlaylist :one -SELECT id, name, is_smart, smart_rules, created_at, updated_at FROM playlists WHERE id = ? LIMIT 1 +SELECT id, name, is_smart, smart_rules, smart_snapshot_at, created_at, updated_at FROM playlists WHERE id = ? LIMIT 1 ` func (q *Queries) GetPlaylist(ctx context.Context, id int64) (Playlist, error) { @@ -262,6 +264,7 @@ func (q *Queries) GetPlaylist(ctx context.Context, id int64) (Playlist, error) { &i.Name, &i.IsSmart, &i.SmartRules, + &i.SmartSnapshotAt, &i.CreatedAt, &i.UpdatedAt, ) diff --git a/backend/database/sql/sqlcgen/release_groups.sql.go b/backend/database/sql/sqlcgen/release_groups.sql.go index 07df736..d191b73 100644 --- a/backend/database/sql/sqlcgen/release_groups.sql.go +++ b/backend/database/sql/sqlcgen/release_groups.sql.go @@ -109,6 +109,17 @@ SELECT COALESCE(rg.original_year, rg.year) AS year, COALESCE(rg.year, 0) AS release_year, COALESCE(ac.text, fallback_ac.text, '') as artist_name, + -- primary (first-credited) album artist's MBID, for linking the + -- artist name to its detail page. Empty when the album has no + -- MB-tagged album-artist credit. + CAST(COALESCE(( + SELECT a.mbid + FROM artist_credit_artist aca_p + JOIN artists a ON a.id = aca_p.artist_id + WHERE aca_p.credit_id = rg.album_artist_credit_id + ORDER BY aca_p.id + LIMIT 1 + ), '') AS TEXT) as artist_mbid, COALESCE(ca.file_path, '') as cover_art_path FROM release_groups rg JOIN artist_credit ac ON rg.album_artist_credit_id = ac.id @@ -131,6 +142,7 @@ type GetAlbumsByArtistRow struct { Year sql.NullInt64 ReleaseYear int64 ArtistName string + ArtistMbid string CoverArtPath string } @@ -149,6 +161,7 @@ func (q *Queries) GetAlbumsByArtist(ctx context.Context, artistID int64) ([]GetA &i.Year, &i.ReleaseYear, &i.ArtistName, + &i.ArtistMbid, &i.CoverArtPath, ); err != nil { return nil, err @@ -171,6 +184,17 @@ SELECT COALESCE(rg.original_year, rg.year) AS year, COALESCE(rg.year, 0) AS release_year, COALESCE(ac.text, fallback_ac.text, '') as artist_name, + -- primary (first-credited) album artist's MBID, for linking the + -- artist name to its detail page. Empty when the album has no + -- MB-tagged album-artist credit. + CAST(COALESCE(( + SELECT a.mbid + FROM artist_credit_artist aca_p + JOIN artists a ON a.id = aca_p.artist_id + WHERE aca_p.credit_id = rg.album_artist_credit_id + ORDER BY aca_p.id + LIMIT 1 + ), '') AS TEXT) as artist_mbid, COALESCE(ca.file_path, '') as cover_art_path FROM release_groups rg JOIN artist_credit ac ON rg.album_artist_credit_id = ac.id @@ -205,6 +229,7 @@ type GetAlbumsByArtistByLibraryRow struct { Year sql.NullInt64 ReleaseYear int64 ArtistName string + ArtistMbid string CoverArtPath string } @@ -223,6 +248,7 @@ func (q *Queries) GetAlbumsByArtistByLibrary(ctx context.Context, arg GetAlbumsB &i.Year, &i.ReleaseYear, &i.ArtistName, + &i.ArtistMbid, &i.CoverArtPath, ); err != nil { return nil, err @@ -250,6 +276,17 @@ SELECT COALESCE(rg.year, 0) AS release_year, rg.mbid, COALESCE(ac.text, fallback_ac.text, '') as artist_name, + -- primary (first-credited) album artist's MBID, for linking the + -- artist name to its detail page. Empty when the album has no + -- MB-tagged album-artist credit. + CAST(COALESCE(( + SELECT a.mbid + FROM artist_credit_artist aca_p + JOIN artists a ON a.id = aca_p.artist_id + WHERE aca_p.credit_id = rg.album_artist_credit_id + ORDER BY aca_p.id + LIMIT 1 + ), '') AS TEXT) as artist_mbid, COALESCE(ca.file_path, '') as cover_art_path FROM release_groups rg LEFT JOIN artist_credit ac ON rg.album_artist_credit_id = ac.id @@ -271,6 +308,7 @@ type GetAllAlbumsWithDetailsRow struct { ReleaseYear int64 Mbid sql.NullString ArtistName string + ArtistMbid string CoverArtPath string } @@ -290,6 +328,7 @@ func (q *Queries) GetAllAlbumsWithDetails(ctx context.Context) ([]GetAllAlbumsWi &i.ReleaseYear, &i.Mbid, &i.ArtistName, + &i.ArtistMbid, &i.CoverArtPath, ); err != nil { return nil, err @@ -317,6 +356,17 @@ SELECT COALESCE(rg.year, 0) AS release_year, rg.mbid, COALESCE(ac.text, fallback_ac.text, '') as artist_name, + -- primary (first-credited) album artist's MBID, for linking the + -- artist name to its detail page. Empty when the album has no + -- MB-tagged album-artist credit. + CAST(COALESCE(( + SELECT a.mbid + FROM artist_credit_artist aca_p + JOIN artists a ON a.id = aca_p.artist_id + WHERE aca_p.credit_id = rg.album_artist_credit_id + ORDER BY aca_p.id + LIMIT 1 + ), '') AS TEXT) as artist_mbid, COALESCE(ca.file_path, '') as cover_art_path FROM release_groups rg LEFT JOIN artist_credit ac ON rg.album_artist_credit_id = ac.id @@ -345,6 +395,7 @@ type GetAllAlbumsWithDetailsByLibraryRow struct { ReleaseYear int64 Mbid sql.NullString ArtistName string + ArtistMbid string CoverArtPath string } @@ -364,6 +415,7 @@ func (q *Queries) GetAllAlbumsWithDetailsByLibrary(ctx context.Context, libraryI &i.ReleaseYear, &i.Mbid, &i.ArtistName, + &i.ArtistMbid, &i.CoverArtPath, ); err != nil { return nil, err diff --git a/backend/database/sql/sqlcgen/tagging_candidates.sql.go b/backend/database/sql/sqlcgen/tagging_candidates.sql.go new file mode 100644 index 0000000..812612d --- /dev/null +++ b/backend/database/sql/sqlcgen/tagging_candidates.sql.go @@ -0,0 +1,51 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 +// source: tagging_candidates.sql + +package sqlcgen + +import ( + "context" +) + +const deleteTaggingCandidates = `-- name: DeleteTaggingCandidates :exec +DELETE FROM tagging_candidates +WHERE group_key = ? +` + +func (q *Queries) DeleteTaggingCandidates(ctx context.Context, groupKey string) error { + _, err := q.db.ExecContext(ctx, deleteTaggingCandidates, groupKey) + return err +} + +const getTaggingCandidates = `-- name: GetTaggingCandidates :one +SELECT candidates FROM tagging_candidates +WHERE group_key = ? +LIMIT 1 +` + +func (q *Queries) GetTaggingCandidates(ctx context.Context, groupKey string) (string, error) { + row := q.db.QueryRowContext(ctx, getTaggingCandidates, groupKey) + var candidates string + err := row.Scan(&candidates) + return candidates, err +} + +const upsertTaggingCandidates = `-- name: UpsertTaggingCandidates :exec +INSERT INTO tagging_candidates (group_key, candidates, computed_at) +VALUES (?, ?, CURRENT_TIMESTAMP) +ON CONFLICT(group_key) DO UPDATE SET + candidates = excluded.candidates, + computed_at = excluded.computed_at +` + +type UpsertTaggingCandidatesParams struct { + GroupKey string + Candidates string +} + +func (q *Queries) UpsertTaggingCandidates(ctx context.Context, arg UpsertTaggingCandidatesParams) error { + _, err := q.db.ExecContext(ctx, upsertTaggingCandidates, arg.GroupKey, arg.Candidates) + return err +} diff --git a/backend/database/sql/sqlcgen/tagging_items.sql.go b/backend/database/sql/sqlcgen/tagging_items.sql.go index 0609da0..514b391 100644 --- a/backend/database/sql/sqlcgen/tagging_items.sql.go +++ b/backend/database/sql/sqlcgen/tagging_items.sql.go @@ -127,7 +127,7 @@ SELECT ti.library_id, COALESCE(lb.name, '') AS library_name, COALESCE(lb.path, '') AS library_path, - CAST(COALESCE((SELECT af.file_path FROM audio_files af WHERE af.group_key = ti.group_key LIMIT 1), '') AS TEXT) AS sample_file_path, + CAST(COALESCE((SELECT af.file_path FROM audio_files af WHERE af.group_key = ti.group_key AND af.group_key != '' LIMIT 1), '') AS TEXT) AS sample_file_path, ti.track_count, ti.album_name, ti.album_artist, @@ -543,7 +543,7 @@ SELECT ti.library_id, COALESCE(lb.name, '') AS library_name, COALESCE(lb.path, '') AS library_path, - CAST(COALESCE((SELECT af.file_path FROM audio_files af WHERE af.group_key = ti.group_key LIMIT 1), '') AS TEXT) AS sample_file_path, + CAST(COALESCE((SELECT af.file_path FROM audio_files af WHERE af.group_key = ti.group_key AND af.group_key != '' LIMIT 1), '') AS TEXT) AS sample_file_path, ti.track_count, ti.album_name, ti.album_artist, diff --git a/backend/database/tagging_items_test.go b/backend/database/tagging_items_test.go index 693f1bf..139d1dd 100644 --- a/backend/database/tagging_items_test.go +++ b/backend/database/tagging_items_test.go @@ -283,6 +283,56 @@ func TestCountPendingTaggingItems_UsesPartialIndex(t *testing.T) { } } +// TestListPendingFolders_SampleFilePathUsesIndex guards the folder-list +// query's per-row sample_file_path subquery against regressing to a +// full table scan of audio_files. The `AND af.group_key != ”` guard +// is load-bearing: without it SQLite can't prove the partial index +// idx_audio_files_group_key (WHERE group_key != ”) applies, and the +// subquery degrades to O(folders * audio_files) — the difference +// between the review list loading instantly and taking a minute. +func TestListPendingFolders_SampleFilePathUsesIndex(t *testing.T) { + t.Parallel() + + db := database.NewTestDB(t) + + rows, err := db.QueryContext(` + EXPLAIN QUERY PLAN + SELECT + ti.group_key, + CAST(COALESCE((SELECT af.file_path FROM audio_files af + WHERE af.group_key = ti.group_key AND af.group_key != '' LIMIT 1), '') AS TEXT) + FROM tagging_items ti + `) + if err != nil { + t.Fatalf("explain: %v", err) + } + + defer func() { _ = rows.Close() }() + + var plan strings.Builder + + for rows.Next() { + var id, parent, notused int + + var detail string + + if scanErr := rows.Scan(&id, &parent, ¬used, &detail); scanErr != nil { + t.Fatalf("scan: %v", scanErr) + } + + plan.WriteString(detail) + plan.WriteString("\n") + } + + if !strings.Contains(plan.String(), "idx_audio_files_group_key") { + t.Errorf( + "sample_file_path subquery no longer uses idx_audio_files_group_key "+ + "(would full-scan audio_files per folder):\n%s", + plan.String(), + ) + } +} + func TestGetTaggingItemAndListAudioFilesInGroup(t *testing.T) { t.Parallel() diff --git a/backend/database/testhelper.go b/backend/database/testhelper.go index 9af2dd7..0c083b3 100644 --- a/backend/database/testhelper.go +++ b/backend/database/testhelper.go @@ -75,10 +75,13 @@ func NewTestDB(t *testing.T) *DB { t.Cleanup(func() { _ = db.Close() }) return &DB{ - db: db, - Ctx: ctx, - Queries: queries, - logger: slog.Default(), + db: db, + Ctx: ctx, + // The in-memory test DB shares one connection, so reads and + // writes use the same handle; ReadQueries aliases Queries. + Queries: queries, + ReadQueries: queries, + logger: slog.Default(), } } diff --git a/backend/events/events.go b/backend/events/events.go index efbd395..cbe1908 100644 --- a/backend/events/events.go +++ b/backend/events/events.go @@ -90,4 +90,22 @@ const ( // Explore / search index events. const ( IndexStatusChanged = "IndexStatusChanged" + + // ArtistDiscographyReady fires (payload: artist MBID string) after a + // lazy background discography fetch persists into the index, so the + // artist detail page can re-fetch its top tracks / top releases + // without the initial request having blocked on a live fetch. + ArtistDiscographyReady = "ArtistDiscographyReady" + + // ArtistSimilarReady fires (payload: artist MBID string) after a lazy + // background similar-artists fetch persists into similar_artist_map, + // so the artist detail page can re-fetch that section without the + // initial request having blocked on a live LB labs call. + ArtistSimilarReady = "ArtistSimilarReady" + + // AlbumReleasesReady fires (payload: release-group MBID string) after a + // lazy background BrowseReleases fetch populates the response cache, so + // the album detail page can re-fetch its versions / tracklist without + // the initial request having blocked on a live MusicBrainz browse. + AlbumReleasesReady = "AlbumReleasesReady" ) diff --git a/backend/explore/autotagclient.go b/backend/explore/autotagclient.go index 7cfe0a3..f55db38 100644 --- a/backend/explore/autotagclient.go +++ b/backend/explore/autotagclient.go @@ -95,22 +95,52 @@ func (c *AutotagClient) LookupReleaseGroup( }, nil } -// LookupArtist returns the artist's sort name when available, -// falling back to the display name. Used by the resolver when -// constructing Lucene-style fallback queries. -func (c *AutotagClient) LookupArtist( - ctx context.Context, mbid string, -) (string, error) { - a, err := c.inner.LookupArtist(ctx, mbid) +// SearchRecordings delegates to the wrapped client and projects hits +// into autotag's minimal recording shape. Length is millisecond- +// aligned to match local audio_files. +func (c *AutotagClient) SearchRecordings( + ctx context.Context, query string, limit int, +) ([]autotag.MBRecordingHit, int, error) { + recs, total, err := c.inner.SearchRecordings(ctx, query, limit) if err != nil { - return "", err + return nil, 0, err } - if a.SortName != "" { - return a.SortName, nil + out := make([]autotag.MBRecordingHit, 0, len(recs)) + for _, rec := range recs { + out = append(out, autotag.MBRecordingHit{ + MBID: rec.MBID, + Title: rec.Title, + ArtistCredit: rec.ArtistCredit, + LengthMillis: int64(rec.Length), + }) } - return a.Name, nil + return out, total, nil +} + +// LookupRecordingReleases returns the releases a recording appears on, +// as slim references the resolver ranks to pick a representative +// release. +func (c *AutotagClient) LookupRecordingReleases( + ctx context.Context, recordingMBID string, +) ([]autotag.MBReleaseRef, error) { + refs, err := c.inner.LookupRecordingReleases(ctx, recordingMBID) + if err != nil { + return nil, err + } + + out := make([]autotag.MBReleaseRef, 0, len(refs)) + for _, r := range refs { + out = append(out, autotag.MBReleaseRef{ + MBID: r.MBID, + Title: r.Title, + Status: r.Status, + Date: r.Date, + }) + } + + return out, nil } func exploreToAutotagRelease(rel MBRelease) autotag.MBRelease { diff --git a/backend/explore/diskfree_unix.go b/backend/explore/diskfree_unix.go new file mode 100644 index 0000000..85d600d --- /dev/null +++ b/backend/explore/diskfree_unix.go @@ -0,0 +1,19 @@ +//go:build unix + +package explore + +import "golang.org/x/sys/unix" + +// diskFreeBytes returns the free bytes available to the current user +// on the filesystem containing path. ok is false when unknown. +func diskFreeBytes(path string) (free uint64, ok bool) { + var st unix.Statfs_t + + if err := unix.Statfs(path, &st); err != nil { + return 0, false + } + + // Bavail/Bsize integer types vary by platform. + //nolint:unconvert,gosec + return st.Bavail * uint64(st.Bsize), true +} diff --git a/backend/explore/diskfree_windows.go b/backend/explore/diskfree_windows.go new file mode 100644 index 0000000..efd2ff1 --- /dev/null +++ b/backend/explore/diskfree_windows.go @@ -0,0 +1,22 @@ +//go:build windows + +package explore + +import "golang.org/x/sys/windows" + +// diskFreeBytes returns the free bytes available to the current user +// on the volume containing path. ok is false when unknown. +func diskFreeBytes(path string) (free uint64, ok bool) { + p, err := windows.UTF16PtrFromString(path) + if err != nil { + return 0, false + } + + var freeToCaller, total, totalFree uint64 + + if err := windows.GetDiskFreeSpaceEx(p, &freeToCaller, &total, &totalFree); err != nil { + return 0, false + } + + return freeToCaller, true +} diff --git a/backend/explore/dumpcatalog.go b/backend/explore/dumpcatalog.go new file mode 100644 index 0000000..2a2a0c4 --- /dev/null +++ b/backend/explore/dumpcatalog.go @@ -0,0 +1,1079 @@ +package explore + +import ( + "archive/tar" + "bufio" + "context" + "encoding/csv" + "errors" + "fmt" + "io" + "sort" + "strings" + + "github.com/klauspost/compress/zstd" +) + +// Stages 2+3 of the dump import. Stage 2 picks per-entity popularity +// floors from the aggregated listen counts (top-N by listen count, +// with an absolute minimum). Stage 3 streams the MusicBrainz +// canonical dump (~2GB tar.zst of CSVs) and keeps only rows above the +// floors, assembling them directly into explore_index. Nothing except +// the final index rows touches disk; the canonical stream is cheap +// enough to simply restart after an interruption. + +const ( + // Entity budgets: the floors are chosen as "the listen count of + // the Nth most-listened entity", clamped to minListenFloor. + keepRecordings = 1_500_000 + keepReleases = 2_000_000 + keepArtists = 250_000 + keepReleaseGroup = 400_000 + + // minListenFloor cuts long-tail noise even when a budget isn't + // reached (small dumps, sparse entities). + minListenFloor = 10 + + // releaseMinListenFloor is the (lower) floor for releases, which + // are not indexed themselves but roll up into release groups. + releaseMinListenFloor = 5 + + // dumpAssembleBatch is the number of index rows per upsert + // transaction during assembly. + dumpAssembleBatch = 2000 + + // canonicalProgressRows controls progress reporting during the + // canonical CSV scan (~30M rows total). + canonicalProgressRows = 2_000_000 +) + +// Per-artist discography coverage (S2). The global budgets above keep +// the most-listened entities overall; on their own they leave a +// moderately popular artist with little or nothing on their detail page, +// forcing a slow live ListenBrainz fetch on first view. S2 guarantees +// each browse-likely artist a graded slice of their own top tracks and +// release groups, selected offline from the same dump — no API calls. +const ( + // perArtistArtistBudget is how many top artists (by listen count) + // receive graded per-artist coverage. Library artists are always + // covered in full regardless of rank (see markLibraryArtists). + perArtistArtistBudget = 10_000 + + // Rank boundaries (1-based) within the top perArtistArtistBudget + // artists. Tier A is the most-listened; everything from tier B's + // boundary down to perArtistArtistBudget is tier C. + perArtistTierA = 1_000 + perArtistTierB = 4_000 + + // Per-tier track/release-group budgets. These are caps that count + // globally-kept entities toward the total, so a superstar whose + // catalogue is already in the global set adds ~nothing. + perArtistTierATrack = 50 + perArtistTierBTrack = 25 + perArtistTierCTrack = 12 + perArtistTierARG = 15 + perArtistTierBRG = 10 + perArtistTierCRG = 5 + + // Library artists get effectively their full discography, capped to + // bound memory for pathologically prolific credits. + perArtistLibraryTrack = 500 + perArtistLibraryRG = 100 + + // keepRecordingsSecondary bounds the recording listen counts retained + // through the canonical scan for per-artist selection (a superset of + // the global kept set). Set well above keepRecordings so a top-10K + // artist's deep cuts still qualify as candidates. + keepRecordingsSecondary = 5_000_000 +) + +// uuid16 is a parsed MBID. +type uuid16 [16]byte + +// keptSets holds the entities that survived the popularity threshold, +// with their listen counts. +type keptSets struct { + recordings map[uuid16]uint32 + releases map[uuid16]uint32 + artists map[uuid16]uint32 + + recFloor uint32 + relFloor uint32 + artFloor uint32 + + // Per-artist coverage (S2). recSecondary holds recording listen + // counts down to a lower floor than recordings (a superset of it), + // so a target artist's sub-global-floor tracks stay selectable. + // trackBudget/rgBudget give the per-artist top-N cap for each browse + // target (top perArtistArtistBudget artists by listens, plus every + // library artist). + recSecondary map[uuid16]uint32 + trackBudget map[uuid16]int + rgBudget map[uuid16]int +} + +// computeThreshold picks per-entity floors and builds the kept sets. +// This is the "statistical analysis" step: listen counts follow a +// power law, so budget-based cutoffs (keep the top N) adapt to the +// actual distribution instead of hardcoding a magic listen count. +func (imp *dumpImporter) computeThreshold(counts map[mbidKey]uint32) *keptSets { + var recVals, relVals, artVals []uint32 + + var recTotal, relTotal, artTotal uint64 + + for k, v := range counts { + switch k[0] { + case countKindRecording: + recVals = append(recVals, v) + recTotal += uint64(v) + case countKindRelease: + relVals = append(relVals, v) + relTotal += uint64(v) + case countKindArtist: + artVals = append(artVals, v) + artTotal += uint64(v) + } + } + + ks := &keptSets{ + recFloor: floorForBudget(recVals, keepRecordings, minListenFloor), + relFloor: floorForBudget(relVals, keepReleases, releaseMinListenFloor), + artFloor: floorForBudget(artVals, keepArtists, minListenFloor), + + recordings: make(map[uuid16]uint32, keepRecordings), + releases: make(map[uuid16]uint32, keepReleases), + artists: make(map[uuid16]uint32, keepArtists), + + recSecondary: make(map[uuid16]uint32, keepRecordingsSecondary), + trackBudget: make(map[uuid16]int), + rgBudget: make(map[uuid16]int), + } + + // Per-artist coverage (S2): retain recording counts down to a lower + // secondary floor, and derive rank-based artist tier floors so each + // of the top perArtistArtistBudget artists gets a graded track/RG + // budget. artVals is sorted descending so rank N is at index N-1. + recSecFloor := floorForBudget(recVals, keepRecordingsSecondary, minListenFloor) + + sort.Slice(artVals, func(i, j int) bool { return artVals[i] > artVals[j] }) + artTierAFloor := rankFloor(artVals, perArtistTierA) + artTierBFloor := rankFloor(artVals, perArtistTierB) + artTierCFloor := rankFloor(artVals, perArtistArtistBudget) + + var recKept, relKept, artKept uint64 + + for k, v := range counts { + var id uuid16 + + copy(id[:], k[1:]) + + switch { + case k[0] == countKindRecording && v >= ks.recFloor: + ks.recordings[id] = v + ks.recSecondary[id] = v + recKept += uint64(v) + case k[0] == countKindRecording && v >= recSecFloor: + ks.recSecondary[id] = v + case k[0] == countKindRelease && v >= ks.relFloor: + ks.releases[id] = v + relKept += uint64(v) + case k[0] == countKindArtist && v >= ks.artFloor: + ks.artists[id] = v + artKept += uint64(v) + + if v >= artTierCFloor { + ks.trackBudget[id], ks.rgBudget[id] = tierBudget( + v, artTierAFloor, artTierBFloor, + ) + } + } + } + + // Library artists are always covered in full — override any rank tier + // and include those below the global artist floor. + imp.markLibraryArtists(ks) + + imp.logger.Info("dump import: popularity thresholds chosen", + "recordingFloor", ks.recFloor, + "recordings", len(ks.recordings), + "recordingCoverage", coveragePct(recKept, recTotal), + "releaseFloor", ks.relFloor, + "releases", len(ks.releases), + "artistFloor", ks.artFloor, + "artists", len(ks.artists), + "artistCoverage", coveragePct(artKept, artTotal), + "secondaryFloor", recSecFloor, + "secondaryRecordings", len(ks.recSecondary), + "targetArtists", len(ks.trackBudget), + ) + + return ks +} + +// rankFloor returns the listen count at the given 1-based rank in a slice +// already sorted in descending order, clamped to the slice length. Used +// to turn artist-rank tier boundaries into concrete listen-count floors +// that adapt to each dump's distribution. +func rankFloor(sortedDesc []uint32, rank int) uint32 { + if len(sortedDesc) == 0 { + return 0 + } + + if rank > len(sortedDesc) { + rank = len(sortedDesc) + } + + return sortedDesc[rank-1] +} + +// tierBudget maps an artist's listen count to its per-artist track and +// release-group budgets via the rank-derived tier floors. +func tierBudget(listens, tierAFloor, tierBFloor uint32) (int, int) { + switch { + case listens >= tierAFloor: + return perArtistTierATrack, perArtistTierARG + case listens >= tierBFloor: + return perArtistTierBTrack, perArtistTierBRG + default: + return perArtistTierCTrack, perArtistTierCRG + } +} + +// markLibraryArtists grants full per-artist coverage to every library +// artist, overriding any rank-based tier and including artists below the +// global artist floor so their discography is covered even when globally +// obscure. +func (imp *dumpImporter) markLibraryArtists(ks *keptSets) { + n := 0 + + for _, s := range imp.si.getLibraryArtistMBIDs() { + var id uuid16 + + if !parseUUID(s, id[:]) { + continue + } + + ks.trackBudget[id] = perArtistLibraryTrack + ks.rgBudget[id] = perArtistLibraryRG + n++ + } + + imp.logger.Info("dump import: library artists granted full coverage", "artists", n) +} + +// floorForBudget returns the listen count of the budget-th largest +// value (so keeping everything >= floor yields ~budget entries), +// clamped below by minFloor. +func floorForBudget(vals []uint32, budget int, minFloor uint32) uint32 { + if len(vals) == 0 { + return minFloor + } + + if len(vals) <= budget { + return minFloor + } + + sort.Slice(vals, func(i, j int) bool { return vals[i] > vals[j] }) + + floor := vals[budget-1] + if floor < minFloor { + floor = minFloor + } + + return floor +} + +func coveragePct(kept, total uint64) string { + if total == 0 { + return "0%" + } + + return fmt.Sprintf("%.1f%%", float64(kept)/float64(total)*100) +} + +// --------------------------------------------------------------------------- +// Canonical dump scan +// --------------------------------------------------------------------------- + +// keptRecordingRow is a canonical-dump row that survived the filter. +type keptRecordingRow struct { + mbid uuid16 + name string + artistName string + artistMBID string + releaseMBID uuid16 + releaseName string + listens uint32 +} + +// releaseInfo carries the display fields for a kept release, used to +// title its release group. +type releaseInfo struct { + name string + artistName string + artistMBID string +} + +// rgTarget is a release's redirect target. +type rgTarget struct { + rg uuid16 + canonical uuid16 +} + +// canonicalScan is the in-RAM result of streaming the canonical dump. +type canonicalScan struct { + recordings []keptRecordingRow + releaseInfos map[uuid16]releaseInfo + releaseToRG map[uuid16]rgTarget + artistNames map[uuid16]string + + // artistTracks accumulates each target artist's top recordings for + // S2 coverage; merged into recordings before assembly. + artistTracks *perArtistTracks +} + +// perArtistTracks holds, per target artist, that artist's top recordings +// by listen count. Populated during the canonical scan. +type perArtistTracks struct { + byArtist map[uuid16]*artistTopN +} + +func newPerArtistTracks() *perArtistTracks { + return &perArtistTracks{byArtist: make(map[uuid16]*artistTopN)} +} + +func (p *perArtistTracks) add(artist uuid16, budget int, row keptRecordingRow) { + a := p.byArtist[artist] + if a == nil { + a = &artistTopN{n: budget, inSet: make(map[uuid16]struct{}, budget)} + p.byArtist[artist] = a + } + + a.add(row) +} + +// artistTopN keeps the top-n recordings for a single artist by listen +// count, deduplicated by recording MBID. n is small (tens to a few +// hundred), so a linear min-scan on eviction is cheaper than the overhead +// of container/heap. +type artistTopN struct { + n int + rows []keptRecordingRow + inSet map[uuid16]struct{} +} + +func (a *artistTopN) add(row keptRecordingRow) { + if _, dup := a.inSet[row.mbid]; dup { + return + } + + if len(a.rows) < a.n { + a.rows = append(a.rows, row) + a.inSet[row.mbid] = struct{}{} + + return + } + + minIdx := 0 + + for i := 1; i < len(a.rows); i++ { + if a.rows[i].listens < a.rows[minIdx].listens { + minIdx = i + } + } + + if row.listens <= a.rows[minIdx].listens { + return + } + + delete(a.inSet, a.rows[minIdx].mbid) + a.rows[minIdx] = row + a.inSet[row.mbid] = struct{}{} +} + +// rgCandidate is a release group offered to an artist's top-K selection. +type rgCandidate struct { + rg uuid16 + listens uint32 +} + +// artistTopRG keeps the top-n release groups for a single artist by +// rolled-up listen count, deduplicated by release-group MBID. +type artistTopRG struct { + n int + rgs []rgCandidate + inSet map[uuid16]struct{} +} + +func (a *artistTopRG) add(rg uuid16, listens uint32) { + if _, dup := a.inSet[rg]; dup { + return + } + + if len(a.rgs) < a.n { + a.rgs = append(a.rgs, rgCandidate{rg: rg, listens: listens}) + a.inSet[rg] = struct{}{} + + return + } + + minIdx := 0 + + for i := 1; i < len(a.rgs); i++ { + if a.rgs[i].listens < a.rgs[minIdx].listens { + minIdx = i + } + } + + if listens <= a.rgs[minIdx].listens { + return + } + + delete(a.inSet, a.rgs[minIdx].rg) + a.rgs[minIdx] = rgCandidate{rg: rg, listens: listens} + a.inSet[rg] = struct{}{} +} + +// scanCanonicalDump streams the canonical dump and filters it against +// the kept sets. Member order inside the tar doesn't matter — the two +// CSVs populate independent maps that are joined during assembly. +func (imp *dumpImporter) scanCanonicalDump( + ctx context.Context, url string, ks *keptSets, +) (*canonicalScan, error) { + stream := newResumableReader(ctx, imp.httpClient, url, 0) + + defer func() { _ = stream.Close() }() + + zr, err := zstd.NewReader(bufio.NewReaderSize(stream, 1<<20)) + if err != nil { + return nil, fmt.Errorf("canonical zstd: %w", err) + } + + defer zr.Close() + + scan := &canonicalScan{ + releaseInfos: make(map[uuid16]releaseInfo, len(ks.releases)), + releaseToRG: make(map[uuid16]rgTarget, len(ks.releases)), + artistNames: make(map[uuid16]string, len(ks.artists)), + artistTracks: newPerArtistTracks(), + } + + sawData, sawRedirect := false, false + tr := tar.NewReader(zr) + + for { + if err := ctx.Err(); err != nil { + return nil, err + } + + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + + if err != nil { + return nil, fmt.Errorf("canonical tar: %w", err) + } + + if hdr.Typeflag != tar.TypeReg { + continue + } + + var member io.Reader = tr + + // Individual members may themselves be zstd-compressed. + if strings.HasSuffix(hdr.Name, ".zst") { + mzr, err := zstd.NewReader(member) + if err != nil { + return nil, fmt.Errorf("canonical member zstd: %w", err) + } + + member = mzr.IOReadCloser() + } + + switch { + case strings.Contains(hdr.Name, "canonical_musicbrainz_data.csv"): + if err := imp.scanCanonicalData(ctx, member, ks, scan); err != nil { + return nil, err + } + + sawData = true + case strings.Contains(hdr.Name, "canonical_release_redirect.csv"): + if err := imp.scanReleaseRedirect(ctx, member, ks, scan); err != nil { + return nil, err + } + + sawRedirect = true + } + } + + if !sawData || !sawRedirect { + return nil, fmt.Errorf("%w: canonical dump missing expected CSVs (data=%t redirect=%t)", + ErrDumpFormat, sawData, sawRedirect) + } + + return scan, nil +} + +// canonicalDataColumns maps the columns of canonical_musicbrainz_data.csv. +type canonicalDataColumns struct { + artistMBIDs int + artistCreditName int + releaseMBID int + releaseName int + recordingMBID int + recordingName int +} + +// defaultCanonicalDataColumns matches the documented column order. +func defaultCanonicalDataColumns() canonicalDataColumns { + return canonicalDataColumns{ + artistMBIDs: 2, + artistCreditName: 3, + releaseMBID: 4, + releaseName: 5, + recordingMBID: 6, + recordingName: 7, + } +} + +func (imp *dumpImporter) scanCanonicalData( + ctx context.Context, r io.Reader, ks *keptSets, scan *canonicalScan, +) error { + cr := csv.NewReader(bufio.NewReaderSize(r, 1<<20)) + cr.ReuseRecord = true + cr.FieldsPerRecord = -1 + + cols := defaultCanonicalDataColumns() + rows := 0 + + for { + rec, err := cr.Read() + if errors.Is(err, io.EOF) { + break + } + + if err != nil { + return fmt.Errorf("canonical data csv: %w", err) + } + + rows++ + + if rows == 1 && looksLikeHeader(rec) { + cols = headerColumns(rec, cols) + + continue + } + + if rows%canonicalProgressRows == 0 { + if err := ctx.Err(); err != nil { + return err + } + + imp.logger.Info("dump import: canonical scan progress", + "rows", rows, + "kept", len(scan.recordings), + ) + imp.setStageProgress(dumpStageCatalog, rows, 0) + } + + maxCol := cols.recordingName + if cols.recordingMBID > maxCol { + maxCol = cols.recordingMBID + } + + if len(rec) <= maxCol { + continue + } + + var recMBID uuid16 + + if !parseUUID(rec[cols.recordingMBID], recMBID[:]) { + continue + } + + var relMBID uuid16 + + relOK := parseUUID(rec[cols.releaseMBID], relMBID[:]) + + artistMBIDs := parsePGStringArray(rec[cols.artistMBIDs]) + creditName := rec[cols.artistCreditName] + + // Artist names: a single-artist credit names that artist. + if len(artistMBIDs) == 1 { + var artMBID uuid16 + + if parseUUID(artistMBIDs[0], artMBID[:]) { + if _, kept := ks.artists[artMBID]; kept { + if _, seen := scan.artistNames[artMBID]; !seen { + scan.artistNames[artMBID] = creditName + } + } + } + } + + // Release display info for release-group titling. + if relOK { + if _, kept := ks.releases[relMBID]; kept { + if _, seen := scan.releaseInfos[relMBID]; !seen { + firstArtist := "" + if len(artistMBIDs) > 0 { + firstArtist = artistMBIDs[0] + } + + scan.releaseInfos[relMBID] = releaseInfo{ + name: rec[cols.releaseName], + artistName: creditName, + artistMBID: firstArtist, + } + } + } + } + + globalListens, keptGlobal := ks.recordings[recMBID] + secListens, keptSecondary := ks.recSecondary[recMBID] + + // Below even the secondary floor: neither globally kept nor a + // per-artist candidate, so there's nothing to do. + if !keptGlobal && !keptSecondary { + continue + } + + firstArtist := "" + if len(artistMBIDs) > 0 { + firstArtist = artistMBIDs[0] + } + + // Build the row once (globalListens == secListens for global-kept + // recordings, since the global set is a subset of the secondary). + row := keptRecordingRow{ + mbid: recMBID, + name: strings.Clone(rec[cols.recordingName]), + artistName: strings.Clone(creditName), + artistMBID: firstArtist, + releaseMBID: relMBID, + releaseName: strings.Clone(rec[cols.releaseName]), + listens: secListens, + } + + // S2: offer this recording to its primary artist's top-N when + // that artist is a browse target. + if keptSecondary && firstArtist != "" { + var faID uuid16 + + if parseUUID(firstArtist, faID[:]) { + if budget, target := ks.trackBudget[faID]; target { + scan.artistTracks.add(faID, budget, row) + } + } + } + + // Global-kept recordings are written unconditionally. + if keptGlobal { + row.listens = globalListens + scan.recordings = append(scan.recordings, row) + } + } + + imp.logger.Info("dump import: canonical data scanned", + "rows", rows, + "keptRecordings", len(scan.recordings), + "artistNames", len(scan.artistNames), + ) + + return nil +} + +func (imp *dumpImporter) scanReleaseRedirect( + ctx context.Context, r io.Reader, ks *keptSets, scan *canonicalScan, +) error { + cr := csv.NewReader(bufio.NewReaderSize(r, 1<<20)) + cr.ReuseRecord = true + cr.FieldsPerRecord = -1 + + // Documented order: release_mbid, canonical_release_mbid, + // release_group_mbid. + relCol, canonCol, rgCol := 0, 1, 2 + rows := 0 + + for { + rec, err := cr.Read() + if errors.Is(err, io.EOF) { + break + } + + if err != nil { + return fmt.Errorf("release redirect csv: %w", err) + } + + rows++ + + if rows == 1 && looksLikeHeader(rec) { + for i, name := range rec { + switch strings.TrimSpace(name) { + case "release_mbid": + relCol = i + case "canonical_release_mbid": + canonCol = i + case "release_group_mbid": + rgCol = i + } + } + + continue + } + + if rows%canonicalProgressRows == 0 { + if err := ctx.Err(); err != nil { + return err + } + } + + if len(rec) <= rgCol || len(rec) <= relCol || len(rec) <= canonCol { + continue + } + + var rel uuid16 + + if !parseUUID(rec[relCol], rel[:]) { + continue + } + + if _, kept := ks.releases[rel]; !kept { + continue + } + + var target rgTarget + + if !parseUUID(rec[rgCol], target.rg[:]) { + continue + } + + if !parseUUID(rec[canonCol], target.canonical[:]) { + target.canonical = rel + } + + scan.releaseToRG[rel] = target + } + + imp.logger.Info("dump import: release redirects scanned", + "rows", rows, + "keptReleases", len(scan.releaseToRG), + ) + + return nil +} + +// looksLikeHeader reports whether a CSV record is a header row (no +// parseable UUIDs, contains a known column name). +func looksLikeHeader(rec []string) bool { + for _, f := range rec { + switch strings.TrimSpace(f) { + case "recording_mbid", "release_mbid", "artist_mbids", "release_group_mbid": + return true + } + } + + return false +} + +// headerColumns resolves column indexes from a header row, falling +// back to the documented defaults for any missing name. +func headerColumns(rec []string, fallback canonicalDataColumns) canonicalDataColumns { + cols := fallback + + for i, name := range rec { + switch strings.TrimSpace(name) { + case "artist_mbids": + cols.artistMBIDs = i + case "artist_credit_name": + cols.artistCreditName = i + case "release_mbid": + cols.releaseMBID = i + case "release_name": + cols.releaseName = i + case "recording_mbid": + cols.recordingMBID = i + case "recording_name": + cols.recordingName = i + } + } + + return cols +} + +// parsePGStringArray parses the artist_mbids CSV field, tolerating +// Postgres array syntax ({a,b}), JSON-ish lists (['a', 'b']), and bare +// single values. +func parsePGStringArray(s string) []string { + s = strings.TrimSpace(s) + if s == "" { + return nil + } + + if len(s) >= 2 { + first, last := s[0], s[len(s)-1] + if (first == '{' && last == '}') || (first == '[' && last == ']') { + s = s[1 : len(s)-1] + } + } + + if s == "" { + return nil + } + + parts := strings.Split(s, ",") + out := make([]string, 0, len(parts)) + + for _, p := range parts { + p = strings.Trim(strings.TrimSpace(p), `"'`) + if p != "" { + out = append(out, p) + } + } + + return out +} + +// --------------------------------------------------------------------------- +// Assembly into explore_index +// --------------------------------------------------------------------------- + +// assembleIndex writes the filtered catalog into explore_index via the +// standard upsert path (idempotent — safe to re-run after a crash). +func (imp *dumpImporter) assembleIndex( + ctx context.Context, ks *keptSets, scan *canonicalScan, +) error { + batch := make([]SearchIndexResult, 0, dumpAssembleBatch) + + flush := func() error { + if len(batch) == 0 { + return nil + } + + if err := ctx.Err(); err != nil { + return err + } + + imp.si.upsertBatch(batch) + batch = batch[:0] + + return nil + } + + // S2: fold each target artist's top recordings into the kept set. + imp.mergePerArtistTracks(ks, scan) + + // Recordings. + written := 0 + + for _, row := range scan.recordings { + caa := row.releaseMBID + if target, ok := scan.releaseToRG[row.releaseMBID]; ok { + caa = target.canonical + } + + batch = append(batch, SearchIndexResult{ + EntityType: "recording", + MBID: formatUUID(row.mbid[:]), + Title: row.name, + ArtistName: row.artistName, + ArtistMBID: row.artistMBID, + Popularity: int(row.listens), + ReleaseName: row.releaseName, + CAAReleaseMBID: formatUUID(caa[:]), + }) + + written++ + + if len(batch) >= dumpAssembleBatch { + if err := flush(); err != nil { + return err + } + + if written%500_000 == 0 { + imp.logger.Info("dump import: assembling recordings", "written", written) + imp.setStageProgress(dumpStageCatalog, written, len(scan.recordings)) + } + } + } + + if err := flush(); err != nil { + return err + } + + // Release groups: roll release listen counts up to the redirect + // target, keep the top keepReleaseGroup, and title each group + // after its most-listened kept release. + type rgAgg struct { + listens uint32 + bestRel uuid16 + bestCnt uint32 + canonical uuid16 + } + + rgs := make(map[uuid16]*rgAgg, len(scan.releaseToRG)) + + for rel, target := range scan.releaseToRG { + cnt := ks.releases[rel] + + agg := rgs[target.rg] + if agg == nil { + agg = &rgAgg{} + rgs[target.rg] = agg + } + + agg.listens += cnt + + if cnt >= agg.bestCnt { + agg.bestCnt = cnt + agg.bestRel = rel + agg.canonical = target.canonical + } + } + + rgFloorVals := make([]uint32, 0, len(rgs)) + for _, agg := range rgs { + rgFloorVals = append(rgFloorVals, agg.listens) + } + + rgFloor := floorForBudget(rgFloorVals, keepReleaseGroup, releaseMinListenFloor) + + // S2: keep each target artist's top release groups even below the + // global RG floor. Attributed via the best release's artist credit. + keepRG := make(map[uuid16]struct{}) + + rgPickers := make(map[uuid16]*artistTopRG) + + for rg, agg := range rgs { + info, ok := scan.releaseInfos[agg.bestRel] + if !ok { + continue + } + + var aID uuid16 + + if !parseUUID(info.artistMBID, aID[:]) { + continue + } + + budget, target := ks.rgBudget[aID] + if !target { + continue + } + + p := rgPickers[aID] + if p == nil { + p = &artistTopRG{n: budget, inSet: make(map[uuid16]struct{}, budget)} + rgPickers[aID] = p + } + + p.add(rg, agg.listens) + } + + for _, p := range rgPickers { + for _, c := range p.rgs { + keepRG[c.rg] = struct{}{} + } + } + + rgWritten := 0 + rgFromPerArtist := 0 + + for rg, agg := range rgs { + if agg.listens < rgFloor { + if _, keep := keepRG[rg]; !keep { + continue + } + + rgFromPerArtist++ + } + + info, ok := scan.releaseInfos[agg.bestRel] + if !ok { + // No kept release carries display info for this group. + continue + } + + batch = append(batch, SearchIndexResult{ + EntityType: "release_group", + MBID: formatUUID(rg[:]), + Title: info.name, + ArtistName: info.artistName, + ArtistMBID: info.artistMBID, + Popularity: int(agg.listens), + CAAReleaseMBID: formatUUID(agg.canonical[:]), + }) + + rgWritten++ + + if len(batch) >= dumpAssembleBatch { + if err := flush(); err != nil { + return err + } + } + } + + if err := flush(); err != nil { + return err + } + + // Artists (only those whose name is derivable from a solo credit; + // the metadata patch pass fills the rest from ListenBrainz). + artWritten := 0 + + for mbid, name := range scan.artistNames { + batch = append(batch, SearchIndexResult{ + EntityType: "artist", + MBID: formatUUID(mbid[:]), + Title: name, + ArtistName: name, + ArtistMBID: formatUUID(mbid[:]), + Popularity: int(ks.artists[mbid]), + }) + + artWritten++ + + if len(batch) >= dumpAssembleBatch { + if err := flush(); err != nil { + return err + } + } + } + + if err := flush(); err != nil { + return err + } + + imp.logger.Info("dump import: index assembled", + "recordings", written, + "releaseGroups", rgWritten, + "releaseGroupsPerArtist", rgFromPerArtist, + "artists", artWritten, + ) + + return nil +} + +// mergePerArtistTracks folds the S2 per-artist top-N recordings into the +// kept recordings, skipping any already covered by the global floor so no +// row is written twice (upsert dedupes by MBID as a backstop anyway). +func (imp *dumpImporter) mergePerArtistTracks(ks *keptSets, scan *canonicalScan) { + added := 0 + + for _, a := range scan.artistTracks.byArtist { + for _, row := range a.rows { + if _, global := ks.recordings[row.mbid]; global { + continue + } + + scan.recordings = append(scan.recordings, row) + added++ + } + } + + imp.logger.Info("dump import: per-artist track coverage", + "targetArtists", len(scan.artistTracks.byArtist), + "recordingsAdded", added, + ) +} diff --git a/backend/explore/dumpcounts.go b/backend/explore/dumpcounts.go new file mode 100644 index 0000000..a8ce318 --- /dev/null +++ b/backend/explore/dumpcounts.go @@ -0,0 +1,584 @@ +package explore + +import ( + "archive/tar" + "bufio" + "bytes" + "context" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "strings" + "sync" + + "github.com/parquet-go/parquet-go" +) + +// Stage 1 of the dump import: stream the ListenBrainz spark listens +// dump (a plain tar of ~128MB parquet files) and aggregate listen +// counts per recording, release, and artist MBID. Nothing is written +// to disk except counts.bin — each parquet member is buffered in RAM, +// parsed, and discarded. The aggregate map lives in RAM (~40M entities +// ≈ 2GB) and is flushed atomically with the stream byte offset so an +// interrupted import resumes without re-downloading processed data. + +const ( + // countKindRecording etc. tag entries in the counts map/file. + countKindRecording = byte(1) + countKindRelease = byte(2) + countKindArtist = byte(3) + + // countsFlushEveryMembers controls checkpoint frequency. Each + // flush rewrites counts.bin (~1GB by the end), so this trades + // checkpoint I/O against re-download on crash (~150 members ≈ + // 19GB of stream progress). + countsFlushEveryMembers = 150 + + // countsProgressEveryMembers controls progress log frequency. + countsProgressEveryMembers = 50 + + // parquetParseWorkers is the number of concurrent parquet + // decoders. Bounded to limit RAM: each worker holds one + // ~128MB member buffer. + parquetParseWorkers = 3 + + // maxParquetMemberSize guards against unexpected dump format + // changes blowing out RAM. + maxParquetMemberSize = 1 << 30 + + // countsFileMagic identifies + versions the counts file format. + countsFileMagic = "YJCNTS01" +) + +// ErrDumpFormat is returned when dump contents don't match the +// expected format. +var ErrDumpFormat = errors.New("unexpected dump format") + +// mbidKey is a parsed UUID plus an entity-kind tag, used as the counts +// map key. 17 bytes instead of a 36-byte string keeps the ~40M-entry +// map around 2GB. +type mbidKey [17]byte + +func makeMBIDKey(kind byte, mbid string) (mbidKey, bool) { + var k mbidKey + + k[0] = kind + + if !parseUUID(mbid, k[1:]) { + return k, false + } + + return k, true +} + +// parseUUID parses a canonical 36-char UUID string into 16 bytes. +// Returns false for anything malformed. +func parseUUID(s string, out []byte) bool { + if len(s) != 36 || s[8] != '-' || s[13] != '-' || s[18] != '-' || s[23] != '-' { + return false + } + + j := 0 + + for i := 0; i < 36; i++ { + if i == 8 || i == 13 || i == 18 || i == 23 { + continue + } + + hi := hexNibble(s[i]) + i++ + + lo := hexNibble(s[i]) + if hi == 0xFF || lo == 0xFF { + return false + } + + out[j] = hi<<4 | lo + j++ + } + + return true +} + +func hexNibble(c byte) byte { + switch { + case c >= '0' && c <= '9': + return c - '0' + case c >= 'a' && c <= 'f': + return c - 'a' + 10 + case c >= 'A' && c <= 'F': + return c - 'A' + 10 + default: + return 0xFF + } +} + +func formatUUID(b []byte) string { + const hexdigits = "0123456789abcdef" + + out := make([]byte, 36) + j := 0 + + for i := range 16 { + if i == 4 || i == 6 || i == 8 || i == 10 { + out[j] = '-' + j++ + } + + out[j] = hexdigits[b[i]>>4] + out[j+1] = hexdigits[b[i]&0x0F] + j += 2 + } + + return string(out) +} + +// countsState is the checkpointed stage-1 state: the counts map plus +// the stream position it corresponds to. +type countsState struct { + // SparkURL pins the dump being processed so a resume never mixes + // two different dumps. + SparkURL string `json:"sparkUrl"` + + // Offset is the byte offset of the next unprocessed tar member + // header (exact — includes the padding of the previous member). + Offset int64 `json:"offset"` + + // MemberIdx is the index of the next unprocessed parquet member + // (logging only; Offset is authoritative for resume). + MemberIdx int `json:"memberIdx"` + + // Done marks stage 1 complete. + Done bool `json:"done"` + + counts map[mbidKey]uint32 +} + +// sparkListenRow is the projection of the spark listens parquet schema +// that the aggregator reads. All other columns are skipped. +type sparkListenRow struct { + RecordingMBID string `parquet:"recording_mbid,optional"` + ReleaseMBID string `parquet:"release_mbid,optional"` + ArtistMBIDs []string `parquet:"artist_credit_mbids,optional,list"` +} + +type countParseJob struct { + idx int + endOffset int64 // exact offset of the next member header + buf []byte +} + +type countParseResult struct { + idx int + endOffset int64 + deltas map[mbidKey]uint32 + err error +} + +// aggregateListenCounts runs stage 1 to completion (or ctx cancel), +// checkpointing to the staging counts file as it goes. +func (imp *dumpImporter) aggregateListenCounts(ctx context.Context, st *countsState) error { + if st.counts == nil { + st.counts = make(map[mbidKey]uint32, 1<<20) + } + + stream := newResumableReader(ctx, imp.httpClient, st.SparkURL, st.Offset) + + defer func() { _ = stream.Close() }() + + buffered := bufio.NewReaderSize(stream, 1<<20) + tr := tar.NewReader(buffered) + + // consumedOffset is the absolute stream position of everything the + // tar reader has consumed: bytes delivered by HTTP minus bytes + // still sitting in the bufio buffer. + consumedOffset := func() int64 { + return stream.Offset - int64(buffered.Buffered()) + } + + jobs := make(chan countParseJob) + results := make(chan countParseResult, parquetParseWorkers) + applierDone := make(chan struct{}) + bufPool := sync.Pool{New: func() any { return []byte(nil) }} + + var workerWG sync.WaitGroup + + for range parquetParseWorkers { + workerWG.Add(1) + + go func() { + defer workerWG.Done() + + for job := range jobs { + deltas, err := parseListenParquet(job.buf) + // Buffer reuse across members is intentional. + bufPool.Put(job.buf[:0]) //nolint:staticcheck + + results <- countParseResult{ + idx: job.idx, + endOffset: job.endOffset, + deltas: deltas, + err: err, + } + } + }() + } + + // The applier merges results into st in member order, so every + // checkpoint is a contiguous prefix of the stream. It owns + // st.counts, st.Offset, and st.MemberIdx until applierDone closes; + // on error it keeps draining results so nothing deadlocks. + var applyErr error + + go func() { + defer close(applierDone) + + pending := make(map[int]countParseResult) + next := st.MemberIdx + lastFlushed := st.MemberIdx + + for res := range results { + if applyErr != nil { + continue + } + + pending[res.idx] = res + + for { + r, ok := pending[next] + if !ok { + break + } + + delete(pending, next) + + if r.err != nil { + applyErr = r.err + + break + } + + for k, v := range r.deltas { + st.counts[k] += v + } + + next++ + st.MemberIdx = next + st.Offset = r.endOffset + + if next-lastFlushed >= countsFlushEveryMembers { + if err := imp.writeCountsFile(st); err != nil { + applyErr = err + + break + } + + lastFlushed = next + + imp.logCountsProgress(next, r.endOffset, stream.Size, len(st.counts)) + + if err := imp.checkDiskHeadroom(); err != nil { + applyErr = err + + break + } + } else if next%countsProgressEveryMembers == 0 { + imp.logCountsProgress(next, r.endOffset, stream.Size, len(st.counts)) + } + } + } + }() + + memberIdx := st.MemberIdx + readErr := error(nil) + +readLoop: + for { + if err := ctx.Err(); err != nil { + readErr = err + + break + } + + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + + if err != nil { + readErr = fmt.Errorf("listens tar: %w", err) + + break + } + + if hdr.Typeflag != tar.TypeReg || !strings.HasSuffix(hdr.Name, ".parquet") { + continue + } + + if hdr.Size > maxParquetMemberSize { + readErr = fmt.Errorf("%w: parquet member %s is %d bytes", ErrDumpFormat, hdr.Name, hdr.Size) + + break + } + + buf, _ := bufPool.Get().([]byte) + if cap(buf) < int(hdr.Size) { + buf = make([]byte, hdr.Size) + } + + buf = buf[:hdr.Size] + + if _, err := io.ReadFull(tr, buf); err != nil { + readErr = fmt.Errorf("listens tar member read: %w", err) + + break + } + + // Exact next-header offset: position after the entry data plus + // the entry's block padding. Correct even when the next member + // uses PAX extension headers (those start at its header offset). + endOffset := consumedOffset() + tarPadding(hdr.Size) + + select { + case jobs <- countParseJob{idx: memberIdx, endOffset: endOffset, buf: buf}: + case <-ctx.Done(): + readErr = ctx.Err() + + break readLoop + } + + memberIdx++ + } + + close(jobs) + workerWG.Wait() + close(results) + <-applierDone + + if readErr == nil { + readErr = applyErr + } + + if readErr != nil { + // Best-effort checkpoint of applied progress before bailing, + // so even a cancelled run resumes where it left off. + _ = imp.writeCountsFile(st) + + return readErr + } + + st.Done = true + + if err := imp.writeCountsFile(st); err != nil { + return err + } + + imp.logger.Info("dump import: listen counts complete", + "members", st.MemberIdx, + "entities", len(st.counts), + ) + + return nil +} + +// tarPadding returns the number of zero bytes following a tar entry of +// the given size (entries are padded to 512-byte blocks). +func tarPadding(size int64) int64 { + const block = 512 + + return (block - size%block) % block +} + +// parseListenParquet decodes one parquet member and returns the +// per-entity listen-count deltas. +func parseListenParquet(buf []byte) (map[mbidKey]uint32, error) { + reader := parquet.NewGenericReader[sparkListenRow](bytes.NewReader(buf)) + + defer func() { _ = reader.Close() }() + + deltas := make(map[mbidKey]uint32, 1<<18) + rows := make([]sparkListenRow, 4096) + + for { + n, err := reader.Read(rows) + + for _, row := range rows[:n] { + key, ok := makeMBIDKey(countKindRecording, row.RecordingMBID) + if !ok { + // Unmapped listen — no usable recording MBID. + continue + } + + deltas[key]++ + + if relKey, relOK := makeMBIDKey(countKindRelease, row.ReleaseMBID); relOK { + deltas[relKey]++ + } + + for _, artist := range row.ArtistMBIDs { + if artKey, artOK := makeMBIDKey(countKindArtist, artist); artOK { + deltas[artKey]++ + } + } + } + + if errors.Is(err, io.EOF) { + break + } + + if err != nil { + return nil, fmt.Errorf("parquet read: %w", err) + } + + if n == 0 { + break + } + } + + return deltas, nil +} + +// --------------------------------------------------------------------------- +// counts.bin persistence +// --------------------------------------------------------------------------- + +// writeCountsFile atomically persists the counts map + stream position +// (write to temp file, fsync, rename). +func (imp *dumpImporter) writeCountsFile(st *countsState) error { + tmp := imp.countsPath() + ".tmp" + + f, err := os.Create(tmp) + if err != nil { + return fmt.Errorf("counts file create: %w", err) + } + + w := bufio.NewWriterSize(f, 1<<20) + + meta, err := json.Marshal(st) + if err != nil { + _ = f.Close() + + return fmt.Errorf("counts meta marshal: %w", err) + } + + _, _ = w.WriteString(countsFileMagic) + + var lenBuf [4]byte + + binary.LittleEndian.PutUint32(lenBuf[:], uint32(len(meta))) + _, _ = w.Write(lenBuf[:]) + _, _ = w.Write(meta) + + var rec [21]byte + + for k, v := range st.counts { + copy(rec[:17], k[:]) + binary.LittleEndian.PutUint32(rec[17:], v) + + if _, err := w.Write(rec[:]); err != nil { + _ = f.Close() + + return fmt.Errorf("counts file write: %w", err) + } + } + + if err := w.Flush(); err != nil { + _ = f.Close() + + return fmt.Errorf("counts file flush: %w", err) + } + + if err := f.Sync(); err != nil { + _ = f.Close() + + return fmt.Errorf("counts file sync: %w", err) + } + + if err := f.Close(); err != nil { + return fmt.Errorf("counts file close: %w", err) + } + + if err := os.Rename(tmp, imp.countsPath()); err != nil { + return fmt.Errorf("counts file rename: %w", err) + } + + return nil +} + +// readCountsFile loads a previously checkpointed counts file. Returns +// (nil, nil) when no checkpoint exists. +func (imp *dumpImporter) readCountsFile() (*countsState, error) { + f, err := os.Open(imp.countsPath()) + if errors.Is(err, os.ErrNotExist) { + return nil, nil //nolint:nilnil // no checkpoint is a valid, non-error state + } + + if err != nil { + return nil, fmt.Errorf("counts file open: %w", err) + } + + defer func() { _ = f.Close() }() + + r := bufio.NewReaderSize(f, 1<<20) + + magic := make([]byte, len(countsFileMagic)) + if _, err := io.ReadFull(r, magic); err != nil || string(magic) != countsFileMagic { + return nil, fmt.Errorf("%w: bad counts file header", ErrDumpFormat) + } + + var lenBuf [4]byte + + if _, err := io.ReadFull(r, lenBuf[:]); err != nil { + return nil, fmt.Errorf("counts meta length: %w", err) + } + + meta := make([]byte, binary.LittleEndian.Uint32(lenBuf[:])) + if _, err := io.ReadFull(r, meta); err != nil { + return nil, fmt.Errorf("counts meta read: %w", err) + } + + st := &countsState{} + if err := json.Unmarshal(meta, st); err != nil { + return nil, fmt.Errorf("counts meta unmarshal: %w", err) + } + + st.counts = make(map[mbidKey]uint32, 1<<20) + + var rec [21]byte + + for { + if _, err := io.ReadFull(r, rec[:]); err != nil { + if errors.Is(err, io.EOF) { + break + } + + return nil, fmt.Errorf("counts record read: %w", err) + } + + var k mbidKey + + copy(k[:], rec[:17]) + st.counts[k] = binary.LittleEndian.Uint32(rec[17:]) + } + + return st, nil +} + +func (imp *dumpImporter) logCountsProgress(members int, offset, size int64, entities int) { + pct := float64(0) + if size > 0 { + pct = float64(offset) / float64(size) * 100 + } + + imp.logger.Info("dump import: listen counts progress", + "members", members, + "gb", fmt.Sprintf("%.1f", float64(offset)/(1<<30)), + "pct", fmt.Sprintf("%.1f", pct), + "entities", entities, + ) + + imp.setStageProgress(dumpStageCounts, int(pct), 100) +} diff --git a/backend/explore/dumpimport.go b/backend/explore/dumpimport.go new file mode 100644 index 0000000..30190e2 --- /dev/null +++ b/backend/explore/dumpimport.go @@ -0,0 +1,574 @@ +package explore + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net/http" + "os" + "path/filepath" + "regexp" + "strconv" + "time" + + "yellowjacket/backend/system" +) + +// Dump-based index population. Instead of crawling the ListenBrainz +// API artist-by-artist, the index is built from two MetaBrainz dumps: +// +// 1. The spark listens dump (~170GB, streamed, never stored) yields +// listen counts for every recording/release/artist MBID. +// 2. The MusicBrainz canonical dump (~2GB, streamed) yields names and +// MBIDs, filtered to entities above a popularity floor. +// +// A short API patch pass then fills listener counts (top rows only), +// artist metadata, and the similar-artist map. Total temporary disk +// is one ~1GB counts file, deleted on completion. The pipeline +// resumes from checkpoints after interruption. + +const ( + // dumpImportDoneKey marks a completed import in explore_index_meta. + dumpImportDoneKey = "dump_import_done" + + // listensAppliedSeriesKey stores the dump series number whose listen + // counts are folded into popularity (the high-water-mark for the + // incremental refresh). Set to the full dump's series at import, then + // advanced by each applied incremental. + listensAppliedSeriesKey = "listens_applied_series" + + // releaseToRGInsertBatch bounds how many rows are written per + // transaction when persisting the release→release-group map. + releaseToRGInsertBatch = 10_000 + + // dumpMinStartFreeBytes is the free-disk requirement to begin an + // import (counts file + index growth + headroom). + dumpMinStartFreeBytes = 6 << 30 + + // dumpAbortFreeBytes aborts a running import when free disk + // drops below it. + dumpAbortFreeBytes = 2 << 30 + + // dumpStageAssembled in state.json means the index rows are + // written and only patch passes remain. + dumpStageAssembled = "assembled" +) + +// ErrDiskSpace is returned when free disk falls below the safety floor. +var ErrDiskSpace = errors.New("insufficient free disk space") + +// Dump import stages, mapped to status names shown in the UI. +const ( + dumpStageCounts = iota + dumpStageCatalog + dumpStagePatch + dumpStageListeners +) + +var dumpStageNames = [...]string{ + "Listen Counts", + "Catalog Import", + "Metadata Patch", + "Listener Counts", +} + +// Dump discovery patterns. +var ( + canonicalDirRe = regexp.MustCompile(`^musicbrainz-canonical-dump-\d{8}-\d+$`) + canonicalFileRe = regexp.MustCompile(`^musicbrainz-canonical-dump-.*\.tar\.zst$`) + listensDirRe = regexp.MustCompile(`^listenbrainz-dump-\d+-\d{8}-\d+-full$`) + sparkFileRe = regexp.MustCompile(`^listenbrainz-spark-dump-.*-full\.tar$`) +) + +// Production dump locations (overridable for tests). +const ( + defaultCanonicalBaseURL = "https://data.metabrainz.org/pub/musicbrainz/canonical_data/" + defaultListensBaseURL = "https://data.metabrainz.org/pub/musicbrainz/listenbrainz/fullexport/" +) + +// dumpImportState is the small persistent state file (staging dir). +// The heavyweight stage-1 checkpoint lives in counts.bin. +type dumpImportState struct { + SparkURL string `json:"sparkUrl"` + CanonicalURL string `json:"canonicalUrl"` + Stage string `json:"stage"` +} + +// dumpImporter runs the dump import pipeline. +type dumpImporter struct { + si *SearchIndex + lb *ListenBrainzClient + logger *slog.Logger + + httpClient *http.Client + stagingDir string + + canonicalBaseURL string + listensBaseURL string + + // Disk safety floors (fields so tests can relax them). + minStartFreeBytes uint64 + abortFreeBytes uint64 + + // pendingArtists are kept artists whose names weren't derivable + // from the canonical dump; the metadata patch pass resolves them. + pendingArtists []string +} + +func newDumpImporter(si *SearchIndex, lb *ListenBrainzClient) (*dumpImporter, error) { + dataDir, err := system.GetUserDataDirPath() + if err != nil { + return nil, fmt.Errorf("dump import: data dir: %w", err) + } + + stagingDir := filepath.Join(dataDir, "explore-staging") + if err := os.MkdirAll(stagingDir, 0o755); err != nil { + return nil, fmt.Errorf("dump import: staging dir: %w", err) + } + + return &dumpImporter{ + si: si, + lb: lb, + logger: si.logger, + // No client-level timeout: the listens stream runs for hours. + // Discovery requests use per-request context timeouts, and + // resumableReader recovers from stalled connections. + httpClient: &http.Client{}, + stagingDir: stagingDir, + canonicalBaseURL: defaultCanonicalBaseURL, + listensBaseURL: defaultListensBaseURL, + minStartFreeBytes: dumpMinStartFreeBytes, + abortFreeBytes: dumpAbortFreeBytes, + }, nil +} + +func (imp *dumpImporter) countsPath() string { + return filepath.Join(imp.stagingDir, "counts.bin") +} + +func (imp *dumpImporter) statePath() string { + return filepath.Join(imp.stagingDir, "state.json") +} + +// run executes the pipeline, resuming from any prior checkpoint. +func (imp *dumpImporter) run(ctx context.Context) error { + if err := checkFreeDisk(imp.stagingDir, imp.minStartFreeBytes); err != nil { + return err + } + + state, err := imp.readState() + if err != nil { + return err + } + + // Fast path: rows already assembled, only patch passes remain. + if state.Stage == dumpStageAssembled { + imp.si.MarkReadyIfPopulated() + imp.runPatchPasses(ctx) + + if err := ctx.Err(); err != nil { + return err + } + + return imp.finalize() + } + + // Stage 1: listen counts from the spark listens dump. + counts, err := imp.readCountsFile() + if err != nil { + imp.logger.Warn("dump import: discarding unreadable counts checkpoint", "error", err) + + counts = nil + } + + if counts == nil { + sparkURL, err := discoverDumpFile( + ctx, imp.httpClient, imp.listensBaseURL, listensDirRe, sparkFileRe, + ) + if err != nil { + return err + } + + imp.logger.Info("dump import: starting", "listensDump", sparkURL) + + counts = &countsState{SparkURL: sparkURL} + } else if !counts.Done { + imp.logger.Info("dump import: resuming listen counts", + "offset", counts.Offset, + "members", counts.MemberIdx, + "entities", len(counts.counts), + ) + } + + // Record which dump series this import is baselined on, so the + // incremental refresh knows where to resume applying daily deltas. + // Written early (before assembly) so it survives a crash-and-resume; + // incrementals only apply once dumpImportDoneKey confirms completion. + imp.recordDumpSeries(counts.SparkURL) + + imp.setStageProgress(dumpStageCounts, 0, 100) + + if !counts.Done { + if err := imp.aggregateListenCounts(ctx, counts); err != nil { + return err + } + } + + imp.si.setTierStatus(dumpStageNames[dumpStageCounts], "complete", 100, 100) + + // Stage 2: popularity thresholds (in RAM, deterministic). + kept := imp.computeThreshold(counts.counts) + + // Free the full counts map; only the kept sets are needed now. + counts.counts = nil + + // Stage 3: canonical dump scan + assembly. Restartable: the + // scan is a cheap 2GB stream and assembly is an idempotent + // upsert, so no intra-stage checkpoint is needed. + if state.CanonicalURL == "" { + state.CanonicalURL, err = discoverDumpFile( + ctx, imp.httpClient, imp.canonicalBaseURL, canonicalDirRe, canonicalFileRe, + ) + if err != nil { + return err + } + + if err := imp.writeState(state); err != nil { + return err + } + } + + imp.setStageProgress(dumpStageCatalog, 0, 0) + + scan, err := imp.scanCanonicalDump(ctx, state.CanonicalURL, kept) + if err != nil { + return err + } + + if err := imp.checkDiskHeadroom(); err != nil { + return err + } + + // One-time reset when migrating from the legacy API-crawled + // index: its popularity values are on a different scale (the LB + // API includes MLHD+ history) and would permanently outrank + // dump-derived counts via the highest-wins upsert. Re-imports + // (dump→dump) skip this — listen counts only grow. + if !imp.si.hasMeta(dumpImportDoneKey) { + if _, err := imp.si.db.ExecContext("DELETE FROM explore_index"); err == nil { + imp.logger.Info("dump import: cleared legacy index for consistent popularity scale") + } + } + + if err := imp.assembleIndex(ctx, kept, scan); err != nil { + return err + } + + // Persist the release→release-group map (otherwise in-memory only) + // so incremental dumps can roll per-release listen deltas up to their + // album without an API call. Kept in lockstep with the index it was + // just built from. + imp.persistReleaseToRG(ctx, scan.releaseToRG) + + imp.si.setTierStatus(dumpStageNames[dumpStageCatalog], "complete", 0, 0) + + // Artists that need names from the metadata patch pass. + for mbid := range kept.artists { + if _, ok := scan.artistNames[mbid]; !ok { + imp.pendingArtists = append(imp.pendingArtists, formatUUID(mbid[:])) + } + } + + state.Stage = dumpStageAssembled + if err := imp.writeState(state); err != nil { + return err + } + + imp.si.MarkReadyIfPopulated() + imp.si.refreshStatusCounts() + + // Stage 4: API patch passes (idempotent). + imp.runPatchPasses(ctx) + + if err := ctx.Err(); err != nil { + return err + } + + return imp.finalize() +} + +// finalize records completion and removes all staging data. +func (imp *dumpImporter) finalize() error { + imp.si.setMeta(dumpImportDoneKey, time.Now().UTC().Format(time.RFC3339)) + + // Retire the legacy tier-crawl freshness keys. + _, _ = imp.si.db.ExecContext( + `DELETE FROM explore_index_meta + WHERE key IN ('tier1_built', 'tier2_built', 'tier3_built', 'tier4_built')`, + ) + + if err := os.RemoveAll(imp.stagingDir); err != nil { + imp.logger.Warn("dump import: staging cleanup failed", "error", err) + } + + imp.si.setTierStatus(dumpStageNames[dumpStagePatch], "complete", 0, 0) + imp.si.setTierStatus(dumpStageNames[dumpStageListeners], "complete", 0, 0) + imp.si.refreshStatusCounts() + + // The imported catalog changed which rows are popular, so refresh the + // champion tier used for generic short-prefix searches. + imp.si.scheduleChampionRebuild() + + imp.logger.Info("dump import: complete") + + return nil +} + +// dumpSeriesRe extracts the monotonic series number NNNN from a dump +// URL or directory name (e.g. "listenbrainz-spark-dump-2593-…"). +var dumpSeriesRe = regexp.MustCompile(`listenbrainz-(?:spark-)?dump-(\d+)-`) + +// parseDumpSeries pulls the series number out of a dump URL/name. +func parseDumpSeries(url string) (int, bool) { + m := dumpSeriesRe.FindStringSubmatch(url) + if m == nil { + return 0, false + } + + n, err := strconv.Atoi(m[1]) + if err != nil { + return 0, false + } + + return n, true +} + +// recordDumpSeries stores the baseline series number for this import. +func (imp *dumpImporter) recordDumpSeries(sparkURL string) { + series, ok := parseDumpSeries(sparkURL) + if !ok { + imp.logger.Warn("dump import: could not parse dump series", "url", sparkURL) + + return + } + + imp.si.setMeta(listensAppliedSeriesKey, strconv.Itoa(series)) +} + +// persistReleaseToRG replaces the release_to_rg table with the mapping +// captured during this import, so it always reflects the just-built +// index. Idempotent: a full rebuild clears and repopulates it. +func (imp *dumpImporter) persistReleaseToRG(ctx context.Context, m map[uuid16]rgTarget) { + if len(m) == 0 { + return + } + + if _, err := imp.si.db.ExecContext("DELETE FROM release_to_rg"); err != nil { + imp.logger.Warn("dump import: clear release_to_rg failed", "error", err) + + return + } + + written := 0 + pending := 0 + + tx, err := imp.si.db.BeginTx() + if err != nil { + imp.logger.Warn("dump import: begin release_to_rg tx failed", "error", err) + + return + } + + for rel, target := range m { + if ctx.Err() != nil { + _ = tx.Rollback() + + return + } + + if _, err := tx.Exec( + "INSERT OR REPLACE INTO release_to_rg (release_mbid, rg_mbid) VALUES (?, ?)", + formatUUID(rel[:]), formatUUID(target.rg[:]), + ); err != nil { + imp.logger.Warn("dump import: insert release_to_rg failed", "error", err) + + continue + } + + written++ + pending++ + + if pending >= releaseToRGInsertBatch { + if err := tx.Commit(); err != nil { + imp.logger.Warn("dump import: commit release_to_rg batch failed", "error", err) + + return + } + + pending = 0 + + tx, err = imp.si.db.BeginTx() + if err != nil { + imp.logger.Warn("dump import: begin release_to_rg tx failed", "error", err) + + return + } + } + } + + if err := tx.Commit(); err != nil { + imp.logger.Warn("dump import: commit release_to_rg failed", "error", err) + + return + } + + imp.logger.Info("dump import: persisted release→release-group map", "rows", written) +} + +func (imp *dumpImporter) readState() (*dumpImportState, error) { + state := &dumpImportState{} + + data, err := os.ReadFile(imp.statePath()) + if errors.Is(err, os.ErrNotExist) { + return state, nil + } + + if err != nil { + return nil, fmt.Errorf("dump import state read: %w", err) + } + + if err := json.Unmarshal(data, state); err != nil { + // Corrupt state: start over rather than fail permanently. + return &dumpImportState{}, nil + } + + return state, nil +} + +func (imp *dumpImporter) writeState(state *dumpImportState) error { + data, err := json.Marshal(state) + if err != nil { + return fmt.Errorf("dump import state marshal: %w", err) + } + + tmp := imp.statePath() + ".tmp" + if err := os.WriteFile(tmp, data, 0o644); err != nil { + return fmt.Errorf("dump import state write: %w", err) + } + + if err := os.Rename(tmp, imp.statePath()); err != nil { + return fmt.Errorf("dump import state rename: %w", err) + } + + return nil +} + +// setStageProgress reports stage progress to the UI status feed. +func (imp *dumpImporter) setStageProgress(stage, completed, total int) { + imp.si.setTierStatus(dumpStageNames[stage], "running", total, completed) +} + +// checkDiskHeadroom aborts the import when free disk is critically low. +func (imp *dumpImporter) checkDiskHeadroom() error { + return checkFreeDisk(imp.stagingDir, imp.abortFreeBytes) +} + +// checkFreeDisk returns ErrDiskSpace when the volume holding path has +// less than minBytes free. Unknown free space (unsupported platform) +// passes. +func checkFreeDisk(path string, minBytes uint64) error { + free, ok := diskFreeBytes(path) + if !ok { + return nil + } + + if free < minBytes { + return fmt.Errorf("%w: %d MB free, need %d MB", + ErrDiskSpace, free>>20, minBytes>>20) + } + + return nil +} + +// --------------------------------------------------------------------------- +// SearchIndex integration +// --------------------------------------------------------------------------- + +// runDumpBuild is the build entrypoint called from StartBuild's +// goroutine. It replaces the legacy tier crawl. +func (si *SearchIndex) runDumpBuild(ctx context.Context) { + si.MarkReadyIfPopulated() + + // The catalog dump is authoritative and only grows; it is imported + // once and never re-crawled on a timer. Popularity freshness comes + // from incremental dumps, and new releases from lazy per-artist + // fetches — not from re-running this multi-GB import. + if si.hasMeta(dumpImportDoneKey) { + si.logger.Info("search index: dump import already complete, skipping") + si.refreshStatusCounts() + + return + } + + si.mu.Lock() + si.buildStatus = IndexStatus{ + Building: true, + Tiers: []TierStatus{ + {Name: dumpStageNames[dumpStageCounts], State: "pending"}, + {Name: dumpStageNames[dumpStageCatalog], State: "pending"}, + {Name: dumpStageNames[dumpStagePatch], State: "pending"}, + {Name: dumpStageNames[dumpStageListeners], State: "pending"}, + }, + } + si.mu.Unlock() + si.refreshStatusCounts() + + // Patch passes use a dedicated rate limiter so background API + // calls never compete with interactive search/browse requests. + var indexLB *ListenBrainzClient + + if si.lb != nil { + indexLB = NewListenBrainzClient( + NewRateLimiterN(indexerRate), si.lb.cache, si.logger.WithGroup("indexer"), + ) + } + + imp, err := newDumpImporter(si, indexLB) + if err != nil { + si.logger.Error("search index: dump import init failed", "error", err) + + return + } + + start := time.Now() + + if err := imp.run(ctx); err != nil { + if errors.Is(err, context.Canceled) || ctx.Err() != nil { + si.logger.Info("search index: dump import paused (will resume)", + "elapsed", time.Since(start).Round(time.Second), + ) + } else { + si.logger.Error("search index: dump import failed", "error", err) + si.setTierError(dumpStageNames[dumpStageCounts], err.Error()) + } + + return + } + + si.mu.Lock() + si.buildStatus.Building = false + si.mu.Unlock() + + // Fold the local library into the freshly-imported catalog: owned + // entities below the dump's popularity floor are inserted, and + // dump-seeded rows that match the library are flagged in_library. + // Deep discographies stay lazy (fetched when an artist page opens). + si.PopulateLocalCrossReferences() + + si.refreshStatusCounts() + si.logger.Info("search index: dump import finished", + "elapsed", time.Since(start).Round(time.Second), + ) +} diff --git a/backend/explore/dumpimport_test.go b/backend/explore/dumpimport_test.go new file mode 100644 index 0000000..a0cfe6c --- /dev/null +++ b/backend/explore/dumpimport_test.go @@ -0,0 +1,925 @@ +package explore + +import ( + "archive/tar" + "bytes" + "context" + "encoding/csv" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + "time" + + "github.com/klauspost/compress/zstd" + "github.com/parquet-go/parquet-go" + + "yellowjacket/backend/database" +) + +// Fixed MBIDs for fixtures. +const ( + recA = "11111111-1111-1111-1111-111111111111" + recB = "22222222-2222-2222-2222-222222222222" + recC = "33333333-3333-3333-3333-333333333333" + relA = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa" + relB = "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb" + rgA = "cccccccc-cccc-cccc-cccc-cccccccccccc" + rgB = "dddddddd-dddd-dddd-dddd-dddddddddddd" + artA = "eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee" + artB = "ffffffff-ffff-ffff-ffff-ffffffffffff" +) + +func testLogger() *slog.Logger { + return slog.New(slog.DiscardHandler) +} + +// --------------------------------------------------------------------------- +// Unit tests: parsing helpers +// --------------------------------------------------------------------------- + +func TestParseUUIDRoundTrip(t *testing.T) { + var buf [16]byte + + if !parseUUID(recA, buf[:]) { + t.Fatalf("parseUUID rejected valid UUID %s", recA) + } + + if got := formatUUID(buf[:]); got != recA { + t.Fatalf("round trip = %q, want %q", got, recA) + } + + invalid := []string{ + "", "not-a-uuid", + "11111111-1111-1111-1111-11111111111", // too short + "11111111-1111-1111-1111-1111111111111", // too long + "1111111101111-1111-1111-111111111111", // bad dash + "gggggggg-1111-1111-1111-111111111111", // bad hex + } + + for _, s := range invalid { + if parseUUID(s, buf[:]) { + t.Errorf("parseUUID accepted invalid input %q", s) + } + } +} + +func TestParsePGStringArray(t *testing.T) { + cases := []struct { + in string + want []string + }{ + {"{" + artA + "}", []string{artA}}, + {"{" + artA + "," + artB + "}", []string{artA, artB}}, + {`{"` + artA + `","` + artB + `"}`, []string{artA, artB}}, + {"['" + artA + "', '" + artB + "']", []string{artA, artB}}, + {artA, []string{artA}}, + {"", nil}, + {"{}", nil}, + } + + for _, c := range cases { + got := parsePGStringArray(c.in) + if len(got) != len(c.want) { + t.Errorf("parsePGStringArray(%q) = %v, want %v", c.in, got, c.want) + + continue + } + + for i := range got { + if got[i] != c.want[i] { + t.Errorf("parsePGStringArray(%q)[%d] = %q, want %q", c.in, i, got[i], c.want[i]) + } + } + } +} + +func TestFloorForBudget(t *testing.T) { + vals := []uint32{100, 90, 80, 70, 60, 50, 40, 30, 20, 5} + + if got := floorForBudget(vals, 3, 10); got != 80 { + t.Errorf("budget 3: floor = %d, want 80", got) + } + + // Budget larger than data → min floor. + if got := floorForBudget(vals, 100, 10); got != 10 { + t.Errorf("budget 100: floor = %d, want 10", got) + } + + // Floor clamped up to minFloor. + if got := floorForBudget(vals, 10, 10); got != 10 { + t.Errorf("clamp: floor = %d, want 10", got) + } + + if got := floorForBudget(nil, 5, 7); got != 7 { + t.Errorf("empty: floor = %d, want 7", got) + } +} + +func TestRankFloor(t *testing.T) { + desc := []uint32{100, 90, 80, 70, 60} + + cases := []struct { + rank int + want uint32 + }{ + {1, 100}, + {3, 80}, + {5, 60}, + {99, 60}, // clamped to the last element + } + + for _, c := range cases { + if got := rankFloor(desc, c.rank); got != c.want { + t.Errorf("rankFloor(rank=%d) = %d, want %d", c.rank, got, c.want) + } + } + + if got := rankFloor(nil, 3); got != 0 { + t.Errorf("rankFloor(empty) = %d, want 0", got) + } +} + +func TestTierBudget(t *testing.T) { + const aFloor, bFloor = uint32(1000), uint32(100) + + cases := []struct { + listens uint32 + wantTrack, wantRG int + }{ + {2000, perArtistTierATrack, perArtistTierARG}, // tier A + {1000, perArtistTierATrack, perArtistTierARG}, // exactly on A floor + {500, perArtistTierBTrack, perArtistTierBRG}, // tier B + {100, perArtistTierBTrack, perArtistTierBRG}, // exactly on B floor + {10, perArtistTierCTrack, perArtistTierCRG}, // tier C + } + + for _, c := range cases { + gotTrack, gotRG := tierBudget(c.listens, aFloor, bFloor) + if gotTrack != c.wantTrack || gotRG != c.wantRG { + t.Errorf("tierBudget(%d) = (%d, %d), want (%d, %d)", + c.listens, gotTrack, gotRG, c.wantTrack, c.wantRG) + } + } +} + +// mkMBID builds a distinct uuid16 from a single byte, for heap tests. +func mkMBID(b byte) uuid16 { + var id uuid16 + + id[0] = b + + return id +} + +func TestArtistTopNBoundedAndDeduped(t *testing.T) { + a := &artistTopN{n: 3, inSet: make(map[uuid16]struct{})} + + // Add five distinct recordings; only the top 3 by listens survive. + for i, listens := range []uint32{10, 50, 30, 5, 40} { + a.add(keptRecordingRow{mbid: mkMBID(byte(i + 1)), listens: listens}) + } + + if len(a.rows) != 3 { + t.Fatalf("len = %d, want 3 (bounded)", len(a.rows)) + } + + got := map[uint32]bool{} + for _, r := range a.rows { + got[r.listens] = true + } + + for _, want := range []uint32{50, 40, 30} { + if !got[want] { + t.Errorf("expected top listens %d retained, have %v", want, got) + } + } + + if got[10] || got[5] { + t.Errorf("evicted entries survived: %v", got) + } + + // Re-adding an existing MBID is a no-op, even with a higher count. + before := len(a.rows) + + a.add(keptRecordingRow{mbid: mkMBID(2), listens: 9999}) + + if len(a.rows) != before { + t.Errorf("duplicate MBID grew the set: %d != %d", len(a.rows), before) + } + + if _, dupHigh := got[9999]; dupHigh { + t.Error("duplicate MBID should not have been re-ranked") + } +} + +func TestArtistTopRGBoundedAndDeduped(t *testing.T) { + a := &artistTopRG{n: 2, inSet: make(map[uuid16]struct{})} + + a.add(mkMBID(1), 100) + a.add(mkMBID(2), 200) + a.add(mkMBID(3), 50) // below both, dropped + a.add(mkMBID(1), 999) // duplicate, ignored + + if len(a.rgs) != 2 { + t.Fatalf("len = %d, want 2", len(a.rgs)) + } + + for _, c := range a.rgs { + if c.listens == 50 { + t.Error("sub-threshold RG was kept") + } + + if c.rg == mkMBID(1) && c.listens != 100 { + t.Errorf("duplicate RG re-ranked to %d", c.listens) + } + } +} + +// --------------------------------------------------------------------------- +// Fixture builders +// --------------------------------------------------------------------------- + +// sparkFixtureRow mimics the real spark listens schema: the aggregator +// must project just recording/release/artist MBIDs out of it. +type sparkFixtureRow struct { + ListenedAt int64 `parquet:"listened_at"` + UserID int64 `parquet:"user_id"` + ArtistName string `parquet:"artist_name,optional"` + RecordingMBID string `parquet:"recording_mbid,optional"` + ReleaseMBID string `parquet:"release_mbid,optional"` + ArtistMBIDs []string `parquet:"artist_credit_mbids,optional,list"` +} + +func makeParquet(t *testing.T, rows []sparkFixtureRow) []byte { + t.Helper() + + var buf bytes.Buffer + + w := parquet.NewGenericWriter[sparkFixtureRow](&buf) + + if _, err := w.Write(rows); err != nil { + t.Fatalf("parquet write: %v", err) + } + + if err := w.Close(); err != nil { + t.Fatalf("parquet close: %v", err) + } + + return buf.Bytes() +} + +func makeTar(t *testing.T, members map[string][]byte, order []string) []byte { + t.Helper() + + var buf bytes.Buffer + + tw := tar.NewWriter(&buf) + + for _, name := range order { + data := members[name] + hdr := &tar.Header{ + Name: name, + Mode: 0o644, + Size: int64(len(data)), + Typeflag: tar.TypeReg, + } + + if err := tw.WriteHeader(hdr); err != nil { + t.Fatalf("tar header: %v", err) + } + + if _, err := tw.Write(data); err != nil { + t.Fatalf("tar write: %v", err) + } + } + + if err := tw.Close(); err != nil { + t.Fatalf("tar close: %v", err) + } + + return buf.Bytes() +} + +func zstdCompress(t *testing.T, data []byte) []byte { + t.Helper() + + var buf bytes.Buffer + + zw, err := zstd.NewWriter(&buf) + if err != nil { + t.Fatalf("zstd writer: %v", err) + } + + if _, err := zw.Write(data); err != nil { + t.Fatalf("zstd write: %v", err) + } + + if err := zw.Close(); err != nil { + t.Fatalf("zstd close: %v", err) + } + + return buf.Bytes() +} + +func csvBytes(t *testing.T, rows [][]string) []byte { + t.Helper() + + var buf bytes.Buffer + + w := csv.NewWriter(&buf) + if err := w.WriteAll(rows); err != nil { + t.Fatalf("csv write: %v", err) + } + + return buf.Bytes() +} + +// listensOf builds n identical listen rows for a recording. +func listensOf(n int, recording, release string, artists []string) []sparkFixtureRow { + rows := make([]sparkFixtureRow, n) + for i := range rows { + rows[i] = sparkFixtureRow{ + ListenedAt: 1700000000 + int64(i), + UserID: int64(i), + ArtistName: "Fixture Artist", + RecordingMBID: recording, + ReleaseMBID: release, + ArtistMBIDs: artists, + } + } + + return rows +} + +// canonicalDataCSV builds a canonical_musicbrainz_data.csv fixture. +func canonicalDataCSV(t *testing.T) []byte { + t.Helper() + + rows := [][]string{ + { + "id", "artist_credit_id", "artist_mbids", "artist_credit_name", + "release_mbid", "release_name", "recording_mbid", "recording_name", + "combined_lookup", "score", + }, + {"1", "10", "{" + artA + "}", "Solo Star", relA, "Big Album", recA, "Hit Song", "x", "1"}, + {"2", "10", "{" + artA + "}", "Solo Star", relA, "Big Album", recB, "Deep Cut", "x", "1"}, + { + "3", "11", "{" + artA + "," + artB + "}", "Solo Star feat. Guest", + relB, "Duet Album", recC, "Duet Song", "x", "1", + }, + } + + return csvBytes(t, rows) +} + +// canonicalRedirectCSV builds a canonical_release_redirect.csv fixture. +func canonicalRedirectCSV(t *testing.T) []byte { + t.Helper() + + rows := [][]string{ + {"release_mbid", "canonical_release_mbid", "release_group_mbid"}, + {relA, relA, rgA}, + {relB, relB, rgB}, + } + + return csvBytes(t, rows) +} + +// serveDumps returns an httptest server presenting MetaBrainz-style +// listing pages and Range-capable dump files. +func serveDumps(t *testing.T, sparkTar, canonicalTarZst []byte) *httptest.Server { + t.Helper() + + const ( + listensDir = "listenbrainz-dump-1-20260101-000003-full" + sparkFile = "listenbrainz-spark-dump-1-20260101-000003-full.tar" + canonicalDir = "musicbrainz-canonical-dump-20260101-080003" + canonicalTar = "musicbrainz-canonical-dump-20260101-080003.tar.zst" + ) + + modTime := time.Now() + mux := http.NewServeMux() + + mux.HandleFunc("/listens/", func(w http.ResponseWriter, r *http.Request) { + switch strings.TrimPrefix(r.URL.Path, "/listens/") { + case "": + _, _ = fmt.Fprintf(w, `%s/`, listensDir, listensDir) + case listensDir + "/": + _, _ = fmt.Fprintf(w, `%s`, sparkFile, sparkFile) + case listensDir + "/" + sparkFile: + http.ServeContent(w, r, sparkFile, modTime, bytes.NewReader(sparkTar)) + default: + http.NotFound(w, r) + } + }) + + mux.HandleFunc("/canonical/", func(w http.ResponseWriter, r *http.Request) { + switch strings.TrimPrefix(r.URL.Path, "/canonical/") { + case "": + _, _ = fmt.Fprintf(w, `%s/`, canonicalDir, canonicalDir) + case canonicalDir + "/": + _, _ = fmt.Fprintf(w, `%s`, canonicalTar, canonicalTar) + case canonicalDir + "/" + canonicalTar: + http.ServeContent(w, r, canonicalTar, modTime, bytes.NewReader(canonicalTarZst)) + default: + http.NotFound(w, r) + } + }) + + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + + return srv +} + +func testImporter(t *testing.T, si *SearchIndex, srv *httptest.Server) *dumpImporter { + t.Helper() + + return &dumpImporter{ + si: si, + lb: nil, // patch passes skipped in tests + logger: testLogger(), + httpClient: srv.Client(), + stagingDir: t.TempDir(), + canonicalBaseURL: srv.URL + "/canonical/", + listensBaseURL: srv.URL + "/listens/", + } +} + +func fixtureSparkTar(t *testing.T) []byte { + t.Helper() + + // Member 1: recA is popular (12 listens). Member 2: recB has 11, + // recC has 12 (multi-artist credit). Totals: artA = 35, artB = 12. + member1 := makeParquet(t, listensOf(12, recA, relA, []string{artA})) + member2 := makeParquet(t, append( + listensOf(11, recB, relA, []string{artA}), + listensOf(12, recC, relB, []string{artA, artB})..., + )) + + prefix := "listenbrainz-spark-dump-1-20260101-000003-full/listens/" + + return makeTar(t, + map[string][]byte{ + prefix + "1.parquet": member1, + prefix + "2.parquet": member2, + }, + []string{prefix + "1.parquet", prefix + "2.parquet"}, + ) +} + +func fixtureCanonicalTarZst(t *testing.T) []byte { + t.Helper() + + prefix := "musicbrainz-canonical-dump-20260101-080003/" + + raw := makeTar(t, + map[string][]byte{ + prefix + "canonical_musicbrainz_data.csv": canonicalDataCSV(t), + prefix + "canonical_release_redirect.csv": canonicalRedirectCSV(t), + prefix + "canonical_recording_redirect.csv": {}, + }, + []string{ + prefix + "canonical_release_redirect.csv", + prefix + "canonical_musicbrainz_data.csv", + prefix + "canonical_recording_redirect.csv", + }, + ) + + return zstdCompress(t, raw) +} + +// --------------------------------------------------------------------------- +// Stage tests +// --------------------------------------------------------------------------- + +func TestAggregateListenCounts(t *testing.T) { + srv := serveDumps(t, fixtureSparkTar(t), nil) + + db := database.NewTestDB(t) + si := NewSearchIndex(db, nil, nil, testLogger()) + imp := testImporter(t, si, srv) + + sparkURL, err := discoverDumpFile( + context.Background(), imp.httpClient, imp.listensBaseURL, listensDirRe, sparkFileRe, + ) + if err != nil { + t.Fatalf("discover: %v", err) + } + + st := &countsState{SparkURL: sparkURL} + if err := imp.aggregateListenCounts(context.Background(), st); err != nil { + t.Fatalf("aggregate: %v", err) + } + + assertCount := func(kind byte, mbid string, want uint32) { + t.Helper() + + key, ok := makeMBIDKey(kind, mbid) + if !ok { + t.Fatalf("bad fixture mbid %s", mbid) + } + + if got := st.counts[key]; got != want { + t.Errorf("count(kind=%d, %s) = %d, want %d", kind, mbid, got, want) + } + } + + assertCount(countKindRecording, recA, 12) + assertCount(countKindRecording, recB, 11) + assertCount(countKindRecording, recC, 12) + assertCount(countKindRelease, relA, 23) + assertCount(countKindRelease, relB, 12) + assertCount(countKindArtist, artA, 35) + assertCount(countKindArtist, artB, 12) + + if !st.Done { + t.Error("state not marked done") + } + + // The checkpoint file round-trips. + loaded, err := imp.readCountsFile() + if err != nil { + t.Fatalf("read counts file: %v", err) + } + + if loaded == nil || !loaded.Done || len(loaded.counts) != len(st.counts) { + t.Fatalf("checkpoint mismatch: %+v", loaded) + } +} + +func TestAggregateResumeFromOffset(t *testing.T) { + sparkTar := fixtureSparkTar(t) + srv := serveDumps(t, sparkTar, nil) + + db := database.NewTestDB(t) + si := NewSearchIndex(db, nil, nil, testLogger()) + imp := testImporter(t, si, srv) + + sparkURL := srv.URL + "/listens/listenbrainz-dump-1-20260101-000003-full/listenbrainz-spark-dump-1-20260101-000003-full.tar" + + // Full run for reference. + full := &countsState{SparkURL: sparkURL} + if err := imp.aggregateListenCounts(context.Background(), full); err != nil { + t.Fatalf("full aggregate: %v", err) + } + + // Simulate a checkpoint taken after member 1: offset = header + // block + padded member-1 size (fixture names are short, so the + // header is a single 512-byte block). + member1 := makeParquet(t, listensOf(12, recA, relA, []string{artA})) + offset := int64(512) + (int64(len(member1))+511)/512*512 + + key, _ := makeMBIDKey(countKindRecording, recA) + relKey, _ := makeMBIDKey(countKindRelease, relA) + artKey, _ := makeMBIDKey(countKindArtist, artA) + + resumed := &countsState{ + SparkURL: sparkURL, + Offset: offset, + MemberIdx: 1, + counts: map[mbidKey]uint32{ + key: 12, + relKey: 12, + artKey: 12, + }, + } + + if err := imp.aggregateListenCounts(context.Background(), resumed); err != nil { + t.Fatalf("resumed aggregate: %v", err) + } + + if len(resumed.counts) != len(full.counts) { + t.Fatalf("resumed entities = %d, want %d", len(resumed.counts), len(full.counts)) + } + + for k, want := range full.counts { + if got := resumed.counts[k]; got != want { + t.Errorf("resumed count %s = %d, want %d (double count?)", formatUUID(k[1:]), got, want) + } + } +} + +func TestResumableReaderReconnects(t *testing.T) { + payload := bytes.Repeat([]byte("0123456789abcdef"), 4096) // 64KB + + // A flaky server that truncates every response to 10KB, forcing + // the reader to reconnect with Range requests. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + offset := int64(0) + if rng := r.Header.Get("Range"); rng != "" { + _, _ = fmt.Sscanf(rng, "bytes=%d-", &offset) + } + + chunk := payload[offset:min(offset+10240, int64(len(payload)))] + + w.Header().Set("Content-Range", + fmt.Sprintf("bytes %d-%d/%d", offset, offset+int64(len(chunk))-1, len(payload))) + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write(chunk) + })) + t.Cleanup(srv.Close) + + r := newResumableReader(context.Background(), srv.Client(), srv.URL, 0) + + got, err := io.ReadAll(r) + if err != nil { + t.Fatalf("read: %v", err) + } + + if !bytes.Equal(got, payload) { + t.Fatalf("payload mismatch: got %d bytes, want %d", len(got), len(payload)) + } +} + +// --------------------------------------------------------------------------- +// End-to-end +// --------------------------------------------------------------------------- + +func TestDumpImportEndToEnd(t *testing.T) { + srv := serveDumps(t, fixtureSparkTar(t), fixtureCanonicalTarZst(t)) + + db := database.NewTestDB(t) + si := NewSearchIndex(db, nil, nil, testLogger()) + imp := testImporter(t, si, srv) + stagingDir := imp.stagingDir + + // A legacy API-crawled row with inflated popularity must be + // cleared by the first dump import (scale consistency). + legacyMBID := "99999999-9999-9999-9999-999999999999" + + si.upsertBatch([]SearchIndexResult{{ + EntityType: "recording", + MBID: legacyMBID, + Title: "Legacy Row", + ArtistName: "Old Crawl", + ArtistMBID: artA, + Popularity: 123_456_789, + }}) + + if err := imp.run(context.Background()); err != nil { + t.Fatalf("run: %v", err) + } + + legacyRows, err := db.QueryContext( + "SELECT COUNT(*) FROM explore_index WHERE mbid = ?", legacyMBID, + ) + if err != nil { + t.Fatalf("legacy query: %v", err) + } + + if legacyRows.Next() { + var n int + + _ = legacyRows.Scan(&n) + + if n != 0 { + t.Error("legacy API-crawled row survived the first dump import") + } + } + + _ = legacyRows.Close() + + // Index rows landed with dump-derived popularity. + assertRow := func(mbid, entityType, title string, popularity int) { + t.Helper() + + rows, err := db.QueryContext( + "SELECT title, popularity FROM explore_index WHERE mbid = ? AND entity_type = ?", + mbid, entityType, + ) + if err != nil { + t.Fatalf("query: %v", err) + } + + defer func() { _ = rows.Close() }() + + if !rows.Next() { + t.Fatalf("no %s row for %s", entityType, mbid) + } + + var gotTitle string + + var gotPop int + + if err := rows.Scan(&gotTitle, &gotPop); err != nil { + t.Fatalf("scan: %v", err) + } + + if gotTitle != title || gotPop != popularity { + t.Errorf("%s %s = (%q, %d), want (%q, %d)", + entityType, mbid, gotTitle, gotPop, title, popularity) + } + } + + assertRow(recA, "recording", "Hit Song", 12) + assertRow(recB, "recording", "Deep Cut", 11) + assertRow(recC, "recording", "Duet Song", 12) + assertRow(rgA, "release_group", "Big Album", 23) + assertRow(rgB, "release_group", "Duet Album", 12) + assertRow(artA, "artist", "Solo Star", 35) + + // artB only ever appears in a multi-artist credit: no name is + // derivable from the dump, so it must be queued for the API + // metadata patch instead of being written nameless. + rows, err := db.QueryContext( + "SELECT COUNT(*) FROM explore_index WHERE mbid = ?", artB, + ) + if err != nil { + t.Fatalf("query artB: %v", err) + } + + if rows.Next() { + var n int + + _ = rows.Scan(&n) + + if n != 0 { + t.Errorf("artB row written without a name source") + } + } + + _ = rows.Close() + + found := false + + for _, mbid := range imp.pendingArtists { + if mbid == artB { + found = true + } + } + + if !found { + t.Errorf("artB not queued for metadata patch: %v", imp.pendingArtists) + } + + // FTS search works end to end. + si.MarkReadyIfPopulated() + + results := si.Search(context.Background(), "hit song", 10) + if len(results) == 0 || results[0].MBID != recA { + t.Fatalf("search for indexed recording failed: %+v", results) + } + + // Completion recorded; staging cleaned up. + if !si.hasMeta(dumpImportDoneKey) { + t.Error("dump_import_done not recorded") + } + + // The incremental refresh baseline was recorded, and the + // release→release-group map was persisted for future rollups. + if _, ok := si.metaInt(listensAppliedSeriesKey); !ok { + t.Error("listens_applied_series baseline not recorded") + } + + var relToRGRows int + + rtrRows, err := db.QueryContext("SELECT COUNT(*) FROM release_to_rg") + if err != nil { + t.Fatalf("query release_to_rg: %v", err) + } + + if rtrRows.Next() { + _ = rtrRows.Scan(&relToRGRows) + } + + _ = rtrRows.Close() + + if relToRGRows == 0 { + t.Error("release_to_rg not populated after import") + } + + if _, err := os.Stat(stagingDir); !os.IsNotExist(err) { + t.Errorf("staging dir not cleaned up: %v", err) + } + + // Re-running is a cheap no-op that doesn't error. + imp2 := testImporter(t, si, srv) + if err := imp2.run(context.Background()); err != nil { + t.Fatalf("second run: %v", err) + } +} + +func TestDumpImportResumesAfterCancel(t *testing.T) { + srv := serveDumps(t, fixtureSparkTar(t), fixtureCanonicalTarZst(t)) + + db := database.NewTestDB(t) + si := NewSearchIndex(db, nil, nil, testLogger()) + imp := testImporter(t, si, srv) + + // Cancelled before it can start streaming: no partial state may + // break the follow-up run. + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + if err := imp.run(ctx); err == nil { + t.Fatal("cancelled run should return an error") + } + + if err := imp.run(context.Background()); err != nil { + t.Fatalf("rerun after cancel: %v", err) + } + + results := si.Search(context.Background(), "hit song", 10) + if len(results) == 0 { + t.Fatal("index empty after resumed run") + } +} + +func TestCheckFreeDisk(t *testing.T) { + dir := t.TempDir() + + if err := checkFreeDisk(dir, 1); err != nil { + t.Errorf("1 byte requirement should pass: %v", err) + } + + if err := checkFreeDisk(dir, 1<<62); err == nil { + t.Error("absurd requirement should fail") + } +} + +func TestDiscoverDumpFilePickNewest(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/": + _, _ = io.WriteString(w, ` + old + new + x`) + case "/musicbrainz-canonical-dump-20260615-080003/": + _, _ = io.WriteString(w, + `f`) + default: + http.NotFound(w, r) + } + }) + + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + + url, err := discoverDumpFile( + context.Background(), srv.Client(), srv.URL+"/", canonicalDirRe, canonicalFileRe, + ) + if err != nil { + t.Fatalf("discover: %v", err) + } + + want := srv.URL + "/musicbrainz-canonical-dump-20260615-080003/musicbrainz-canonical-dump-20260615-080003.tar.zst" + if url != want { + t.Errorf("url = %s, want %s", url, want) + } +} + +func TestListenerCountUpdateDoesNotTouchPopularity(t *testing.T) { + db := database.NewTestDB(t) + si := NewSearchIndex(db, nil, nil, testLogger()) + + si.upsertBatch([]SearchIndexResult{{ + EntityType: "recording", + MBID: recA, + Title: "Hit Song", + ArtistName: "Solo Star", + ArtistMBID: artA, + Popularity: 12, + }}) + + updated := si.updateListenerCounts(map[string]PopularityData{ + recA: {ListenCount: 999_999, ListenerCount: 42}, + }) + if updated != 1 { + t.Fatalf("updated = %d, want 1", updated) + } + + rows, err := db.QueryContext( + "SELECT popularity, listener_count FROM explore_index WHERE mbid = ?", recA, + ) + if err != nil { + t.Fatalf("query: %v", err) + } + + defer func() { _ = rows.Close() }() + + if !rows.Next() { + t.Fatal("row missing") + } + + var pop, listeners int + + if err := rows.Scan(&pop, &listeners); err != nil { + t.Fatalf("scan: %v", err) + } + + if pop != 12 { + t.Errorf("popularity = %d, want 12 (dump scale must stay authoritative)", pop) + } + + if listeners != 42 { + t.Errorf("listener_count = %d, want 42", listeners) + } +} diff --git a/backend/explore/dumpincremental.go b/backend/explore/dumpincremental.go new file mode 100644 index 0000000..651d3a9 --- /dev/null +++ b/backend/explore/dumpincremental.go @@ -0,0 +1,522 @@ +package explore + +import ( + "archive/tar" + "bufio" + "context" + "database/sql" + "errors" + "fmt" + "io" + "net/http" + "regexp" + "sort" + "strconv" + "strings" + "time" +) + +// Incremental listen-count refresh. ListenBrainz publishes a small +// (~250MB) incremental spark dump every day containing only the listens +// submitted since the previous dump, in the same parquet-in-tar format +// as the full dump. This folds those daily deltas into the index's +// popularity numbers additively, so recordings, artists, and albums stay +// fresh without re-streaming the ~170GB full dump and without a single +// ListenBrainz API call. +// +// Album (release-group) popularity is derived locally: the incremental +// gives per-release listen counts, which are rolled up to their release +// group via the release_to_rg map captured during the full import. +// +// Correctness: each dump's deltas plus the high-water-mark advance are +// applied in one transaction, so a crash can't half-apply a dump, and +// the high-water-mark guarantees each dump is applied at most once. The +// sum of the full dump plus every incremental is therefore the exact +// cumulative listen count — the additive model does not drift. + +const ( + // defaultIncrementalBaseURL is where ListenBrainz publishes daily + // incremental listen dumps. + defaultIncrementalBaseURL = "https://data.metabrainz.org/pub/musicbrainz/listenbrainz/incremental/" + + // listensCatchupTsKey records when the incremental refresh last ran + // (RFC3339), gating how often it re-checks for new dumps. + listensCatchupTsKey = "listens_last_catchup" + + // listensCatchupInterval is the default minimum time between refresh + // checks. Album/track popularity is slow-moving and each dump is a + // ~250MB download, so a weekly cadence keeps the index current while + // bounding background network use. + listensCatchupInterval = 7 * 24 * time.Hour + + // deltaInsertBatch bounds rows per multi-row INSERT into the temp + // delta table during a single incremental apply. + deltaInsertBatch = 500 + + // releaseLookupBatch bounds release MBIDs per release_to_rg lookup. + releaseLookupBatch = 500 +) + +var ( + incrementalDirRe = regexp.MustCompile(`^listenbrainz-dump-\d+-\d{8}-\d+-incremental$`) + incrementalFileRe = regexp.MustCompile(`^listenbrainz-spark-dump-.*-incremental\.tar$`) +) + +// incrementalDump identifies one daily incremental dump. +type incrementalDump struct { + series int + url string +} + +// RefreshListenCounts folds any incremental dumps newer than the current +// high-water-mark into the index's popularity numbers. It is a no-op +// when there is no completed baseline import, when a full build is +// running, when the last refresh was within minInterval (pass 0 to +// force), or when offline. Runs synchronously — callers wanting the +// background behaviour should invoke it in a goroutine. +func (si *SearchIndex) RefreshListenCounts(ctx context.Context, minInterval time.Duration) { + if !si.hasMeta(dumpImportDoneKey) { + si.logger.Info("incremental refresh: no baseline import yet, skipping") + + return + } + + si.mu.RLock() + building := si.cancel != nil + si.mu.RUnlock() + + if building { + si.logger.Info("incremental refresh: full build running, skipping") + + return + } + + if minInterval > 0 && si.refreshedWithin(minInterval) { + si.logger.Info("incremental refresh: checked recently, skipping") + + return + } + + hwm, ok := si.metaInt(listensAppliedSeriesKey) + if !ok { + si.logger.Warn("incremental refresh: no baseline series recorded, skipping") + + return + } + + client := &http.Client{} + + dumps, err := discoverIncrementalDumps(ctx, client, defaultIncrementalBaseURL, hwm) + if err != nil { + si.logger.Warn("incremental refresh: discovery failed", "error", err) + + return + } + + // Record the check even when there is nothing to apply, so the + // cadence gate holds regardless of outcome. + defer si.setMeta(listensCatchupTsKey, time.Now().UTC().Format(time.RFC3339)) + + if len(dumps) == 0 { + si.logger.Info("incremental refresh: up to date", "throughSeries", hwm) + + return + } + + si.logger.Info("incremental refresh: applying dumps", + "count", len(dumps), "fromSeries", hwm+1, "toSeries", dumps[len(dumps)-1].series, + ) + + applied := 0 + through := hwm + + for _, d := range dumps { + if ctx.Err() != nil { + break + } + + if err := si.applyIncremental(ctx, client, d); err != nil { + // Stop at the first failure; the high-water-mark is only + // advanced on success, so the next run resumes from here. + si.logger.Warn("incremental refresh: apply failed, stopping", + "series", d.series, "error", err) + + break + } + + applied++ + through = d.series + } + + if applied > 0 { + // Popularity changed, so refresh the champion tier that backs + // generic short-prefix searches. + si.scheduleChampionRebuild() + } + + si.logger.Info("incremental refresh: complete", "applied", applied, "throughSeries", through) +} + +// applyIncremental downloads one incremental dump, aggregates its listen +// counts, rolls release counts up to release groups, and applies the +// deltas atomically together with the high-water-mark advance. +func (si *SearchIndex) applyIncremental( + ctx context.Context, client *http.Client, dump incrementalDump, +) error { + start := time.Now() + + stream := newResumableReader(ctx, client, dump.url, 0) + + defer func() { _ = stream.Close() }() + + counts, err := aggregateTarListens(ctx, stream) + if err != nil { + return fmt.Errorf("aggregate incremental %d: %w", dump.series, err) + } + + rec, art, rel := splitCountsByKind(counts) + rg := si.rollupReleaseDeltas(rel) + + if err := si.commitListenDeltas(dump.series, rec, art, rg); err != nil { + return err + } + + si.logger.Info("incremental refresh: applied dump", + "series", dump.series, + "recordings", len(rec), + "artists", len(art), + "releaseGroups", len(rg), + "elapsed", time.Since(start).Round(time.Millisecond), + ) + + return nil +} + +// splitCountsByKind separates an aggregated counts map into per-kind +// maps of canonical MBID string → delta. +func splitCountsByKind(counts map[mbidKey]uint32) (rec, art, rel map[string]uint32) { + rec = make(map[string]uint32) + art = make(map[string]uint32) + rel = make(map[string]uint32) + + for k, v := range counts { + mbid := formatUUID(k[1:]) + + switch k[0] { + case countKindRecording: + rec[mbid] += v + case countKindArtist: + art[mbid] += v + case countKindRelease: + rel[mbid] += v + } + } + + return rec, art, rel +} + +// rollupReleaseDeltas maps per-release listen deltas to their release +// group via the release_to_rg table and sums per group. Releases not in +// the table (below the index floor, or unknown) contribute nothing. +func (si *SearchIndex) rollupReleaseDeltas(rel map[string]uint32) map[string]uint32 { + rg := make(map[string]uint32) + if len(rel) == 0 { + return rg + } + + mbids := make([]string, 0, len(rel)) + for m := range rel { + mbids = append(mbids, m) + } + + for i := 0; i < len(mbids); i += releaseLookupBatch { + end := min(i+releaseLookupBatch, len(mbids)) + batch := mbids[i:end] + + placeholders := strings.TrimSuffix(strings.Repeat("?,", len(batch)), ",") + + args := make([]any, len(batch)) + for j, m := range batch { + args[j] = m + } + + rows, err := si.db.QueryContext( + "SELECT release_mbid, rg_mbid FROM release_to_rg WHERE release_mbid IN ("+placeholders+")", + args..., + ) + if err != nil { + si.logger.Warn("incremental refresh: release_to_rg lookup failed", "error", err) + + continue + } + + for rows.Next() { + var relMBID, rgMBID string + if err := rows.Scan(&relMBID, &rgMBID); err == nil { + rg[rgMBID] += rel[relMBID] + } + } + + _ = rows.Close() + } + + return rg +} + +// commitListenDeltas applies recording, artist, and release-group deltas +// to explore_index and advances the high-water-mark to series, all in a +// single transaction so the apply is crash-atomic and exactly-once. +func (si *SearchIndex) commitListenDeltas( + series int, rec, art, rg map[string]uint32, +) error { + tx, err := si.db.BeginTx() + if err != nil { + return fmt.Errorf("incremental tx: %w", err) + } + + defer func() { _ = tx.Rollback() }() + + if _, err := tx.Exec( + "CREATE TEMP TABLE IF NOT EXISTS incr_delta (mbid TEXT, kind TEXT, delta INTEGER)", + ); err != nil { + return fmt.Errorf("incremental temp table: %w", err) + } + + if _, err := tx.Exec("DELETE FROM incr_delta"); err != nil { + return fmt.Errorf("incremental temp reset: %w", err) + } + + for _, kd := range []struct { + kind string + deltas map[string]uint32 + }{ + {"recording", rec}, + {"artist", art}, + {"release_group", rg}, + } { + if err := insertDeltas(tx, kd.kind, kd.deltas); err != nil { + return err + } + } + + // Additive apply: bump popularity for every index row that has a + // matching delta. Rows with no delta are untouched; deltas with no + // matching row (entity not indexed) are ignored. + if _, err := tx.Exec(` + UPDATE explore_index + SET popularity = popularity + d.delta + FROM incr_delta d + WHERE d.mbid = explore_index.mbid + AND d.kind = explore_index.entity_type + `); err != nil { + return fmt.Errorf("incremental apply: %w", err) + } + + if _, err := tx.Exec("DELETE FROM incr_delta"); err != nil { + return fmt.Errorf("incremental temp cleanup: %w", err) + } + + if _, err := tx.Exec( + "INSERT OR REPLACE INTO explore_index_meta (key, value) VALUES (?, ?)", + listensAppliedSeriesKey, strconv.Itoa(series), + ); err != nil { + return fmt.Errorf("incremental advance high-water-mark: %w", err) + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("incremental commit: %w", err) + } + + return nil +} + +// insertDeltas bulk-inserts a kind's deltas into the temp table. +func insertDeltas(tx *sql.Tx, kind string, deltas map[string]uint32) error { + if len(deltas) == 0 { + return nil + } + + rowArgs := make([]any, 0, deltaInsertBatch*3) + pending := 0 + + flush := func() error { + if pending == 0 { + return nil + } + + values := strings.TrimSuffix(strings.Repeat("(?,?,?),", pending), ",") + query := "INSERT INTO incr_delta (mbid, kind, delta) VALUES " + values + + if _, err := tx.Exec(query, rowArgs...); err != nil { + return fmt.Errorf("incremental insert deltas: %w", err) + } + + rowArgs = rowArgs[:0] + pending = 0 + + return nil + } + + for mbid, d := range deltas { + rowArgs = append(rowArgs, mbid, kind, int64(d)) + pending++ + + if pending >= deltaInsertBatch { + if err := flush(); err != nil { + return err + } + } + } + + return flush() +} + +// aggregateTarListens streams a tar of parquet listen members and sums +// the per-entity listen counts. Unlike the full-dump path, the whole +// (small) incremental is aggregated in RAM with no checkpointing. +func aggregateTarListens(ctx context.Context, r io.Reader) (map[mbidKey]uint32, error) { + buffered := bufio.NewReaderSize(r, 1<<20) + tr := tar.NewReader(buffered) + counts := make(map[mbidKey]uint32, 1<<18) + + for { + if err := ctx.Err(); err != nil { + return nil, err + } + + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + + if err != nil { + return nil, fmt.Errorf("incremental tar: %w", err) + } + + if hdr.Typeflag != tar.TypeReg || !strings.HasSuffix(hdr.Name, ".parquet") { + continue + } + + if hdr.Size > maxParquetMemberSize { + return nil, fmt.Errorf( + "%w: parquet member %s is %d bytes", ErrDumpFormat, hdr.Name, hdr.Size, + ) + } + + buf := make([]byte, hdr.Size) + if _, err := io.ReadFull(tr, buf); err != nil { + return nil, fmt.Errorf("incremental member read: %w", err) + } + + deltas, err := parseListenParquet(buf) + if err != nil { + return nil, fmt.Errorf("incremental parse: %w", err) + } + + for k, v := range deltas { + counts[k] += v + } + } + + return counts, nil +} + +// discoverIncrementalDumps lists the incremental directory and returns +// the spark-dump URLs for every dump with a series greater than +// sinceSeries, sorted ascending so they apply in chronological order. +func discoverIncrementalDumps( + ctx context.Context, client *http.Client, baseURL string, sinceSeries int, +) ([]incrementalDump, error) { + hrefs, err := listHrefs(ctx, client, baseURL) + if err != nil { + return nil, err + } + + var dumps []incrementalDump + + for _, h := range hrefs { + dir := trimTrailingSlash(h) + if !incrementalDirRe.MatchString(dir) { + continue + } + + series, ok := parseDumpSeries(dir) + if !ok || series <= sinceSeries { + continue + } + + dirURL := baseURL + dir + "/" + + files, err := listHrefs(ctx, client, dirURL) + if err != nil { + continue + } + + for _, f := range files { + if incrementalFileRe.MatchString(f) { + dumps = append(dumps, incrementalDump{series: series, url: dirURL + f}) + + break + } + } + } + + sort.Slice(dumps, func(i, j int) bool { return dumps[i].series < dumps[j].series }) + + return dumps, nil +} + +// metaInt reads an integer-valued explore_index_meta key. +func (si *SearchIndex) metaInt(key string) (int, bool) { + rows, err := si.db.QueryContext("SELECT value FROM explore_index_meta WHERE key = ?", key) + if err != nil { + return 0, false + } + + defer func() { _ = rows.Close() }() + + if !rows.Next() { + return 0, false + } + + var v string + if err := rows.Scan(&v); err != nil { + return 0, false + } + + n, err := strconv.Atoi(v) + if err != nil { + return 0, false + } + + return n, true +} + +// refreshedWithin reports whether the incremental refresh last ran less +// than d ago. +func (si *SearchIndex) refreshedWithin(d time.Duration) bool { + rows, err := si.db.QueryContext( + "SELECT value FROM explore_index_meta WHERE key = ?", listensCatchupTsKey, + ) + if err != nil { + return false + } + + defer func() { _ = rows.Close() }() + + if !rows.Next() { + return false + } + + var v string + if err := rows.Scan(&v); err != nil { + return false + } + + t, err := time.Parse(time.RFC3339, v) + if err != nil { + return false + } + + return time.Since(t) < d +} diff --git a/backend/explore/dumpincremental_test.go b/backend/explore/dumpincremental_test.go new file mode 100644 index 0000000..ed298a4 --- /dev/null +++ b/backend/explore/dumpincremental_test.go @@ -0,0 +1,209 @@ +package explore + +import ( + "bytes" + "context" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "yellowjacket/backend/database" +) + +func TestParseDumpSeries(t *testing.T) { + cases := []struct { + in string + want int + ok bool + }{ + {"https://x/fullexport/listenbrainz-dump-2593-20260712-000004-full/y.tar", 2593, true}, + {"listenbrainz-dump-2594-20260713-000003-incremental", 2594, true}, + {"listenbrainz-spark-dump-2603-20260722-000003-incremental.tar", 2603, true}, + {"nothing-here", 0, false}, + } + + for _, c := range cases { + got, ok := parseDumpSeries(c.in) + if ok != c.ok || (ok && got != c.want) { + t.Errorf("parseDumpSeries(%q) = (%d, %v); want (%d, %v)", c.in, got, ok, c.want, c.ok) + } + } +} + +func popularityOf(t *testing.T, db *database.DB, mbid string) (int, bool) { + t.Helper() + + rows, err := db.QueryContext( + "SELECT popularity FROM explore_index WHERE mbid = ?", mbid, + ) + if err != nil { + t.Fatalf("query popularity: %v", err) + } + + defer func() { _ = rows.Close() }() + + if !rows.Next() { + return 0, false + } + + var pop int + if err := rows.Scan(&pop); err != nil { + t.Fatalf("scan popularity: %v", err) + } + + return pop, true +} + +// applyTar runs the incremental split → rollup → commit path against an +// in-memory tar, mirroring applyIncremental without the HTTP stream. +func applyTar(t *testing.T, si *SearchIndex, series int, tarBytes []byte) { + t.Helper() + + counts, err := aggregateTarListens(context.Background(), bytes.NewReader(tarBytes)) + if err != nil { + t.Fatalf("aggregate: %v", err) + } + + rec, art, rel := splitCountsByKind(counts) + rg := si.rollupReleaseDeltas(rel) + + if err := si.commitListenDeltas(series, rec, art, rg); err != nil { + t.Fatalf("commit series %d: %v", series, err) + } +} + +func TestIncrementalApplyAdditive(t *testing.T) { + db := database.NewTestDB(t) + si := NewSearchIndex(db, nil, nil, testLogger()) + + si.upsertBatch([]SearchIndexResult{ + {EntityType: "recording", MBID: recA, Title: "Rec A", Popularity: 100}, + {EntityType: "artist", MBID: artA, Title: "Art A", Popularity: 100}, + {EntityType: "release_group", MBID: rgA, Title: "RG A", Popularity: 100}, + }) + + if _, err := db.ExecContext( + "INSERT INTO release_to_rg (release_mbid, rg_mbid) VALUES (?, ?)", relA, rgA, + ); err != nil { + t.Fatalf("seed release_to_rg: %v", err) + } + + // 5 listens of recA on relA credited to artA. + tarBytes := makeTar(t, + map[string][]byte{ + "listens/1.parquet": makeParquet(t, listensOf(5, recA, relA, []string{artA})), + }, + []string{"listens/1.parquet"}, + ) + + applyTar(t, si, 2, tarBytes) + + for _, c := range []struct { + mbid string + want int + }{{recA, 105}, {artA, 105}, {rgA, 105}} { + if got, ok := popularityOf(t, db, c.mbid); !ok || got != c.want { + t.Errorf("popularity(%s) = %d (present=%v); want %d", c.mbid, got, ok, c.want) + } + } + + if hwm, ok := si.metaInt(listensAppliedSeriesKey); !ok || hwm != 2 { + t.Errorf("high-water-mark = %d (present=%v); want 2", hwm, ok) + } + + // A second, later dump accumulates additively. + applyTar(t, si, 3, tarBytes) + + if got, _ := popularityOf(t, db, recA); got != 110 { + t.Errorf("popularity(recA) after second dump = %d; want 110", got) + } + + if hwm, _ := si.metaInt(listensAppliedSeriesKey); hwm != 3 { + t.Errorf("high-water-mark after second dump = %d; want 3", hwm) + } +} + +func TestIncrementalIgnoresUnknownAndUnmapped(t *testing.T) { + db := database.NewTestDB(t) + si := NewSearchIndex(db, nil, nil, testLogger()) + + // Only recA is indexed; relA has no release_to_rg mapping. + si.upsertBatch([]SearchIndexResult{ + {EntityType: "recording", MBID: recA, Title: "Rec A", Popularity: 100}, + }) + + // Listens for recB (not indexed) on relB (unmapped) credited to + // artB (not indexed). Nothing should change and no row created. + tarBytes := makeTar(t, + map[string][]byte{ + "listens/1.parquet": makeParquet(t, listensOf(7, recB, relB, []string{artB})), + }, + []string{"listens/1.parquet"}, + ) + + applyTar(t, si, 5, tarBytes) + + if _, ok := popularityOf(t, db, recB); ok { + t.Error("recB should not have been inserted into the index") + } + + if got, _ := popularityOf(t, db, recA); got != 100 { + t.Errorf("popularity(recA) = %d; want 100 (untouched)", got) + } +} + +func TestDiscoverIncrementalDumps(t *testing.T) { + series := []string{"2594", "2595", "2596"} + + mux := http.NewServeMux() + + mux.HandleFunc("/incremental/", func(w http.ResponseWriter, r *http.Request) { + // Only the base listing (exact path); dir listings are handled + // by their own more-specific patterns below. + if r.URL.Path != "/incremental/" { + http.NotFound(w, r) + + return + } + + for _, s := range series { + _, _ = fmt.Fprintf(w, + `dir`+"\n", s, + ) + } + }) + + for _, s := range series { + dir := fmt.Sprintf("/incremental/listenbrainz-dump-%s-20260713-000003-incremental/", s) + file := fmt.Sprintf("listenbrainz-spark-dump-%s-20260713-000003-incremental.tar", s) + + mux.HandleFunc(dir, func(w http.ResponseWriter, _ *http.Request) { + _, _ = fmt.Fprintf(w, `file`, file) + }) + } + + srv := httptest.NewServer(mux) + defer srv.Close() + + dumps, err := discoverIncrementalDumps( + context.Background(), srv.Client(), srv.URL+"/incremental/", 2594, + ) + if err != nil { + t.Fatalf("discover: %v", err) + } + + // Only 2595 and 2596 are newer than the 2594 high-water-mark. + if len(dumps) != 2 { + t.Fatalf("got %d dumps; want 2: %+v", len(dumps), dumps) + } + + if dumps[0].series != 2595 || dumps[1].series != 2596 { + t.Errorf("series order wrong: %d, %d; want 2595, 2596", dumps[0].series, dumps[1].series) + } + + if !strings.Contains(dumps[0].url, "spark-dump-2595") { + t.Errorf("unexpected url: %s", dumps[0].url) + } +} diff --git a/backend/explore/dumppatch.go b/backend/explore/dumppatch.go new file mode 100644 index 0000000..b9b8ec4 --- /dev/null +++ b/backend/explore/dumppatch.go @@ -0,0 +1,279 @@ +package explore + +import ( + "context" + "strings" +) + +// Stage 4 of the dump import: small, idempotent API patch passes that +// fill in what the dumps can't provide. All calls go through the +// shared rate-limited ListenBrainz client and its HTTP cache, so +// re-running after an interruption is cheap. + +const ( + // Listener-count patch budgets: only the most popular rows get + // listener counts (a secondary ranking signal). 1000 MBIDs per + // batched POST call → ~350 API calls total. + listenerPatchArtists = 50_000 + listenerPatchRGs = 100_000 + listenerPatchRecordings = 200_000 + + // popularityBatchSize is the number of MBIDs per LB popularity + // or metadata request. LB accepts up to 1000 per call. + popularityBatchSize = 1000 +) + +// runPatchPasses fills listener counts, artist metadata, and the +// similar-artist map from the ListenBrainz API. +func (imp *dumpImporter) runPatchPasses(ctx context.Context) { + if imp.lb == nil { + return + } + + imp.patchArtistMetadata(ctx) + + if ctx.Err() != nil { + return + } + + imp.patchSimilarArtists(ctx) + + if ctx.Err() != nil { + return + } + + imp.patchListenerCounts(ctx) +} + +// patchArtistMetadata batch-fetches type/country/name for indexed +// artists that are missing them. Also creates rows for kept artists +// whose name wasn't derivable from the canonical dump (multi-artist +// credits only). +func (imp *dumpImporter) patchArtistMetadata(ctx context.Context) { + rows, err := imp.si.db.QueryContext(` + SELECT mbid FROM explore_index + WHERE entity_type = 'artist' + AND (artist_type = '' OR country = '' OR title = '' OR title = mbid) + `) + if err != nil { + return + } + + var mbids []string + + for rows.Next() { + var m string + if err := rows.Scan(&m); err == nil { + mbids = append(mbids, m) + } + } + + _ = rows.Close() + + if len(imp.pendingArtists) > 0 { + mbids = append(mbids, imp.pendingArtists...) + } + + if len(mbids) == 0 { + return + } + + imp.logger.Info("dump import: patching artist metadata", "artists", len(mbids)) + + batches := chunkStrings(mbids, popularityBatchSize) + patched := 0 + + for i, batch := range batches { + if ctx.Err() != nil { + return + } + + meta, err := imp.lb.BatchArtistMetadata(ctx, batch) + if err != nil || len(meta) == 0 { + continue + } + + entries := make([]SearchIndexResult, 0, len(meta)) + + for mbid, m := range meta { + if m.Name == "" { + continue + } + + entries = append(entries, SearchIndexResult{ + EntityType: "artist", + MBID: mbid, + Title: m.Name, + ArtistName: m.Name, + ArtistMBID: mbid, + ArtistType: m.Type, + Country: m.Country, + }) + + // Pre-populate the MB rels cache so on-demand artist + // image resolution skips a MusicBrainz call. + if imp.si.artistImg != nil { + imp.si.artistImg.PreloadArtistRels(mbid, m) + } + } + + imp.si.upsertBatch(entries) + + patched += len(entries) + + imp.setStageProgress(dumpStagePatch, i+1, len(batches)) + } + + imp.logger.Info("dump import: artist metadata patched", "artists", patched) +} + +// patchSimilarArtists refreshes the similar-artist map for library +// artists (one API call per library artist, cached for a week). +func (imp *dumpImporter) patchSimilarArtists(ctx context.Context) { + libraryMBIDs := imp.si.getLibraryArtistMBIDs() + if len(libraryMBIDs) == 0 { + return + } + + for i := 0; i < len(libraryMBIDs); i += similarArtistsBatchSize { + if ctx.Err() != nil { + return + } + + end := min(i+similarArtistsBatchSize, len(libraryMBIDs)) + + grouped := imp.si.fetchSimilarArtistsBatch(ctx, imp.lb, libraryMBIDs[i:end]) + for seed, similar := range grouped { + imp.si.storeSimilarArtists(seed, similar) + + // Flag indexed similar artists for personalized ranking. + for _, s := range similar { + _, _ = imp.si.db.ExecContext( + "UPDATE explore_index SET is_similar = 1 WHERE artist_mbid = ?", + s.ArtistMBID, + ) + } + } + } + + imp.logger.Info("dump import: similar artists patched", "libraryArtists", len(libraryMBIDs)) +} + +// patchListenerCounts fills listener_count for the most popular rows +// of each entity type. Popularity (listen count) is NOT overwritten — +// the dump-derived counts stay authoritative so the ranking scale is +// consistent across the whole index. +func (imp *dumpImporter) patchListenerCounts(ctx context.Context) { + kinds := []struct { + entityType string + limit int + fetch func(context.Context, []string) (map[string]PopularityData, error) + }{ + {"artist", listenerPatchArtists, imp.lb.ArtistPopularity}, + {"release_group", listenerPatchRGs, imp.lb.ReleaseGroupPopularity}, + {"recording", listenerPatchRecordings, imp.lb.RecordingPopularity}, + } + + for _, kind := range kinds { + if ctx.Err() != nil { + return + } + + mbids := imp.topMBIDs(kind.entityType, kind.limit) + if len(mbids) == 0 { + continue + } + + batches := chunkStrings(mbids, popularityBatchSize) + filled := 0 + + for i, batch := range batches { + if ctx.Err() != nil { + return + } + + pops, err := kind.fetch(ctx, batch) + if err != nil { + continue + } + + filled += imp.si.updateListenerCounts(pops) + + imp.setStageProgress(dumpStageListeners, i+1, len(batches)) + } + + imp.logger.Info("dump import: listener counts patched", + "entityType", kind.entityType, + "rows", filled, + ) + } +} + +// topMBIDs returns the most popular index MBIDs for an entity type +// that don't have listener counts yet. +func (imp *dumpImporter) topMBIDs(entityType string, limit int) []string { + rows, err := imp.si.db.QueryContext(` + SELECT mbid FROM explore_index + WHERE entity_type = ? AND listener_count = 0 + ORDER BY popularity DESC + LIMIT ? + `, entityType, limit) + if err != nil { + return nil + } + + defer func() { _ = rows.Close() }() + + var mbids []string + + for rows.Next() { + var m string + if err := rows.Scan(&m); err == nil { + mbids = append(mbids, m) + } + } + + return mbids +} + +// updateListenerCounts writes listener counts only (never popularity), +// keeping the dump-derived popularity scale consistent. Returns the +// number of rows updated. +func (si *SearchIndex) updateListenerCounts(updates map[string]PopularityData) int { + if len(updates) == 0 { + return 0 + } + + tx, err := si.db.BeginTx() + if err != nil { + return 0 + } + + defer func() { _ = tx.Rollback() }() + + updated := 0 + + for mbid, data := range updates { + if data.ListenerCount <= 0 { + continue + } + + res, err := tx.Exec( + `UPDATE explore_index + SET listener_count = ? + WHERE mbid = ? AND listener_count < ?`, + data.ListenerCount, strings.ToLower(mbid), data.ListenerCount, + ) + if err != nil { + continue + } + + if n, err := res.RowsAffected(); err == nil { + updated += int(n) + } + } + + _ = tx.Commit() + + return updated +} diff --git a/backend/explore/dumpstream.go b/backend/explore/dumpstream.go new file mode 100644 index 0000000..62ca61d --- /dev/null +++ b/backend/explore/dumpstream.go @@ -0,0 +1,329 @@ +package explore + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "regexp" + "sort" + "strconv" + "time" +) + +// Streaming helpers for the MetaBrainz dump imports. Dumps are never +// written to disk: the HTTP body is decoded (tar / zstd+tar) in flight. +// resumableReader reconnects with HTTP Range requests on transient +// failures, which also lets the listens import resume across app +// restarts from a checkpointed byte offset. + +const ( + // maxStreamRetries is the number of consecutive failed reconnect + // attempts before a stream read gives up. The counter resets + // whenever bytes are successfully delivered. + maxStreamRetries = 8 + + // streamRetryBaseDelay is the initial reconnect backoff; it + // doubles per consecutive failure. + streamRetryBaseDelay = 2 * time.Second + + // dumpDiscoveryTimeout bounds the small directory-listing + // requests (not the multi-hour stream requests). + dumpDiscoveryTimeout = 30 * time.Second +) + +// ErrDumpDiscovery is returned when a dump directory listing does not +// contain the expected entries. +var ErrDumpDiscovery = errors.New("dump discovery failed") + +// ErrDumpStream is returned when a dump stream fails permanently. +var ErrDumpStream = errors.New("dump stream failed") + +var hrefRe = regexp.MustCompile(`href="([^"?/][^"?]*)"`) + +// listHrefs fetches an Apache-style index page and returns the href +// values (directory entries end with a trailing slash). +func listHrefs(ctx context.Context, client *http.Client, url string) ([]string, error) { + reqCtx, cancel := context.WithTimeout(ctx, dumpDiscoveryTimeout) + defer cancel() + + req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("dump listing request: %w", err) + } + + req.Header.Set("User-Agent", lbUserAgent) + + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("dump listing fetch: %w", err) + } + + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf( + "%w: listing %s returned HTTP %d", ErrDumpDiscovery, url, resp.StatusCode, + ) + } + + body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20)) + if err != nil { + return nil, fmt.Errorf("dump listing read: %w", err) + } + + var hrefs []string + + for _, m := range hrefRe.FindAllStringSubmatch(string(body), -1) { + hrefs = append(hrefs, m[1]) + } + + return hrefs, nil +} + +// discoverDumpFile walks a dump base directory, finds subdirectories +// matching dirRe (newest first, lexicographically — MetaBrainz dump +// directory names embed sortable timestamps), and returns the full URL +// of the first file inside matching fileRe. Directories that don't +// contain a matching file (e.g. partial uploads) are skipped. +func discoverDumpFile( + ctx context.Context, + client *http.Client, + baseURL string, + dirRe, fileRe *regexp.Regexp, +) (string, error) { + hrefs, err := listHrefs(ctx, client, baseURL) + if err != nil { + return "", err + } + + var dirs []string + + for _, h := range hrefs { + trimmed := trimTrailingSlash(h) + if dirRe.MatchString(trimmed) { + dirs = append(dirs, trimmed) + } + } + + if len(dirs) == 0 { + return "", fmt.Errorf("%w: no dump directories under %s", ErrDumpDiscovery, baseURL) + } + + sort.Sort(sort.Reverse(sort.StringSlice(dirs))) + + for _, dir := range dirs { + dirURL := baseURL + dir + "/" + + files, err := listHrefs(ctx, client, dirURL) + if err != nil { + continue + } + + for _, f := range files { + if fileRe.MatchString(f) { + return dirURL + f, nil + } + } + } + + return "", fmt.Errorf("%w: no matching dump file under %s", ErrDumpDiscovery, baseURL) +} + +func trimTrailingSlash(s string) string { + if len(s) > 0 && s[len(s)-1] == '/' { + return s[:len(s)-1] + } + + return s +} + +// resumableReader is an io.Reader over an HTTP resource that survives +// connection failures by reconnecting with a Range request at the +// current offset. Offset is the absolute position of the next byte to +// deliver, so callers can checkpoint it and construct a new +// resumableReader later to resume a partially-processed stream. +type resumableReader struct { + ctx context.Context + client *http.Client + url string + + // Offset is the absolute byte position of the next read. + Offset int64 + + // Size is the total resource size, learned from the first + // response. -1 until known. + Size int64 + + body io.ReadCloser + retries int +} + +func newResumableReader( + ctx context.Context, client *http.Client, url string, offset int64, +) *resumableReader { + return &resumableReader{ + ctx: ctx, + client: client, + url: url, + Offset: offset, + Size: -1, + } +} + +func (r *resumableReader) Read(p []byte) (int, error) { + for { + if err := r.ctx.Err(); err != nil { + return 0, err + } + + if r.body == nil { + if err := r.connect(); err != nil { + return 0, err + } + } + + n, err := r.body.Read(p) + r.Offset += int64(n) + + if n > 0 { + r.retries = 0 + } + + switch { + case err == nil: + return n, nil + case errors.Is(err, io.EOF): + // A server that closes early looks like EOF; only + // trust it when we've seen the advertised size. + if r.Size >= 0 && r.Offset < r.Size { + r.closeBody() + + if retryErr := r.backoff(err); retryErr != nil { + return n, retryErr + } + + if n > 0 { + return n, nil + } + + continue + } + + return n, io.EOF + default: + r.closeBody() + + if retryErr := r.backoff(err); retryErr != nil { + return n, retryErr + } + + if n > 0 { + return n, nil + } + } + } +} + +// backoff sleeps with exponential backoff, or returns a terminal error +// once the retry budget is exhausted. +func (r *resumableReader) backoff(cause error) error { + r.retries++ + if r.retries > maxStreamRetries { + return fmt.Errorf( + "%w: %s after %d retries: %w", ErrDumpStream, r.url, maxStreamRetries, cause, + ) + } + + delay := streamRetryBaseDelay << (r.retries - 1) + + select { + case <-r.ctx.Done(): + return r.ctx.Err() + case <-time.After(delay): + return nil + } +} + +func (r *resumableReader) connect() error { + req, err := http.NewRequestWithContext(r.ctx, http.MethodGet, r.url, nil) + if err != nil { + return fmt.Errorf("dump stream request: %w", err) + } + + req.Header.Set("User-Agent", lbUserAgent) + + if r.Offset > 0 { + req.Header.Set("Range", "bytes="+strconv.FormatInt(r.Offset, 10)+"-") + } + + resp, err := r.client.Do(req) + if err != nil { + return r.backoff(err) + } + + switch resp.StatusCode { + case http.StatusPartialContent: + if r.Size < 0 { + r.Size = parseContentRangeTotal(resp.Header.Get("Content-Range")) + } + + r.body = resp.Body + + return nil + case http.StatusOK: + if r.Size < 0 && resp.ContentLength > 0 { + r.Size = resp.ContentLength + } + + // Server ignored the Range header: discard the prefix so + // the caller still reads from the requested offset. + if r.Offset > 0 { + if _, err := io.CopyN(io.Discard, resp.Body, r.Offset); err != nil { + _ = resp.Body.Close() + + return r.backoff(err) + } + } + + r.body = resp.Body + + return nil + default: + _ = resp.Body.Close() + + return r.backoff(fmt.Errorf("%w: HTTP %d from %s", ErrDumpStream, resp.StatusCode, r.url)) + } +} + +func (r *resumableReader) closeBody() { + if r.body != nil { + _ = r.body.Close() + r.body = nil + } +} + +// Close releases the underlying HTTP body, if any. +func (r *resumableReader) Close() error { + r.closeBody() + + return nil +} + +// parseContentRangeTotal extracts the total size from a Content-Range +// header ("bytes 100-199/12345"). Returns -1 if unavailable. +func parseContentRangeTotal(v string) int64 { + for i := len(v) - 1; i >= 0; i-- { + if v[i] == '/' { + total, err := strconv.ParseInt(v[i+1:], 10, 64) + if err != nil { + return -1 + } + + return total + } + } + + return -1 +} diff --git a/backend/explore/eval/eval.go b/backend/explore/eval/eval.go new file mode 100644 index 0000000..a273b2b --- /dev/null +++ b/backend/explore/eval/eval.go @@ -0,0 +1,115 @@ +// Package eval is the search-ranking evaluation harness. It turns +// "this query feels wrong" into a number that goes up or down, so a +// ranking change can be validated against a frozen set of labelled +// queries instead of tuned by anecdote. +// +// The harness is deliberately decoupled from the explore package: it +// knows nothing about MusicBrainz, ListenBrainz, or the search index. +// A caller adapts whatever ranking function it wants to measure to the +// Ranker interface, loads a fixture set, and runs Evaluate. The +// explore package wires its real index Search to this in an +// integration test (see explore/eval_harness_test.go). +package eval + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + "strings" +) + +// ErrNoFixtures is returned when a fixture file contains zero queries. +var ErrNoFixtures = errors.New("eval: fixture set is empty") + +// Result is one ranked search hit, reduced to the only two fields the +// harness needs to decide whether it matches an expectation. +type Result struct { + EntityType string `json:"entityType"` + MBID string `json:"mbid"` +} + +// Ranker produces an ordered result list for a query. Best result +// first. Implemented by adapting a real search function. +type Ranker interface { + Rank(query string, limit int) []Result +} + +// RankerFunc adapts a plain function to the Ranker interface. +type RankerFunc func(query string, limit int) []Result + +// Rank calls the underlying function. +func (f RankerFunc) Rank(query string, limit int) []Result { + return f(query, limit) +} + +// Expected is one acceptable result for a fixture query. Grade is the +// graded-relevance weight used by nDCG (higher = more relevant); it +// defaults to 1 when omitted. Type is optional — when set, a ranked +// result must match both MBID and entity type to count as a hit. +type Expected struct { + Type string `json:"type,omitempty"` + MBID string `json:"mbid"` + Grade int `json:"grade,omitempty"` +} + +// Fixture is a single labelled query: the input plus the result(s) a +// user should get. Every edge case ever hand-fixed in the ranker +// belongs here so it can never silently regress. +type Fixture struct { + Query string `json:"query"` + Note string `json:"note,omitempty"` + Expect []Expected `json:"expect"` +} + +// LoadFixtures reads a JSON fixture file from disk. +func LoadFixtures(path string) ([]Fixture, error) { + f, err := os.Open(path) //nolint:gosec // path is a test fixture, not user input + if err != nil { + return nil, fmt.Errorf("eval: open fixtures: %w", err) + } + + defer func() { _ = f.Close() }() + + return ParseFixtures(f) +} + +// ParseFixtures decodes a JSON fixture set from a reader. +func ParseFixtures(r io.Reader) ([]Fixture, error) { + var fixtures []Fixture + + if err := json.NewDecoder(r).Decode(&fixtures); err != nil { + return nil, fmt.Errorf("eval: decode fixtures: %w", err) + } + + if len(fixtures) == 0 { + return nil, ErrNoFixtures + } + + return fixtures, nil +} + +// matches reports whether a ranked result satisfies an expectation. +// MBID match is required; entity type is checked only when the +// expectation pins one. +func (e Expected) matches(r Result) bool { + if !strings.EqualFold(e.MBID, r.MBID) { + return false + } + + if e.Type != "" && !strings.EqualFold(e.Type, r.EntityType) { + return false + } + + return true +} + +// grade returns the graded-relevance weight, defaulting to 1. +func (e Expected) grade() int { + if e.Grade <= 0 { + return 1 + } + + return e.Grade +} diff --git a/backend/explore/eval/metrics.go b/backend/explore/eval/metrics.go new file mode 100644 index 0000000..d7e7ec2 --- /dev/null +++ b/backend/explore/eval/metrics.go @@ -0,0 +1,228 @@ +package eval + +import ( + "fmt" + "math" + "sort" + "strconv" + "strings" +) + +// QueryScore holds the per-query metrics for one fixture. +type QueryScore struct { + Query string + Note string + + // BestRank is the 1-based rank of the highest-placed expected + // result, or 0 if none of the expected results appear in topK. + BestRank int + + ReciprocalRank float64 + PrecisionAtK float64 + NDCGAtK float64 +} + +// Hit reports whether any expected result landed in topK. +func (q QueryScore) Hit() bool { + return q.BestRank > 0 +} + +// Report aggregates per-query scores into the numbers you watch across +// a ranking change: mean reciprocal rank, mean precision@k, mean +// nDCG@k, plus the list of queries that missed entirely. +type Report struct { + K int + NumQueries int + + MRR float64 + MeanPAtK float64 + MeanNDCG float64 + HitRate float64 // fraction of queries with any expected result in topK + Top1Rate float64 // fraction whose best expected result is rank 1 + PerQuery []QueryScore +} + +// Evaluate runs every fixture through the ranker and aggregates the +// results into a Report. topK bounds how deep a result can be and +// still count (a result at rank 20 helps no one). +func Evaluate(r Ranker, fixtures []Fixture, topK int) Report { + if topK <= 0 { + topK = 5 + } + + report := Report{K: topK, NumQueries: len(fixtures)} + + for _, fx := range fixtures { + ranked := r.Rank(fx.Query, topK) + report.PerQuery = append(report.PerQuery, scoreQuery(fx, ranked, topK)) + } + + for _, q := range report.PerQuery { + report.MRR += q.ReciprocalRank + report.MeanPAtK += q.PrecisionAtK + report.MeanNDCG += q.NDCGAtK + + if q.Hit() { + report.HitRate++ + } + + if q.BestRank == 1 { + report.Top1Rate++ + } + } + + if n := float64(len(fixtures)); n > 0 { + report.MRR /= n + report.MeanPAtK /= n + report.MeanNDCG /= n + report.HitRate /= n + report.Top1Rate /= n + } + + return report +} + +// scoreQuery computes the metrics for a single fixture against a ranked +// result list. +func scoreQuery(fx Fixture, ranked []Result, topK int) QueryScore { + score := QueryScore{Query: fx.Query, Note: fx.Note} + + limit := min(topK, len(ranked)) + + relevantInK := 0 + + for i := range limit { + if !anyMatch(fx.Expect, ranked[i]) { + continue + } + + relevantInK++ + + if score.BestRank == 0 { + score.BestRank = i + 1 + score.ReciprocalRank = 1.0 / float64(i+1) + } + } + + score.PrecisionAtK = float64(relevantInK) / float64(topK) + score.NDCGAtK = ndcg(fx.Expect, ranked, topK) + + return score +} + +// anyMatch reports whether a result satisfies any expectation. +func anyMatch(expected []Expected, r Result) bool { + for _, e := range expected { + if e.matches(r) { + return true + } + } + + return false +} + +// ndcg computes normalized discounted cumulative gain at k using graded +// relevance. Returns 0 when there are no expected results. +func ndcg(expected []Expected, ranked []Result, k int) float64 { + ideal := idealDCG(expected, k) + if ideal == 0 { + return 0 + } + + limit := min(k, len(ranked)) + + dcg := 0.0 + + for i := range limit { + g := matchedGrade(expected, ranked[i]) + if g == 0 { + continue + } + + dcg += gain(g, i) + } + + return dcg / ideal +} + +// matchedGrade returns the relevance grade for a result, or 0 if it +// matches no expectation. +func matchedGrade(expected []Expected, r Result) int { + for _, e := range expected { + if e.matches(r) { + return e.grade() + } + } + + return 0 +} + +// idealDCG is the DCG of the best possible ordering: every expected +// result, sorted by grade descending, placed at the front. +func idealDCG(expected []Expected, k int) float64 { + grades := make([]int, 0, len(expected)) + for _, e := range expected { + grades = append(grades, e.grade()) + } + + sort.Sort(sort.Reverse(sort.IntSlice(grades))) + + limit := min(k, len(grades)) + + ideal := 0.0 + for i := range limit { + ideal += gain(grades[i], i) + } + + return ideal +} + +// gain is the discounted gain of a grade at 0-based position i. +func gain(grade, i int) float64 { + return (math.Pow(2, float64(grade)) - 1) / math.Log2(float64(i+2)) +} + +// Format renders a Report as a human-readable table for test output. +func (r Report) Format() string { + var b strings.Builder + + fmt.Fprintf(&b, "ranking eval — %d queries @k=%d\n", r.NumQueries, r.K) + fmt.Fprintf(&b, " MRR %.3f\n", r.MRR) + fmt.Fprintf(&b, " P@%d %.3f\n", r.K, r.MeanPAtK) + fmt.Fprintf(&b, " nDCG@%d %.3f\n", r.K, r.MeanNDCG) + fmt.Fprintf(&b, " hit rate %.3f\n", r.HitRate) + fmt.Fprintf(&b, " top-1 rate %.3f\n", r.Top1Rate) + + misses := r.Misses() + if len(misses) > 0 { + b.WriteString(" misses:\n") + + for _, m := range misses { + fmt.Fprintf(&b, " %-40q rank=%s\n", m.Query, rankLabel(m.BestRank)) + } + } + + return b.String() +} + +// Misses returns the queries whose best expected result was absent +// from topK or buried below rank 1 — the regression watch-list. +func (r Report) Misses() []QueryScore { + var out []QueryScore + + for _, q := range r.PerQuery { + if q.BestRank != 1 { + out = append(out, q) + } + } + + return out +} + +func rankLabel(rank int) string { + if rank == 0 { + return "absent" + } + + return strconv.Itoa(rank) +} diff --git a/backend/explore/eval/metrics_test.go b/backend/explore/eval/metrics_test.go new file mode 100644 index 0000000..d5cb986 --- /dev/null +++ b/backend/explore/eval/metrics_test.go @@ -0,0 +1,193 @@ +package eval + +import ( + "math" + "strings" + "testing" +) + +// rankerFromIDs builds a Ranker that returns a fixed ordering keyed by +// query, for deterministic metric tests. +func rankerFromIDs(table map[string][]Result) Ranker { + return RankerFunc(func(query string, limit int) []Result { + out := table[query] + if limit < len(out) { + out = out[:limit] + } + + return out + }) +} + +func approx(a, b float64) bool { + return math.Abs(a-b) < 1e-9 +} + +func TestReciprocalRank(t *testing.T) { + tests := []struct { + name string + ranked []Result + expect []Expected + wantRR float64 + wantPos int + }{ + { + name: "top result", + ranked: []Result{{MBID: "a"}, {MBID: "b"}}, + expect: []Expected{{MBID: "a"}}, + wantRR: 1.0, + wantPos: 1, + }, + { + name: "third result", + ranked: []Result{{MBID: "x"}, {MBID: "y"}, {MBID: "a"}}, + expect: []Expected{{MBID: "a"}}, + wantRR: 1.0 / 3.0, + wantPos: 3, + }, + { + name: "absent", + ranked: []Result{{MBID: "x"}, {MBID: "y"}}, + expect: []Expected{{MBID: "a"}}, + wantRR: 0, + wantPos: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fx := Fixture{Query: "q", Expect: tt.expect} + got := scoreQuery(fx, tt.ranked, 5) + + if !approx(got.ReciprocalRank, tt.wantRR) { + t.Errorf("RR = %v, want %v", got.ReciprocalRank, tt.wantRR) + } + + if got.BestRank != tt.wantPos { + t.Errorf("BestRank = %d, want %d", got.BestRank, tt.wantPos) + } + }) + } +} + +func TestPrecisionAtK(t *testing.T) { + fx := Fixture{ + Query: "q", + Expect: []Expected{{MBID: "a"}, {MBID: "b"}}, + } + ranked := []Result{{MBID: "a"}, {MBID: "x"}, {MBID: "b"}, {MBID: "y"}} + + got := scoreQuery(fx, ranked, 4) + + // 2 relevant out of k=4. + if !approx(got.PrecisionAtK, 0.5) { + t.Errorf("P@4 = %v, want 0.5", got.PrecisionAtK) + } +} + +func TestNDCGRespectsOrdering(t *testing.T) { + expect := []Expected{{MBID: "a", Grade: 3}, {MBID: "b", Grade: 1}} + + // Ideal ordering: high-grade result first. + good := scoreQuery( + Fixture{Query: "q", Expect: expect}, + []Result{{MBID: "a"}, {MBID: "b"}, {MBID: "z"}}, + 5, + ) + + // Worse ordering: high-grade result buried below an irrelevant one. + bad := scoreQuery( + Fixture{Query: "q", Expect: expect}, + []Result{{MBID: "z"}, {MBID: "b"}, {MBID: "a"}}, + 5, + ) + + if !approx(good.NDCGAtK, 1.0) { + t.Errorf("ideal ordering nDCG = %v, want 1.0", good.NDCGAtK) + } + + if bad.NDCGAtK >= good.NDCGAtK { + t.Errorf("worse ordering nDCG %v should be < ideal %v", bad.NDCGAtK, good.NDCGAtK) + } +} + +func TestTypeMustMatchWhenPinned(t *testing.T) { + fx := Fixture{ + Query: "q", + Expect: []Expected{{Type: "artist", MBID: "a"}}, + } + + // Same MBID but wrong entity type — must not count. + wrongType := scoreQuery(fx, []Result{{EntityType: "recording", MBID: "a"}}, 5) + if wrongType.Hit() { + t.Error("result with wrong entity type counted as a hit") + } + + rightType := scoreQuery(fx, []Result{{EntityType: "artist", MBID: "a"}}, 5) + if !rightType.Hit() { + t.Error("result with matching entity type did not count") + } +} + +func TestEvaluateAggregates(t *testing.T) { + fixtures := []Fixture{ + {Query: "hit-top", Expect: []Expected{{MBID: "a"}}}, + {Query: "hit-second", Expect: []Expected{{MBID: "a"}}}, + {Query: "miss", Expect: []Expected{{MBID: "a"}}}, + } + + r := rankerFromIDs(map[string][]Result{ + "hit-top": {{MBID: "a"}}, + "hit-second": {{MBID: "x"}, {MBID: "a"}}, + "miss": {{MBID: "x"}, {MBID: "y"}}, + }) + + report := Evaluate(r, fixtures, 5) + + // MRR = (1 + 1/2 + 0) / 3. + wantMRR := (1.0 + 0.5 + 0.0) / 3.0 + if !approx(report.MRR, wantMRR) { + t.Errorf("MRR = %v, want %v", report.MRR, wantMRR) + } + + // 2 of 3 queries surfaced the result somewhere in topK. + if !approx(report.HitRate, 2.0/3.0) { + t.Errorf("HitRate = %v, want %v", report.HitRate, 2.0/3.0) + } + + // Only 1 of 3 had it at rank 1. + if !approx(report.Top1Rate, 1.0/3.0) { + t.Errorf("Top1Rate = %v, want %v", report.Top1Rate, 1.0/3.0) + } + + if len(report.Misses()) != 2 { + t.Errorf("Misses = %d, want 2", len(report.Misses())) + } +} + +func TestParseFixtures(t *testing.T) { + const doc = `[ + {"query": "radiohead", "expect": [{"type": "artist", "mbid": "abc"}]}, + {"query": "ok computer", "note": "album not band", "expect": [{"mbid": "def", "grade": 2}]} + ]` + + fixtures, err := ParseFixtures(strings.NewReader(doc)) + if err != nil { + t.Fatalf("ParseFixtures: %v", err) + } + + if len(fixtures) != 2 { + t.Fatalf("got %d fixtures, want 2", len(fixtures)) + } + + if fixtures[0].Expect[0].MBID != "abc" { + t.Errorf("MBID = %q, want abc", fixtures[0].Expect[0].MBID) + } +} + +func TestParseFixturesEmpty(t *testing.T) { + _, err := ParseFixtures(strings.NewReader(`[]`)) + if err == nil { + t.Fatal("expected ErrNoFixtures, got nil") + } +} diff --git a/backend/explore/eval/testdata/eval_queries.json b/backend/explore/eval/testdata/eval_queries.json new file mode 100644 index 0000000..d1a92de --- /dev/null +++ b/backend/explore/eval/testdata/eval_queries.json @@ -0,0 +1,32 @@ +[ + { + "query": "radiohead", + "note": "single-word artist name — should resolve to the artist, not an album titled similarly", + "expect": [{ "type": "artist", "mbid": "a74b1b7f-71a5-4011-9441-d0b5e4122711" }] + }, + { + "query": "the beatles", + "note": "common-word prefix must not let the article dominate", + "expect": [{ "type": "artist", "mbid": "b10bbbfc-cf9e-42e0-be17-e2c3e1d2600d" }] + }, + { + "query": "abbey road", + "note": "album title — should rank the release group above any track of the same name", + "expect": [{ "type": "release_group", "mbid": "" }] + }, + { + "query": "calling you blue october", + "note": "composite title+artist query — recording should win even if the artist is unindexed", + "expect": [{ "type": "recording", "mbid": "" }] + }, + { + "query": "the teenagers", + "note": "regression: must rank The Teenagers above The Beatles despite far lower popularity", + "expect": [{ "type": "artist", "mbid": "" }] + }, + { + "query": "beyonce", + "note": "diacritic folding (migration 37): unaccented query must find the accented artist Beyoncé", + "expect": [{ "type": "artist", "mbid": "859d0860-d480-4efd-970c-c05d5f1776b8" }] + } +] diff --git a/backend/explore/eval_harness_test.go b/backend/explore/eval_harness_test.go new file mode 100644 index 0000000..60dee32 --- /dev/null +++ b/backend/explore/eval_harness_test.go @@ -0,0 +1,150 @@ +package explore + +import ( + "context" + "log/slog" + "testing" + + "yellowjacket/backend/database" + "yellowjacket/backend/explore/eval" +) + +// seedIndexRow inserts one explore_index row. The FTS triggers keep +// explore_index_fts in sync automatically. +func seedIndexRow( + t *testing.T, + db *database.DB, + entityType, mbid, title, artist string, + popularity int, +) { + t.Helper() + + _, err := db.ExecContext(` + INSERT INTO explore_index + (entity_type, mbid, title, artist_name, artist_mbid, popularity, listener_count) + VALUES (?, ?, ?, ?, '', ?, ?) + `, entityType, mbid, title, artist, popularity, popularity/10) + if err != nil { + t.Fatalf("seed %s/%s: %v", entityType, mbid, err) + } +} + +// newTestIndex builds a SearchIndex over a seeded in-memory DB. lb and +// artistImg are nil because Search touches neither. +func newTestIndex(t *testing.T, db *database.DB) *SearchIndex { + t.Helper() + + idx := NewSearchIndex(db, nil, nil, slog.Default()) + idx.MarkReadyIfPopulated() + + return idx +} + +// TestEvalHarnessIndexRanking is the end-to-end wiring of the eval +// harness against the real FTS index Search. It seeds a controlled +// corpus where the correct answer is known, then asserts the harness +// reports a perfect score — proving both the index ranking and the +// harness plumbing on a case we fully control. +func TestEvalHarnessIndexRanking(t *testing.T) { + db := database.NewTestDB(t) + + // Popular exact-match artist should beat a more obscure namesake + // and an unrelated album. + seedIndexRow(t, db, "artist", "rh", "Radiohead", "Radiohead", 5_000_000) + seedIndexRow(t, db, "artist", "radio-obscure", "Radio Birdman", "Radio Birdman", 40_000) + seedIndexRow(t, db, "release_group", "okc", "OK Computer", "Radiohead", 2_000_000) + seedIndexRow(t, db, "artist", "beatles", "The Beatles", "The Beatles", 9_000_000) + seedIndexRow(t, db, "artist", "teenagers", "The Teenagers", "The Teenagers", 60_000) + + idx := newTestIndex(t, db) + + ranker := eval.RankerFunc(func(query string, limit int) []eval.Result { + hits := idx.Search(context.Background(), query, limit) + out := make([]eval.Result, 0, len(hits)) + + for _, h := range hits { + out = append(out, eval.Result{EntityType: h.EntityType, MBID: h.MBID}) + } + + return out + }) + + fixtures := []eval.Fixture{ + { + Query: "radiohead", + Note: "popular exact artist match", + Expect: []eval.Expected{{Type: "artist", MBID: "rh"}}, + }, + { + Query: "the teenagers", + Note: "low-popularity exact match must beat high-popularity article match", + Expect: []eval.Expected{{Type: "artist", MBID: "teenagers"}}, + }, + } + + report := eval.Evaluate(ranker, fixtures, 5) + t.Log("\n" + report.Format()) + + if report.HitRate < 1.0 { + t.Errorf("expected every query to surface its result, got hit rate %.3f", report.HitRate) + } + + if report.Top1Rate < 1.0 { + t.Errorf("expected every result at rank 1, got top-1 rate %.3f:\n%s", + report.Top1Rate, report.Format()) + } +} + +// TestExploreFTSDiacriticFolding proves migration 37: an unaccented +// query must find an accented title (and vice versa) now that +// explore_index_fts folds diacritics. +func TestExploreFTSDiacriticFolding(t *testing.T) { + db := database.NewTestDB(t) + + seedIndexRow(t, db, "artist", "bey", "Beyoncé", "Beyoncé", 8_000_000) + seedIndexRow(t, db, "artist", "bjork", "Björk", "Björk", 3_000_000) + + idx := newTestIndex(t, db) + + cases := []struct { + query string + wantMBID string + }{ + {"beyonce", "bey"}, // unaccented query → accented title + {"beyoncé", "bey"}, // accented query still works + {"bjork", "bjork"}, // ö → o folding + {"björk", "bjork"}, // accented query still works + } + + for _, tc := range cases { + t.Run(tc.query, func(t *testing.T) { + hits := idx.Search(context.Background(), tc.query, 5) + if len(hits) == 0 { + t.Fatalf("query %q returned no hits", tc.query) + } + + if hits[0].MBID != tc.wantMBID { + t.Errorf("query %q: top hit = %q, want %q", tc.query, hits[0].MBID, tc.wantMBID) + } + }) + } +} + +// TestEvalFixtureFileParses guards the checked-in fixture file so a +// malformed edit fails fast rather than silently skipping queries. +func TestEvalFixtureFileParses(t *testing.T) { + fixtures, err := eval.LoadFixtures("eval/testdata/eval_queries.json") + if err != nil { + t.Fatalf("load fixtures: %v", err) + } + + for i, fx := range fixtures { + if fx.Query == "" { + t.Errorf("fixture %d has empty query", i) + } + + if len(fx.Expect) == 0 { + t.Errorf("fixture %q has no expectations", fx.Query) + } + } +} diff --git a/backend/explore/explore.go b/backend/explore/explore.go index 6e57892..419361e 100644 --- a/backend/explore/explore.go +++ b/backend/explore/explore.go @@ -9,7 +9,11 @@ import ( "sync" "time" + "github.com/wailsapp/wails/v2/pkg/runtime" + "golang.org/x/sync/singleflight" + "yellowjacket/backend/database" + "yellowjacket/backend/events" ) // Service is the Wails-bound service for the explore feature. @@ -20,6 +24,7 @@ import ( type Service struct { mb *MusicBrainzClient lb *ListenBrainzClient + lrclib *LRCLibClient cache *Cache index *SearchIndex artProxy *CoverArtProxy @@ -29,6 +34,27 @@ type Service struct { db *database.DB logger *slog.Logger ctx context.Context + + // searchMu guards searchCancel, which cancels the currently + // in-flight SearchLocal so a superseded query releases the shared + // SQLite connection immediately instead of running to completion. + searchMu sync.Mutex + searchCancel context.CancelFunc + + // discogSF collapses concurrent lazy discography fetches for the same + // artist (the detail page fires top-tracks and top-releases at once) + // into a single background fetch + one ArtistDiscographyReady event. + discogSF singleflight.Group + + // similarSF collapses concurrent lazy similar-artist fetches for the + // same artist into a single LB labs call + one ArtistSimilarReady event. + similarSF singleflight.Group + + // releasesSF collapses concurrent lazy BrowseReleases fetches for the + // same release group (e.g. the album page firing while a prefetch is + // already in flight) into one MusicBrainz browse + one + // AlbumReleasesReady event. + releasesSF singleflight.Group } // NewExploreService creates a Service backed by the given @@ -52,6 +78,7 @@ func NewExploreService(logger *slog.Logger, db *database.DB) *Service { mbBackgroundLimiter := NewRateLimiter() mb := NewMusicBrainzClient(cache, mbSearchLimiter, logger.WithGroup("musicbrainz")) lb := NewListenBrainzClient(lbLimiter, cache, logger.WithGroup("listenbrainz")) + lrclib := NewLRCLibClient(cache, logger.WithGroup("lrclib")) artProxy := NewCoverArtProxy(db, caaLimiter) artistImg := NewArtistImageProvider( db, cache, mbBackgroundLimiter, logger.WithGroup("artist-image"), @@ -66,6 +93,7 @@ func NewExploreService(logger *slog.Logger, db *database.DB) *Service { return &Service{ mb: mb, lb: lb, + lrclib: lrclib, cache: cache, index: index, artProxy: artProxy, @@ -104,10 +132,14 @@ func (e *Service) StartIndexBuild() { e.index.StartBuild(e.ctx) } -// IndexNewArtists indexes only library artists not yet in the search -// index. Lightweight post-scan path — skips the full tier machinery. -func (e *Service) IndexNewArtists() { - e.index.IndexNewArtists(e.ctx) +// RefreshListenCounts folds any newly-published incremental listen dumps +// into the index's popularity numbers, in the background. No-op when +// offline, when a full build is running, when there is no baseline +// import, or when the last refresh was within the weekly cadence. Fully +// local — downloads the small daily dumps but makes no ListenBrainz API +// calls. +func (e *Service) RefreshListenCounts() { + go e.index.RefreshListenCounts(e.ctx, listensCatchupInterval) } // StopIndexBuild cancels the background search index build. @@ -133,6 +165,28 @@ func (e *Service) PopulateLocalCrossReferences() { e.index.PopulateLocalCrossReferences() } +// PopulateLocalCrossReferencesIfNeeded runs the library→index sync only +// when it has not run since the last library change. Use it on the +// unchanged-library launch path so the write-heavy re-sync is skipped in +// steady state; the scan-completion path calls the unconditional form. +func (e *Service) PopulateLocalCrossReferencesIfNeeded() { + if e.index.hasMeta(localXrefReadyKey) { + return + } + + e.index.PopulateLocalCrossReferences() +} + +// InvalidateLibrarySync clears the "ready" markers guarding the gated +// library-sync steps so they re-run on the next launch. Call after a +// mutation that changes owned content outside a scan (e.g. removing a +// library), which would otherwise leave stale in_library flags and +// orphaned lyric-index rows. +func (e *Service) InvalidateLibrarySync() { + e.index.deleteMeta(localXrefReadyKey) + e.index.deleteMeta(lyricsIndexReadyKey) +} + // GetIndexStatus returns the current search index build status. func (e *Service) GetIndexStatus() IndexStatus { return e.index.GetIndexStatus() @@ -145,43 +199,73 @@ func (e *Service) InvalidateIndexDiscographies() { e.index.InvalidateDiscographies() } -// --------------------------------------------------------------------------- -// MusicBrainz search -// --------------------------------------------------------------------------- +// beginSearch cancels any SearchLocal still in flight and returns a +// fresh context (plus its cancel func) scoped to the new search. +// Callers must defer the returned cancel so the context's resources +// are released once the search returns. +func (e *Service) beginSearch() (context.Context, context.CancelFunc) { + e.searchMu.Lock() + defer e.searchMu.Unlock() -// SearchArtists queries MusicBrainz for artists matching the query. -func (e *Service) SearchArtists(query string) ([]MBArtist, error) { - artists, _, err := e.mb.SearchArtists(e.ctx, query, mbSearchLimit) + if e.searchCancel != nil { + e.searchCancel() + } - return artists, err + ctx, cancel := context.WithCancel(e.ctx) + e.searchCancel = cancel + + return ctx, cancel } -// SearchReleaseGroups queries MusicBrainz for release groups matching the query. -func (e *Service) SearchReleaseGroups(query string) ([]MBReleaseGroup, error) { - rgs, _, err := e.mb.SearchReleaseGroups(e.ctx, query, mbSearchLimit) - - return rgs, err -} - -// SearchRecordings queries MusicBrainz for recordings matching the query. -func (e *Service) SearchRecordings(query string) ([]MBRecording, error) { - recs, _, err := e.mb.SearchRecordings(e.ctx, query, mbSearchLimit) - - return recs, err -} - -// SearchLocal queries only the local FTS5 index and returns results -// instantly with no network calls. Returns nil if the index isn't -// ready. The frontend calls this in parallel with Search() to show -// instant results while the full pipeline runs. +// SearchLocal queries only the local FTS5 index and returns fully +// ranked results instantly with no network calls. This is the +// primary interactive search path: now that the index is populated +// from the MetaBrainz dumps it covers essentially every popular +// entity, so the frontend drives search entirely from here. The +// old MusicBrainz network pipeline (Search) is retained for a future +// opt-in "search online" affordance but is no longer called on the +// hot path. +// +// Returns nil if the index has no hits for the query, so the caller +// can fall back to whatever owned-library matches it already has. func (e *Service) SearchLocal(query string) *MBSearchResult { - indexHits := e.index.Search(query, indexSearchLimit) - if len(indexHits) == 0 { + searchStart := time.Now() + + // Abandon any search still running: the query has changed, so its + // results are already stale. Cancelling interrupts its in-flight + // FTS query and frees the single shared SQLite connection for this + // one, instead of letting superseded searches back up behind it. + ctx, cancel := e.beginSearch() + defer cancel() + + sqlStart := time.Now() + indexHits := e.index.Search(ctx, query, indexSearchLimit) + sqlDur := time.Since(sqlStart) + + // Superseded mid-query: drop this result so the caller's version + // check isn't racing a partial one, and skip the post-processing. + if ctx.Err() != nil { return nil } + if len(indexHits) == 0 { + e.logger.Info("searchlocal complete (no hits)", + "query", query, + "sql", sqlDur.Round(time.Microsecond), + "total", time.Since(searchStart).Round(time.Microsecond), + ) + + return nil + } + + postStart := time.Now() + var result MBSearchResult - mergeIndexHits(&result, indexHits) + + mergeIndexHits(query, &result, indexHits) + e.resolveRecordingReleaseGroups(result.Recordings) + + mergeDur := time.Since(postStart) // Remove special-purpose artists from local results too. if len(result.Artists) > 0 { @@ -195,9 +279,17 @@ func (e *Service) SearchLocal(query string) *MBSearchResult { result.Artists = filtered } - // Cap counts but skip the minBlendedScore filter — index hits - // use scalePopularity scores that shouldn't be compared to - // blended MB+LB scores. + // Boost exact/substring name matches so a precise query ranks the + // obvious entity first, mirroring the MB pipeline's Phase 5. + boostStart := time.Now() + + e.boostNameMatches(query, &result) + + boostDur := time.Since(boostStart) + + // Cap counts but skip the minBlendedScore filter: this path has no + // MB results to calibrate against, so we surface whatever local + // matches exist rather than dropping low-popularity ones. if len(result.Artists) > maxResults { result.Artists = result.Artists[:maxResults] } @@ -210,6 +302,28 @@ func (e *Service) SearchLocal(query string) *MBSearchResult { result.Recordings = result.Recordings[:maxResults] } + // Resolve the top-result cards via intent scoring (index-only — + // no network), same as the MB pipeline's Phase 7. + topStart := time.Now() + result.TopResults = e.resolveTopResults(query, &result) + topDur := time.Since(topStart) + + postDur := time.Since(postStart) + + e.logger.Info("searchlocal complete", + "query", query, + "hits", len(indexHits), + "artists", len(result.Artists), + "releaseGroups", len(result.ReleaseGroups), + "recordings", len(result.Recordings), + "sql", sqlDur.Round(time.Microsecond), + "post", postDur.Round(time.Microsecond), + "post.merge", mergeDur.Round(time.Microsecond), + "post.boost", boostDur.Round(time.Microsecond), + "post.topResults", topDur.Round(time.Microsecond), + "total", time.Since(searchStart).Round(time.Microsecond), + ) + return &result } @@ -256,6 +370,7 @@ func (e *Service) LookupReleaseGroup(mbid string) (*MBReleaseGroup, error) { MBID: mbid, Title: indexed.Title, ArtistCredit: indexed.ArtistName, + ArtistMBID: indexed.ArtistMBID, Popularity: indexed.Popularity, ListenerCount: indexed.ListenerCount, PrimaryType: indexed.PrimaryType, @@ -265,11 +380,19 @@ func (e *Service) LookupReleaseGroup(mbid string) (*MBReleaseGroup, error) { LocalID: indexed.LocalReleaseGroupID, } - // Background: fetch full MB data if secondary_types is empty. - // After the first visit this will populate on the next request. - if indexed.SecondaryTypes == "" { + // Background: fetch full MB data once per RG to fill in fields + // the dump doesn't carry (notably secondary_types, used by the + // album page's version scorer). Gated on DiscogFetched — not on + // "secondary_types == ''" — so a genuinely studio album (which + // has no secondary types) is looked up once and marked, instead + // of re-firing on every page visit. The result is written back + // to the index so the next visit reads it locally. + if !indexed.DiscogFetched { go func() { - _, _ = e.mb.LookupReleaseGroup(e.ctx, mbid) + fetched, err := e.mb.LookupReleaseGroup(e.ctx, mbid) + if err == nil && fetched != nil { + e.index.PersistReleaseGroupLookup(fetched) + } }() } @@ -310,6 +433,7 @@ func (e *Service) BrowseReleaseGroups(artistMBID string) ([]MBReleaseGroup, erro MBID: r.MBID, Title: r.Title, ArtistCredit: r.ArtistName, + ArtistMBID: r.ArtistMBID, Popularity: r.Popularity, ListenerCount: r.ListenerCount, PrimaryType: r.PrimaryType, @@ -335,17 +459,15 @@ func (e *Service) BrowseReleaseGroups(artistMBID string) ([]MBReleaseGroup, erro return out, nil } - rgs, err := e.mb.BrowseReleaseGroups(e.ctx, artistMBID) - if err != nil { - return nil, err - } + // Not indexed yet. Don't block on a live MusicBrainz browse: the top + // sections already trigger EnsureArtistDiscography (via + // TopReleaseGroupsForArtist), which fetches the same release groups + // from ListenBrainz and writes them into the index. Kick that off (or + // join the in-flight fetch) and return empty — the ArtistDiscographyReady + // event signals the caller to re-read this section from the index. + e.ensureDiscographyAsync(artistMBID) - // Tier 5: organic growth — index this discography. - artistName := e.resolveArtistName(artistMBID, rgs) - - go e.index.AddFromCache(artistName, artistMBID, rgs) - - return rgs, nil + return nil, nil } // resolveArtistName picks the best available artist name for a list @@ -356,10 +478,30 @@ func (e *Service) BrowseReleaseGroups(artistMBID string) ([]MBReleaseGroup, erro // 2. The local explore_index (if the artist was previously indexed) // 3. A LookupArtist call to MB (last resort) // 4. The MBID itself (worst case fallback) +// +// looksLikeFeaturingCredit reports whether a credit string carries a +// "featuring" clause — i.e. it names a collaboration rather than a +// single artist. Only true "featuring" markers count; "&", "x", and +// "," are excluded because they appear inside real artist names. +func looksLikeFeaturingCredit(credit string) bool { + lower := strings.ToLower(credit) + for _, sep := range []string{" feat. ", " feat ", " featuring ", " ft. ", " ft "} { + if strings.Contains(lower, sep) { + return true + } + } + + return false +} + func (e *Service) resolveArtistName(artistMBID string, rgs []MBReleaseGroup) string { - // Try first non-empty ArtistCredit from the release groups. + // Try the first single-artist credit from the release groups. A + // credit carrying a "featuring" clause names a collaboration, not + // the artist whose page this is, so skip those and fall through to + // the index / MB lookup for the canonical single-artist name — this + // is what AddFromCache stamps onto every release group. for _, rg := range rgs { - if rg.ArtistCredit != "" { + if rg.ArtistCredit != "" && !looksLikeFeaturingCredit(rg.ArtistCredit) { return rg.ArtistCredit } } @@ -385,16 +527,30 @@ func (e *Service) resolveArtistName(artistMBID string, rgs []MBReleaseGroup) str } // BrowseReleases fetches releases for a given release group MBID. +// +// Local-first, non-blocking: a warm response cache is served instantly; +// on a miss the request does NOT block on a live MusicBrainz browse +// (which pulls every version's full tracklist and can take seconds). +// Instead it kicks off a background fetch and returns empty — the +// AlbumReleasesReady event signals the caller to re-fetch once the cache +// is warm. func (e *Service) BrowseReleases(releaseGroupMBID string) ([]MBRelease, error) { - releases, err := e.mb.BrowseReleases(e.ctx, releaseGroupMBID) - if err != nil { - return nil, err + if releases, ok := e.mb.BrowseReleasesCached(releaseGroupMBID); ok { + e.markReleasesInLibrary(releases) + + return releases, nil } - // Collect all recording MBIDs across all releases and check them - // against the local library in a single query. Populates the - // InLibrary flag on each track so the tracklist renderer can - // show the library-status indicator without a per-track roundtrip. + e.ensureReleasesAsync(releaseGroupMBID) + + return nil, nil +} + +// markReleasesInLibrary collects all recording MBIDs across all releases +// and checks them against the local library in a single query, setting the +// InLibrary flag on each track so the tracklist renderer can show the +// library-status indicator without a per-track roundtrip. +func (e *Service) markReleasesInLibrary(releases []MBRelease) { var trackMBIDs []string for _, rel := range releases { @@ -405,82 +561,202 @@ func (e *Service) BrowseReleases(releaseGroupMBID string) ([]MBRelease, error) { } } - if len(trackMBIDs) > 0 { - found := e.libMBID.CheckMBIDs(trackMBIDs) + if len(trackMBIDs) == 0 { + return + } - for i := range releases { - for j := range releases[i].Tracks { - mbid := releases[i].Tracks[j].MBID - if _, ok := found[mbid]; ok { - releases[i].Tracks[j].InLibrary = true - } + found := e.libMBID.CheckMBIDs(trackMBIDs) + + for i := range releases { + for j := range releases[i].Tracks { + if _, ok := found[releases[i].Tracks[j].MBID]; ok { + releases[i].Tracks[j].InLibrary = true } } } +} - return releases, nil +// ensureReleasesAsync fetches a release group's releases + tracklists into +// the response cache in the background and emits AlbumReleasesReady when +// done. Concurrent calls for the same release group (e.g. a prefetch +// already in flight when the user opens the album) collapse into one +// MusicBrainz browse + one event via the singleflight. +func (e *Service) ensureReleasesAsync(releaseGroupMBID string) { + if releaseGroupMBID == "" { + return + } + + go func() { + _, _, _ = e.releasesSF.Do(releaseGroupMBID, func() (any, error) { + _, err := e.mb.BrowseReleases(e.ctx, releaseGroupMBID) + if err == nil && e.ctx != nil { + runtime.EventsEmit(e.ctx, events.AlbumReleasesReady, releaseGroupMBID) + } + + return nil, nil + }) + }() +} + +// PrefetchReleases warms the local response cache for a set of release +// groups in the background so opening any of them is instant. Called from +// the artist page once its top-releases / discography render — album +// navigation almost always originates there. Already-cached groups are +// skipped; a cap bounds how many live fetches a single artist view can +// trigger so the MusicBrainz rate limiter isn't flooded. +func (e *Service) PrefetchReleases(releaseGroupMBIDs []string) { + const maxPrefetch = 8 + + fired := 0 + + for _, mbid := range releaseGroupMBIDs { + if mbid == "" { + continue + } + + if _, ok := e.mb.BrowseReleasesCached(mbid); ok { + continue + } + + e.ensureReleasesAsync(mbid) + + fired++ + if fired >= maxPrefetch { + return + } + } } // --------------------------------------------------------------------------- // ListenBrainz // --------------------------------------------------------------------------- -// TopRecordingsForArtist returns the most-listened recordings for an artist. -func (e *Service) TopRecordingsForArtist(artistMBID string) ([]LBTopRecording, error) { - // Try the local index first (instant, no API call). - if indexed := e.index.TopRecordingsByArtist(artistMBID, 50); len(indexed) > 0 { - out := make([]LBTopRecording, len(indexed)) - for i, r := range indexed { - out[i] = LBTopRecording{ - RecordingMBID: r.MBID, - ArtistName: r.ArtistName, - TrackName: r.Title, - TotalListenCount: r.Popularity, - CAAReleaseMBID: r.CAAReleaseMBID, - ReleaseName: r.ReleaseName, - Length: r.Duration, - InLibrary: r.InLibrary || r.LocalRecordingID > 0, - LocalID: r.LocalRecordingID, - } +// topRecordingsToWire projects indexed recordings to the wire type. +func topRecordingsToWire(indexed []SearchIndexResult) []LBTopRecording { + out := make([]LBTopRecording, len(indexed)) + for i, r := range indexed { + out[i] = LBTopRecording{ + RecordingMBID: r.MBID, + ArtistName: r.ArtistName, + TrackName: r.Title, + TotalListenCount: r.Popularity, + CAAReleaseMBID: r.CAAReleaseMBID, + ReleaseName: r.ReleaseName, + Length: r.Duration, + InLibrary: r.InLibrary || r.LocalRecordingID > 0, + LocalID: r.LocalRecordingID, } - - return out, nil } - // Fall back to LB API. - return e.lb.TopRecordingsForArtist(e.ctx, artistMBID) + return out } -// TopReleaseGroupsForArtist returns the most-listened release groups for an artist. -func (e *Service) TopReleaseGroupsForArtist(artistMBID string) ([]LBTopReleaseGroup, error) { - // Try the local index first (instant, no API call). - if indexed := e.index.TopReleaseGroupsByArtist(artistMBID, 50); len(indexed) > 0 { - out := make([]LBTopReleaseGroup, len(indexed)) - for i, r := range indexed { - out[i] = LBTopReleaseGroup{ - ReleaseGroupMBID: r.MBID, - Title: r.Title, - ArtistName: r.ArtistName, - TotalListenCount: r.Popularity, - Type: r.PrimaryType, - Date: r.ReleaseDate, - CAAReleaseMBID: r.CAAReleaseMBID, - InLibrary: r.InLibrary || r.LocalReleaseGroupID > 0, - LocalID: r.LocalReleaseGroupID, - } - } +// TopRecordingsForArtist returns the most-listened recordings for an +// artist. Serves instantly from the local index when available; when the +// artist isn't indexed yet it returns empty immediately and fetches the +// discography in the background, emitting ArtistDiscographyReady so the +// caller can re-fetch — the request never blocks on a live fetch. +func (e *Service) TopRecordingsForArtist(artistMBID string) ([]LBTopRecording, error) { + if indexed := e.index.TopRecordingsByArtist(artistMBID, 50); len(indexed) > 0 { + out := topRecordingsToWire(indexed) + e.resolveTopRecordingReleaseGroups(out) return out, nil } - // Fall back to LB API. - return e.lb.TopReleaseGroupsForArtist(e.ctx, artistMBID) + e.ensureDiscographyAsync(artistMBID) + + return nil, nil +} + +// resolveTopRecordingReleaseGroups fills each top recording's parent +// release group (from its CAA release MBID) in a single local index +// query, so a top-track row can link to its album page with the track +// highlighted. +func (e *Service) resolveTopRecordingReleaseGroups(recs []LBTopRecording) { + var caaMBIDs []string + + for i := range recs { + if recs[i].CAAReleaseMBID != "" { + caaMBIDs = append(caaMBIDs, recs[i].CAAReleaseMBID) + } + } + + if len(caaMBIDs) == 0 { + return + } + + rgByCAA := e.index.ReleaseGroupMBIDsForCAAReleaseMBIDs(caaMBIDs) + + for i := range recs { + if rg, ok := rgByCAA[recs[i].CAAReleaseMBID]; ok { + recs[i].ReleaseGroupMBID = rg + } + } +} + +// topReleaseGroupsToWire projects indexed release groups to the wire type. +func topReleaseGroupsToWire(indexed []SearchIndexResult) []LBTopReleaseGroup { + out := make([]LBTopReleaseGroup, len(indexed)) + for i, r := range indexed { + out[i] = LBTopReleaseGroup{ + ReleaseGroupMBID: r.MBID, + Title: r.Title, + ArtistName: r.ArtistName, + TotalListenCount: r.Popularity, + Type: r.PrimaryType, + Date: r.ReleaseDate, + CAAReleaseMBID: r.CAAReleaseMBID, + InLibrary: r.InLibrary || r.LocalReleaseGroupID > 0, + LocalID: r.LocalReleaseGroupID, + } + } + + return out +} + +// TopReleaseGroupsForArtist returns the most-listened release groups for +// an artist. Same non-blocking contract as TopRecordingsForArtist: index +// hit is instant, a miss kicks off a background discography fetch and +// returns empty, and ArtistDiscographyReady signals when to re-fetch. +func (e *Service) TopReleaseGroupsForArtist(artistMBID string) ([]LBTopReleaseGroup, error) { + if indexed := e.index.TopReleaseGroupsByArtist(artistMBID, 50); len(indexed) > 0 { + return topReleaseGroupsToWire(indexed), nil + } + + e.ensureDiscographyAsync(artistMBID) + + return nil, nil +} + +// ensureDiscographyAsync fetches an artist's discography into the index in +// the background and emits ArtistDiscographyReady when done. Concurrent +// calls for the same artist collapse into one fetch and one event via the +// singleflight, so the detail page firing both top sections at once costs +// a single ListenBrainz round-trip. +func (e *Service) ensureDiscographyAsync(artistMBID string) { + if artistMBID == "" { + return + } + + go func() { + _, _, _ = e.discogSF.Do(artistMBID, func() (any, error) { + e.index.EnsureArtistDiscography(e.ctx, artistMBID) + + if e.ctx != nil { + runtime.EventsEmit(e.ctx, events.ArtistDiscographyReady, artistMBID) + } + + return nil, nil + }) + }() } // SimilarArtists returns artists similar to the given artist MBID. func (e *Service) SimilarArtists(artistMBID string) ([]LBSimilarArtist, error) { // Try the pre-computed similar_artist_map first (instant, no API call). - // This is populated during Tier 4 for library artists and their network. + // Populated by the dump patch pass for library artists and, on first + // view, by the lazy persist below for everyone else. rows, err := e.db.QueryContext(` SELECT similar_artist_mbid, similar_artist_name, score FROM similar_artist_map @@ -504,8 +780,37 @@ func (e *Service) SimilarArtists(artistMBID string) ([]LBSimilarArtist, error) { } } - // Fall back to LB labs API. - return e.lb.SimilarArtists(e.ctx, artistMBID) + // Not cached — don't block on the live LB labs call. Fetch and persist + // in the background, then emit ArtistSimilarReady so the caller re-reads + // this section from similar_artist_map. Later views are served locally. + e.ensureSimilarArtistsAsync(artistMBID) + + return nil, nil +} + +// ensureSimilarArtistsAsync fetches an artist's similar artists from the LB +// labs API into similar_artist_map in the background and emits +// ArtistSimilarReady when done. Concurrent calls for the same artist +// collapse into one fetch and one event via the singleflight. +func (e *Service) ensureSimilarArtistsAsync(artistMBID string) { + if artistMBID == "" { + return + } + + go func() { + _, _, _ = e.similarSF.Do(artistMBID, func() (any, error) { + similar, err := e.lb.SimilarArtists(e.ctx, artistMBID) + if err == nil { + e.index.PersistSimilarArtists(artistMBID, similar) + + if e.ctx != nil { + runtime.EventsEmit(e.ctx, events.ArtistSimilarReady, artistMBID) + } + } + + return nil, nil + }) + }() } // GetArtistPlayCount returns the total LB listen count for an artist. @@ -789,557 +1094,32 @@ func (e *Service) GetArtistImages(names []string) map[string]string { return result } -// Search concurrently queries MusicBrainz for artists, release -// groups, and recordings matching the query, then boosts results -// using ListenBrainz popularity data. The final score blends -// text relevance (60%) with log-scaled listen counts (40%). -// -// If any sub-search or popularity lookup fails the error is logged -// and the remaining results are still returned — popularity -// failures degrade to MB-only ordering. -func (e *Service) Search(query string) (*MBSearchResult, error) { - searchStart := time.Now() - - // Build the Lucene query: AND terms with wildcard on last. - luceneQuery := buildLuceneQuery(query) - // For RGs, also search by artist credit so that "queen" returns - // albums BY Queen, not just titles containing "queen". - rgQuery := buildLuceneQueryWithArtist(query, "releasegroup", "artist") - // Recordings search by title only — the OR with artist caused - // double-match inflation where tracks by "Queen" with "queen" in - // the title got artificially boosted over more popular results. - // The local index handles artist→recording discovery via - // popularity-weighted FTS across title + artist_name + aliases. - recQuery := buildLuceneQuery(query) - - e.logger.Info("search started", "query", query, "lucene", luceneQuery) - - // Phase 0: query local popularity index (instant, no API calls). - p0Start := time.Now() - indexHits := e.index.Search(query, indexSearchLimit) //nolint:mnd - p0Dur := time.Since(p0Start) - - e.logger.Info("search phase 0 complete (index)", - "query", query, - "hits", len(indexHits), - "elapsed", p0Dur, - ) - - // Phase 1: concurrent MB search (3 goroutines) with a deadline - // so a slow MusicBrainz server doesn't hold up the whole search. - // - // First pass uses a small limit to discover total match counts. - // If MB reports many matches, a second pass re-fetches with a - // larger limit so the ranking pipeline has better material. - p1Start := time.Now() - - mbCtx, mbCancel := context.WithTimeout(e.ctx, searchMBTimeout) - defer mbCancel() - - var ( - result MBSearchResult - mu sync.Mutex - wg sync.WaitGroup - ) - - type mbInitial struct { - artists []MBArtist - rgs []MBReleaseGroup - recordings []MBRecording - artistN int - rgN int - recN int - } - - var initial mbInitial - - type searchFunc struct { - name string - fn func() - } - - searches := []searchFunc{ - { - name: "artists", - fn: func() { - t := time.Now() - artists, total, err := e.mb.SearchArtists(mbCtx, luceneQuery, mbSearchLimit) - - e.logger.Info("search MB sub-call", - "entity", "artists", - "elapsed", time.Since(t).Round(time.Millisecond), - "results", len(artists), - "totalMatches", total, - "cached", err == nil && time.Since(t) < 5*time.Millisecond, - ) - - if err != nil { - e.logger.Warn("search sub-call failed", - "entity", "artists", - "query", query, - "error", err, - ) - - return - } - - mu.Lock() - initial.artists = artists - initial.artistN = total - mu.Unlock() - }, - }, - { - name: "releaseGroups", - fn: func() { - t := time.Now() - rgs, total, err := e.mb.SearchReleaseGroups(mbCtx, rgQuery, mbSearchLimit) - - e.logger.Info("search MB sub-call", - "entity", "releaseGroups", - "elapsed", time.Since(t).Round(time.Millisecond), - "results", len(rgs), - "totalMatches", total, - "cached", err == nil && time.Since(t) < 5*time.Millisecond, - ) - - if err != nil { - e.logger.Warn("search sub-call failed", - "entity", "releaseGroups", - "query", query, - "error", err, - ) - - return - } - - mu.Lock() - initial.rgs = rgs - initial.rgN = total - mu.Unlock() - }, - }, - { - name: "recordings", - fn: func() { - t := time.Now() - recs, total, err := e.mb.SearchRecordings(mbCtx, recQuery, mbSearchLimit) - - e.logger.Info("search MB sub-call", - "entity", "recordings", - "elapsed", time.Since(t).Round(time.Millisecond), - "results", len(recs), - "totalMatches", total, - "cached", err == nil && time.Since(t) < 5*time.Millisecond, - ) - - if err != nil { - e.logger.Warn("search sub-call failed", - "entity", "recordings", - "query", query, - "error", err, - ) - - return - } - - mu.Lock() - initial.recordings = recs - initial.recN = total - mu.Unlock() - }, - }, - } - - wg.Add(len(searches)) - - for _, s := range searches { - go func() { - defer wg.Done() - - s.fn() - }() - } - - wg.Wait() - - result.Artists = initial.artists - result.ReleaseGroups = initial.rgs - result.Recordings = initial.recordings - - p1Dur := time.Since(p1Start) - - e.logger.Info( - "search phase 1 complete (MB)", - "query", - query, - "artists", - len(result.Artists), - "releaseGroups", - len(result.ReleaseGroups), - "recordings", - len(result.Recordings), - "expanded", - len(result.Artists) > mbSearchLimit || len(result.ReleaseGroups) > mbSearchLimit || - len(result.Recordings) > mbSearchLimit, - "elapsed", - p1Dur.Round(time.Millisecond), - ) - - // Phases 2+3: when the index is ready, use cached popularity - // from the index to rerank MB results (no API calls). - // When the index isn't ready, fall back to live LB API calls. - p2Start := time.Now() - indexReady := e.index.IsReady() - - // Phase 2a: resolve artist popularity and library membership. - artistMBIDs := make([]string, 0, len(result.Artists)) - for _, a := range result.Artists { - if a.MBID != "" { - artistMBIDs = append(artistMBIDs, a.MBID) - } - } - - artistPop := make(map[string]int) - libMBIDs := make(map[string]bool) - simScores := make(map[string]int) - - if indexReady { - // Fast path: use local index data only — no API call. - // The batch includes popularity, in_library, and similarity scores. - batch := e.index.GetPopularityBatch(artistMBIDs) - if batch != nil { - for mbid, pop := range batch.Popularity { - artistPop[mbid] = pop - } - - libMBIDs = batch.InLibrary - simScores = batch.SimilarityScores - } - - // Fill in missing artist popularity from LB synchronously - // (with a tight timeout). Without this, artists not yet - // indexed get popularity 0 and the rerank can't - // differentiate them from each other, producing nonsense - // ordering for result sets where MB gave every candidate - // the same text relevance score. - var missingPop []string - - for _, mbid := range artistMBIDs { - if artistPop[mbid] <= 0 { - missingPop = append(missingPop, mbid) - } - } - - if len(missingPop) > 0 { - popCtx, popCancel := context.WithTimeout(e.ctx, searchSlowPathTimeout) - - pop, err := e.lb.ArtistPopularity(popCtx, missingPop) - - popCancel() - - if err == nil && pop != nil { - for mbid, data := range pop { - if data.ListenCount > 0 { - artistPop[mbid] = data.ListenCount - } - } - - go e.index.BackfillPopularity(pop) - } - } - } else { - // Slow path: fetch from LB API with a tight timeout - // so a hung LB server doesn't stall the search. - popCtx, popCancel := context.WithTimeout(e.ctx, 2*time.Second) - pop, _ := e.lb.ArtistPopularity(popCtx, artistMBIDs) - - popCancel() - - if pop != nil { - artistPop = listenCounts(pop) - - go e.index.BackfillPopularity(pop) - } - - // Still need library/similar membership from the index. - batch := e.index.GetPopularityBatch(artistMBIDs) - if batch != nil { - libMBIDs = batch.InLibrary - simScores = batch.SimilarityScores - } - } - - // Mark popularity and library status on artists for downstream use. - for i := range result.Artists { - if pop, ok := artistPop[result.Artists[i].MBID]; ok && pop > 0 { - result.Artists[i].HasPopularity = true - result.Artists[i].Popularity = pop - } - - if libMBIDs[result.Artists[i].MBID] { - result.Artists[i].InLibrary = true - } - } - - rerankArtistsPersonalized(result.Artists, artistPop, libMBIDs, simScores) - - // Phase 2b: rerank release groups and recordings. - if indexReady { - e.boostWithIndexPopularityRGsAndRecs(&result) - } else { - // Slow path: LB popularity + cross-reference in parallel. - slowCtx, slowCancel := context.WithTimeout(e.ctx, searchSlowPathTimeout) - - var wgSlow sync.WaitGroup - wgSlow.Add(2) //nolint:mnd - - // Leg 1: LB popularity for RGs and recordings. - go func() { - defer wgSlow.Done() - - e.boostWithPopularityRGsAndRecs(&result) - }() - - // Leg 2: cross-reference artist discographies. - go func() { - defer wgSlow.Done() - - if slowCtx.Err() == nil { - e.crossReferenceAlbums(slowCtx, query, &result) - } - }() - - wgSlow.Wait() - slowCancel() - } - - p2Dur := time.Since(p2Start) - - e.logger.Info("search phase 2-3 complete (rerank)", - "query", query, - "indexReady", indexReady, - "elapsed", p2Dur.Round(time.Millisecond), - ) - - // Phase 4: merge local index hits into results, dedup by MBID. - mergeIndexHits(&result, indexHits) - - // Phase 5: boost exact/substring name matches so a search for - // "the teenagers" ranks "The Teenagers" above "The Beatles" - // even when The Beatles have vastly more listens. - e.boostNameMatches(query, &result) - - // Phase 6: filter low-scoring results and cap counts. - filterAndCap(&result) - - // Phase 7: resolve top result cards via intent scoring. - result.TopResults = e.resolveTopResults(query, &result) - - totalDur := time.Since(searchStart) - - e.logger.Info("search completed", - "query", query, - "artists", len(result.Artists), - "releaseGroups", len(result.ReleaseGroups), - "recordings", len(result.Recordings), - "total", totalDur.Round(time.Millisecond), - "phase0", p0Dur.Round(time.Millisecond), - "phase1_mb", p1Dur.Round(time.Millisecond), - "phase2_rerank", p2Dur.Round(time.Millisecond), - ) - - return &result, nil -} - -// --------------------------------------------------------------------------- -// Cross-reference search -// --------------------------------------------------------------------------- - -const ( - // crossRefArtists is the number of top artists whose - // discographies are searched for matching albums. - crossRefArtists = 3 - - // crossRefMinRatio is the minimum fuzzy match ratio (0–1) - // for an album title to be considered a match. - crossRefMinRatio = 0.4 -) - -// crossReferenceAlbums browses the discographies of the top N -// artists and fuzzy-matches the query against album titles. -// Matched albums not already in result.ReleaseGroups are injected -// at the front. This handles queries like "for you tatsuro" -// where MB text search can't associate the title with the artist. -func (e *Service) crossReferenceAlbums(ctx context.Context, query string, result *MBSearchResult) { - if len(result.Artists) == 0 { - return - } - - limit := crossRefArtists - if limit > len(result.Artists) { - limit = len(result.Artists) - } - - topArtists := result.Artists[:limit] - queryLower := strings.ToLower(strings.TrimSpace(query)) - - // Build a set of release group MBIDs already in results. - existing := make(map[string]bool, len(result.ReleaseGroups)) - for _, rg := range result.ReleaseGroups { - existing[rg.MBID] = true - } - - // Browse discographies concurrently. - type match struct { - rg MBReleaseGroup - ratio float64 - } - - var ( - matches []match - mu sync.Mutex - wg sync.WaitGroup - ) - - wg.Add(limit) - - for _, artist := range topArtists { - go func(a MBArtist) { - defer wg.Done() - - rgs, err := e.mb.BrowseReleaseGroups(ctx, a.MBID) - if err != nil { - e.logger.Warn("cross-reference browse failed", - "artist", a.Name, - "mbid", a.MBID, - "error", err, - ) - - return - } - - for _, rg := range rgs { - if existing[rg.MBID] { - continue - } - - ratio := fuzzyMatchRatio(queryLower, strings.ToLower(rg.Title)) - if ratio >= crossRefMinRatio { - mu.Lock() - - matches = append(matches, match{rg: rg, ratio: ratio}) - - mu.Unlock() - } - } - }(artist) - } - - wg.Wait() - - if len(matches) == 0 { - return - } - - // Sort by match ratio descending. - sort.SliceStable(matches, func(i, j int) bool { - return matches[i].ratio > matches[j].ratio - }) - - // Inject at the front of release groups. - injected := make([]MBReleaseGroup, 0, len(matches)) - - for _, m := range matches { - if !existing[m.rg.MBID] { - injected = append(injected, m.rg) - existing[m.rg.MBID] = true - } - } - - if len(injected) > 0 { - result.ReleaseGroups = append(injected, result.ReleaseGroups...) - - e.logger.Info("cross-reference injected albums", - "count", len(injected), - "topMatch", injected[0].Title, - ) - } -} - -// fuzzyMatchRatio computes a similarity score between query and -// title. It checks: -// 1. Whether the title appears as a substring of the query (or -// vice versa) — handles "for you tatsuro" containing "for you" -// 2. Word overlap ratio as a fallback -// -// Returns 0–1 where 1 is a perfect match. -func fuzzyMatchRatio(query, title string) float64 { - if query == title { - return 1.0 - } - - // Substring containment: "for you tatsuro" contains "for you". - // Use both character ratio and word ratio, take the higher one. - if strings.Contains(query, title) || strings.Contains(title, query) { - shorter := len(title) - longer := len(query) - - if shorter > longer { - shorter, longer = longer, shorter - } - - charRatio := float64(shorter) / float64(longer) - - // Also check word-level ratio for short titles in long queries. - titleWords := strings.Fields(title) - queryWords := strings.Fields(query) - - wordRatio := float64(len(titleWords)) / float64(len(queryWords)) - if len(titleWords) > len(queryWords) { - wordRatio = float64(len(queryWords)) / float64(len(titleWords)) - } - - if wordRatio > charRatio { - return wordRatio - } - - return charRatio - } - - // Word overlap: count how many query words appear in the title. - queryWords := strings.Fields(query) - titleWords := strings.Fields(title) - - if len(queryWords) == 0 || len(titleWords) == 0 { - return 0 - } - - titleSet := make(map[string]bool, len(titleWords)) - for _, w := range titleWords { - titleSet[w] = true - } - - hits := 0 - - for _, w := range queryWords { - if titleSet[w] { - hits++ - } - } - - return float64(hits) / float64(len(queryWords)) -} - // --------------------------------------------------------------------------- // Index result merging // --------------------------------------------------------------------------- +// Index-hit relevance approximation tiers. MB results derive their +// relevance from MusicBrainz's Lucene score; index hits have none, so +// we approximate it from how cleanly the query matches the title or +// artist credit. The floor is non-zero because the FTS index already +// matched *something* (a per-word or alias hit). +const ( + indexRelevanceFloor = 0.15 + indexRelExact = 1.0 + indexRelPrefix = 0.8 + indexRelWord = 0.6 + indexRelSubstring = 0.4 +) + // mergeIndexHits injects local popularity index results into the -// MBSearchResult. Index hits for entity types not already present -// (by MBID) are prepended so they appear first — they come from -// the most popular albums/tracks globally and deserve prominence. -func mergeIndexHits(result *MBSearchResult, hits []SearchIndexResult) { +// MBSearchResult. Index hits for entity types not already present (by +// MBID) are scored on the SAME blended scale as the reranked MB results +// (text relevance + log-popularity + personalization), then merged and +// re-sorted by that score. This replaces an earlier approach that +// scored index hits with a half-scaled popularity number and blindly +// prepended them — two incompatible scales that the downstream +// minBlendedScore filter and final sort then compared as if equal. +func mergeIndexHits(query string, result *MBSearchResult, hits []SearchIndexResult) { if len(hits) == 0 { return } @@ -1360,6 +1140,36 @@ func mergeIndexHits(result *MBSearchResult, hits []SearchIndexResult) { recMBIDs[r.MBID] = true } + // Per-entity-type max popularity over the combined population + // (existing MB results + incoming index hits) so the blended score + // normalizes popularity consistently across both sources. This max + // may differ slightly from the per-list max the MB rerank used; the + // single comparable scale is worth that minor drift. + maxArtistPop, maxRGPop, maxRecPop := 0, 0, 0 + + for _, a := range result.Artists { + maxArtistPop = max(maxArtistPop, a.Popularity) + } + + for _, rg := range result.ReleaseGroups { + maxRGPop = max(maxRGPop, rg.Popularity) + } + + for _, r := range result.Recordings { + maxRecPop = max(maxRecPop, r.Popularity) + } + + for _, h := range hits { + switch h.EntityType { + case "artist": + maxArtistPop = max(maxArtistPop, h.Popularity) + case "release_group": + maxRGPop = max(maxRGPop, h.Popularity) + case "recording": + maxRecPop = max(maxRecPop, h.Popularity) + } + } + // Collect new entries from index that MB didn't return. var ( newArtists []MBArtist @@ -1371,7 +1181,7 @@ func mergeIndexHits(result *MBSearchResult, hits []SearchIndexResult) { switch h.EntityType { case "artist": if !artistMBIDs[h.MBID] { - score := int(float64(scalePopularity(h.Popularity)) * 0.5) + inLib := h.InLibrary || h.LocalArtistID > 0 newArtists = append(newArtists, MBArtist{ MBID: h.MBID, @@ -1380,12 +1190,14 @@ func mergeIndexHits(result *MBSearchResult, hits []SearchIndexResult) { Country: h.Country, Disambiguation: h.Disambiguation, SortName: h.SortName, - Score: score, - HasPopularity: h.Popularity > 0, - Popularity: h.Popularity, - ListenerCount: h.ListenerCount, - InLibrary: h.InLibrary || h.LocalArtistID > 0, - LocalID: h.LocalArtistID, + Score: indexHitBlendedScore( + query, h.Title, "", h.Popularity, maxArtistPop, inLib, h.IsSimilar, + ), + HasPopularity: h.Popularity > 0, + Popularity: h.Popularity, + ListenerCount: h.ListenerCount, + InLibrary: inLib, + LocalID: h.LocalArtistID, }) artistMBIDs[h.MBID] = true @@ -1393,7 +1205,7 @@ func mergeIndexHits(result *MBSearchResult, hits []SearchIndexResult) { case "release_group": if !rgMBIDs[h.MBID] { - score := int(float64(scalePopularity(h.Popularity)) * 0.5) + inLib := h.InLibrary || h.LocalReleaseGroupID > 0 var secondary []string if h.SecondaryTypes != "" { @@ -1401,16 +1213,19 @@ func mergeIndexHits(result *MBSearchResult, hits []SearchIndexResult) { } newRGs = append(newRGs, MBReleaseGroup{ - MBID: h.MBID, - Title: h.Title, - ArtistCredit: h.ArtistName, - Score: score, + MBID: h.MBID, + Title: h.Title, + ArtistCredit: h.ArtistName, + ArtistMBID: h.ArtistMBID, + Score: indexHitBlendedScore( + query, h.Title, h.ArtistName, h.Popularity, maxRGPop, inLib, h.IsSimilar, + ), Popularity: h.Popularity, ListenerCount: h.ListenerCount, PrimaryType: h.PrimaryType, SecondaryTypes: secondary, FirstReleaseDate: h.ReleaseDate, - InLibrary: h.InLibrary || h.LocalReleaseGroupID > 0, + InLibrary: inLib, LocalID: h.LocalReleaseGroupID, }) rgMBIDs[h.MBID] = true @@ -1418,18 +1233,23 @@ func mergeIndexHits(result *MBSearchResult, hits []SearchIndexResult) { case "recording": if !recMBIDs[h.MBID] { - score := int(float64(scalePopularity(h.Popularity)) * 0.5) + inLib := h.InLibrary || h.LocalRecordingID > 0 newRecs = append(newRecs, MBRecording{ - MBID: h.MBID, - Title: h.Title, - Length: h.Duration, - ArtistCredit: h.ArtistName, - Score: score, - Popularity: h.Popularity, - ListenerCount: h.ListenerCount, - InLibrary: h.InLibrary || h.LocalRecordingID > 0, - LocalID: h.LocalRecordingID, + MBID: h.MBID, + Title: h.Title, + Length: h.Duration, + ArtistCredit: h.ArtistName, + ArtistMBID: h.ArtistMBID, + Score: indexHitBlendedScore( + query, h.Title, h.ArtistName, h.Popularity, maxRecPop, inLib, h.IsSimilar, + ), + Popularity: h.Popularity, + ListenerCount: h.ListenerCount, + CAAReleaseMBID: h.CAAReleaseMBID, + ReleaseName: h.ReleaseName, + InLibrary: inLib, + LocalID: h.LocalRecordingID, }) recMBIDs[h.MBID] = true @@ -1437,38 +1257,82 @@ func mergeIndexHits(result *MBSearchResult, hits []SearchIndexResult) { } } - // Prepend index hits so they appear before MB-only results. - // The subsequent reranking and filtering passes will sort - // everything by blended score. + // Merge and re-sort each list by the now-comparable blended Score. if len(newArtists) > 0 { - result.Artists = append(newArtists, result.Artists...) + result.Artists = append(result.Artists, newArtists...) + sort.SliceStable(result.Artists, func(i, j int) bool { + return result.Artists[i].Score > result.Artists[j].Score + }) } if len(newRGs) > 0 { - result.ReleaseGroups = append(newRGs, result.ReleaseGroups...) + result.ReleaseGroups = append(result.ReleaseGroups, newRGs...) + sort.SliceStable(result.ReleaseGroups, func(i, j int) bool { + return result.ReleaseGroups[i].Score > result.ReleaseGroups[j].Score + }) } if len(newRecs) > 0 { - result.Recordings = append(newRecs, result.Recordings...) + result.Recordings = append(result.Recordings, newRecs...) + sort.SliceStable(result.Recordings, func(i, j int) bool { + return result.Recordings[i].Score > result.Recordings[j].Score + }) } } -// scalePopularity maps a raw LB listen count to a 0–100 score -// comparable with MB/blended scores. Uses log scaling. -func scalePopularity(listens int) int { - if listens <= 0 { - return 0 +// indexHitBlendedScore scores a local index hit on the same 0–100 +// blended scale the MB rerank uses, so merged results sort and filter +// consistently regardless of source. +func indexHitBlendedScore( + query, title, artist string, + pop, maxPop int, + inLibrary, isSimilar bool, +) int { + rel := indexHitRelevance(query, title, artist) + + personal := 0.0 + + switch { + case inLibrary: + personal = personalInLibrary + case isSimilar: + personal = personalSimilar } - // log10(1M) ≈ 6, log10(10M) ≈ 7. Scale so 1M+ listens → ~80-100. - const scale = 15.0 // tuned so ~100K listens → ~75, ~1M → ~90 + return int(blendedScoreFull(rel, pop, maxPop, personal) * 100) //nolint:mnd +} - score := int(math.Log10(float64(listens)) * scale) - if score > 100 { //nolint:mnd - score = 100 +// indexHitRelevance approximates a 0..1 text-relevance for an index hit +// from how cleanly the query matches its title or artist credit, taking +// the stronger of the two fields. +func indexHitRelevance(query, title, artist string) float64 { + q := normalizeForMatch(query) + if q == "" { + return indexRelevanceFloor } - return score + best := indexRelevanceFloor + + for _, field := range [2]string{title, artist} { + if field == "" { + continue + } + + f := normalizeForMatch(field) + + switch { + case f == q: + best = max(best, indexRelExact) + case strings.HasPrefix(f, q): + best = max(best, indexRelPrefix) + case containsWord(f, q): + best = max(best, indexRelWord) + case strings.Contains(f, q): + best = max(best, indexRelSubstring) + } + } + + return best } // --------------------------------------------------------------------------- @@ -1554,36 +1418,15 @@ func filterAndCap(result *MBSearchResult) { } // --------------------------------------------------------------------------- -// Popularity-boosted reranking +// Search result limits and thresholds // --------------------------------------------------------------------------- const ( - // mbSearchLimit is the initial limit passed to each MB search call. - // The pipeline may re-fetch with a larger limit (up to mbSearchMaxLimit) - // when MB reports many total matches. - mbSearchLimit = 25 - - // mbSearchMaxLimit caps the expanded fetch. MB's API maximum is 100. - mbSearchMaxLimit = 75 //nolint:unused // referenced by deferred MB search rework - // indexSearchLimit is the number of results to fetch from the local - // popularity index (Phase 0). Larger than maxResults because - // results are filtered and the index is the primary search domain. + // popularity index. Larger than maxResults because results are + // filtered and the index is the primary search domain. indexSearchLimit = 60 - // searchMBTimeout is the maximum time to wait for MusicBrainz - // API responses during interactive search. If MB is slow, - // results degrade to index-only rather than blocking the user. - searchMBTimeout = 3 * time.Second - - // searchSlowPathTimeout caps the total time spent on the slow - // path (LB popularity + cross-referencing). When the index - // isn't ready, these API calls can stack up — especially - // cross-referencing, which browses 3 artist discographies via - // MB and can hit 429 retries. The timeout ensures search - // returns within a reasonable window. - searchSlowPathTimeout = 3 * time.Second - // maxResults caps each entity slice after filtering. maxResults = 15 @@ -1653,335 +1496,6 @@ var mbSpecialPurposeArtists = map[string]bool{ "89ad4ac3-39f7-470e-963a-56509c546377": true, // Various Artists (regular MBID, same issue) } -// boostWithIndexPopularity reranks MB search results using -// popularity data from the local search index. No API calls — -// just SQLite lookups. This is the fast path used when the index -// is ready. -// -//nolint:unused // referenced by deferred MB search rework. -func (e *Service) boostWithIndexPopularity(result *MBSearchResult) { - // Collect all MBIDs across all entity types. - allMBIDs := make([]string, 0, - len(result.Artists)+len(result.ReleaseGroups)+len(result.Recordings)) - - for _, a := range result.Artists { - if a.MBID != "" { - allMBIDs = append(allMBIDs, a.MBID) - } - } - - for _, rg := range result.ReleaseGroups { - if rg.MBID != "" { - allMBIDs = append(allMBIDs, rg.MBID) - } - } - - for _, r := range result.Recordings { - if r.MBID != "" { - allMBIDs = append(allMBIDs, r.MBID) - } - } - - // Single batch query for all popularity + in_library data. - batch := e.index.GetPopularityBatch(allMBIDs) - if batch == nil { - return - } - - // Build per-entity maps from the batch result. - artistPop := make(map[string]int, len(result.Artists)) - for i, a := range result.Artists { - if pop, ok := batch.Popularity[a.MBID]; ok { - artistPop[a.MBID] = pop - result.Artists[i].HasPopularity = true - result.Artists[i].Popularity = pop - } - - if batch.InLibrary[a.MBID] { - result.Artists[i].InLibrary = true - } - } - - rerankArtistsPersonalized(result.Artists, artistPop, batch.InLibrary, batch.SimilarityScores) - - rgPop := make(map[string]int, len(result.ReleaseGroups)) - for i, rg := range result.ReleaseGroups { - if pop, ok := batch.Popularity[rg.MBID]; ok { - rgPop[rg.MBID] = pop - result.ReleaseGroups[i].Popularity = pop - } - - if batch.InLibrary[rg.MBID] { - result.ReleaseGroups[i].InLibrary = true - } - } - - rerankReleaseGroupsPersonalized( - result.ReleaseGroups, - rgPop, - batch.InLibrary, - batch.SimilarityScores, - ) - - recPop := make(map[string]int, len(result.Recordings)) - for i, r := range result.Recordings { - if pop, ok := batch.Popularity[r.MBID]; ok { - recPop[r.MBID] = pop - result.Recordings[i].Popularity = pop - } - - if batch.InLibrary[r.MBID] { - result.Recordings[i].InLibrary = true - } - } - - rerankRecordingsPersonalized(result.Recordings, recPop, batch.InLibrary, batch.SimilarityScores) -} - -// boostWithIndexPopularityRGsAndRecs reranks release groups and -// recordings using index popularity. Artists are handled separately -// via the always-on LB API lookup. -func (e *Service) boostWithIndexPopularityRGsAndRecs(result *MBSearchResult) { - allMBIDs := make([]string, 0, - len(result.ReleaseGroups)+len(result.Recordings)) - - for _, rg := range result.ReleaseGroups { - if rg.MBID != "" { - allMBIDs = append(allMBIDs, rg.MBID) - } - } - - for _, r := range result.Recordings { - if r.MBID != "" { - allMBIDs = append(allMBIDs, r.MBID) - } - } - - if len(allMBIDs) == 0 { - return - } - - batch := e.index.GetPopularityBatch(allMBIDs) - if batch == nil { - batch = &PopularityBatchResult{ - Popularity: map[string]int{}, - InLibrary: map[string]bool{}, - } - } - - // Collect MBIDs the index had no popularity for. For result sets - // where every candidate has identical MB relevance (e.g. many - // covers of the same song), missing popularity means the rerank - // has no signal to pick between them — so fall back to the LB - // popularity API for just the missing entries. This keeps the - // common path cache-only while correctness-critical cases get - // a ~1 round-trip to LB. - missingRecs := make([]string, 0) - - for _, r := range result.Recordings { - if r.MBID == "" { - continue - } - - if _, ok := batch.Popularity[r.MBID]; !ok { - missingRecs = append(missingRecs, r.MBID) - } - } - - missingRGs := make([]string, 0) - - for _, rg := range result.ReleaseGroups { - if rg.MBID == "" { - continue - } - - if _, ok := batch.Popularity[rg.MBID]; !ok { - missingRGs = append(missingRGs, rg.MBID) - } - } - - if len(missingRecs) > 0 || len(missingRGs) > 0 { - e.fillMissingPopularity(batch, missingRecs, missingRGs) - } - - rgPop := make(map[string]int, len(result.ReleaseGroups)) - for i, rg := range result.ReleaseGroups { - if pop, ok := batch.Popularity[rg.MBID]; ok { - rgPop[rg.MBID] = pop - result.ReleaseGroups[i].Popularity = pop - } - - if batch.InLibrary[rg.MBID] { - result.ReleaseGroups[i].InLibrary = true - } - } - - rerankReleaseGroups(result.ReleaseGroups, rgPop) - - recPop := make(map[string]int, len(result.Recordings)) - for i, r := range result.Recordings { - if pop, ok := batch.Popularity[r.MBID]; ok { - recPop[r.MBID] = pop - result.Recordings[i].Popularity = pop - } - - if batch.InLibrary[r.MBID] { - result.Recordings[i].InLibrary = true - } - } - - rerankRecordings(result.Recordings, recPop) -} - -// fillMissingPopularity fetches LB popularity for recordings and -// release groups that weren't in the local index, merging the -// results back into batch.Popularity. Also backfills the index in -// the background so subsequent searches hit the cache. Runs the -// two LB POST calls concurrently and bounds the total wait to -// searchSlowPathTimeout so a slow LB response can't block search. -func (e *Service) fillMissingPopularity( - batch *PopularityBatchResult, - missingRecs []string, - missingRGs []string, -) { - ctx, cancel := context.WithTimeout(e.ctx, searchSlowPathTimeout) - defer cancel() - - var ( - recPop map[string]PopularityData - rgPop map[string]PopularityData - wg sync.WaitGroup - ) - - if len(missingRecs) > 0 { - wg.Add(1) - - go func() { - defer wg.Done() - - pop, err := e.lb.RecordingPopularity(ctx, missingRecs) - if err != nil { - e.logger.Debug("search: fill missing recording popularity failed", - "count", len(missingRecs), "error", err) - - return - } - - recPop = pop - }() - } - - if len(missingRGs) > 0 { - wg.Add(1) - - go func() { - defer wg.Done() - - pop, err := e.lb.ReleaseGroupPopularity(ctx, missingRGs) - if err != nil { - e.logger.Debug("search: fill missing RG popularity failed", - "count", len(missingRGs), "error", err) - - return - } - - rgPop = pop - }() - } - - wg.Wait() - - // Merge LB results into the batch map so the subsequent rerank - // picks them up without needing a second lookup path. - for mbid, data := range recPop { - batch.Popularity[mbid] = data.ListenCount - if batch.ListenerCount != nil { - batch.ListenerCount[mbid] = data.ListenerCount - } - } - - for mbid, data := range rgPop { - batch.Popularity[mbid] = data.ListenCount - if batch.ListenerCount != nil { - batch.ListenerCount[mbid] = data.ListenerCount - } - } - - // Backfill the index in the background so next time this query - // runs, the index has the answer and we skip the LB round-trip. - if len(recPop) > 0 { - go e.index.BackfillPopularity(recPop) - } - - if len(rgPop) > 0 { - go e.index.BackfillPopularity(rgPop) - } -} - -// boostWithPopularityRGsAndRecs fetches LB popularity for release -// groups and recordings only (artist popularity is fetched separately -// in the main search path). Runs two concurrent POST calls. -func (e *Service) boostWithPopularityRGsAndRecs(result *MBSearchResult) { - recordingMBIDs := make([]string, len(result.Recordings)) - for i, r := range result.Recordings { - recordingMBIDs[i] = r.MBID - } - - rgMBIDs := make([]string, len(result.ReleaseGroups)) - for i, rg := range result.ReleaseGroups { - rgMBIDs[i] = rg.MBID - } - - // Fetch popularity concurrently (2 POST calls). - var ( - recordingPopData map[string]PopularityData - rgPopData map[string]PopularityData - wg sync.WaitGroup - ) - - wg.Add(2) //nolint:mnd - - go func() { - defer wg.Done() - - pop, err := e.lb.RecordingPopularity(e.ctx, recordingMBIDs) - if err != nil { - e.logger.Warn("popularity lookup failed", "entity", "recording", "error", err) - - return - } - - recordingPopData = pop - }() - - go func() { - defer wg.Done() - - pop, err := e.lb.ReleaseGroupPopularity(e.ctx, rgMBIDs) - if err != nil { - e.logger.Warn("popularity lookup failed", "entity", "releaseGroup", "error", err) - - return - } - - rgPopData = pop - }() - - wg.Wait() - - // Backfill index with popularity data for future searches. - if recordingPopData != nil { - go e.index.BackfillPopularity(recordingPopData) - } - - if rgPopData != nil { - go e.index.BackfillPopularity(rgPopData) - } - - rerankRecordings(result.Recordings, listenCounts(recordingPopData)) - rerankReleaseGroups(result.ReleaseGroups, listenCounts(rgPopData)) -} - // boostNameMatches re-sorts artists and release groups so that // exact or substring name matches rank above results that only // matched on common words like "the". Without this, a search @@ -2035,11 +1549,13 @@ func (e *Service) boostNameMatches(query string, result *MBSearchResult) { } } -// disambiguateSameNameArtists resolves ordering among artists -// that share the exact same name as the query by fetching their -// LB popularity. This is a targeted micro-lookup (typically 2-6 -// MBIDs) that only fires when the index fast path couldn't -// meaningfully differentiate same-named artists. +// disambiguateSameNameArtists resolves ordering among artists that +// share the exact same name as the query, by popularity descending. +// It runs entirely from the local index — no network: index-sourced +// artist rows already carry their listen count, and a single +// GetPopularityBatch fills any that don't. Keeping this offline is +// what lets SearchLocal honour its "no network calls" contract; a live +// lookup here previously stalled the hot search path. func (e *Service) disambiguateSameNameArtists(query string, artists []MBArtist) { // Find the contiguous block of tier-0 same-name artists at the front. var sameNameEnd int @@ -2056,7 +1572,8 @@ func (e *Service) disambiguateSameNameArtists(query string, artists []MBArtist) return // 0 or 1 same-name artists — nothing to disambiguate } - // Collect MBIDs for the targeted LB lookup. + // Collect MBIDs so the index can fill popularity for any rows that + // don't already carry it (e.g. artists sourced from the MB pipeline). mbids := make([]string, 0, sameNameEnd) for i := range sameNameEnd { if artists[i].MBID != "" { @@ -2068,14 +1585,25 @@ func (e *Service) disambiguateSameNameArtists(query string, artists []MBArtist) return } - pop, err := e.lb.ArtistPopularity(e.ctx, mbids) - if err != nil || len(pop) == 0 { - return + // Index-only popularity lookup — no network. + var indexPop map[string]int + if batch := e.index.GetPopularityBatch(mbids); batch != nil { + indexPop = batch.Popularity } - // Re-sort the same-name block by LB popularity descending. + // Prefer the index popularity, falling back to whatever listen count + // the artist row already carries when the index has none. + popOf := func(a MBArtist) int { + if p, ok := indexPop[a.MBID]; ok && p > 0 { + return p + } + + return a.Popularity + } + + // Re-sort the same-name block by popularity descending. sort.SliceStable(artists[:sameNameEnd], func(i, j int) bool { - return pop[artists[i].MBID].ListenCount > pop[artists[j].MBID].ListenCount + return popOf(artists[i]) > popOf(artists[j]) }) } @@ -2135,181 +1663,6 @@ func rgMatchTier(query, title, artistCredit string) int { return 4 } -// rerankArtists sorts artists by blended score and updates their -// Score field to the new value (0–100 scale). -// -//nolint:unused // referenced by deferred MB search rework. -func rerankArtists(artists []MBArtist, pop map[string]int, libraryMBIDs map[string]bool) { - rerankArtistsPersonalized(artists, pop, libraryMBIDs, nil) -} - -func rerankArtistsPersonalized( - artists []MBArtist, - pop map[string]int, - inLib map[string]bool, - simScores map[string]int, -) { - if len(artists) == 0 { - return - } - - maxPop := maxListenCount(pop) - maxSim := maxSimScoreVal(simScores) - - sort.SliceStable(artists, func(i, j int) bool { - si := blendedScoreFull( - float64(artists[i].Score)/100.0, - pop[artists[i].MBID], - maxPop, - personalScore(artists[i].MBID, inLib, simScores, maxSim), - ) - sj := blendedScoreFull( - float64(artists[j].Score)/100.0, - pop[artists[j].MBID], - maxPop, - personalScore(artists[j].MBID, inLib, simScores, maxSim), - ) - - return si > sj - }) - - for i := range artists { - s := blendedScoreFull( - float64(artists[i].Score)/100.0, - pop[artists[i].MBID], - maxPop, - personalScore(artists[i].MBID, inLib, simScores, maxSim), - ) - artists[i].Score = int(s * 100) - } -} - -// rerankRecordings sorts recordings by blended score and updates -// their Score field. -func rerankRecordings(recordings []MBRecording, pop map[string]int) { - rerankRecordingsPersonalized(recordings, pop, nil, nil) -} - -func rerankRecordingsPersonalized( - recordings []MBRecording, - pop map[string]int, - inLib map[string]bool, - simScores map[string]int, -) { - if len(recordings) == 0 { - return - } - - maxPop := maxListenCount(pop) - maxSim := maxSimScoreVal(simScores) - - sort.SliceStable(recordings, func(i, j int) bool { - si := blendedScoreFull( - float64(recordings[i].Score)/100.0, - pop[recordings[i].MBID], - maxPop, - personalScore(recordings[i].MBID, inLib, simScores, maxSim), - ) - sj := blendedScoreFull( - float64(recordings[j].Score)/100.0, - pop[recordings[j].MBID], - maxPop, - personalScore(recordings[j].MBID, inLib, simScores, maxSim), - ) - - return si > sj - }) - - for i := range recordings { - s := blendedScoreFull( - float64(recordings[i].Score)/100.0, - pop[recordings[i].MBID], - maxPop, - personalScore(recordings[i].MBID, inLib, simScores, maxSim), - ) - recordings[i].Score = int(s * 100) - } -} - -// rerankReleaseGroups sorts release groups by blended score -// (text relevance + popularity + personalization) and updates their Score field. -func rerankReleaseGroups(rgs []MBReleaseGroup, pop map[string]int) { - rerankReleaseGroupsPersonalized(rgs, pop, nil, nil) -} - -func rerankReleaseGroupsPersonalized( - rgs []MBReleaseGroup, - pop map[string]int, - inLib map[string]bool, - simScores map[string]int, -) { - if len(rgs) == 0 { - return - } - - maxPop := maxListenCount(pop) - maxSim := maxSimScoreVal(simScores) - - sort.SliceStable(rgs, func(i, j int) bool { - si := blendedScoreFull( - float64(rgs[i].Score)/100.0, - pop[rgs[i].MBID], - maxPop, - personalScore(rgs[i].MBID, inLib, simScores, maxSim), - ) - sj := blendedScoreFull( - float64(rgs[j].Score)/100.0, - pop[rgs[j].MBID], - maxPop, - personalScore(rgs[j].MBID, inLib, simScores, maxSim), - ) - - return si > sj - }) - - for i := range rgs { - s := blendedScoreFull( - float64(rgs[i].Score)/100.0, - pop[rgs[i].MBID], - maxPop, - personalScore(rgs[i].MBID, inLib, simScores, maxSim), - ) - rgs[i].Score = int(s * 100) - } -} - -// maxSimScoreVal returns the highest similarity score in the map. -func maxSimScoreVal(scores map[string]int) int { - maxVal := 0 - for _, v := range scores { - if v > maxVal { - maxVal = v - } - } - - return maxVal -} - -// personalScore returns the personalization signal (0.0–1.0) for an MBID. -// Uses similarity scores from similar_artist_map, scaled by the max score -// in the batch so the most similar artist gets the full personalSimilar weight. -func personalScore( - mbid string, - inLib map[string]bool, - simScores map[string]int, - maxSimScore int, -) float64 { - if inLib[mbid] { - return personalInLibrary - } - - if score, ok := simScores[mbid]; ok && score > 0 && maxSimScore > 0 { - return personalSimilar * (float64(score) / float64(maxSimScore)) - } - - return 0.0 -} - // --------------------------------------------------------------------------- // Top Results — intent-scored cards // --------------------------------------------------------------------------- @@ -2407,10 +1760,34 @@ func (e *Service) resolveTopResults(query string, result *MBSearchResult) []TopR } // Stage 1: gather candidates. + clicksStart := time.Now() clicks := e.getSearchClicks(q) - exactMatches := e.index.ExactMatches(q, topResultsExactCap) + clicksDur := time.Since(clicksStart) + exactStart := time.Now() + exactMatches := e.index.ExactMatches(q, topResultsExactCap) + exactDur := time.Since(exactStart) + + if clicksDur+exactDur > 100*time.Millisecond { + e.logger.Info("search top results: slow candidate gather", + "query", query, + "getSearchClicks", clicksDur.Round(time.Microsecond), + "exactMatches", exactDur.Round(time.Microsecond), + ) + } + + gatherStart := time.Now() candidates := e.gatherTopCandidates(q, result, exactMatches, clicks) + gatherDur := time.Since(gatherStart) + + if gatherDur > 100*time.Millisecond { + e.logger.Info("search top results: slow gatherTopCandidates", + "query", query, + "candidates", len(candidates), + "elapsed", gatherDur.Round(time.Microsecond), + ) + } + if len(candidates) == 0 { return nil } @@ -2538,9 +1915,80 @@ func (e *Service) resolveTopResults(query string, result *MBSearchResult) []TopR ) } + rgResolveStart := time.Now() + + e.resolveTopResultReleaseGroups(selected) + + if d := time.Since(rgResolveStart); d > 100*time.Millisecond { + e.logger.Info("search top results: slow RG resolve", + "query", query, + "selected", len(selected), + "elapsed", d.Round(time.Microsecond), + ) + } + return selected } +// resolveTopResultReleaseGroups resolves each recording top-result's parent +// release group (from its CAA release MBID) so a track click can open the +// album page with the track highlighted — the same behaviour as clicking a +// track anywhere else. Uses a single local index query; recordings whose +// release can't be resolved simply keep an empty ReleaseGroupMBID and fall +// back to a name-only navigation on the frontend. +func (e *Service) resolveTopResultReleaseGroups(results []TopResult) { + var caaMBIDs []string + + for i := range results { + if results[i].EntityType == "recording" && results[i].CAAReleaseMBID != "" { + caaMBIDs = append(caaMBIDs, results[i].CAAReleaseMBID) + } + } + + if len(caaMBIDs) == 0 { + return + } + + rgByCAA := e.index.ReleaseGroupMBIDsForCAAReleaseMBIDs(caaMBIDs) + + for i := range results { + if results[i].EntityType != "recording" { + continue + } + + if rg, ok := rgByCAA[results[i].CAAReleaseMBID]; ok { + results[i].ReleaseGroupMBID = rg + } + } +} + +// resolveRecordingReleaseGroups fills each recording's parent release +// group (from its CAA release MBID) in a single local index query, so +// a track shown in search results can link to its album page with the +// track highlighted. Recordings whose release can't be resolved keep +// an empty ReleaseGroupMBID and fall back to non-linked text. +func (e *Service) resolveRecordingReleaseGroups(recordings []MBRecording) { + var caaMBIDs []string + + for i := range recordings { + if recordings[i].CAAReleaseMBID != "" { + caaMBIDs = append(caaMBIDs, recordings[i].CAAReleaseMBID) + } + } + + if len(caaMBIDs) == 0 { + return + } + + rgByCAA := e.index.ReleaseGroupMBIDsForCAAReleaseMBIDs(caaMBIDs) + + for i := range recordings { + if rg, ok := rgByCAA[recordings[i].CAAReleaseMBID]; ok { + recordings[i].ReleaseGroupMBID = rg + } + } +} + // topCandidate is a single scored candidate flowing through the // top-results pipeline. qualityScore is the per-candidate signal // without category bias; finalScore is qualityScore multiplied by @@ -2671,6 +2119,7 @@ func (e *Service) gatherTopCandidates( MBID: rg.MBID, Name: rg.Title, ArtistCredit: rg.ArtistCredit, + ArtistMBID: rg.ArtistMBID, PrimaryType: rg.PrimaryType, Year: year, InLibrary: rg.InLibrary, @@ -2715,6 +2164,7 @@ func (e *Service) gatherTopCandidates( MBID: rg.MBID, Name: rg.Title, ArtistCredit: rg.ArtistCredit, + ArtistMBID: rg.ArtistMBID, PrimaryType: rg.PrimaryType, Year: year, InLibrary: rg.InLibrary, @@ -2736,12 +2186,15 @@ func (e *Service) gatherTopCandidates( quality := e.scoreRecordingCandidate(q, &r, clicks) add(topCandidate{ topResult: TopResult{ - EntityType: "recording", - MBID: r.MBID, - Name: r.Title, - ArtistCredit: r.ArtistCredit, - Length: r.Length, - InLibrary: r.InLibrary, + EntityType: "recording", + MBID: r.MBID, + Name: r.Title, + ArtistCredit: r.ArtistCredit, + ArtistMBID: r.ArtistMBID, + Length: r.Length, + CAAReleaseMBID: r.CAAReleaseMBID, + ReleaseName: r.ReleaseName, + InLibrary: r.InLibrary, }, category: "recording", qualityScore: quality, @@ -2789,12 +2242,15 @@ func (e *Service) gatherTopCandidates( add(topCandidate{ topResult: TopResult{ - EntityType: "recording", - MBID: r.MBID, - Name: r.Title, - ArtistCredit: r.ArtistCredit, - Length: r.Length, - InLibrary: r.InLibrary, + EntityType: "recording", + MBID: r.MBID, + Name: r.Title, + ArtistCredit: r.ArtistCredit, + ArtistMBID: r.ArtistMBID, + Length: r.Length, + CAAReleaseMBID: r.CAAReleaseMBID, + ReleaseName: r.ReleaseName, + InLibrary: r.InLibrary, }, category: "recording", qualityScore: quality, @@ -2833,6 +2289,7 @@ func (e *Service) gatherTopCandidates( MBID: m.MBID, Name: m.Title, ArtistCredit: m.ArtistName, + ArtistMBID: m.ArtistMBID, PrimaryType: m.PrimaryType, Year: year, InLibrary: m.InLibrary || m.LocalReleaseGroupID > 0, @@ -2847,6 +2304,7 @@ func (e *Service) gatherTopCandidates( MBID: m.MBID, Name: m.Title, ArtistCredit: m.ArtistName, + ArtistMBID: m.ArtistMBID, Length: m.Duration, InLibrary: m.InLibrary || m.LocalRecordingID > 0, }, @@ -3230,23 +2688,13 @@ func priorConfidence(p intentPrior) float64 { return maxVal - secondMax } -// normLog returns log10(n+1) / log10(maxScale+1), clamped to [0, 1]. -// maxScale is a fixed reference point so the function is stable -// across queries — different from popRank which normalizes to a -// dynamic per-query max. +// normLog normalizes a count to [0,1] against a fixed reference scale, +// so the value is stable across queries (unlike the blended rerank, +// which normalizes against each result set's dynamic max). func normLog(n int) float64 { - if n <= 0 { - return 0 - } - const maxScale = 50_000_000 // top-tier artists have ~10-150M listens - v := math.Log10(float64(n)+1) / math.Log10(maxScale+1) //nolint:mnd - if v > 1.0 { - return 1.0 - } - - return v + return logNormalize(n, maxScale) } // clickFeature returns the additive feature contribution from a @@ -3483,176 +2931,38 @@ func (e *Service) RecordSearchClick(query, mbid, entityType string) { `, q, mbid, entityType) } -// blendedScore computes relevanceWeight*relevance + popularityWeight*logPop. -// relevance is 0–1. listenCount is raw; maxListenCount is the -// maximum in the result set (for normalization). -// -//nolint:unused // referenced by deferred MB search rework. -func blendedScore(relevance float64, listenCount, maxListenCount int) float64 { - return blendedScoreFull(relevance, listenCount, maxListenCount, 0.0) -} - // blendedScoreFull computes the weighted blend of relevance, popularity, -// and personalization. personalization is 0.0–1.0. +// and personalization. personalization is 0.0–1.0. Popularity is +// normalized against the result set's max (floored to 100K so a single +// modestly-popular result doesn't get an inflated score). func blendedScoreFull( relevance float64, listenCount, maxListenCount int, personalization float64, ) float64 { - effectiveMax := maxListenCount - if effectiveMax < 100_000 { //nolint:mnd - effectiveMax = 100_000 - } + effectiveMax := max(maxListenCount, 100_000) //nolint:mnd - logPop := math.Log10(float64(listenCount)+1) / math.Log10(float64(effectiveMax)+1) + logPop := logNormalize(listenCount, effectiveMax) return relevanceWeight*relevance + popularityWeight*logPop + personalizationWeight*personalization } -// dynamicSearchLimit computes the number of results to request from -// MB based on the total match count. Returns at least mbSearchLimit -// and at most mbSearchMaxLimit. Aims for ~15% of total matches so -// the ranking pipeline has enough candidates to surface popular -// results that MB's text relevance alone would bury. -// -//nolint:unused // referenced by deferred MB search rework. -func dynamicSearchLimit(totalMatches int) int { - if totalMatches <= mbSearchLimit { - return mbSearchLimit +// logNormalize maps a count to [0,1] on a log10 scale relative to a +// reference maximum: log10(n+1) / log10(ref+1), clamped. Shared by the +// blended rerank (dynamic per-result-set ref) and normLog (fixed ref). +func logNormalize(n, ref int) float64 { + if n <= 0 || ref <= 0 { + return 0 } - // 15% of total matches, but floor to mbSearchLimit and - // cap to mbSearchMaxLimit (and MB's API max of 100). - want := totalMatches * 15 / 100 //nolint:mnd - if want < mbSearchLimit { - want = mbSearchLimit + v := math.Log10(float64(n)+1) / math.Log10(float64(ref)+1) + if v > 1.0 { + return 1.0 } - if want > mbSearchMaxLimit { - want = mbSearchMaxLimit - } - - return want -} - -// maxListenCount returns the highest listen count in the map. -func maxListenCount(pop map[string]int) int { - maxVal := 0 - - for _, v := range pop { - if v > maxVal { - maxVal = v - } - } - - return maxVal -} - -// listenCounts extracts a simple mbid→listenCount map from PopularityData. -func listenCounts(pop map[string]PopularityData) map[string]int { - out := make(map[string]int, len(pop)) - for mbid, d := range pop { - out[mbid] = d.ListenCount - } - - return out + return v } // --------------------------------------------------------------------------- // Lucene query building // --------------------------------------------------------------------------- - -// luceneSpecialChars are characters that have special meaning in -// Lucene query syntax and must be escaped in user input. -var luceneSpecialChars = strings.NewReplacer( //nolint:gochecknoglobals - `\`, `\\`, - `+`, `\+`, - `-`, `\-`, - `!`, `\!`, - `(`, `\(`, - `)`, `\)`, - `{`, `\{`, - `}`, `\}`, - `[`, `\[`, - `]`, `\]`, - `^`, `\^`, - `"`, `\"`, - `~`, `\~`, - `*`, `\*`, - `?`, `\?`, - `:`, `\:`, - `/`, `\/`, -) - -// buildLuceneQuery converts a user's search input into a Lucene -// AND query with a wildcard on the last term for type-ahead. -// -// Examples: -// -// "radiohead" → "radiohead*" -// "the teenagers" → "the AND teenagers*" -// "florence machine" → "florence AND machine*" -// "ac/dc" → "ac\/dc*" -// -// This eliminates the common-word pollution problem: "the teenagers" -// no longer matches "The Beatles" (which only contains "the"). -// The trailing wildcard enables prefix matching as the user types. -func buildLuceneQuery(input string) string { - words := strings.Fields(strings.TrimSpace(input)) - if len(words) == 0 { - return "" - } - - // Escape special Lucene characters in each word. - for i, w := range words { - words[i] = luceneSpecialChars.Replace(w) - } - - if len(words) == 1 { - return words[0] + "*" - } - - // AND all terms, wildcard on the last (type-ahead). - var b strings.Builder - - for i, w := range words { - if i > 0 { - b.WriteString(" AND ") - } - - b.WriteString(w) - - if i == len(words)-1 { - b.WriteByte('*') - } - } - - return b.String() -} - -// buildLuceneQueryWithArtist builds a Lucene query that searches -// both the entity's own field (title) and the artist credit field. -// For "queen": (releasegroup:queen* OR artist:queen*) -// This ensures searches return results BY the artist, not just -// results with the query in the title. -func buildLuceneQueryWithArtist(input, entityField, artistField string) string { - words := strings.Fields(strings.TrimSpace(input)) - if len(words) == 0 { - return "" - } - - for i, w := range words { - words[i] = luceneSpecialChars.Replace(w) - } - - // Build the base query terms. - base := buildLuceneQuery(input) - - // Single word: (field:word* OR artist:word*) - if len(words) == 1 { - return "(" + entityField + ":" + base + " OR " + artistField + ":" + base + ")" - } - - // Multi-word: (field:(term1 AND term2*) OR artist:(term1 AND term2*)) - return "(" + entityField + ":(" + base + ") OR " + artistField + ":(" + base + "))" -} diff --git a/backend/explore/fuzzy.go b/backend/explore/fuzzy.go new file mode 100644 index 0000000..a5c5b17 --- /dev/null +++ b/backend/explore/fuzzy.go @@ -0,0 +1,58 @@ +package explore + +import "strings" + +// Character-bigram similarity, used by the search index's typo-tolerant +// rescue pass (see fuzzyRescue). A name is represented as the set of its +// sliding 2-rune windows; a single-character typo alters only the one or +// two bigrams that span it, so most of the set survives. Overlap between +// two such sets (Dice coefficient) therefore stays high across a +// misspelling, where prefix matching collapses to nothing. + +// fuzzyNormalize lowercases and collapses runs of whitespace to a single +// space so bigram sets are stable across casing and spacing noise. +func fuzzyNormalize(s string) string { + return strings.Join(strings.Fields(strings.ToLower(s)), " ") +} + +// fuzzyBigrams returns the set of character bigrams of s after +// normalization. Returns nil for inputs shorter than two runes, which +// have no bigram and can't be scored. +func fuzzyBigrams(s string) map[string]struct{} { + runes := []rune(fuzzyNormalize(s)) + if len(runes) < 2 { + return nil + } + + set := make(map[string]struct{}, len(runes)) + + for i := 0; i+1 < len(runes); i++ { + set[string(runes[i:i+2])] = struct{}{} + } + + return set +} + +// diceCoefficient is 2·|A∩B| / (|A|+|B|), a similarity in [0, 1] where 1 +// is an identical bigram set and 0 is disjoint. Iterating the smaller +// set keeps the intersection count cheap. +func diceCoefficient(a, b map[string]struct{}) float64 { + if len(a) == 0 || len(b) == 0 { + return 0 + } + + small, large := a, b + if len(large) < len(small) { + small, large = large, small + } + + intersection := 0 + + for bg := range small { + if _, ok := large[bg]; ok { + intersection++ + } + } + + return 2 * float64(intersection) / float64(len(a)+len(b)) +} diff --git a/backend/explore/fuzzy_test.go b/backend/explore/fuzzy_test.go new file mode 100644 index 0000000..dd0fa88 --- /dev/null +++ b/backend/explore/fuzzy_test.go @@ -0,0 +1,43 @@ +package explore + +import "testing" + +func TestDiceCoefficientTypoTolerance(t *testing.T) { + tests := []struct { + name string + a, b string + wantMin float64 // score must be at least this + wantMax float64 // ...and at most this + }{ + {name: "identical", a: "beatles", b: "beatles", wantMin: 1.0, wantMax: 1.0}, + {name: "single typo", a: "beetles", b: "beatles", wantMin: 0.5, wantMax: 0.9}, + {name: "missing char", a: "nirvna", b: "nirvana", wantMin: 0.5, wantMax: 0.95}, + // Longer-name typo: a single substitution stays comfortably above + // the rescue threshold, which is the realistic case (artist and + // album names are rarely as short as five characters). + {name: "longer name typo", a: "metalica", b: "metallica", wantMin: 0.5, wantMax: 0.95}, + // Adjacent transposition in a short word is bigrams' known weak + // spot — it breaks most windows, so it scores below threshold. + // Documented, not a bug: bigram overlap targets substitution, + // insertion, and deletion typos. + {name: "short transposition", a: "raido", b: "radio", wantMin: 0.0, wantMax: 0.34}, + {name: "case and space", a: "The Beatles", b: "the beatles", wantMin: 1.0, wantMax: 1.0}, + {name: "unrelated", a: "beatles", b: "metallica", wantMin: 0.0, wantMax: 0.34}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := diceCoefficient(fuzzyBigrams(tc.a), fuzzyBigrams(tc.b)) + if got < tc.wantMin || got > tc.wantMax { + t.Errorf("diceCoefficient(%q, %q) = %.3f, want in [%.2f, %.2f]", + tc.a, tc.b, got, tc.wantMin, tc.wantMax) + } + }) + } +} + +func TestFuzzyBigramsShortInput(t *testing.T) { + if got := fuzzyBigrams("a"); got != nil { + t.Errorf("fuzzyBigrams(%q) = %v, want nil", "a", got) + } +} diff --git a/backend/explore/lrclib.go b/backend/explore/lrclib.go new file mode 100644 index 0000000..c00c66e --- /dev/null +++ b/backend/explore/lrclib.go @@ -0,0 +1,189 @@ +package explore + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "strconv" + "time" +) + +const ( + // lrclibBaseURL is the LRCLIB lyrics API. LRCLIB is a free, + // community-maintained lyrics database (plain + synced) with no + // auth required. + lrclibBaseURL = "https://lrclib.net" + + // lrclibUserAgent identifies the app per LRCLIB's guidelines. + lrclibUserAgent = "YellowJacket (https://github.com/yellowjacket)" + + // lrclibRate is the requests-per-second budget for LRCLIB. The + // service has no hard published limit but asks clients to be + // gentle; this keeps the library backfill polite. + lrclibRate = 3 +) + +// ErrLyricsNotFound is returned when LRCLIB has no match for the +// requested track. +var ErrLyricsNotFound = errors.New("lyrics not found") + +// Lyrics holds the plain and (optional) time-synced lyrics for a +// track, plus whether the track is marked instrumental. +type Lyrics struct { + Plain string `json:"plain"` + Synced string `json:"synced"` + Instrumental bool `json:"instrumental"` +} + +// LRCLibClient is a thin, rate-limited, cached HTTP client for the +// LRCLIB lyrics API. +type LRCLibClient struct { + http *http.Client + limiter *RateLimiter + cache *Cache + logger *slog.Logger + baseURL string // overridable in tests +} + +// NewLRCLibClient creates an LRCLIB client sharing the given cache. +func NewLRCLibClient(cache *Cache, logger *slog.Logger) *LRCLibClient { + return &LRCLibClient{ + http: &http.Client{Timeout: 20 * time.Second}, + limiter: NewRateLimiterN(lrclibRate), + cache: cache, + logger: logger, + baseURL: lrclibBaseURL, + } +} + +// lrclibResponse is the wire shape of LRCLIB's /api/get response. +type lrclibResponse struct { + ID int64 `json:"id"` + TrackName string `json:"trackName"` + ArtistName string `json:"artistName"` + AlbumName string `json:"albumName"` + Duration float64 `json:"duration"` + Instrumental bool `json:"instrumental"` + PlainLyrics string `json:"plainLyrics"` + SyncedLyrics string `json:"syncedLyrics"` +} + +// GetLyrics fetches lyrics for a track by artist, title, album, and +// duration (seconds; pass 0 if unknown). LRCLIB matches on the +// metadata with a small duration tolerance. Returns ErrLyricsNotFound +// when no match exists. Successful and negative results are both +// cached so a repeated backfill doesn't re-hit the network. +func (c *LRCLibClient) GetLyrics( + ctx context.Context, artist, title, album string, durationSec int, +) (*Lyrics, error) { + if artist == "" || title == "" { + return nil, ErrLyricsNotFound + } + + q := url.Values{} + q.Set("artist_name", artist) + q.Set("track_name", title) + + if album != "" { + q.Set("album_name", album) + } + + if durationSec > 0 { + q.Set("duration", strconv.Itoa(durationSec)) + } + + reqURL := c.baseURL + "/api/get?" + q.Encode() + cacheKey := "lrclib:get:" + q.Encode() + + if data, ok := c.cache.Get(cacheKey); ok { + return decodeCachedLyrics(data) + } + + body, status, err := c.doGet(ctx, reqURL) + if err != nil { + return nil, fmt.Errorf("lrclib get: %w", err) + } + + if status == http.StatusNotFound { + // Cache the miss as a sentinel so we don't re-request it. + c.cache.Set(cacheKey, []byte(lyricsMissSentinel), cacheTTLSearch, "", "lyrics") + + return nil, ErrLyricsNotFound + } + + if status < 200 || status >= 300 { + return nil, fmt.Errorf("lrclib get: %w: status %d", ErrListenBrainzHTTP, status) + } + + var wire lrclibResponse + if err := json.Unmarshal(body, &wire); err != nil { + return nil, fmt.Errorf("lrclib get unmarshal: %w", err) + } + + lyrics := &Lyrics{ + Plain: wire.PlainLyrics, + Synced: wire.SyncedLyrics, + Instrumental: wire.Instrumental, + } + + // Persist the normalized result (not the raw wire body) so the + // cached shape matches what callers expect. + if encoded, err := json.Marshal(lyrics); err == nil { + c.cache.Set(cacheKey, encoded, cacheTTLSearch, "", "lyrics") + } + + return lyrics, nil +} + +// lyricsMissSentinel marks a cached "no lyrics found" result. +const lyricsMissSentinel = "\x00miss" + +// decodeCachedLyrics interprets a cached LRCLIB payload, mapping the +// miss sentinel back to ErrLyricsNotFound. +func decodeCachedLyrics(data []byte) (*Lyrics, error) { + if string(data) == lyricsMissSentinel { + return nil, ErrLyricsNotFound + } + + var lyrics Lyrics + if err := json.Unmarshal(data, &lyrics); err != nil { + return nil, fmt.Errorf("lrclib cache decode: %w", err) + } + + return &lyrics, nil +} + +// doGet performs a rate-limited GET and returns the body and status. +// Unlike the ListenBrainz client, a 404 is a normal "no lyrics" +// outcome, so the status is returned rather than folded into an error. +func (c *LRCLibClient) doGet(ctx context.Context, reqURL string) ([]byte, int, error) { + if err := c.limiter.Wait(ctx); err != nil { + return nil, 0, fmt.Errorf("rate limiter: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL, nil) + if err != nil { + return nil, 0, err + } + + req.Header.Set("User-Agent", lrclibUserAgent) + + resp, err := c.http.Do(req) + if err != nil { + return nil, 0, err + } + + defer func() { _ = resp.Body.Close() }() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, resp.StatusCode, err + } + + return body, resp.StatusCode, nil +} diff --git a/backend/explore/lrclib_test.go b/backend/explore/lrclib_test.go new file mode 100644 index 0000000..5f68a24 --- /dev/null +++ b/backend/explore/lrclib_test.go @@ -0,0 +1,127 @@ +package explore + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" + + "yellowjacket/backend/database" +) + +// newTestLRCLib builds an LRCLIB client whose HTTP calls hit the given +// test server, sharing a real (in-memory) cache so caching behaviour is +// exercised. +func newTestLRCLib(t *testing.T, handler http.HandlerFunc) (*LRCLibClient, *httptest.Server) { + t.Helper() + + srv := httptest.NewServer(handler) + t.Cleanup(srv.Close) + + db := database.NewTestDB(t) + cache := NewCache(db, testLogger()) + + c := NewLRCLibClient(cache, testLogger()) + // Point the client at the test server instead of the real API by + // overriding its transport to rewrite the host. + c.http = srv.Client() + c.baseURL = srv.URL + + return c, srv +} + +func TestLRCLibGetLyrics(t *testing.T) { + t.Parallel() + + var calls int + + c, _ := newTestLRCLib(t, func(w http.ResponseWriter, r *http.Request) { + calls++ + + if r.URL.Path != "/api/get" { + http.Error(w, "not found", http.StatusNotFound) + + return + } + + if r.URL.Query().Get("track_name") == "Missing" { + http.Error(w, "not found", http.StatusNotFound) + + return + } + + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "id": 42, + "trackName": "Yesterday", + "artistName": "The Beatles", + "albumName": "Help!", + "duration": 125, + "instrumental": false, + "plainLyrics": "Yesterday, all my troubles seemed so far away", + "syncedLyrics": "[00:00.00] Yesterday" + }`)) + }) + + t.Run("hit returns plain + synced lyrics", func(t *testing.T) { + lyrics, err := c.GetLyrics(context.Background(), "The Beatles", "Yesterday", "Help!", 125) + if err != nil { + t.Fatalf("GetLyrics: %v", err) + } + + if lyrics.Plain == "" || lyrics.Synced == "" { + t.Errorf("expected plain and synced lyrics, got %+v", lyrics) + } + + if lyrics.Instrumental { + t.Error("expected non-instrumental") + } + }) + + t.Run("second identical request is served from cache", func(t *testing.T) { + before := calls + + if _, err := c.GetLyrics( + context.Background(), + "The Beatles", + "Yesterday", + "Help!", + 125, + ); err != nil { + t.Fatalf("GetLyrics: %v", err) + } + + if calls != before { + t.Errorf("expected cache hit (no new HTTP call), calls went %d → %d", before, calls) + } + }) + + t.Run("404 maps to ErrLyricsNotFound", func(t *testing.T) { + _, err := c.GetLyrics(context.Background(), "Nobody", "Missing", "", 0) + if !errors.Is(err, ErrLyricsNotFound) { + t.Errorf("expected ErrLyricsNotFound, got %v", err) + } + }) + + t.Run("missing artist/title short-circuits without a request", func(t *testing.T) { + before := calls + + if _, err := c.GetLyrics( + context.Background(), + "", + "Yesterday", + "", + 0, + ); !errors.Is( + err, + ErrLyricsNotFound, + ) { + t.Errorf("expected ErrLyricsNotFound, got %v", err) + } + + if calls != before { + t.Error("expected no HTTP call for empty artist") + } + }) +} diff --git a/backend/explore/lyrics.go b/backend/explore/lyrics.go new file mode 100644 index 0000000..5ed116b --- /dev/null +++ b/backend/explore/lyrics.go @@ -0,0 +1,210 @@ +package explore + +import ( + "context" + "errors" + + "yellowjacket/backend/database" +) + +// LyricsResult is a single lyric-search hit, mapped from the DB layer +// into the camelCase shape the frontend consumes. +type LyricsResult struct { + RecordingID int64 `json:"recordingId"` + FilePath string `json:"filePath"` + LengthMs int64 `json:"lengthMs"` + Title string `json:"title"` + Artist string `json:"artist"` + Album string `json:"album"` +} + +// TrackLyrics is the stored or freshly-fetched lyrics for one track. +// Source is "embedded" (from the file's tags / library DB), "lrclib" +// (fetched on demand), or "" when none are available. +type TrackLyrics struct { + Plain string `json:"plain"` + Synced string `json:"synced"` + Instrumental bool `json:"instrumental"` + Source string `json:"source"` +} + +const ( + // lyricsSearchLimit caps lyric-search hits returned to the UI. + lyricsSearchLimit = 30 + + // lyricsBackfillBatch is how many missing-lyrics recordings the + // background backfill processes per pass. + lyricsBackfillBatch = 200 + + // lyricsBackfillMaxPasses bounds a single backfill run so it can't + // loop forever on a huge library; the next launch resumes where + // this one left off (candidates with lyrics now filled are skipped). + lyricsBackfillMaxPasses = 25 +) + +// SearchLyrics finds library tracks whose lyrics contain the given +// fragment, ranked by relevance. Pure local FTS — no network. +func (e *Service) SearchLyrics(query string) []LyricsResult { + hits, err := e.db.SearchLyrics(query, lyricsSearchLimit) + if err != nil { + e.logger.Warn("lyrics search failed", "query", query, "err", err) + + return nil + } + + out := make([]LyricsResult, 0, len(hits)) + for _, h := range hits { + out = append(out, LyricsResult{ + RecordingID: h.RecordingID, + FilePath: h.FilePath, + LengthMs: h.LengthMilliseconds, + Title: h.Title, + Artist: h.Artist, + Album: h.Album, + }) + } + + return out +} + +// GetTrackLyrics returns lyrics for a recording. If the library +// already has them (from embedded tags) they're returned as-is; +// otherwise it fetches from LRCLIB, persists them (updating the FTS +// index), and returns them. Never returns an error to the frontend — +// a miss just yields an empty result. +func (e *Service) GetTrackLyrics(recordingID int64) TrackLyrics { + stored, err := e.db.GetRecordingLyrics(recordingID) + if err == nil && stored != "" { + return TrackLyrics{Plain: stored, Source: "embedded"} + } + + lookup, err := e.db.RecordingLyricLookup(recordingID) + if err != nil || lookup == nil { + return TrackLyrics{} + } + + fetched := e.fetchAndStoreLyrics(e.ctx, *lookup) + if fetched == nil { + return TrackLyrics{} + } + + return TrackLyrics{ + Plain: fetched.Plain, + Synced: fetched.Synced, + Instrumental: fetched.Instrumental, + Source: "lrclib", + } +} + +// RebuildLyricsIndex rebuilds the FTS lyrics index from the current +// library. Cheap; safe to call after every scan. +func (e *Service) RebuildLyricsIndex() { + if err := e.db.RebuildLyricsIndex(); err != nil { + e.logger.Warn("lyrics index rebuild failed", "err", err) + + return + } + + e.index.setMeta(lyricsIndexReadyKey, "1") + e.logger.Info("lyrics index rebuilt") +} + +// RebuildLyricsIndexIfNeeded rebuilds the lyrics FTS only when it has not +// been built since the last library change. The backfill keeps the index +// in sync incrementally thereafter, so on an unchanged library the full +// rebuild is redundant; the scan-completion path calls the unconditional +// form. +func (e *Service) RebuildLyricsIndexIfNeeded() { + if e.index.hasMeta(lyricsIndexReadyKey) { + return + } + + e.RebuildLyricsIndex() +} + +// BackfillLibraryLyrics fetches lyrics from LRCLIB for library tracks +// that don't have them, in the background. Idempotent and bounded — +// each recording is tried once (a miss is cached), and a run stops +// after a fixed number of passes, resuming on the next launch. +func (e *Service) BackfillLibraryLyrics() { + go e.backfillLibraryLyrics(e.ctx) +} + +func (e *Service) backfillLibraryLyrics(ctx context.Context) { + total := 0 + + for range lyricsBackfillMaxPasses { + if ctx.Err() != nil { + return + } + + candidates, err := e.db.RecordingsMissingLyrics(lyricsBackfillBatch) + if err != nil { + e.logger.Warn("lyrics backfill: query failed", "err", err) + + return + } + + if len(candidates) == 0 { + break + } + + filled := 0 + + for _, c := range candidates { + if ctx.Err() != nil { + return + } + + if e.fetchAndStoreLyrics(ctx, c) != nil { + filled++ + total++ + } + } + + // If a whole batch produced no stored lyrics, every remaining + // candidate is a cached miss with no new data — stop early + // rather than spinning through identical misses. + if filled == 0 { + break + } + } + + if total > 0 { + e.logger.Info("lyrics backfill complete", "filled", total) + } +} + +// fetchAndStoreLyrics looks up a single candidate on LRCLIB and, on a +// non-instrumental hit with plain lyrics, persists it to the recording +// (which also updates the FTS index). Returns the fetched lyrics, or +// nil on any miss/error. Instrumental hits are recorded as an empty +// lyrics string so they still count as "resolved" and aren't retried. +func (e *Service) fetchAndStoreLyrics( + ctx context.Context, c database.LyricsCandidate, +) *Lyrics { + durationSec := int(c.LengthMilliseconds / 1000) //nolint:mnd + + lyrics, err := e.lrclib.GetLyrics(ctx, c.Artist, c.Title, c.Album, durationSec) + if err != nil { + if !errors.Is(err, ErrLyricsNotFound) { + e.logger.Debug("lyrics fetch failed", + "artist", c.Artist, "title", c.Title, "err", err, + ) + } + + return nil + } + + if lyrics.Instrumental || lyrics.Plain == "" { + return nil + } + + if err := e.db.SetRecordingLyrics(c.RecordingID, lyrics.Plain); err != nil { + e.logger.Warn("lyrics store failed", "recordingId", c.RecordingID, "err", err) + + return nil + } + + return lyrics +} diff --git a/backend/explore/mergeindexhits_test.go b/backend/explore/mergeindexhits_test.go new file mode 100644 index 0000000..7f5f09f --- /dev/null +++ b/backend/explore/mergeindexhits_test.go @@ -0,0 +1,121 @@ +package explore + +import "testing" + +// TestMergeIndexHitsBlendedScale is the regression test for the +// merge→filter scale mismatch. An exact-match, in-library album that +// exists only in the local index (not returned by MB) with very low +// popularity must survive filterAndCap and outrank a weakly-matching MB +// result — because index hits are now scored on the same blended scale. +// +// Under the previous behaviour the index hit was scored as +// scalePopularity(50)*0.5 ≈ 12, below minBlendedScore (15), so +// filterAndCap dropped it entirely. +func TestMergeIndexHitsBlendedScale(t *testing.T) { + // A weakly-matching MB release group, already reranked: ~0.35 + // relevance, no popularity → blended Score ≈ 35. + result := MBSearchResult{ + ReleaseGroups: []MBReleaseGroup{ + {MBID: "mb-weak", Title: "Live At Wembley", Score: 35, Popularity: 0}, + }, + } + + // An exact-match, in-library album from the local index with only + // 50 listens. + hits := []SearchIndexResult{ + { + EntityType: "release_group", + MBID: "idx-exact", + Title: "Abbey Road", + ArtistName: "The Beatles", + Popularity: 50, + InLibrary: true, + }, + } + + mergeIndexHits("abbey road", &result, hits) + + if got := result.ReleaseGroups[0].MBID; got != "idx-exact" { + t.Fatalf("expected exact in-library index hit to rank first, got %q (score %d)", + got, result.ReleaseGroups[0].Score) + } + + idxScore := result.ReleaseGroups[0].Score + if idxScore < minBlendedScore { + t.Errorf("index hit score %d below minBlendedScore %d — would be filtered out", + idxScore, minBlendedScore) + } + + // It must survive the filter that previously dropped it. + filterAndCap(&result) + + if len(result.ReleaseGroups) == 0 || result.ReleaseGroups[0].MBID != "idx-exact" { + t.Fatalf("index hit did not survive filterAndCap: %+v", result.ReleaseGroups) + } +} + +// TestMergeIndexHitsSortsRecordings guards the recordings path, which +// previously had no post-merge sort: a high-scoring index recording was +// merged but its position depended on prepend order, not score. +func TestMergeIndexHitsSortsRecordings(t *testing.T) { + result := MBSearchResult{ + Recordings: []MBRecording{ + {MBID: "mb-a", Title: "Filler", Score: 60, Popularity: 100}, + {MBID: "mb-b", Title: "Filler Two", Score: 20, Popularity: 50}, + }, + } + + hits := []SearchIndexResult{ + { + EntityType: "recording", + MBID: "idx-exact", + Title: "Calling You", + Popularity: 100_000, + InLibrary: true, + }, + } + + mergeIndexHits("calling you", &result, hits) + + // Exact + in-library + decent popularity should land on top, and + // the whole list must be in descending Score order. + if result.Recordings[0].MBID != "idx-exact" { + t.Errorf("exact in-library recording should rank first, got %q", result.Recordings[0].MBID) + } + + for i := 1; i < len(result.Recordings); i++ { + if result.Recordings[i-1].Score < result.Recordings[i].Score { + t.Errorf("recordings not sorted by score descending: %+v", result.Recordings) + + break + } + } +} + +func TestIndexHitRelevanceTiers(t *testing.T) { + const q = "abbey road" + + exact := indexHitRelevance(q, "Abbey Road", "") + prefix := indexHitRelevance(q, "Abbey Road Sessions", "") + word := indexHitRelevance(q, "Live: Abbey Road Medley", "") + none := indexHitRelevance(q, "Something Unrelated", "") + + if !(exact > prefix && prefix > word && word >= indexRelevanceFloor) { + t.Errorf("relevance tiers not ordered: exact=%v prefix=%v word=%v", exact, prefix, word) + } + + if exact != indexRelExact { + t.Errorf("exact relevance = %v, want %v", exact, indexRelExact) + } + + // A non-matching title still gets the floor (FTS matched something). + if none != indexRelevanceFloor { + t.Errorf("non-match relevance = %v, want floor %v", none, indexRelevanceFloor) + } + + // Artist-credit match should count when the title doesn't. + artistMatch := indexHitRelevance("the beatles", "Abbey Road", "The Beatles") + if artistMatch != indexRelExact { + t.Errorf("artist-credit exact match = %v, want %v", artistMatch, indexRelExact) + } +} diff --git a/backend/explore/musicbrainz.go b/backend/explore/musicbrainz.go index 467520b..037058d 100644 --- a/backend/explore/musicbrainz.go +++ b/backend/explore/musicbrainz.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "log/slog" + "strings" "time" "unicode" @@ -18,6 +19,12 @@ const ( // cacheTTLEntity is the TTL for lookup/browse results (entity data // changes rarely). cacheTTLEntity = 7 * 24 * time.Hour + // cacheTTLReleases is the TTL for a release group's releases + + // tracklists. This data is effectively immutable once published, so + // it's cached far longer than other entities: it's the local store + // that keeps an album page's cold fetch a once-per-quarter event + // rather than a weekly one. + cacheTTLReleases = 90 * 24 * time.Hour ) // MusicBrainzClient wraps the musicbrainzws2 library with a local @@ -360,15 +367,27 @@ func (c *MusicBrainzClient) LookupRelease( return &out, nil } -// BrowseReleases fetches the releases for a given release group -// MBID, including media/track information. Cached for 7 days. -func (c *MusicBrainzClient) BrowseReleases( - ctx context.Context, releaseGroupMBID string, -) ([]MBRelease, error) { - cacheKey := "mb:browse:releases:" + releaseGroupMBID +// MBRecordingRelease is a slim reference to one release a recording +// appears on — enough for the autotagger to pick a representative +// release and then LookupRelease it in full. +type MBRecordingRelease struct { + MBID string `json:"mbid"` + Title string `json:"title"` + Status string `json:"status"` + Date string `json:"date"` +} + +// LookupRecordingReleases fetches the releases a recording appears on +// (id / title / status / date only). Used by the autotag recording- +// search path to resolve a picked recording to a concrete release. +// Cached for 7 days. +func (c *MusicBrainzClient) LookupRecordingReleases( + ctx context.Context, recordingMBID string, +) ([]MBRecordingRelease, error) { + cacheKey := "mb:lookup:recording-releases:" + recordingMBID if data, ok := c.cache.Get(cacheKey); ok { - var out []MBRelease + var out []MBRecordingRelease if err := json.Unmarshal(data, &out); err == nil { return out, nil } @@ -378,6 +397,77 @@ func (c *MusicBrainzClient) BrowseReleases( return nil, err } + c.logger.Info("musicbrainz lookup recording releases", "mbid", recordingMBID) + + rec, err := c.mb.LookupRecording( + ctx, + mbtypes.MBID(recordingMBID), + musicbrainzws2.IncludesFilter{Includes: []string{"releases"}}, + ) + if err != nil { + return nil, err + } + + out := make([]MBRecordingRelease, 0, len(rec.Releases)) + for _, rel := range rec.Releases { + out = append(out, MBRecordingRelease{ + MBID: string(rel.ID), + Title: rel.Title, + Status: rel.Status, + Date: rel.Date.String(), + }) + } + + c.cacheJSON(cacheKey, out, cacheTTLEntity, recordingMBID, "recording") + + return out, nil +} + +// BrowseReleases fetches the releases for a given release group +// MBID, including media/track information. Cached for 7 days. +// browseReleasesCacheKey returns the response-cache key for a release +// group's releases. +func browseReleasesCacheKey(releaseGroupMBID string) string { + return "mb:browse:releases:" + releaseGroupMBID +} + +// BrowseReleasesCached returns a release group's releases from the local +// response cache only, never hitting the network. The bool reports +// whether a fresh (unexpired) cache entry was found. Used by the album +// page's local-first path so a cold fetch can be deferred to the +// background instead of blocking the request. +func (c *MusicBrainzClient) BrowseReleasesCached( + releaseGroupMBID string, +) ([]MBRelease, bool) { + data, ok := c.cache.Get(browseReleasesCacheKey(releaseGroupMBID)) + if !ok { + return nil, false + } + + var out []MBRelease + if err := json.Unmarshal(data, &out); err != nil { + return nil, false + } + + return out, true +} + +// BrowseReleases fetches all releases (with recordings + media) for a +// release group, serving from the local response cache when warm and +// otherwise hitting MusicBrainz and caching the result. +func (c *MusicBrainzClient) BrowseReleases( + ctx context.Context, releaseGroupMBID string, +) ([]MBRelease, error) { + cacheKey := browseReleasesCacheKey(releaseGroupMBID) + + if out, ok := c.BrowseReleasesCached(releaseGroupMBID); ok { + return out, nil + } + + if err := c.limiter.Wait(ctx); err != nil { + return nil, err + } + c.logger.Info("musicbrainz browse releases", "releaseGroupMBID", releaseGroupMBID, ) @@ -395,7 +485,7 @@ func (c *MusicBrainzClient) BrowseReleases( out := convertReleases(result.Releases) - c.cacheJSON(cacheKey, out, cacheTTLEntity, releaseGroupMBID, "release-group") + c.cacheJSON(cacheKey, out, cacheTTLReleases, releaseGroupMBID, "release-group") return out, nil } @@ -534,7 +624,19 @@ func convertRelease(r musicbrainzws2.Release) MBRelease { } for _, m := range r.Media { + // Skip video media outright — DVD/Blu-ray bonus discs + // inflate track counts and wreck track-count-based scoring + // (beets ignores video/data tracks for the same reason). + if isVideoFormat(m.Format) { + continue + } + for _, t := range m.Tracks { + // Same for individual video recordings on audio media. + if t.Recording.IsVideo { + continue + } + // Use the recording MBID, not the track MBID. Tracks // and recordings have distinct MBIDs in MusicBrainz: // a track is the placement of a recording on a specific @@ -565,6 +667,25 @@ func convertRelease(r musicbrainzws2.Release) MBRelease { return rel } +// isVideoFormat reports whether a medium's format string names a +// video carrier. "DVD-Audio" stays audio; bare "DVD", "DVD-Video", +// "Blu-ray", "HD-DVD", "VHS", "VCD"/"SVCD" are video. +func isVideoFormat(format string) bool { + f := strings.ToLower(format) + + if strings.Contains(f, "dvd-audio") { + return false + } + + for _, v := range []string{"dvd", "blu-ray", "bluray", "hd-dvd", "vhs", "vcd"} { + if strings.Contains(f, v) { + return true + } + } + + return false +} + func convertReleases(releases []musicbrainzws2.Release) []MBRelease { out := make([]MBRelease, len(releases)) for i, r := range releases { diff --git a/backend/explore/ranker.go b/backend/explore/ranker.go new file mode 100644 index 0000000..276d491 --- /dev/null +++ b/backend/explore/ranker.go @@ -0,0 +1,195 @@ +package explore + +import ( + "encoding/json" + "fmt" + "io" + "math" +) + +// This file sketches a learned ranking model that replaces the +// hand-tuned scoring constants (the fw* feature weights, tierBonus, +// rgTierBonus, the intent-prior multipliers) with weights that can be +// trained from the user's own click history. +// +// Status: NOT yet wired into Search(). The integration point is the +// candidate scorers in the top-results pipeline +// (scoreArtistCandidate / scoreReleaseGroupCandidate / +// scoreRecordingCandidate) and the main rerank. Swap those additive +// constant sums for RankFeatures + LinearModel.Score once the eval +// harness has real fixtures to prove the change is a win. +// +// Why a linear model and not something fancier: it needs no scale, no +// GPU, trains in microseconds on a desktop's worth of clicks, and — +// crucially — personalises to ONE user. That is the lever a +// multi-million-user system gets from aggregate logs; we get the same +// shape of signal from a single user's repeated intent. +// +// Training prerequisite (important): logistic training needs negatives, +// i.e. results that were SHOWN but NOT clicked. search_clicks records +// only clicks today. To train properly, log impressions too (the +// MBIDs shown for a query); clicked rows are positive labels, the rest +// of the shown set are negatives. Until impressions are logged, use +// DefaultModel(), whose weights reproduce the current behaviour. + +// RankFeatures is the feature vector for a single candidate. Every +// field is normalised to roughly [0,1] so weights are comparable. +type RankFeatures struct { + // Textual match strength against the query (mutually exclusive + // tiers collapsed to a single 0..1 magnitude: exact=1.0, + // prefix=0.6, whole-word=0.4, substring=0.2, none=0). + NameMatch float64 + + // Artist-credit match strength, same scale. Lets "abbey road + // beatles" reward the album credited to The Beatles. + ArtistMatch float64 + + // Log-scaled popularity and listener count, both via normLog so + // they share the fixed reference scale. + LogPopularity float64 + LogListeners float64 + + // Personalisation signals. + InLibrary float64 // 1.0 when owned, else 0 + IsSimilar float64 // 0..1 similarity to an owned artist + + // Recency-decayed per-query click signal for this candidate. + ClickRate float64 +} + +// rankFeatureCount is the number of features (excluding bias). Used by +// the gradient step to iterate fields generically. +const rankFeatureCount = 7 + +// asSlice returns the features in a stable order so Score and Update +// agree on indexing. +func (f RankFeatures) asSlice() [rankFeatureCount]float64 { + return [rankFeatureCount]float64{ + f.NameMatch, + f.ArtistMatch, + f.LogPopularity, + f.LogListeners, + f.InLibrary, + f.IsSimilar, + f.ClickRate, + } +} + +// LinearModel scores a candidate as bias + Σ wᵢ·featureᵢ. For ranking +// the raw score is what matters; the logistic squashing is used only +// during training to produce a probability for the gradient. +type LinearModel struct { + Bias float64 `json:"bias"` + Weights [rankFeatureCount]float64 `json:"weights"` +} + +// DefaultModel returns weights that reproduce the current hand-tuned +// scorer, so swapping the model in with no training leaves behaviour +// unchanged. The values mirror the fw* constants in explore.go. +func DefaultModel() LinearModel { + return LinearModel{ + Bias: 0, + Weights: [rankFeatureCount]float64{ + 1.00, // NameMatch ~ fwExactTitle / fwPrefixTitle blend + 0.90, // ArtistMatch ~ fwExactArtist + 0.80, // LogPopularity ~ fwListenLog + 0.60, // LogListeners ~ fwListenerLog + 0.50, // InLibrary ~ fwInLibrary + 0.20, // IsSimilar ~ fwSimilar + 0.30, // ClickRate ~ click feature cap + }, + } +} + +// Score returns the un-squashed ranking score for a candidate. Higher +// is better. This is the value to sort by. +func (m LinearModel) Score(f RankFeatures) float64 { + score := m.Bias + fs := f.asSlice() + + for i := range fs { + score += m.Weights[i] * fs[i] + } + + return score +} + +// Probability squashes the score to (0,1) via the logistic function. +// Used during training to compute the gradient. +func (m LinearModel) Probability(f RankFeatures) float64 { + return 1.0 / (1.0 + math.Exp(-m.Score(f))) +} + +// Sample is one training example: a candidate's features and whether +// the user clicked it (1.0) or saw-but-skipped it (0.0). +type Sample struct { + Features RankFeatures + Label float64 +} + +// Update performs one logistic-regression SGD step toward the label. +// learningRate is typically ~0.05. Returns the pre-update prediction +// so callers can track convergence. +func (m *LinearModel) Update(s Sample, learningRate float64) float64 { + pred := m.Probability(s.Features) + err := s.Label - pred + fs := s.Features.asSlice() + + m.Bias += learningRate * err + + for i := range fs { + m.Weights[i] += learningRate * err * fs[i] + } + + return pred +} + +// Train runs SGD over the samples for the given number of epochs. A +// few hundred clicks over a handful of epochs converges fine; this is +// cheap enough to run on startup or after a search session. +func Train(model *LinearModel, samples []Sample, epochs int, learningRate float64) { + for range epochs { + for _, s := range samples { + model.Update(s, learningRate) + } + } +} + +// matchStrength collapses the mutually-exclusive textual match tiers +// used throughout the current scorers into a single magnitude, so the +// model has one weight to learn instead of four overlapping constants. +func matchStrength(exact, prefix, wholeWord, substring bool) float64 { + switch { + case exact: + return 1.0 + case prefix: + return 0.6 + case wholeWord: + return 0.4 + case substring: + return 0.2 + default: + return 0.0 + } +} + +// SaveModel writes the model as JSON. Callers persist this to disk or +// the index meta table; the model is small (8 floats). +func SaveModel(w io.Writer, m LinearModel) error { + if err := json.NewEncoder(w).Encode(m); err != nil { + return fmt.Errorf("ranker: encode model: %w", err) + } + + return nil +} + +// LoadModel reads a model previously written by SaveModel. +func LoadModel(r io.Reader) (LinearModel, error) { + var m LinearModel + + if err := json.NewDecoder(r).Decode(&m); err != nil { + return LinearModel{}, fmt.Errorf("ranker: decode model: %w", err) + } + + return m, nil +} diff --git a/backend/explore/ranker_test.go b/backend/explore/ranker_test.go new file mode 100644 index 0000000..8bf215d --- /dev/null +++ b/backend/explore/ranker_test.go @@ -0,0 +1,102 @@ +package explore + +import ( + "bytes" + "math" + "testing" +) + +func TestDefaultModelScoreOrdering(t *testing.T) { + m := DefaultModel() + + // A popular, in-library exact match must outscore an obscure + // substring match. + strong := RankFeatures{NameMatch: 1.0, LogPopularity: 0.9, InLibrary: 1.0} + weak := RankFeatures{NameMatch: 0.2, LogPopularity: 0.1} + + if m.Score(strong) <= m.Score(weak) { + t.Errorf("strong candidate %.3f should outscore weak %.3f", + m.Score(strong), m.Score(weak)) + } +} + +func TestUpdateMovesTowardLabel(t *testing.T) { + m := DefaultModel() + + // A candidate the user repeatedly clicks should see its predicted + // probability rise after training on positive labels. + f := RankFeatures{NameMatch: 0.4, LogPopularity: 0.2} + + before := m.Probability(f) + + for range 50 { + m.Update(Sample{Features: f, Label: 1.0}, 0.1) + } + + after := m.Probability(f) + + if after <= before { + t.Errorf("probability should rise toward positive label: before=%.4f after=%.4f", + before, after) + } +} + +func TestUpdateLearnsNegative(t *testing.T) { + m := DefaultModel() + + f := RankFeatures{NameMatch: 0.9, LogPopularity: 0.9} + + before := m.Probability(f) + + // Shown repeatedly, never clicked — probability should fall. + for range 50 { + m.Update(Sample{Features: f, Label: 0.0}, 0.1) + } + + after := m.Probability(f) + + if after >= before { + t.Errorf("probability should fall toward negative label: before=%.4f after=%.4f", + before, after) + } +} + +func TestMatchStrengthTiers(t *testing.T) { + if matchStrength(true, false, false, false) != 1.0 { + t.Error("exact match should be 1.0") + } + + if matchStrength(false, false, false, false) != 0.0 { + t.Error("no match should be 0.0") + } + + // Tiers must be strictly ordered. + exact := matchStrength(true, false, false, false) + prefix := matchStrength(false, true, false, false) + word := matchStrength(false, false, true, false) + sub := matchStrength(false, false, false, true) + + if !(exact > prefix && prefix > word && word > sub) { + t.Errorf("tiers not strictly ordered: %v %v %v %v", exact, prefix, word, sub) + } +} + +func TestModelRoundTrip(t *testing.T) { + m := DefaultModel() + m.Bias = 0.123 + m.Weights[0] = 0.777 + + var buf bytes.Buffer + if err := SaveModel(&buf, m); err != nil { + t.Fatalf("SaveModel: %v", err) + } + + got, err := LoadModel(&buf) + if err != nil { + t.Fatalf("LoadModel: %v", err) + } + + if math.Abs(got.Bias-m.Bias) > 1e-9 || math.Abs(got.Weights[0]-m.Weights[0]) > 1e-9 { + t.Errorf("round trip mismatch: got %+v want %+v", got, m) + } +} diff --git a/backend/explore/searchindex.go b/backend/explore/searchindex.go index b71024c..86f9988 100644 --- a/backend/explore/searchindex.go +++ b/backend/explore/searchindex.go @@ -3,16 +3,17 @@ package explore import ( "context" "encoding/json" + "errors" "fmt" "log/slog" - "math" - "net/http" + "sort" + "strconv" "strings" "sync" - "sync/atomic" "time" "github.com/wailsapp/wails/v2/pkg/runtime" + "golang.org/x/sync/singleflight" "yellowjacket/backend/database" "yellowjacket/backend/events" @@ -20,36 +21,64 @@ import ( // Index build parameters. const ( - // indexTier1Interval is the minimum time between Tier 1 - // (sitewide top lists) refreshes. Cheap — 12 API calls. - indexTier1Interval = 7 * 24 * time.Hour - - // indexTier2Interval is the minimum time between Tier 2/4 - // (discography) refreshes. Incremental — only new artists. - indexTier2Interval = 30 * 24 * time.Hour - - // indexTopArtists is the number of artists to fetch per range - // from the LB sitewide endpoint. - indexTopArtists = 1000 - - // indexMaxRGs is the ceiling for release groups per artist. - // Top-popularity artists get their full discography. + // indexMaxRGs is the number of release groups fetched per + // artist by the API discography path (new library artists). indexMaxRGs = 50 - // indexMinRGs is the floor for release groups per artist. - // Even the least popular indexed artist gets a couple of albums. - indexMinRGs = 2 - - // indexMaxRecs is the ceiling for recordings per artist. + // indexMaxRecs is the number of recordings fetched per artist + // by the API discography path (new library artists). indexMaxRecs = 200 - // indexMinRecs is the floor for recordings per artist. - indexMinRecs = 5 - // indexMinPopularity is the minimum listen count for an entry // to be indexed. Cuts noise from long-tail entries. indexMinPopularity = 50 + // championPopThreshold is the popularity floor for a row to join + // the champion FTS. Rows below it (unless owned) can never win a + // generic short-prefix query, whose ranking is popularity-dominated, + // so excluding them is lossless for those queries. ~90k rows. + championPopThreshold = 10000 + + // genericMaxTokenLen classifies a query as "generic" when its + // longest token is at most this many runes. Such queries ("the", + // "a", "u2") match a huge slice of the index with no selective term, + // so they route to the champion tier; anything with a longer token + // is selective enough that the full index is already fast. + genericMaxTokenLen = 3 + + // Typo-tolerant rescue (see fuzzyRescue). When ordinary prefix-FTS + // retrieval returns fewer than fuzzyRescueThinHits rows — usually a + // misspelled query that shares no token prefix with any indexed name + // — a bigram-similarity pass over the champion set kicks in. + // fuzzyRescueMinLen guards against very short queries (too few + // bigrams to discriminate); fuzzyRescueMinScore is the minimum bigram + // Dice coefficient for a candidate to count as a plausible match. + fuzzyRescueThinHits = 3 + fuzzyRescueMinLen = 4 + fuzzyRescueMinScore = 0.34 + + // prefixCacheTTL bounds how long a memoised generic-query result is + // served before it is recomputed; prefixCacheMaxEntries caps the map. + prefixCacheTTL = 10 * time.Minute + prefixCacheMaxEntries = 512 + + // championBuiltKey marks in explore_index_meta that the champion FTS + // has been populated at least once, so it is trusted across restarts. + championBuiltKey = "champion_built" + + // localXrefReadyKey marks that PopulateLocalCrossReferences has synced + // the current library into the index at least once. The unchanged- + // library launch path skips the (write-heavy) re-sync when it is set; + // a scan re-runs the sync unconditionally, and an explore_index wipe + // clears explore_index_meta with it. + localXrefReadyKey = "local_xref_ready" + + // lyricsIndexReadyKey marks that the lyrics FTS has been rebuilt from + // the library at least once. It is kept in sync incrementally by the + // LRCLIB backfill thereafter, so the unchanged-library launch path + // skips the redundant full rebuild when it is set. + lyricsIndexReadyKey = "lyrics_index_ready" + // indexBatchSize is the number of rows per INSERT transaction. indexBatchSize = 100 @@ -57,20 +86,10 @@ const ( // indexer's dedicated rate limiter (LB allows 30/10s). indexerRate = 3 - // indexProgressInterval is how often to log progress. - indexProgressInterval = 100 - // indexSimilarPerArtist is how many similar artists to store - // per library artist in similar_artist_map and to consider - // for Tier 4 discography expansion. + // per library artist in similar_artist_map. indexSimilarPerArtist = 20 - // indexPopularityExponent controls how steeply the per-artist - // budget scales with popularity. Lower = steeper curve. - // 0.3 means an artist with 1/10th the listens of the max gets - // ~50% of the budget, not 10%. - indexPopularityExponent = 0.3 - // labsBaseURL is the base URL for the ListenBrainz labs API. labsBaseURL = "https://labs.api.listenbrainz.org" @@ -119,11 +138,14 @@ type SearchIndexResult struct { InLibrary bool `json:"inLibrary"` IsSimilar bool `json:"isSimilar"` - // DiscogFetched marks an artist row as having had its full - // discography (release groups + recordings) fetched by the - // indexer pipeline. Only set on artist entity_type entries. - // Used by indexedArtistMBIDs() to skip already-processed - // artists in tier 2/3. + // DiscogFetched marks a row as having had its full MusicBrainz + // metadata fetched. For artist rows it means the discography + // (release groups + recordings) was pulled by the indexer + // pipeline, and indexedArtistMBIDs() uses it to skip already- + // processed artists in tier 2/3. For release_group rows it means + // LookupReleaseGroup already did its one-time MB enrichment (e.g. + // secondary_types), so the album page's background lookup fires at + // most once per RG instead of on every visit. DiscogFetched bool `json:"-"` // Local library cross-reference (0 if not owned). @@ -149,14 +171,13 @@ type lbSitewideArtist struct { } // SearchIndex maintains a local SQLite FTS5 index of popular -// albums and tracks from ListenBrainz. The index is built in the -// background on startup across multiple tiers: +// albums and tracks. The index is populated in the background from +// the MetaBrainz dumps (see dumpimport.go) and patched incrementally +// via the ListenBrainz API: // -// - Tier 1: sitewide top lists (instant, <5s) -// - Tier 2: sitewide artists' full discographies (background, ~16min) -// - Tier 3: library artists' full discographies (background, ~4min) -// - Tier 4: similar artists to library artists (background, ~24min) -// - Tier 5: organic growth from user browsing (ongoing, free) +// - Initial build: listens dump (popularity) + canonical dump (catalog) +// - Post-scan: API discographies for new library artists +// - Ongoing: organic growth from user browsing (AddFromCache) type SearchIndex struct { db *database.DB lb *ListenBrainzClient @@ -167,14 +188,36 @@ type SearchIndex struct { cancel context.CancelFunc done chan struct{} - mu sync.RWMutex - ready bool - maxListens int // highest artist listen count seen, for scaling + mu sync.RWMutex + ready bool + + // championReady is set once the champion FTS (see + // RebuildChampionIndex) has been populated; championBuilding guards + // against overlapping rebuilds. Both are protected by mu. + championReady bool + championBuilding bool + + // prefixCache memoises the raw index hits for generic short-prefix + // queries — the expensive ones — so repeats are served from memory. + prefixCacheMu sync.Mutex + prefixCache map[string]prefixCacheEntry + + // discogSF dedupes concurrent lazy discography fetches for the same + // artist — the detail page fires its top-tracks, top-releases, and + // similar-artists requests in parallel, so without this the same + // artist would be fetched several times at once on first view. + discogSF singleflight.Group // Build status tracking — read by GetIndexStatus for the UI. buildStatus IndexStatus } +// prefixCacheEntry is one memoised generic-query result. +type prefixCacheEntry struct { + hits []SearchIndexResult + created time.Time +} + // TierStatus represents the state of a single index tier. type TierStatus struct { Name string `json:"name"` @@ -245,105 +288,109 @@ func (si *SearchIndex) SetContext(ctx context.Context) { }() } -// IndexNewArtists indexes only library artists that are not yet in the -// search index. This is the lightweight post-scan path — no tier -// machinery, no freshness checks, no sitewide/similar artist logic. -// Just finds library artists with MBIDs missing from the index and -// fetches their discographies + images. -func (si *SearchIndex) IndexNewArtists(ctx context.Context) { - si.mu.Lock() - if si.cancel != nil { - // Full build already running — it will pick up new artists. - si.mu.Unlock() - +// EnsureArtistDiscography lazily fetches an artist's top release groups +// and recordings from ListenBrainz and persists them into the index — +// the first time the artist is actually needed (e.g. their detail page +// opens). It is a no-op once the artist is marked discog_fetched, so +// each artist costs at most one set of API calls, ever; every later +// view is served from the index with no network. This replaces the old +// post-scan sweep that eagerly fetched every library artist on startup. +// +// Concurrent calls for the same artist collapse into one via discogSF, +// since the detail page fires several requests for the same artist at +// once. Safe to call synchronously from a request handler: bounded work +// on its own rate-limited client. +func (si *SearchIndex) EnsureArtistDiscography(ctx context.Context, artistMBID string) { + if si.lb == nil || artistMBID == "" { return } - si.done = make(chan struct{}) - si.mu.Unlock() + if si.artistDiscogFetched(artistMBID) { + return + } - buildCtx, cancel := context.WithCancel(ctx) + _, _, _ = si.discogSF.Do(artistMBID, func() (any, error) { + // Re-check under the singleflight: a sibling call may have just + // finished fetching this artist while we were queued. + if si.artistDiscogFetched(artistMBID) { + return nil, nil + } - si.mu.Lock() - si.cancel = cancel - si.mu.Unlock() + indexLB := NewListenBrainzClient( + NewRateLimiterN(indexerRate), si.lb.cache, si.logger.WithGroup("indexer"), + ) - go func() { - defer func() { - si.mu.Lock() - si.cancel = nil - si.mu.Unlock() + si.indexOneArtist(ctx, indexLB, lbSitewideArtist{ + ArtistMBID: artistMBID, + ArtistName: si.artistDisplayName(artistMBID), + }) - close(si.done) - - // Mark ready if we indexed anything, so search works - // while the full tier build is pending. - si.MarkReadyIfPopulated() - }() - - si.indexNewLibraryArtists(buildCtx) - }() + return nil, nil + }) } -// indexNewLibraryArtists finds library artists with MBIDs that are not -// in the index and fetches their discographies. -func (si *SearchIndex) indexNewLibraryArtists(ctx context.Context) { - indexed := si.indexedArtistMBIDs() - libraryMBIDs := si.getLibraryArtistMBIDs() +// artistDiscogFetched reports whether the artist row already has its +// discography fetched, so EnsureArtistDiscography can skip the network. +func (si *SearchIndex) artistDiscogFetched(mbid string) bool { + rows, err := si.db.QueryContext( + "SELECT 1 FROM explore_index WHERE entity_type = 'artist' AND mbid = ? AND discog_fetched = 1 LIMIT 1", + mbid, + ) + if err != nil { + return false + } - var newArtists []lbSitewideArtist + defer func() { _ = rows.Close() }() - for _, mbid := range libraryMBIDs { - if !indexed[mbid] { - // Look up the artist name from the DB. + return rows.Next() +} + +// artistDisplayName resolves a human-readable name for an artist MBID, +// preferring the index title, then the local library, then the MBID +// itself. Used to seed the discography fetch's artist entry. +func (si *SearchIndex) artistDisplayName(mbid string) string { + for _, q := range []string{ + "SELECT title FROM explore_index WHERE entity_type = 'artist' AND mbid = ? AND title != '' AND title != mbid LIMIT 1", + "SELECT name FROM artists WHERE mbid = ? AND name != '' LIMIT 1", + } { + rows, err := si.db.QueryContext(q, mbid) + if err != nil { + continue + } + + if rows.Next() { var name string - - rows, err := si.db.QueryContext( - "SELECT name FROM artists WHERE mbid = ? LIMIT 1", mbid, - ) - if err != nil { - continue - } - - if !rows.Next() { + if err := rows.Scan(&name); err == nil && name != "" { _ = rows.Close() - continue + return name } + } - if err := rows.Scan(&name); err != nil { - _ = rows.Close() + _ = rows.Close() + } - continue - } + return mbid +} - _ = rows.Close() +// PersistSimilarArtists stores ListenBrainz similar-artist results into +// the similar_artist_map so future lookups are served locally instead of +// re-hitting the labs API on every artist-page view. +func (si *SearchIndex) PersistSimilarArtists(sourceMBID string, similar []LBSimilarArtist) { + if len(similar) == 0 { + return + } - newArtists = append(newArtists, lbSitewideArtist{ - ArtistMBID: mbid, - ArtistName: name, - }) + wire := make([]lbSimilarArtistWire, len(similar)) + for i, s := range similar { + wire[i] = lbSimilarArtistWire{ + ArtistMBID: s.ArtistMBID, + Name: s.Name, + Score: int(s.Score), } } - if len(newArtists) == 0 { - si.logger.Info("search index: no new library artists to index") - - return - } - - si.logger.Info("search index: indexing new library artists", - "count", len(newArtists), - ) - - indexLimiter := NewRateLimiterN(indexerRate) - indexLB := NewListenBrainzClient(indexLimiter, si.lb.cache, si.logger.WithGroup("indexer")) - - si.indexArtistDiscographies(ctx, indexLB, newArtists, "new-artists", true) - - si.logger.Info("search index: new library artists indexed", - "count", len(newArtists), - ) + si.storeSimilarArtists(sourceMBID, wire) } // StartBuild launches the background index build goroutine. @@ -373,9 +420,13 @@ func (si *SearchIndex) StartBuild(ctx context.Context) { si.mu.Unlock() close(si.done) + + // The index rows (and their popularities) may have changed, so + // refresh the champion tier once the build settles. + si.scheduleChampionRebuild() }() - si.build(buildCtx) + si.runDumpBuild(buildCtx) }() } @@ -462,7 +513,7 @@ func (si *SearchIndex) refreshStatusCounts() { var lastBuilt string metaRow, err := si.db.QueryContext( - "SELECT value FROM explore_index_meta WHERE key = 'tier5_built'", + "SELECT value FROM explore_index_meta WHERE key = 'dump_import_done'", ) if err == nil { defer func() { _ = metaRow.Close() }() @@ -510,9 +561,7 @@ func (si *SearchIndex) setTierStatus(name, state string, total, completed int) { si.emitStatus() } -// setTierError marks a tier as errored. -// -//nolint:unused // kept for future per-tier failure surfacing. +// setTierError marks a build stage as errored. func (si *SearchIndex) setTierError(name, errMsg string) { si.mu.Lock() @@ -542,12 +591,6 @@ func (si *SearchIndex) emitStatus() { status.Building = si.cancel != nil si.mu.RUnlock() - si.logger.Info("emitting index status event", - "building", status.Building, - "tiers", len(status.Tiers), - "ready", status.Ready, - ) - runtime.EventsEmit(si.runtimeCtx, events.IndexStatusChanged, status) } @@ -785,7 +828,7 @@ func (si *SearchIndex) LookupReleaseGroupByMBID(mbid string) *SearchIndexResult rows, err := si.db.QueryContext( `SELECT title, artist_name, artist_mbid, popularity, listener_count, primary_type, secondary_types, release_date, - in_library, COALESCE(local_release_group_id, 0) + in_library, COALESCE(local_release_group_id, 0), discog_fetched FROM explore_index WHERE mbid = ? AND entity_type = 'release_group' LIMIT 1`, mbid, @@ -808,7 +851,7 @@ func (si *SearchIndex) LookupReleaseGroupByMBID(mbid string) *SearchIndexResult if err := rows.Scan( &r.Title, &r.ArtistName, &r.ArtistMBID, &r.Popularity, &r.ListenerCount, &r.PrimaryType, &r.SecondaryTypes, &r.ReleaseDate, - &r.InLibrary, &r.LocalReleaseGroupID, + &r.InLibrary, &r.LocalReleaseGroupID, &r.DiscogFetched, ); err != nil { return nil } @@ -816,6 +859,28 @@ func (si *SearchIndex) LookupReleaseGroupByMBID(mbid string) *SearchIndexResult return &r } +// PersistReleaseGroupLookup writes the MB-enriched metadata for a +// single release group back into the index and marks it DiscogFetched +// so LookupReleaseGroup's background enrichment runs at most once per +// RG. Popularity is left at 0 here — upsertBatch keeps the higher of +// old/new, so the dump-provided popularity is never clobbered. +func (si *SearchIndex) PersistReleaseGroupLookup(rg *MBReleaseGroup) { + if rg == nil || rg.MBID == "" { + return + } + + si.upsertBatch([]SearchIndexResult{{ + EntityType: "release_group", + MBID: rg.MBID, + Title: rg.Title, + ArtistName: rg.ArtistCredit, + PrimaryType: rg.PrimaryType, + SecondaryTypes: strings.Join(rg.SecondaryTypes, ","), + ReleaseDate: rg.FirstReleaseDate, + DiscogFetched: true, + }}) +} + // TopRecordingsByArtist returns the most popular recordings for an // artist MBID from the index, ordered by popularity descending. // Falls back to returning entries without popularity data if there @@ -958,6 +1023,12 @@ func (si *SearchIndex) ExactMatches(query string, perCategory int) []SearchIndex perCategory = 3 } + // UNION of two equality lookups rather than `LOWER(title) = ? OR + // LOWER(artist_name) = ?`. The OR form forces SQLite to scan all + // ~240k rows; splitting into two equalities lets each branch seek + // its partial expression index (idx_explore_title_lower / + // idx_explore_artist_lower). Ordering is done in Go below since + // an ORDER BY here would also defeat the index seek. rows, err := si.db.QueryContext(` SELECT entity_type, mbid, title, artist_name, artist_mbid, popularity, listener_count, duration, primary_type, @@ -968,9 +1039,18 @@ func (si *SearchIndex) ExactMatches(query string, perCategory int) []SearchIndex COALESCE(local_release_group_id, 0), COALESCE(local_recording_id, 0) FROM explore_index - WHERE (LOWER(title) = ? OR LOWER(artist_name) = ?) - AND popularity > 0 - ORDER BY entity_type, popularity DESC + WHERE LOWER(title) = ? AND popularity > 0 + UNION + SELECT entity_type, mbid, title, artist_name, artist_mbid, + popularity, listener_count, duration, primary_type, + secondary_types, release_date, caa_release_mbid, + release_name, artist_type, country, disambiguation, + sort_name, in_library, is_similar, + COALESCE(local_artist_id, 0), + COALESCE(local_release_group_id, 0), + COALESCE(local_recording_id, 0) + FROM explore_index + WHERE LOWER(artist_name) = ? AND popularity > 0 `, q, q) if err != nil { return nil @@ -978,14 +1058,7 @@ func (si *SearchIndex) ExactMatches(query string, perCategory int) []SearchIndex defer func() { _ = rows.Close() }() - // Group by entity type and cap at perCategory each, ordered by - // popularity desc because the SQL `ORDER BY entity_type, popularity DESC` - // gives us entity-type buckets already sorted within each. - buckets := map[string][]SearchIndexResult{ - "artist": nil, - "release_group": nil, - "recording": nil, - } + var matches []SearchIndexResult for rows.Next() { var r SearchIndexResult @@ -1005,9 +1078,8 @@ func (si *SearchIndex) ExactMatches(query string, perCategory int) []SearchIndex // and release groups, match on either title or artist name // — that way "miley cyrus" surfaces both the artist and // her recordings. - qLower := strings.ToLower(q) - titleMatch := strings.ToLower(r.Title) == qLower - artistMatch := strings.ToLower(r.ArtistName) == qLower + titleMatch := strings.ToLower(r.Title) == q + artistMatch := strings.ToLower(r.ArtistName) == q if r.EntityType == "artist" && !titleMatch { continue @@ -1017,6 +1089,24 @@ func (si *SearchIndex) ExactMatches(query string, perCategory int) []SearchIndex continue } + matches = append(matches, r) + } + + // Sort by popularity desc so the per-category cap keeps the most + // popular entries (the old SQL ORDER BY guaranteed this; UNION + // does not preserve order). + sort.SliceStable(matches, func(i, j int) bool { + return matches[i].Popularity > matches[j].Popularity + }) + + // Group by entity type, capping at perCategory each. + buckets := map[string][]SearchIndexResult{ + "artist": nil, + "release_group": nil, + "recording": nil, + } + + for _, r := range matches { bucket := buckets[r.EntityType] if len(bucket) >= perCategory { continue @@ -1037,7 +1127,7 @@ func (si *SearchIndex) ExactMatches(query string, perCategory int) []SearchIndex // Search queries the local FTS5 index and returns matches ordered // by relevance (popularity-blended). Returns nil when the index // hasn't finished its initial build. -func (si *SearchIndex) Search(query string, limit int) []SearchIndexResult { +func (si *SearchIndex) Search(ctx context.Context, query string, limit int) []SearchIndexResult { if !si.IsReady() { return nil } @@ -1051,7 +1141,310 @@ func (si *SearchIndex) Search(query string, limit int) []SearchIndexResult { return nil } - rows, err := si.db.QueryContext(` + generic := isGenericQuery(query) + + // Generic short-prefix queries are the slow ones; serve repeats from + // the in-memory cache and route the rest through the champion tier. + if generic { + if hits, ok := si.prefixCacheGet(ftsQuery, limit); ok { + return hits + } + } + + hits := si.runSearch(ctx, ftsQuery, limit, generic) + + // Typo-tolerant fallback: a thin result usually means the query is + // misspelled and matched no token prefix. Rescue with bigram + // similarity over the champion set before giving up. Skipped once a + // search is superseded (ctx cancelled) — the result is discarded + // anyway. + if len(hits) < fuzzyRescueThinHits && ctx.Err() == nil { + hits = si.appendFuzzyRescue(ctx, query, limit, hits) + } + + // Cache only fully-formed generic results — never a partial list from + // a superseded (cancelled) query. + if generic && ctx.Err() == nil { + si.prefixCachePut(ftsQuery, limit, hits) + } + + return hits +} + +// appendFuzzyRescue runs the bigram rescue pass and merges its hits into +// the existing (thin) result, de-duplicated by MBID and preserving the +// original hits first. Returns the original slice unchanged when rescue +// finds nothing new. +func (si *SearchIndex) appendFuzzyRescue( + ctx context.Context, + query string, + limit int, + hits []SearchIndexResult, +) []SearchIndexResult { + rescued := si.fuzzyRescue(ctx, query, limit) + if len(rescued) == 0 { + return hits + } + + seen := make(map[string]struct{}, len(hits)+len(rescued)) + for _, h := range hits { + seen[h.MBID] = struct{}{} + } + + for _, r := range rescued { + if _, dup := seen[r.MBID]; dup { + continue + } + + seen[r.MBID] = struct{}{} + hits = append(hits, r) + + if len(hits) >= limit { + break + } + } + + return hits +} + +// fuzzyRescue is the typo-tolerant fallback for Search. It scans the +// champion set (high-popularity + owned rows) and scores each candidate +// by character-bigram overlap (Dice) against its title and artist name, +// so a misspelled query still surfaces its intended entity even though it +// shares no token prefix with any indexed name. Returns up to `limit` +// hits above fuzzyRescueMinScore, ordered by fuzzy score with popularity +// as a tie-break. Bounded work that fires only on the rare thin query, +// so it never touches the fast common path. +func (si *SearchIndex) fuzzyRescue( + ctx context.Context, + query string, + limit int, +) []SearchIndexResult { + qSet := fuzzyBigrams(query) + if len([]rune(fuzzyNormalize(query))) < fuzzyRescueMinLen || len(qSet) == 0 { + return nil + } + + // Champion membership predicate, mirrored from rebuildChampionIndex: + // a row below the popularity floor that isn't owned can't be a + // meaningful popular match, so scoring it would only add noise. + rows, err := si.db.QueryContextWith(ctx, ` + SELECT id, title, artist_name, popularity + FROM explore_index + WHERE popularity >= ? OR in_library = 1 + `, championPopThreshold) + if err != nil { + if !errors.Is(err, context.Canceled) { + si.logger.Warn("fuzzy rescue scan error", "error", err) + } + + return nil + } + + defer func() { _ = rows.Close() }() + + type scored struct { + id int64 + score float64 + pop int + } + + var candidates []scored + + scanned := 0 + + for rows.Next() { + // Periodically honour cancellation: a superseded search shouldn't + // keep scoring tens of thousands of rows. + scanned++ + if scanned%4096 == 0 && ctx.Err() != nil { + return nil + } + + var ( + id int64 + title string + artist string + pop int + ) + + if err := rows.Scan(&id, &title, &artist, &pop); err != nil { + continue + } + + score := diceCoefficient(qSet, fuzzyBigrams(title)) + if s := diceCoefficient(qSet, fuzzyBigrams(artist)); s > score { + score = s + } + + if score >= fuzzyRescueMinScore { + candidates = append(candidates, scored{id: id, score: score, pop: pop}) + } + } + + if len(candidates) == 0 { + return nil + } + + // Best bigram match first; popularity breaks near-ties so the more + // prominent entity wins when two names are equally close. + sort.SliceStable(candidates, func(i, j int) bool { + if candidates[i].score != candidates[j].score { + return candidates[i].score > candidates[j].score + } + + return candidates[i].pop > candidates[j].pop + }) + + if len(candidates) > limit { + candidates = candidates[:limit] + } + + ids := make([]int64, len(candidates)) + for i, c := range candidates { + ids[i] = c.id + } + + return si.rowsByIDs(ctx, ids) +} + +// rowsByIDs loads full index rows for the given explore_index ids, +// returned in the same order as `ids` (rows are keyed by id in a map, +// then re-emitted in input order). Used by fuzzyRescue to hydrate the +// lightweight candidate ids it scored into full search results. +func (si *SearchIndex) rowsByIDs(ctx context.Context, ids []int64) []SearchIndexResult { + if len(ids) == 0 { + return nil + } + + placeholders := make([]string, len(ids)) + args := make([]any, len(ids)) + + for i, id := range ids { + placeholders[i] = "?" + args[i] = id + } + + query := ` + SELECT id, entity_type, mbid, title, artist_name, artist_mbid, + popularity, listener_count, duration, primary_type, + secondary_types, release_date, caa_release_mbid, + release_name, artist_type, country, disambiguation, + sort_name, in_library, is_similar, + COALESCE(local_artist_id, 0), + COALESCE(local_release_group_id, 0), + COALESCE(local_recording_id, 0) + FROM explore_index + WHERE id IN (` + strings.Join(placeholders, ",") + `)` + + rows, err := si.db.QueryContextWith(ctx, query, args...) + if err != nil { + if !errors.Is(err, context.Canceled) { + si.logger.Warn("fuzzy rescue hydrate error", "error", err) + } + + return nil + } + + defer func() { _ = rows.Close() }() + + byID := make(map[int64]SearchIndexResult, len(ids)) + + for rows.Next() { + var ( + id int64 + r SearchIndexResult + ) + + if err := rows.Scan( + &id, &r.EntityType, &r.MBID, &r.Title, &r.ArtistName, &r.ArtistMBID, + &r.Popularity, &r.ListenerCount, &r.Duration, &r.PrimaryType, + &r.SecondaryTypes, &r.ReleaseDate, &r.CAAReleaseMBID, + &r.ReleaseName, &r.ArtistType, &r.Country, &r.Disambiguation, + &r.SortName, &r.InLibrary, &r.IsSimilar, + &r.LocalArtistID, &r.LocalReleaseGroupID, &r.LocalRecordingID, + ); err != nil { + continue + } + + byID[id] = r + } + + out := make([]SearchIndexResult, 0, len(ids)) + + for _, id := range ids { + if r, ok := byID[id]; ok { + out = append(out, r) + } + } + + return out +} + +// runSearch dispatches an FTS query to the champion tier when the query +// is generic and the champion index is ready, falling back to the full +// index when the champion result is thin (few high-popularity matches) +// or the champion index isn't built yet. +func (si *SearchIndex) runSearch( + ctx context.Context, + ftsQuery string, + limit int, + generic bool, +) []SearchIndexResult { + if generic { + if si.championIsReady() { + cStart := time.Now() + hits := si.queryFTS(ctx, "explore_champion_fts", ftsQuery, limit) + + if len(hits) >= limit { + si.logger.Info("search path: champion", + "fts", ftsQuery, + "rows", len(hits), + "elapsed", time.Since(cStart).Round(time.Millisecond), + ) + + return hits + } + + // Thin champion result: the full index is authoritative, and a + // query the champion couldn't fill matches few rows there too. + si.logger.Info("search path: champion thin -> full", + "fts", ftsQuery, + "champion_rows", len(hits), + "champion_elapsed", time.Since(cStart).Round(time.Millisecond), + ) + } else { + // First generic query before the champion exists — build it in + // the background so the next one is fast. + si.logger.Info("search path: champion not ready -> full", "fts", ftsQuery) + si.scheduleChampionRebuild() + } + } + + fStart := time.Now() + hits := si.queryFTS(ctx, "explore_index_fts", ftsQuery, limit) + + if generic { + si.logger.Info("search path: full", + "fts", ftsQuery, + "rows", len(hits), + "elapsed", time.Since(fStart).Round(time.Millisecond), + ) + } + + return hits +} + +// queryFTS runs the popularity-blended ranking query against the named +// FTS table (the full index or the champion tier — identical schema, so +// the only difference is how many rows MATCH). +func (si *SearchIndex) queryFTS( + ctx context.Context, + ftsTable, ftsQuery string, + limit int, +) []SearchIndexResult { + // ftsTable is a trusted in-package constant, never user input. + sqlText := fmt.Sprintf(` SELECT i.entity_type, i.mbid, i.title, i.artist_name, i.artist_mbid, i.popularity, i.listener_count, i.duration, i.primary_type, i.secondary_types, i.release_date, @@ -1062,20 +1455,26 @@ func (si *SearchIndex) Search(query string, limit int) []SearchIndexResult { COALESCE(i.local_release_group_id, 0), COALESCE(i.local_recording_id, 0) FROM explore_index i - JOIN explore_index_fts f ON f.rowid = i.id - WHERE explore_index_fts MATCH ? - ORDER BY bm25(explore_index_fts, 3.0, 1.0, 0.5) + JOIN %[1]s f ON f.rowid = i.id + WHERE %[1]s MATCH ? + ORDER BY bm25(%[1]s, 3.0, 1.0, 0.5) - (ln(i.popularity + 1) * 1.5) - (i.in_library * 3.0) - (i.is_similar * 1.5) LIMIT ? - `, ftsQuery, limit) + `, ftsTable) + + rows, err := si.db.QueryContextWith(ctx, sqlText, ftsQuery, limit) if err != nil { - si.logger.Warn("search index query error", - "query", query, - "ftsQuery", ftsQuery, - "error", err, - ) + // A superseded search has its context cancelled on purpose; that + // is expected shutdown, not a query fault, so don't warn on it. + if !errors.Is(err, context.Canceled) { + si.logger.Warn("search index query error", + "table", ftsTable, + "ftsQuery", ftsQuery, + "error", err, + ) + } return nil } @@ -1107,6 +1506,176 @@ func (si *SearchIndex) Search(query string, limit int) []SearchIndexResult { return results } +// isGenericQuery reports whether a query has no selective (long) token +// and therefore matches a broad slice of the index — the case the +// champion tier and prefix cache exist to accelerate. +func isGenericQuery(query string) bool { + words := splitWords(query) + if len(words) == 0 { + return false + } + + for _, w := range words { + if len([]rune(w)) > genericMaxTokenLen { + return false + } + } + + return true +} + +// --------------------------------------------------------------------------- +// Champion tier +// --------------------------------------------------------------------------- + +// championIsReady reports whether the champion FTS is populated. +func (si *SearchIndex) championIsReady() bool { + si.mu.RLock() + defer si.mu.RUnlock() + + return si.championReady +} + +// scheduleChampionRebuild rebuilds the champion FTS in the background, +// unless one is already running or the main index isn't ready yet. The +// rebuild runs in a single write transaction, so concurrent searches +// keep reading the previous champion snapshot (WAL) until it commits. +func (si *SearchIndex) scheduleChampionRebuild() { + si.mu.Lock() + if si.championBuilding || !si.ready { + si.mu.Unlock() + + return + } + + si.championBuilding = true + si.mu.Unlock() + + go func() { + // Use a stable context, not any per-search one, so a superseded + // search can't cancel the rebuild midway. + ctx := si.runtimeCtx + if ctx == nil { + ctx = context.Background() + } + + start := time.Now() + err := si.rebuildChampionIndex(ctx) + + si.mu.Lock() + si.championBuilding = false + // Keep a previously-good champion usable if a refresh failed; mark + // ready on success. + si.championReady = si.championReady || err == nil + si.mu.Unlock() + + // The champion set changed, so any memoised generic results are + // stale. + si.clearPrefixCache() + + if err != nil { + si.logger.Warn("champion index rebuild failed", "error", err) + + return + } + + si.setMeta(championBuiltKey, time.Now().UTC().Format(time.RFC3339)) + si.logger.Info("champion index rebuilt", + "elapsed", time.Since(start).Round(time.Millisecond), + ) + }() +} + +// rebuildChampionIndex repopulates the champion FTS from the high- +// popularity and owned rows of explore_index. A row below the +// popularity floor can never place in a generic query's top results +// (ranking there is popularity-dominated), so dropping it is lossless +// for those queries; owned rows are always kept for their in-library +// boost. +func (si *SearchIndex) rebuildChampionIndex(ctx context.Context) error { + tx, err := si.db.BeginTx() + if err != nil { + return fmt.Errorf("begin champion rebuild: %w", err) + } + + defer func() { _ = tx.Rollback() }() + + if _, err := tx.ExecContext(ctx, + `INSERT INTO explore_champion_fts(explore_champion_fts) VALUES('delete-all')`, + ); err != nil { + return fmt.Errorf("clear champion fts: %w", err) + } + + if _, err := tx.ExecContext(ctx, ` + INSERT INTO explore_champion_fts(rowid, title, artist_name, aliases) + SELECT id, title, artist_name, aliases + FROM explore_index + WHERE popularity >= ? OR in_library = 1 + `, championPopThreshold); err != nil { + return fmt.Errorf("populate champion fts: %w", err) + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit champion rebuild: %w", err) + } + + return nil +} + +// --------------------------------------------------------------------------- +// Prefix result cache +// --------------------------------------------------------------------------- + +func prefixCacheKey(ftsQuery string, limit int) string { + return strconv.Itoa(limit) + "\x00" + ftsQuery +} + +func (si *SearchIndex) prefixCacheGet(ftsQuery string, limit int) ([]SearchIndexResult, bool) { + si.prefixCacheMu.Lock() + defer si.prefixCacheMu.Unlock() + + key := prefixCacheKey(ftsQuery, limit) + + entry, ok := si.prefixCache[key] + if !ok { + return nil, false + } + + if time.Since(entry.created) > prefixCacheTTL { + delete(si.prefixCache, key) + + return nil, false + } + + return entry.hits, true +} + +func (si *SearchIndex) prefixCachePut(ftsQuery string, limit int, hits []SearchIndexResult) { + si.prefixCacheMu.Lock() + defer si.prefixCacheMu.Unlock() + + if si.prefixCache == nil { + si.prefixCache = make(map[string]prefixCacheEntry) + } + + // Generic keys are few and cheap to recompute; when the cap is hit, + // drop everything rather than track per-entry LRU state. + if len(si.prefixCache) >= prefixCacheMaxEntries { + si.prefixCache = make(map[string]prefixCacheEntry) + } + + si.prefixCache[prefixCacheKey(ftsQuery, limit)] = prefixCacheEntry{ + hits: hits, + created: time.Now(), + } +} + +func (si *SearchIndex) clearPrefixCache() { + si.prefixCacheMu.Lock() + si.prefixCache = nil + si.prefixCacheMu.Unlock() +} + // --------------------------------------------------------------------------- // FTS query building // --------------------------------------------------------------------------- @@ -1159,643 +1728,6 @@ func isWordChar(r rune) bool { r >= 0x80 } -// --------------------------------------------------------------------------- -// Background build — orchestrator -// --------------------------------------------------------------------------- - -func (si *SearchIndex) build(ctx context.Context) { - start := time.Now() - - si.logger.Info("search index build starting") - - // Initialize tier status for the UI. - si.mu.Lock() - si.buildStatus = IndexStatus{ - Building: true, - Tiers: []TierStatus{ - {Name: "Sitewide Top Lists", State: "pending"}, - {Name: "Sitewide Discographies", State: "pending"}, - {Name: "Library Artists", State: "pending"}, - {Name: "Similar Artists", State: "pending"}, - {Name: "Popularity Backfill", State: "pending"}, - }, - } - si.mu.Unlock() - - // Mark ready from existing rows so search works during the build. - si.MarkReadyIfPopulated() - - indexLimiter := NewRateLimiterN(indexerRate) - indexLB := NewListenBrainzClient(indexLimiter, si.lb.cache, si.logger.WithGroup("indexer")) - - // Tier 1: sitewide instant — refresh weekly (12 calls, <5s). - tier1Fresh := si.isMetaFresh("tier1_built", indexTier1Interval) - - var sitewideArtists []lbSitewideArtist - - if tier1Fresh { - si.logger.Info("search index: Tier 1 fresh, loading cached artists") - si.setTierStatus("Sitewide Top Lists", "skipped", 0, 0) - - sitewideArtists = si.loadCachedSitewideArtists() - } else { - si.setTierStatus("Sitewide Top Lists", "running", 12, 0) - sitewideArtists = si.buildTier1Sitewide(ctx, indexLB) - - if ctx.Err() != nil { - return - } - - si.setMeta("tier1_built", time.Now().UTC().Format(time.RFC3339)) - si.setTierStatus("Sitewide Top Lists", "complete", 12, 12) - } - - si.mu.Lock() - si.ready = true - si.mu.Unlock() - - si.refreshStatusCounts() - si.logger.Info("search index: Tier 1 complete (sitewide instant)") - - // Tiers 2-4: discographies — refresh monthly, incremental. - // Only fetch discographies for artists not already indexed. - // Each tier's timestamp is tracked independently so progress - // survives app restarts mid-build. - tier2Fresh := si.isMetaFresh("tier2_built", indexTier2Interval) - tier3Fresh := si.isMetaFresh("tier3_built", indexTier2Interval) - tier4Fresh := si.isMetaFresh("tier4_built", indexTier2Interval) - - // Repair pass: runs unconditionally (outside the tier-fresh - // gate) so gaps from previous incomplete runs are healed even - // when the tier timestamps claim the build is fresh. Any - // artist row with discog_fetched=0 — including those created - // by AddFromCache during a frontend visit, or left over from - // a crash mid-build — gets its full discography pulled here. - { - indexedForRepair := si.indexedArtistMBIDs() - if unindexed := si.unindexedArtistEntries(indexedForRepair); len(unindexed) > 0 { - si.logger.Info("search index: repair pass starting", - "unindexedArtists", len(unindexed), - ) - - si.setTierStatus("Repair Discographies", "running", len(unindexed), 0) - si.indexArtistDiscographies(ctx, indexLB, unindexed, "Repair", false) - - if ctx.Err() != nil { - return - } - - si.setTierStatus("Repair Discographies", "complete", len(unindexed), len(unindexed)) - si.refreshStatusCounts() - si.logger.Info("search index: repair pass complete", - "artists", len(unindexed), - ) - } else { - si.setTierStatus("Repair Discographies", "skipped", 0, 0) - } - } - - if tier2Fresh && tier3Fresh && tier4Fresh { - si.logger.Info("search index: discographies fresh, skipping Tiers 2-4") - } else { - indexed := si.indexedArtistMBIDs() - - var libraryMBIDs []string - - // Tier 2: sitewide artists' discographies (incremental). - if tier2Fresh { - si.logger.Info("search index: Tier 2 fresh, skipping") - si.setTierStatus("Sitewide Discographies", "skipped", 0, 0) - } else { - newSitewide := filterUnindexed(sitewideArtists, indexed) - - si.logger.Info("search index: Tier 2 starting", - "total", len(sitewideArtists), - "alreadyIndexed", len(sitewideArtists)-len(newSitewide), - "new", len(newSitewide), - ) - - si.setTierStatus("Sitewide Discographies", "running", len(newSitewide), 0) - si.indexArtistDiscographies(ctx, indexLB, newSitewide, "Tier 2", false) - - if ctx.Err() != nil { - return - } - - si.setMeta("tier2_built", time.Now().UTC().Format(time.RFC3339)) - si.setTierStatus( - "Sitewide Discographies", - "complete", - len(newSitewide), - len(newSitewide), - ) - si.refreshStatusCounts() - si.logger.Info("search index: Tier 2 complete (sitewide discographies)") - } - - // Tier 3: library artists' discographies (incremental). - if tier3Fresh { - si.logger.Info("search index: Tier 3 fresh, skipping") - si.setTierStatus("Library Artists", "skipped", 0, 0) - } else { - si.setTierStatus("Library Artists", "running", 0, 0) - indexed = si.indexedArtistMBIDs() - libraryMBIDs = si.buildTier3Library(ctx, indexLB, sitewideArtists, indexed) - - if ctx.Err() != nil { - return - } - - si.setMeta("tier3_built", time.Now().UTC().Format(time.RFC3339)) - si.setTierStatus("Library Artists", "complete", len(libraryMBIDs), len(libraryMBIDs)) - si.logger.Info("search index: Tier 3 complete (library discographies)") - } - - // Tier 4: similar artists (incremental). - if tier4Fresh { - si.logger.Info("search index: Tier 4 fresh, skipping") - si.setTierStatus("Similar Artists", "skipped", 0, 0) - } else { - if libraryMBIDs == nil { - // Tier 3 was skipped, load library MBIDs for Tier 4. - libraryMBIDs = si.getLibraryArtistMBIDs() - } - - indexed = si.indexedArtistMBIDs() - si.setTierStatus("Similar Artists", "running", len(libraryMBIDs), 0) - si.buildTier4Similar(ctx, indexLB, libraryMBIDs, indexed) - - if ctx.Err() != nil { - return - } - - si.setMeta("tier4_built", time.Now().UTC().Format(time.RFC3339)) - si.setTierStatus("Similar Artists", "complete", len(libraryMBIDs), len(libraryMBIDs)) - si.refreshStatusCounts() - si.logger.Info("search index: Tier 4 complete (similar artists)") - } - } - - // Tier 5: backfill popularity for entities with missing data. - tier5Fresh := si.isMetaFresh("tier5_built", indexTier1Interval) - if tier5Fresh { - si.setTierStatus("Popularity Backfill", "skipped", 0, 0) - } else { - si.setTierStatus("Popularity Backfill", "running", 0, 0) - si.buildTier5Popularity(ctx, indexLB) - - if ctx.Err() != nil { - return - } - - si.setMeta("tier5_built", time.Now().UTC().Format(time.RFC3339)) - si.setTierStatus("Popularity Backfill", "complete", 0, 0) - si.logger.Info("search index: Tier 5 complete (popularity backfill)") - } - - si.mu.Lock() - si.buildStatus.Building = false - si.mu.Unlock() - - // Populate local library cross-reference columns so every - // read path has O(1) access to "do I own this?" - si.PopulateLocalCrossReferences() - - si.refreshStatusCounts() - si.logger.Info("search index build complete", "elapsed", time.Since(start).Round(time.Second)) -} - -// --------------------------------------------------------------------------- -// Tier 1: sitewide instant -// --------------------------------------------------------------------------- - -// buildTier1Sitewide fetches top artists, recordings, and release -// groups across all time ranges and inserts them. Returns the -// deduplicated artist list for Tier 2. -func (si *SearchIndex) buildTier1Sitewide( - ctx context.Context, - lb *ListenBrainzClient, -) []lbSitewideArtist { - ranges := []string{"all_time", "this_year", "this_month", "this_week"} - artistMap := make(map[string]lbSitewideArtist) - - for _, r := range ranges { - if ctx.Err() != nil { - break - } - - // Artists. - artists, err := si.fetchSitewideArtists(ctx, r) - if err != nil { - si.logger.Warn("search index: sitewide artists failed", "range", r, "error", err) - - continue - } - - for _, a := range artists { - if _, exists := artistMap[a.ArtistMBID]; !exists { - artistMap[a.ArtistMBID] = a - } - } - - // Recordings. - recs := si.fetchSitewideRecordings(ctx, lb, r) - si.upsertSearchResults(recs) - - // Release groups. - rgs := si.fetchSitewideReleaseGroups(ctx, lb, r) - si.upsertSearchResults(rgs) - } - - // Insert all artists and track max popularity. - artists := make([]lbSitewideArtist, 0, len(artistMap)) - - maxL := 0 - - for _, a := range artistMap { - artists = append(artists, a) - - if a.ListenCount > maxL { - maxL = a.ListenCount - } - } - - si.mu.Lock() - si.maxListens = maxL - si.mu.Unlock() - - si.upsertArtists(artists) - - si.logger.Info("search index: Tier 1 indexed", - "artists", len(artists), - ) - - return artists -} - -func (si *SearchIndex) fetchSitewideArtists( - ctx context.Context, timeRange string, -) ([]lbSitewideArtist, error) { - url := fmt.Sprintf( - "%s/1/stats/sitewide/artists?count=%d&range=%s", - listenBrainzBaseURL, indexTopArtists, timeRange, - ) - - req, err := newLBRequest(ctx, url) - if err != nil { - return nil, err - } - - resp, err := si.lb.http.Do(req) - if err != nil { - return nil, err - } - - defer func() { _ = resp.Body.Close() }() - - var envelope struct { - Payload struct { - Artists []lbSitewideArtist `json:"artists"` - } `json:"payload"` - } - - if err := json.NewDecoder(resp.Body).Decode(&envelope); err != nil { - return nil, err - } - - return envelope.Payload.Artists, nil -} - -func (si *SearchIndex) fetchSitewideRecordings( - ctx context.Context, lb *ListenBrainzClient, timeRange string, -) []SearchIndexResult { - url := fmt.Sprintf( - "%s/1/stats/sitewide/recordings?count=%d&range=%s", - listenBrainzBaseURL, indexTopArtists, timeRange, - ) - - body, err := lb.doGet(ctx, url) - if err != nil { - si.logger.Warn("search index: sitewide recordings failed", - "range", timeRange, "error", err, - ) - - return nil - } - - var envelope struct { - Payload struct { - Recordings []struct { - RecordingMBID string `json:"recording_mbid"` - TrackName string `json:"track_name"` - ArtistName string `json:"artist_name"` - ArtistMBIDs []string `json:"artist_mbids"` - ListenCount int `json:"listen_count"` - } `json:"recordings"` - } `json:"payload"` - } - - if err := json.Unmarshal(body, &envelope); err != nil { - si.logger.Warn("search index: sitewide recordings unmarshal", - "range", timeRange, "error", err, - ) - - return nil - } - - var results []SearchIndexResult - - for _, r := range envelope.Payload.Recordings { - if r.ListenCount < indexMinPopularity { - continue - } - - artistMBID := "" - if len(r.ArtistMBIDs) > 0 { - artistMBID = r.ArtistMBIDs[0] - } - - results = append(results, SearchIndexResult{ - EntityType: "recording", - MBID: r.RecordingMBID, - Title: r.TrackName, - ArtistName: r.ArtistName, - ArtistMBID: artistMBID, - // Popularity intentionally 0 — backfilled by Tier 5 with the - // uncapped total_listen_count from the popularity API. - }) - } - - return results -} - -func (si *SearchIndex) fetchSitewideReleaseGroups( - ctx context.Context, lb *ListenBrainzClient, timeRange string, -) []SearchIndexResult { - url := fmt.Sprintf( - "%s/1/stats/sitewide/release-groups?count=%d&range=%s", - listenBrainzBaseURL, indexTopArtists, timeRange, - ) - - body, err := lb.doGet(ctx, url) - if err != nil { - si.logger.Warn("search index: sitewide release groups failed", - "range", timeRange, "error", err, - ) - - return nil - } - - var envelope struct { - Payload struct { - ReleaseGroups []struct { - ReleaseGroupMBID string `json:"release_group_mbid"` - ReleaseGroupName string `json:"release_group_name"` - ArtistName string `json:"artist_name"` - ArtistMBIDs []string `json:"artist_mbids"` - ListenCount int `json:"listen_count"` - } `json:"release_groups"` - } `json:"payload"` - } - - if err := json.Unmarshal(body, &envelope); err != nil { - si.logger.Warn("search index: sitewide release groups unmarshal", - "range", timeRange, "error", err, - ) - - return nil - } - - var results []SearchIndexResult - - for _, r := range envelope.Payload.ReleaseGroups { - if r.ListenCount < indexMinPopularity { - continue - } - - artistMBID := "" - if len(r.ArtistMBIDs) > 0 { - artistMBID = r.ArtistMBIDs[0] - } - - results = append(results, SearchIndexResult{ - EntityType: "release_group", - MBID: r.ReleaseGroupMBID, - Title: r.ReleaseGroupName, - ArtistName: r.ArtistName, - ArtistMBID: artistMBID, - // Popularity intentionally 0 — backfilled by Tier 5. - }) - } - - return results -} - -// --------------------------------------------------------------------------- -// Tier 2: sitewide artists' full discographies -// --------------------------------------------------------------------------- - -// --------------------------------------------------------------------------- -// Tier 3: library artists' full discographies -// --------------------------------------------------------------------------- - -// buildTier3Library matches local library artist names against -// sitewide artists by name to get MBIDs, then indexes their -// discographies. Returns the resolved MBIDs for Tier 4. -func (si *SearchIndex) buildTier3Library( - ctx context.Context, - lb *ListenBrainzClient, - sitewideArtists []lbSitewideArtist, - indexed map[string]bool, -) []string { - // Build a name→artist map from sitewide (lowercased). - nameMap := make(map[string]lbSitewideArtist, len(sitewideArtists)) - for _, a := range sitewideArtists { - nameMap[strings.ToLower(a.ArtistName)] = a - } - - // Also build from existing index entries (catches organic adds). - rows, err := si.db.QueryContext(` - SELECT DISTINCT artist_name, artist_mbid - FROM explore_index - WHERE entity_type = 'artist' AND artist_mbid != '' - `) - if err == nil { - defer func() { _ = rows.Close() }() - - for rows.Next() { - var name, mbid string - if err := rows.Scan(&name, &mbid); err == nil { - lower := strings.ToLower(name) - if _, exists := nameMap[lower]; !exists { - nameMap[lower] = lbSitewideArtist{ - ArtistMBID: mbid, - ArtistName: name, - } - } - } - } - } - - // Read local library artists — prefer direct MBIDs from tags, - // fall back to name matching against the sitewide/index map. - libRows, err := si.db.QueryContext( - "SELECT DISTINCT name, mbid FROM artists", - ) - if err != nil { - si.logger.Warn("search index: library artists query failed", "error", err) - - return nil - } - - defer func() { _ = libRows.Close() }() - - var matched []lbSitewideArtist - - var resolvedMBIDs []string - - for libRows.Next() { - var name string - - var mbidPtr *string - - if err := libRows.Scan(&name, &mbidPtr); err != nil { - continue - } - - // Direct MBID from tags — most reliable. - if mbidPtr != nil && *mbidPtr != "" { - mbid := *mbidPtr - resolvedMBIDs = append(resolvedMBIDs, mbid) - - if !indexed[mbid] { - matched = append(matched, lbSitewideArtist{ - ArtistMBID: mbid, - ArtistName: name, - }) - } - - continue - } - - // Fall back to name matching. - normalized := strings.ToLower(name) - if idx := strings.Index(normalized, " feat."); idx >= 0 { - normalized = normalized[:idx] - } - - if idx := strings.Index(normalized, " ft."); idx >= 0 { - normalized = normalized[:idx] - } - - normalized = strings.TrimSpace(normalized) - - if a, ok := nameMap[normalized]; ok { - resolvedMBIDs = append(resolvedMBIDs, a.ArtistMBID) - - if !indexed[a.ArtistMBID] { - matched = append(matched, a) - } - } - } - - if len(matched) > 0 { - si.indexArtistDiscographies(ctx, lb, matched, "Tier 3", true) - - // Mark all Tier 3 entries as in_library. - si.markInLibrary(matched) - } - - si.logger.Info("search index: Tier 3 matched", - "libraryArtists", len(resolvedMBIDs), - "newToIndex", len(matched), - ) - - return resolvedMBIDs -} - -// --------------------------------------------------------------------------- -// Tier 4: similar artists to library artists -// --------------------------------------------------------------------------- - -func (si *SearchIndex) buildTier4Similar( - ctx context.Context, - lb *ListenBrainzClient, - libraryMBIDs []string, - indexed map[string]bool, -) { - if len(libraryMBIDs) == 0 { - return - } - - // Fetch similar artists in batches of seeds, fanned out - // concurrently. The labs multi-seed POST form is broken and - // returns mis-grouped results, so fetchSimilarArtistsBatch - // actually makes one GET per seed (see its comment). Batches - // keep the log output bounded. - newArtistMap := make(map[string]lbSitewideArtist) - - for i := 0; i < len(libraryMBIDs); i += similarArtistsBatchSize { - if ctx.Err() != nil { - break - } - - end := i + similarArtistsBatchSize - if end > len(libraryMBIDs) { - end = len(libraryMBIDs) - } - - batch := libraryMBIDs[i:end] - grouped := si.fetchSimilarArtistsBatch(ctx, lb, batch) - - // Persist similarity relationships per seed. - for _, seedMBID := range batch { - similar := grouped[seedMBID] - si.storeSimilarArtists(seedMBID, similar) - - for _, s := range similar { - if !indexed[s.ArtistMBID] { - if _, exists := newArtistMap[s.ArtistMBID]; !exists { - newArtistMap[s.ArtistMBID] = lbSitewideArtist{ - ArtistMBID: s.ArtistMBID, - ArtistName: s.Name, - } - } - } - } - } - - si.logger.Info("search index: Tier 4 similar batch complete", - "batch", (i/similarArtistsBatchSize)+1, - "totalBatches", (len(libraryMBIDs)+similarArtistsBatchSize-1)/similarArtistsBatchSize, - "processed", end, - ) - } - - if len(newArtistMap) == 0 { - return - } - - newArtists := make([]lbSitewideArtist, 0, len(newArtistMap)) - for _, a := range newArtistMap { - newArtists = append(newArtists, a) - } - - si.logger.Info("search index: Tier 4 discovered", - "newArtists", len(newArtists), - ) - - // Index artist rows only (no discography) — similar artists - // are mostly obscure and their per-track data rarely surfaces - // in searches. Discographies are fetched on-demand when the - // user drills into an artist detail view. This saves ~2 API - // calls per artist (~10K total for Tier 4). - si.indexArtistDiscographies(ctx, lb, newArtists, "Tier 4", false) - - // Mark all Tier 4 entries as similar. - si.markSimilar(newArtists) -} - type lbSimilarArtistWire struct { ArtistMBID string `json:"artist_mbid"` Name string `json:"name"` @@ -1880,196 +1812,16 @@ func (si *SearchIndex) fetchSimilarArtistsBatch( // Shared: index artist discographies // --------------------------------------------------------------------------- -// indexArtistDiscographies fetches top release groups and recordings -// for each artist and inserts them into the index. Used by Tiers 2-4. -func (si *SearchIndex) indexArtistDiscographies( - ctx context.Context, - lb *ListenBrainzClient, - artists []lbSitewideArtist, - tier string, - forceMax bool, -) { - if len(artists) == 0 { - return - } - - // Prefetch artist metadata in batches of 1000 — one GET per batch - // instead of one per artist. Populates type, country, and writes - // artist rows with these fields up-front. This runs to completion - // before the discography loop so indexOneArtist can read from - // the batch results via the metadata cache. - si.prefetchArtistMetadata(ctx, lb, artists) - - sem := make(chan struct{}, indexerRate) - - var wg sync.WaitGroup - - var completed atomic.Int32 - - for _, a := range artists { - if ctx.Err() != nil { - break - } - - sem <- struct{}{} - - wg.Add(1) - - go func(artist lbSitewideArtist) { - defer func() { - <-sem - wg.Done() - }() - - si.indexOneArtist(ctx, lb, artist, forceMax) - - n := completed.Add(1) - - // Update tier status for the UI. - tierName := tier // "Tier 2" or "Tier 3" - switch tier { - case "Tier 2": - tierName = "Sitewide Discographies" - case "Tier 3": - tierName = "Library Artists" - } - - si.setTierStatus(tierName, "running", len(artists), int(n)) - - if int(n)%indexProgressInterval == 0 { - si.logger.Info("search index progress", - "tier", tier, - "completed", n, - "total", len(artists), - "pct", fmt.Sprintf("%.0f%%", float64(n)/float64(len(artists))*100), - ) - } - }(a) - } - - wg.Wait() - - si.logger.Info("search index: discographies indexed", - "tier", tier, - "artists", len(artists), - ) -} - -// prefetchArtistMetadata batch-fetches artist metadata from LB's -// /1/metadata/artist/ endpoint and writes artist rows up-front. -// This populates type and country for all artists in a single -// GET per 1000-artist chunk, rather than requiring per-artist -// MB calls. Aliases, disambiguation, and sort_name still come -// from the per-artist MB fetch during image resolution — unless -// we can synthesize a satisfactory cached response. -// -// The prefetch also pre-populates the mb:artist-rels cache with -// a synthesized envelope derived from LB data, so the per-artist -// MB call is skipped entirely for artists where we have LB data. -// Aliases/disambiguation won't be available, but type/country/ -// name and wikidata QID (for image resolution) will be. -func (si *SearchIndex) prefetchArtistMetadata( - ctx context.Context, - lb *ListenBrainzClient, - artists []lbSitewideArtist, -) { - const batchSize = 1000 - - var ( - processed atomic.Int32 - batchWG sync.WaitGroup - ) - - // Build a lookup from mbid to artist name. - nameByMBID := make(map[string]string, len(artists)) - for _, a := range artists { - nameByMBID[a.ArtistMBID] = a.ArtistName - } - - mbids := make([]string, 0, len(artists)) - for _, a := range artists { - if a.ArtistMBID != "" { - mbids = append(mbids, a.ArtistMBID) - } - } - - for i := 0; i < len(mbids); i += batchSize { - if ctx.Err() != nil { - return - } - - end := i + batchSize - if end > len(mbids) { - end = len(mbids) - } - - batch := mbids[i:end] - - batchWG.Add(1) - - go func(chunk []string) { - defer batchWG.Done() - - meta, err := lb.BatchArtistMetadata(ctx, chunk) - if err != nil || len(meta) == 0 { - return - } - - // Upsert artist rows with the LB-sourced fields. - entries := make([]SearchIndexResult, 0, len(meta)) - - for mbid, m := range meta { - name := nameByMBID[mbid] - if name == "" { - name = m.Name - } - - entries = append(entries, SearchIndexResult{ - EntityType: "artist", - MBID: mbid, - Title: name, - ArtistName: name, - ArtistMBID: mbid, - ArtistType: m.Type, - Country: m.Country, - }) - - // Synthesize an MB artist-rels cache entry so - // fetchMBRels skips the per-artist network call. - // Contains only the wikidata URL (for image - // resolution) and the type/country/name that - // GetArtistDetails reads. Aliases and - // disambiguation are empty — those come from - // an on-demand MB lookup later if needed. - if si.artistImg != nil { - si.artistImg.PreloadArtistRels(mbid, m) - } - } - - si.upsertBatch(entries) - processed.Add(int32(len(entries))) - }(batch) - } - - batchWG.Wait() - - si.logger.Info("search index: prefetched artist metadata", - "artists", len(mbids), - "processed", processed.Load(), - ) -} - func (si *SearchIndex) indexOneArtist( ctx context.Context, lb *ListenBrainzClient, artist lbSitewideArtist, - forceMax bool, ) { if ctx.Err() != nil { return } - rgLimit, recLimit := si.scaledLimits(artist.ListenCount, forceMax) + rgLimit, recLimit := indexMaxRGs, indexMaxRecs // Run LB discography fetches and MB artist image resolution // concurrently — they use different rate limiters so they @@ -2276,122 +2028,6 @@ func (si *SearchIndex) fetchTopRecordings( return results } -// --------------------------------------------------------------------------- -// Tier 5: popularity backfill -// --------------------------------------------------------------------------- - -// popularityBatchSize is the number of MBIDs per LB popularity request. -// LB accepts up to 1000 per POST call. -const popularityBatchSize = 1000 - -// buildTier5Popularity batch-queries LB popularity for every entity in the -// index that has popularity = 0, then writes results back via BackfillPopularity. -func (si *SearchIndex) buildTier5Popularity(ctx context.Context, lb *ListenBrainzClient) { - type entityKind struct { - entityType string - fetch func(context.Context, []string) (map[string]PopularityData, error) - } - - kinds := []entityKind{ - {"artist", lb.ArtistPopularity}, - {"release_group", lb.ReleaseGroupPopularity}, - {"recording", lb.RecordingPopularity}, - } - - for _, kind := range kinds { - if ctx.Err() != nil { - return - } - - mbids := si.mbidsWithoutPopularity(kind.entityType) - if len(mbids) == 0 { - si.logger.Info("search index: Tier 5 skipped (no missing popularity)", - "entityType", kind.entityType) - - continue - } - - batches := chunkStrings(mbids, popularityBatchSize) - - si.logger.Info("search index: Tier 5 starting", - "entityType", kind.entityType, - "entities", len(mbids), - "batches", len(batches), - ) - - var filled int - - sem := make(chan struct{}, indexerRate) - - for i, batch := range batches { - if ctx.Err() != nil { - return - } - - sem <- struct{}{} - - pops, err := kind.fetch(ctx, batch) - - <-sem - - if err != nil { - si.logger.Warn("search index: Tier 5 batch failed", - "entityType", kind.entityType, - "batch", i+1, - "error", err, - ) - - continue - } - - si.BackfillPopularity(pops) - filled += len(pops) - - if (i+1)%indexProgressInterval == 0 { - si.logger.Info("search index: Tier 5 progress", - "entityType", kind.entityType, - "batches", fmt.Sprintf("%d/%d", i+1, len(batches)), - "filled", filled, - ) - } - } - - si.logger.Info("search index: Tier 5 entity type done", - "entityType", kind.entityType, - "filled", filled, - "batches", len(batches), - ) - } -} - -// mbidsWithoutPopularity returns all MBIDs in the index for the given -// entity type that have popularity = 0. -func (si *SearchIndex) mbidsWithoutPopularity(entityType string) []string { - rows, err := si.db.QueryContext( - "SELECT mbid FROM explore_index WHERE entity_type = ? AND popularity = 0", - entityType, - ) - if err != nil { - si.logger.Warn("search index: failed to query unpopulated MBIDs", - "entityType", entityType, "error", err) - - return nil - } - - defer func() { _ = rows.Close() }() - - var mbids []string - - for rows.Next() { - var m string - if err := rows.Scan(&m); err == nil { - mbids = append(mbids, m) - } - } - - return mbids -} - // chunkStrings splits a slice into chunks of at most size n. func chunkStrings(s []string, n int) [][]string { var chunks [][]string @@ -2423,48 +2059,6 @@ func chunkStrings(s []string, n int) [][]string { // empty values, and numeric fields use "highest wins" for popularity/ // listener_count/duration so older richer data survives refreshes. -// upsertArtists is a convenience wrapper for Tier 1 sitewide artists. -// Writes them as artist rows with popularity=0 — the actual popularity -// (uncapped total_listen_count) is filled in by Tier 5 via the -// POST popularity API. The sitewide listen_count is a capped -// different metric we don't want to mix in. -func (si *SearchIndex) upsertArtists(artists []lbSitewideArtist) { - batch := make([]SearchIndexResult, 0, indexBatchSize) - - for _, a := range artists { - batch = append(batch, SearchIndexResult{ - EntityType: "artist", - MBID: a.ArtistMBID, - Title: a.ArtistName, - ArtistName: a.ArtistName, - ArtistMBID: a.ArtistMBID, - // Popularity intentionally 0 — backfilled by Tier 5. - }) - - if len(batch) >= indexBatchSize { - si.upsertBatch(batch) - batch = batch[:0] - } - } - - if len(batch) > 0 { - si.upsertBatch(batch) - } -} - -// upsertSearchResults chunks large batches into transactions of -// indexBatchSize and flushes each via upsertBatch. -func (si *SearchIndex) upsertSearchResults(results []SearchIndexResult) { - for i := 0; i < len(results); i += indexBatchSize { - end := i + indexBatchSize - if end > len(results) { - end = len(results) - } - - si.upsertBatch(results[i:end]) - } -} - // upsertBatch writes a batch of SearchIndexResult entries to the index // inside a single transaction. This is the ONE function that all // writes go through. All fields are handled — callers don't need to @@ -2590,77 +2184,6 @@ func (si *SearchIndex) upsertBatch(entries []SearchIndexResult) { // Helpers // --------------------------------------------------------------------------- -// unindexedArtistEntries returns artist rows that exist in the -// index but haven't had their full discography fetched yet -// (discog_fetched = 0). Excludes anything in the indexed set. -// Used by the repair pass to heal gaps from AddFromCache or -// from previous incomplete runs. -func (si *SearchIndex) unindexedArtistEntries(indexed map[string]bool) []lbSitewideArtist { - rows, err := si.db.QueryContext(` - SELECT mbid, title, popularity - FROM explore_index - WHERE entity_type = 'artist' AND discog_fetched = 0 - `) - if err != nil { - return nil - } - - defer func() { _ = rows.Close() }() - - var out []lbSitewideArtist - - for rows.Next() { - var ( - mbid string - name string - pop int - ) - - if err := rows.Scan(&mbid, &name, &pop); err != nil { - continue - } - - if indexed[mbid] { - continue - } - - out = append(out, lbSitewideArtist{ - ArtistMBID: mbid, - ArtistName: name, - ListenCount: pop, - }) - } - - return out -} - -// indexedArtistMBIDs returns artist MBIDs that have had their full -// discography fetched by the indexer pipeline. Used by tier 2/3 to -// skip artists already processed. Excludes artist rows that only -// got into the index via AddFromCache (frontend organic growth) — -// those are missing recordings and need a real indexer pass. -func (si *SearchIndex) indexedArtistMBIDs() map[string]bool { - rows, err := si.db.QueryContext( - "SELECT DISTINCT artist_mbid FROM explore_index WHERE entity_type = 'artist' AND discog_fetched = 1", - ) - if err != nil { - return nil - } - - defer func() { _ = rows.Close() }() - - result := make(map[string]bool) - - for rows.Next() { - var mbid string - if err := rows.Scan(&mbid); err == nil { - result[mbid] = true - } - } - - return result -} - // getLibraryArtistMBIDs returns MBIDs for all library artists that have one. // Used when Tier 3 was skipped but Tier 4 needs the library MBID list. func (si *SearchIndex) getLibraryArtistMBIDs() []string { @@ -2685,9 +2208,10 @@ func (si *SearchIndex) getLibraryArtistMBIDs() []string { return mbids } -func (si *SearchIndex) isMetaFresh(key string, maxAge time.Duration) bool { +// hasMeta reports whether a key exists in explore_index_meta. +func (si *SearchIndex) hasMeta(key string) bool { rows, err := si.db.QueryContext( - "SELECT value FROM explore_index_meta WHERE key = ?", key, + "SELECT 1 FROM explore_index_meta WHERE key = ?", key, ) if err != nil { return false @@ -2695,145 +2219,167 @@ func (si *SearchIndex) isMetaFresh(key string, maxAge time.Duration) bool { defer func() { _ = rows.Close() }() - if !rows.Next() { - return false - } - - var val string - if err := rows.Scan(&val); err != nil { - return false - } - - t, err := time.Parse(time.RFC3339, val) - if err != nil { - return false - } - - return time.Since(t) < maxAge + return rows.Next() } -// loadCachedSitewideArtists reads artist entries from the existing -// index when Tier 1 is fresh and doesn't need re-fetching. -func (si *SearchIndex) loadCachedSitewideArtists() []lbSitewideArtist { - rows, err := si.db.QueryContext(` - SELECT mbid, title, popularity - FROM explore_index - WHERE entity_type = 'artist' - ORDER BY popularity DESC - `) - if err != nil { - return nil - } - - defer func() { _ = rows.Close() }() - - var artists []lbSitewideArtist - - maxL := 0 - - for rows.Next() { - var a lbSitewideArtist - if err := rows.Scan(&a.ArtistMBID, &a.ArtistName, &a.ListenCount); err == nil { - artists = append(artists, a) - - if a.ListenCount > maxL { - maxL = a.ListenCount - } - } - } - - si.mu.Lock() - si.maxListens = maxL - si.mu.Unlock() - - return artists -} - -// filterUnindexed returns artists whose MBIDs are not in the -// indexed set. -func filterUnindexed(artists []lbSitewideArtist, indexed map[string]bool) []lbSitewideArtist { - var out []lbSitewideArtist - - for _, a := range artists { - if !indexed[a.ArtistMBID] { - out = append(out, a) - } - } - - return out -} - -// markInLibrary sets in_library=1 for all index entries whose -// artist_mbid matches one of the given artists. Also populates -// the local_artist_id cross-reference column. -func (si *SearchIndex) markInLibrary(artists []lbSitewideArtist) { - for _, a := range artists { - _, _ = si.db.ExecContext( - `UPDATE explore_index - SET in_library = 1, - local_artist_id = (SELECT id FROM artists WHERE mbid = ?) - WHERE artist_mbid = ?`, - a.ArtistMBID, a.ArtistMBID, - ) - } -} - -// PopulateLocalCrossReferences walks the library tables and updates -// explore_index rows to set local_*_id columns for any MBIDs that -// exist locally. Call after a library scan completes. +// PopulateLocalCrossReferences syncs the local library into the search +// index: every MB-verified library artist, release group, and recording +// is upserted into explore_index, flagged in_library with its local id. +// Call after a library scan completes. +// +// This is the local half of the index — it makes owned content +// searchable regardless of the dump's popularity floor, using only data +// already in the library tables (no API calls). upsertBatch's conflict +// rules mean dump-seeded rows are merely tagged in_library while +// below-floor owned entities are inserted fresh (popularity 0, to be +// filled later by incremental dumps or a lazy discography fetch). +// MBID-less local content is intentionally excluded — the explore index +// is MBID-keyed, and unmatched files are served by the library search. func (si *SearchIndex) PopulateLocalCrossReferences() { - // Artists. - if _, err := si.db.ExecContext(` - UPDATE explore_index - SET local_artist_id = ( - SELECT a.id FROM artists a - WHERE a.mbid = explore_index.mbid - ), - in_library = CASE - WHEN EXISTS (SELECT 1 FROM artists WHERE mbid = explore_index.mbid) - THEN 1 ELSE in_library - END - WHERE entity_type = 'artist' - AND mbid IN (SELECT mbid FROM artists WHERE mbid IS NOT NULL AND mbid != '') - `); err != nil { - si.logger.Warn("cross-ref: update artists failed", "error", err) + entries := si.collectLibraryEntities() + if len(entries) == 0 { + si.logger.Info("library sync: no MB-verified library entities to index") + si.setMeta(localXrefReadyKey, "1") + + return } - // Release groups. - if _, err := si.db.ExecContext(` - UPDATE explore_index - SET local_release_group_id = ( - SELECT rg.id FROM release_groups rg - WHERE rg.mbid = explore_index.mbid - ), - in_library = CASE - WHEN EXISTS (SELECT 1 FROM release_groups WHERE mbid = explore_index.mbid) - THEN 1 ELSE in_library - END - WHERE entity_type = 'release_group' - AND mbid IN (SELECT mbid FROM release_groups WHERE mbid IS NOT NULL AND mbid != '') - `); err != nil { - si.logger.Warn("cross-ref: update release groups failed", "error", err) + const batchSize = 1000 + for i := 0; i < len(entries); i += batchSize { + end := i + batchSize + if end > len(entries) { + end = len(entries) + } + + si.upsertBatch(entries[i:end]) + } + + si.setMeta(localXrefReadyKey, "1") + si.logger.Info("library sync: upserted library entities into index", "count", len(entries)) +} + +// collectLibraryEntities builds index entries for every library artist, +// release group, and recording that carries a MusicBrainz ID. Artist +// credit strings and the primary artist MBID are resolved from the local +// artist_credit tables so no network lookup is needed. +func (si *SearchIndex) collectLibraryEntities() []SearchIndexResult { + var entries []SearchIndexResult + + // Artists. + artistRows, err := si.db.QueryContext( + "SELECT id, name, mbid FROM artists WHERE mbid IS NOT NULL AND mbid != ''", + ) + if err == nil { + for artistRows.Next() { + var ( + id int64 + name, mbid string + ) + + if err := artistRows.Scan(&id, &name, &mbid); err != nil { + continue + } + + entries = append(entries, SearchIndexResult{ + EntityType: "artist", + MBID: mbid, + Title: name, + ArtistName: name, + ArtistMBID: mbid, + InLibrary: true, + LocalArtistID: id, + }) + } + + _ = artistRows.Close() + } else { + si.logger.Warn("library sync: query artists failed", "error", err) + } + + // Release groups. The correlated subquery picks the primary + // credited artist's MBID (if it has one). + rgRows, err := si.db.QueryContext(` + SELECT rg.id, rg.name, rg.mbid, COALESCE(ac.text, ''), + COALESCE(( + SELECT a.mbid FROM artist_credit_artist aca + JOIN artists a ON a.id = aca.artist_id + WHERE aca.credit_id = rg.album_artist_credit_id + AND a.mbid IS NOT NULL AND a.mbid != '' + LIMIT 1 + ), '') + FROM release_groups rg + LEFT JOIN artist_credit ac ON ac.id = rg.album_artist_credit_id + WHERE rg.mbid IS NOT NULL AND rg.mbid != '' + `) + if err == nil { + for rgRows.Next() { + var ( + id int64 + name, mbid, credit, artistMB string + ) + + if err := rgRows.Scan(&id, &name, &mbid, &credit, &artistMB); err != nil { + continue + } + + entries = append(entries, SearchIndexResult{ + EntityType: "release_group", + MBID: mbid, + Title: name, + ArtistName: credit, + ArtistMBID: artistMB, + InLibrary: true, + LocalReleaseGroupID: id, + }) + } + + _ = rgRows.Close() + } else { + si.logger.Warn("library sync: query release groups failed", "error", err) } // Recordings. - if _, err := si.db.ExecContext(` - UPDATE explore_index - SET local_recording_id = ( - SELECT r.id FROM recordings r - WHERE r.mbid = explore_index.mbid - ), - in_library = CASE - WHEN EXISTS (SELECT 1 FROM recordings WHERE mbid = explore_index.mbid) - THEN 1 ELSE in_library - END - WHERE entity_type = 'recording' - AND mbid IN (SELECT mbid FROM recordings WHERE mbid IS NOT NULL AND mbid != '') - `); err != nil { - si.logger.Warn("cross-ref: update recordings failed", "error", err) + recRows, err := si.db.QueryContext(` + SELECT r.id, r.name, r.mbid, COALESCE(ac.text, ''), + COALESCE(( + SELECT a.mbid FROM artist_credit_artist aca + JOIN artists a ON a.id = aca.artist_id + WHERE aca.credit_id = r.artist_credit_id + AND a.mbid IS NOT NULL AND a.mbid != '' + LIMIT 1 + ), '') + FROM recordings r + LEFT JOIN artist_credit ac ON ac.id = r.artist_credit_id + WHERE r.mbid IS NOT NULL AND r.mbid != '' + `) + if err == nil { + for recRows.Next() { + var ( + id int64 + name, mbid, credit, artistMB string + ) + + if err := recRows.Scan(&id, &name, &mbid, &credit, &artistMB); err != nil { + continue + } + + entries = append(entries, SearchIndexResult{ + EntityType: "recording", + MBID: mbid, + Title: name, + ArtistName: credit, + ArtistMBID: artistMB, + InLibrary: true, + LocalRecordingID: id, + }) + } + + _ = recRows.Close() + } else { + si.logger.Warn("library sync: query recordings failed", "error", err) } - si.logger.Info("cross-ref: populated local_*_id columns") + return entries } // storeSimilarArtists persists the similar artist relationships @@ -2867,17 +2413,6 @@ func (si *SearchIndex) storeSimilarArtists(sourceMBID string, similar []lbSimila _ = tx.Commit() } -// markSimilar sets is_similar=1 for all index entries whose -// artist_mbid matches one of the given artists. -func (si *SearchIndex) markSimilar(artists []lbSitewideArtist) { - for _, a := range artists { - _, _ = si.db.ExecContext( - "UPDATE explore_index SET is_similar = 1 WHERE artist_mbid = ?", - a.ArtistMBID, - ) - } -} - // PopularityData holds both listen count and listener count for a // single entity. Used by BackfillPopularity and the popularity // pipeline to pass both metrics together. @@ -2980,11 +2515,11 @@ func (si *SearchIndex) GetSimilarityScores(mbids []string) map[string]int { return result } -// InvalidateDiscographies clears the discography build timestamps -// so the next build re-runs Tiers 2-4. +// InvalidateDiscographies clears the dump-import completion marker so +// the next StartBuild re-runs the full dump import. func (si *SearchIndex) InvalidateDiscographies() { _, _ = si.db.ExecContext( - "DELETE FROM explore_index_meta WHERE key IN ('discog_built', 'tier2_built', 'tier3_built', 'tier4_built')", + "DELETE FROM explore_index_meta WHERE key = 'dump_import_done'", ) } @@ -2997,6 +2532,16 @@ func (si *SearchIndex) setMeta(key, value string) { } } +// deleteMeta removes a key from explore_index_meta, e.g. to invalidate a +// "ready" marker so the next launch re-runs a gated build step. +func (si *SearchIndex) deleteMeta(key string) { + if _, err := si.db.ExecContext( + "DELETE FROM explore_index_meta WHERE key = ?", key, + ); err != nil { + si.logger.Warn("search index: delete meta error", "key", key, "error", err) + } +} + // MarkReadyIfPopulated sets the index as ready for querying if it // already contains data from a previous build. Called eagerly at // service creation so the index is queryable before StartBuild runs. @@ -3006,57 +2551,31 @@ func (si *SearchIndex) MarkReadyIfPopulated() { return } - defer func() { _ = rows.Close() }() - + var count int if rows.Next() { - var count int - if err := rows.Scan(&count); err == nil && count > 0 { - si.mu.Lock() - si.ready = true - si.mu.Unlock() + _ = rows.Scan(&count) + } + // Release the connection before any further query: under a + // single-connection pool (tests) a nested query while these rows are + // open would deadlock. + _ = rows.Close() - si.logger.Info("search index: using existing index", "entries", count) - } + if count == 0 { + return + } + + // Trust a champion index built in a previous run; otherwise build it + // below now that the main index is known-ready. + championBuilt := si.hasMeta(championBuiltKey) + + si.mu.Lock() + si.ready = true + si.championReady = championBuilt + si.mu.Unlock() + + si.logger.Info("search index: using existing index", "entries", count) + + if !championBuilt { + si.scheduleChampionRebuild() } } - -// scaledLimits returns the number of release groups and recordings -// to index for an artist with the given listen count, scaled by -// popularity relative to the most popular artist in the index. -// If forceMax is true, returns the maximum limits regardless of -// popularity (used for library artists). -func (si *SearchIndex) scaledLimits(listenCount int, forceMax bool) (rgs, recs int) { - if forceMax { - return indexMaxRGs, indexMaxRecs - } - - si.mu.RLock() - maxL := si.maxListens - si.mu.RUnlock() - - if maxL <= 0 || listenCount <= 0 { - return indexMinRGs, indexMinRecs - } - - ratio := math.Pow(float64(listenCount)/float64(maxL), indexPopularityExponent) - - rgs = int(float64(indexMinRGs) + ratio*float64(indexMaxRGs-indexMinRGs)) - recs = int(float64(indexMinRecs) + ratio*float64(indexMaxRecs-indexMinRecs)) - - rgs = max(indexMinRGs, min(indexMaxRGs, rgs)) - recs = max(indexMinRecs, min(indexMaxRecs, recs)) - - return rgs, recs -} - -// newLBRequest creates an HTTP GET request with the LB User-Agent. -func newLBRequest(ctx context.Context, url string) (*http.Request, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return nil, err - } - - req.Header.Set("User-Agent", lbUserAgent) - - return req, nil -} diff --git a/backend/explore/types.go b/backend/explore/types.go index 48ad5b8..105869c 100644 --- a/backend/explore/types.go +++ b/backend/explore/types.go @@ -24,6 +24,7 @@ type TopResult struct { MBID string `json:"mbid"` Name string `json:"name"` ArtistCredit string `json:"artistCredit,omitempty"` // for tracks/albums + ArtistMBID string `json:"artistMbid,omitempty"` // for linking the artist subtitle IntentScore float64 `json:"intentScore"` // Artist-specific ArtistType string `json:"artistType,omitempty"` // "Group", "Person" @@ -31,8 +32,14 @@ type TopResult struct { // Album-specific PrimaryType string `json:"primaryType,omitempty"` Year string `json:"year,omitempty"` - // Track-specific - Length int `json:"length,omitempty"` + // Track-specific. ReleaseGroupMBID is resolved (from CAAReleaseMBID) + // so a track click can open its album page with the track highlighted, + // matching how tracks behave everywhere else. ReleaseName is the album + // title used for the album page header. + Length int `json:"length,omitempty"` + CAAReleaseMBID string `json:"caaReleaseMbid,omitempty"` + ReleaseGroupMBID string `json:"releaseGroupMbid,omitempty"` + ReleaseName string `json:"releaseName,omitempty"` // Library status — populated from index cross-reference columns. InLibrary bool `json:"inLibrary"` } @@ -64,8 +71,9 @@ type MBReleaseGroup struct { SecondaryTypes []string `json:"secondaryTypes,omitempty"` FirstReleaseDate string `json:"firstReleaseDate"` ArtistCredit string `json:"artistCredit"` - Score int `json:"-"` // MB search relevance, used for reranking - Popularity int `json:"popularity"` // raw LB listen count (0 if unknown) + ArtistMBID string `json:"artistMbid,omitempty"` // for linking the artist to its detail page + Score int `json:"-"` // MB search relevance, used for reranking + Popularity int `json:"popularity"` // raw LB listen count (0 if unknown) ListenerCount int `json:"listenerCount"` InLibrary bool `json:"inLibrary"` // true if the user owns this album LocalID int64 `json:"localId,omitempty"` // local release_group row ID @@ -85,15 +93,22 @@ type MBRelease struct { // MBRecording is a Wails-friendly projection of a MusicBrainz // recording. type MBRecording struct { - MBID string `json:"mbid"` - Title string `json:"title"` - Length int `json:"length"` - ArtistCredit string `json:"artistCredit"` - Score int `json:"score"` - Popularity int `json:"popularity"` // raw LB listen count (0 if unknown) - ListenerCount int `json:"listenerCount"` - InLibrary bool `json:"inLibrary"` // true if the user owns this recording - LocalID int64 `json:"localId,omitempty"` // local recording row ID + MBID string `json:"mbid"` + Title string `json:"title"` + Length int `json:"length"` + ArtistCredit string `json:"artistCredit"` + ArtistMBID string `json:"artistMbid,omitempty"` // for linking the artist to its detail page + Score int `json:"score"` + Popularity int `json:"popularity"` // raw LB listen count (0 if unknown) + ListenerCount int `json:"listenerCount"` + CAAReleaseMBID string `json:"caaReleaseMbid,omitempty"` // parent release, for album navigation + // ReleaseGroupMBID is resolved from CAAReleaseMBID so a track can + // link to its album page with the track highlighted, matching how + // tracks behave everywhere else. + ReleaseGroupMBID string `json:"releaseGroupMbid,omitempty"` + ReleaseName string `json:"releaseName,omitempty"` // album title + InLibrary bool `json:"inLibrary"` // true if the user owns this recording + LocalID int64 `json:"localId,omitempty"` // local recording row ID } // MBTrack is a Wails-friendly projection of a MusicBrainz track. @@ -119,6 +134,9 @@ type LBTopRecording struct { TrackName string `json:"trackName"` TotalListenCount int `json:"totalListenCount"` CAAReleaseMBID string `json:"caaReleaseMbid"` + // ReleaseGroupMBID is resolved from CAAReleaseMBID so a top-track + // row can link to its album page with the track highlighted. + ReleaseGroupMBID string `json:"releaseGroupMbid,omitempty"` ReleaseName string `json:"releaseName"` Length int `json:"length"` // milliseconds (from LB API) InLibrary bool `json:"inLibrary"` diff --git a/backend/library/artistcredit.go b/backend/library/artistcredit.go new file mode 100644 index 0000000..2ea65f4 --- /dev/null +++ b/backend/library/artistcredit.go @@ -0,0 +1,72 @@ +package library + +import ( + "strings" + + "yellowjacket/backend/metadata" +) + +// featuringSeparators are the credit join phrases that introduce a +// featured (non-primary) artist. Only true "featuring" markers are +// listed: separators like "&", "x", "with", and "," are deliberately +// excluded because they routinely appear inside real artist names +// (e.g. "Simon & Garfunkel", "Tyler, the Creator"). +var featuringSeparators = []string{ + " feat. ", " feat ", " featuring ", " ft. ", " ft ", +} + +// stripFeaturing returns the credit up to its first "featuring" marker, +// yielding the primary-artist portion of a credit string. "Lana Del +// Rey ft. Sean Lennon" becomes "Lana Del Rey"; a credit with no marker +// is returned unchanged (trimmed). Matching is case-insensitive. +func stripFeaturing(credit string) string { + lower := strings.ToLower(credit) + + cut := -1 + + for _, sep := range featuringSeparators { + if i := strings.Index(lower, sep); i >= 0 && (cut < 0 || i < cut) { + cut = i + } + } + + if cut < 0 { + return strings.TrimSpace(credit) + } + + return strings.TrimSpace(credit[:cut]) +} + +// primaryArtist resolves the single canonical artist a track credit +// should map to, plus that artist's MusicBrainz ID. A file tags its +// ARTIST as a full credit string ("Lana Del Rey ft. Sean Lennon") but +// carries only one MUSICBRAINZ_ARTISTID — the primary artist's. Storing +// the whole credit as an artist entity, and stamping the primary MBID on +// it, is what produced duplicate, mis-titled artists (one MBID fanned +// out across many rows); instead we resolve the primary artist's clean +// name here and keep the full credit only as the artist_credit text. +// +// The clean name comes from the album-artist tag when the track resolves +// to the same MBID as the album artist (the common "Album Artist feat. +// Guest" case, where ALBUMARTIST is the authoritative name). Otherwise +// the featured clause is stripped from the credit string. +func primaryArtist(tags *metadata.TrackMetadata) (name, mbid string) { + mbid = tags.ArtistMBID + if mbid == "" { + mbid = tags.AlbumArtistMBID + } + + if tags.ArtistMBID != "" && + tags.ArtistMBID == tags.AlbumArtistMBID && + tags.AlbumArtist != "" { + name = tags.AlbumArtist + } else { + name = stripFeaturing(tags.Artist) + } + + if name == "" { + name = "Unknown Artist" + } + + return name, mbid +} diff --git a/backend/library/artistcredit_test.go b/backend/library/artistcredit_test.go new file mode 100644 index 0000000..2333f8c --- /dev/null +++ b/backend/library/artistcredit_test.go @@ -0,0 +1,126 @@ +package library + +import ( + "testing" + + "yellowjacket/backend/metadata" +) + +func TestStripFeaturing(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + credit string + want string + }{ + {"plain", "Lana Del Rey", "Lana Del Rey"}, + {"ft dot", "Lana Del Rey ft. Sean Lennon", "Lana Del Rey"}, + {"feat dot", "2Pac feat. Nate Dogg", "2Pac"}, + {"featuring", "Beyoncé featuring The Weeknd", "Beyoncé"}, + {"ft no dot", "Drake ft Travis Scott", "Drake"}, + {"case insensitive", "Kanye West FEAT. PARTYNEXTDOOR", "Kanye West"}, + {"ampersand kept", "Simon & Garfunkel", "Simon & Garfunkel"}, + {"comma kept", "Tyler, the Creator", "Tyler, the Creator"}, + {"first marker wins", "A feat. B ft. C", "A"}, + {"trims", " Daft Punk feat. Panda Bear ", "Daft Punk"}, + {"empty", "", ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + if got := stripFeaturing(tt.credit); got != tt.want { + t.Errorf("stripFeaturing(%q) = %q, want %q", tt.credit, got, tt.want) + } + }) + } +} + +func TestPrimaryArtist(t *testing.T) { + t.Parallel() + + const lana = "b7539c32-53e7-4908-bda3-81449c367da6" + + tests := []struct { + name string + tags metadata.TrackMetadata + wantName string + wantMBID string + }{ + { + name: "collab on own album uses clean album artist", + tags: metadata.TrackMetadata{ + Artist: "Lana Del Rey ft. Sean Lennon", + AlbumArtist: "Lana Del Rey", + ArtistMBID: lana, + AlbumArtistMBID: lana, + }, + wantName: "Lana Del Rey", + wantMBID: lana, + }, + { + name: "solo track", + tags: metadata.TrackMetadata{ + Artist: "Lana Del Rey", + AlbumArtist: "Lana Del Rey", + ArtistMBID: lana, + AlbumArtistMBID: lana, + }, + wantName: "Lana Del Rey", + wantMBID: lana, + }, + { + name: "compilation: album artist differs, strip featuring, keep track mbid", + tags: metadata.TrackMetadata{ + Artist: "Some Artist feat. Guest", + AlbumArtist: "Various Artists", + ArtistMBID: "aaaa", + AlbumArtistMBID: "va-mbid", + }, + wantName: "Some Artist", + wantMBID: "aaaa", + }, + { + name: "no album artist, strip featuring", + tags: metadata.TrackMetadata{ + Artist: "Some Artist feat. Guest", + ArtistMBID: "aaaa", + }, + wantName: "Some Artist", + wantMBID: "aaaa", + }, + { + name: "no track mbid falls back to album mbid", + tags: metadata.TrackMetadata{ + Artist: "Solo", + AlbumArtist: "Solo", + AlbumArtistMBID: "album-mbid", + }, + wantName: "Solo", + wantMBID: "album-mbid", + }, + { + name: "empty artist", + tags: metadata.TrackMetadata{}, + wantName: "Unknown Artist", + wantMBID: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + gotName, gotMBID := primaryArtist(&tt.tags) + if gotName != tt.wantName { + t.Errorf("primaryArtist name = %q, want %q", gotName, tt.wantName) + } + + if gotMBID != tt.wantMBID { + t.Errorf("primaryArtist mbid = %q, want %q", gotMBID, tt.wantMBID) + } + }) + } +} diff --git a/backend/library/crud.go b/backend/library/crud.go index abddf36..794d010 100644 --- a/backend/library/crud.go +++ b/backend/library/crud.go @@ -37,6 +37,9 @@ type RemovalHooks struct { StopPlayback func() // CompactQueue reloads queue state after cascade deletes. CompactQueue func() + // PostRemove runs after the removal commits, for cross-cutting + // invalidation (e.g. clearing library-sync "ready" markers). + PostRemove func() } // SetRemovalHooks provides optional hooks for cross-cutting @@ -461,6 +464,12 @@ func (l *Library) RemoveLibrary(id int64) (*RemovalSummary, error) { l.removalHooks.CompactQueue() } + // 22. Post-commit: invalidate library-sync markers so the gated + // index/lyric re-sync runs on the next launch. + if l.removalHooks.PostRemove != nil { + l.removalHooks.PostRemove() + } + summary := &RemovalSummary{ TracksDeleted: tracksDeleted, ArtistsRemoved: artistsRemoved, diff --git a/backend/library/library.go b/backend/library/library.go index 3b6a301..d77cbcb 100644 --- a/backend/library/library.go +++ b/backend/library/library.go @@ -1216,14 +1216,19 @@ func (l *Library) processMetadata( q, cache, metrics, tags, thumbChan, ) - // 2. Get or create artist credit for track artist. - artistName := tags.Artist - if artistName == "" { - artistName = "Unknown Artist" + // 2. Get or create the artist credit for the track. The credit + // text is the full tagged string (e.g. "Lana Del Rey ft. Sean + // Lennon") and is kept only for display; the artist *entity* it + // links to is the primary artist, resolved cleanly by primaryArtist + // so featured-artist credits don't fork into their own bogus artist + // rows (all sharing the primary's single MBID). + creditText := tags.Artist + if creditText == "" { + creditText = "Unknown Artist" } artistCredit, err := l.cachedUpsertArtistCredit( - q, cache, artistName, + q, cache, creditText, ) if err != nil { return 0, fmt.Errorf( @@ -1231,7 +1236,9 @@ func (l *Library) processMetadata( ) } - l.cachedLinkArtist(q, cache, metrics, artistName, artistCredit.ID) + primaryName, primaryMBID := primaryArtist(tags) + + l.cachedLinkArtist(q, cache, metrics, primaryName, artistCredit.ID) // 3. Get or create artist credit for album artist. albumArtistCreditID := l.resolveAlbumArtistCredit( @@ -1289,9 +1296,9 @@ func (l *Library) processMetadata( // 7. Update MusicBrainz IDs (if present in tags). if releaseGroupID.Valid { - l.updateMBIDs(tx, cache, tags, artistName, releaseGroupID.Int64, recording.ID) + l.updateMBIDs(tx, cache, tags, primaryName, primaryMBID, releaseGroupID.Int64, recording.ID) } else { - l.updateMBIDs(tx, cache, tags, artistName, 0, recording.ID) + l.updateMBIDs(tx, cache, tags, primaryName, primaryMBID, 0, recording.ID) } return recording.ID, nil @@ -1306,15 +1313,11 @@ func (l *Library) updateMBIDs( cache *entityCache, tags *metadata.TrackMetadata, artistName string, + artistMBID string, releaseGroupID int64, recordingID int64, ) { - // Artist MBID — prefer album artist, fall back to track artist. - artistMBID := tags.AlbumArtistMBID - if artistMBID == "" { - artistMBID = tags.ArtistMBID - } - + // Artist MBID (the primary artist's, resolved by primaryArtist). if artistMBID != "" { if artist, ok := cache.artists[artistName]; ok { _, _ = tx.ExecContext(l.ctx, diff --git a/backend/library/query.go b/backend/library/query.go index 6590615..077d5f7 100644 --- a/backend/library/query.go +++ b/backend/library/query.go @@ -183,6 +183,7 @@ type Album struct { ID int64 Name string ArtistName string + ArtistMBID string MBID string CoverArtPath string CoverArtSmall string @@ -194,7 +195,7 @@ type Album struct { // GetAllTracks returns an array of track structs of every file in the library. func (l *Library) GetAllTracks() ([]Track, error) { - rows, err := l.db.Queries.GetAllTracksWithFullMetadata( + rows, err := l.db.ReadQueries.GetAllTracksWithFullMetadata( l.ctx, ) if err != nil { @@ -303,7 +304,7 @@ func (l *Library) SearchTracks( // GetAlbumTracks returns all tracks for a given album (release group), ordered by disc and track number. func (l *Library) GetAlbumTracks(albumID int64) ([]Track, error) { - rows, err := l.db.Queries.GetAudioFilesByReleaseGroup(l.ctx, albumID) + rows, err := l.db.ReadQueries.GetAudioFilesByReleaseGroup(l.ctx, albumID) if err != nil { l.logger.Error("could not retrieve album tracks", "albumID", albumID, "error", err) @@ -347,7 +348,7 @@ func (l *Library) GetAlbumTracks(albumID int64) ([]Track, error) { // GetAllAlbums returns all albums with cover art and artist info for the cover grid. func (l *Library) GetAllAlbums() ([]Album, error) { - rows, err := l.db.Queries.GetAllAlbumsWithDetails(l.ctx) + rows, err := l.db.ReadQueries.GetAllAlbumsWithDetails(l.ctx) if err != nil { l.logger.Error("could not retrieve albums", "error", err) @@ -363,6 +364,7 @@ func (l *Library) GetAllAlbums() ([]Album, error) { ID: row.ID, Name: row.Name, ArtistName: row.ArtistName, + ArtistMBID: row.ArtistMbid, } if row.Year.Valid { @@ -392,7 +394,7 @@ func (l *Library) GetAllAlbums() ([]Album, error) { // GetAllArtists returns artists that are credited as album artists, ordered by name. func (l *Library) GetAllArtists() ([]Artist, error) { - rows, err := l.db.Queries.GetAlbumArtists(l.ctx) + rows, err := l.db.ReadQueries.GetAlbumArtists(l.ctx) if err != nil { l.logger.Error( "could not retrieve artists", @@ -489,7 +491,7 @@ func (l *Library) resolveArtistImages(artists []Artist) { func (l *Library) GetAlbumsByArtist( artistID int64, ) ([]Album, error) { - rows, err := l.db.Queries.GetAlbumsByArtist( + rows, err := l.db.ReadQueries.GetAlbumsByArtist( l.ctx, artistID, ) @@ -519,6 +521,7 @@ func (l *Library) GetAlbumsByArtist( ID: row.ID, Name: row.Name, ArtistName: row.ArtistName, + ArtistMBID: row.ArtistMbid, } if row.Year.Valid { @@ -552,7 +555,7 @@ type GenreWithCount struct { func (l *Library) GetTracksByGenre( genreName string, ) ([]Track, error) { - rows, err := l.db.Queries.GetTracksByGenre( + rows, err := l.db.ReadQueries.GetTracksByGenre( l.ctx, genreName, ) if err != nil { @@ -600,7 +603,7 @@ func (l *Library) GetTracksByGenre( func (l *Library) GetAllGenresWithCounts() ( []GenreWithCount, error, ) { - rows, err := l.db.Queries.GetAllGenresWithCounts( + rows, err := l.db.ReadQueries.GetAllGenresWithCounts( l.ctx, ) if err != nil { @@ -630,7 +633,7 @@ func (l *Library) GetAllGenresWithCounts() ( func (l *Library) GetAllTracksByLibrary( libraryID int64, ) ([]Track, error) { - rows, err := l.db.Queries.GetAllTracksWithFullMetadataByLibrary( + rows, err := l.db.ReadQueries.GetAllTracksWithFullMetadataByLibrary( l.ctx, libraryID, ) if err != nil { @@ -687,7 +690,7 @@ func (l *Library) GetAllTracksByLibrary( func (l *Library) GetAllAlbumsByLibrary( libraryID int64, ) ([]Album, error) { - rows, err := l.db.Queries.GetAllAlbumsWithDetailsByLibrary( + rows, err := l.db.ReadQueries.GetAllAlbumsWithDetailsByLibrary( l.ctx, libraryID, ) if err != nil { @@ -715,6 +718,7 @@ func (l *Library) GetAllAlbumsByLibrary( ID: row.ID, Name: row.Name, ArtistName: row.ArtistName, + ArtistMBID: row.ArtistMbid, } if row.Year.Valid { @@ -746,7 +750,7 @@ func (l *Library) GetAllAlbumsByLibrary( func (l *Library) GetAllArtistsByLibrary( libraryID int64, ) ([]Artist, error) { - rows, err := l.db.Queries.GetAlbumArtistsByLibrary( + rows, err := l.db.ReadQueries.GetAlbumArtistsByLibrary( l.ctx, libraryID, ) if err != nil { @@ -792,7 +796,7 @@ func (l *Library) GetAllArtistsByLibrary( func (l *Library) GetAlbumsByArtistByLibrary( artistID, libraryID int64, ) ([]Album, error) { - rows, err := l.db.Queries.GetAlbumsByArtistByLibrary( + rows, err := l.db.ReadQueries.GetAlbumsByArtistByLibrary( l.ctx, sqlcgen.GetAlbumsByArtistByLibraryParams{ ArtistID: artistID, LibraryID: libraryID, @@ -826,6 +830,7 @@ func (l *Library) GetAlbumsByArtistByLibrary( ID: row.ID, Name: row.Name, ArtistName: row.ArtistName, + ArtistMBID: row.ArtistMbid, } if row.Year.Valid { @@ -853,7 +858,7 @@ func (l *Library) GetAlbumsByArtistByLibrary( func (l *Library) GetAllGenresWithCountsByLibrary( libraryID int64, ) ([]GenreWithCount, error) { - rows, err := l.db.Queries.GetAllGenresWithCountsByLibrary( + rows, err := l.db.ReadQueries.GetAllGenresWithCountsByLibrary( l.ctx, libraryID, ) if err != nil { @@ -885,7 +890,7 @@ func (l *Library) GetAllGenresWithCountsByLibrary( func (l *Library) GetTracksByGenreByLibrary( genreName string, libraryID int64, ) ([]Track, error) { - rows, err := l.db.Queries.GetTracksByGenreByLibrary( + rows, err := l.db.ReadQueries.GetTracksByGenreByLibrary( l.ctx, sqlcgen.GetTracksByGenreByLibraryParams{ Name: genreName, LibraryID: libraryID, @@ -939,7 +944,7 @@ func (l *Library) GetTracksByGenreByLibrary( func (l *Library) GetAlbumTracksByLibrary( albumID, libraryID int64, ) ([]Track, error) { - rows, err := l.db.Queries.GetAudioFilesByReleaseGroupByLibrary( + rows, err := l.db.ReadQueries.GetAudioFilesByReleaseGroupByLibrary( l.ctx, sqlcgen.GetAudioFilesByReleaseGroupByLibraryParams{ ReleaseGroupID: albumID, LibraryID: libraryID, @@ -1052,7 +1057,7 @@ type Info struct { // GetAllLibrariesWithTrackCounts returns all libraries with their // audio file counts. Typically 1-5 libraries so the loop is trivial. func (l *Library) GetAllLibrariesWithTrackCounts() ([]Info, error) { - libs, err := l.db.Queries.GetAllLibraries(l.ctx) + libs, err := l.db.ReadQueries.GetAllLibraries(l.ctx) if err != nil { return nil, fmt.Errorf("could not get libraries: %w", err) } @@ -1060,7 +1065,7 @@ func (l *Library) GetAllLibrariesWithTrackCounts() ([]Info, error) { result := make([]Info, 0, len(libs)) for _, lib := range libs { - count, countErr := l.db.Queries.CountAudioFilesByLibrary(l.ctx, lib.ID) + count, countErr := l.db.ReadQueries.CountAudioFilesByLibrary(l.ctx, lib.ID) if countErr != nil { l.logger.Error("could not count tracks for library", "libraryID", lib.ID, "error", countErr) diff --git a/backend/player/buffered_streamer.go b/backend/player/buffered_streamer.go index 1d51f20..f9e01cf 100644 --- a/backend/player/buffered_streamer.go +++ b/backend/player/buffered_streamer.go @@ -19,6 +19,12 @@ import ( // temporarily empty (read-ahead hasn't caught up), Stream returns // silence rather than blocking or signaling end-of-stream. type BufferedStreamer struct { + // srcMu serializes all access to the underlying source. The + // read-ahead goroutine holds it while calling source.Stream; + // callers that need to Seek the source must hold it too (via + // LockSource/UnlockSource) so the non-thread-safe decoder is + // never read and seeked concurrently. + srcMu sync.Mutex mu sync.Mutex source beep.Streamer ring [][2]float64 @@ -89,9 +95,12 @@ func (bs *BufferedStreamer) readAhead() { bs.mu.Unlock() - // Read from source WITHOUT holding the lock so disk I/O - // does not block the speaker goroutine. + // Read from source WITHOUT holding bs.mu so disk I/O does + // not block the speaker goroutine. srcMu is held to keep + // this read from racing a concurrent source Seek. + bs.srcMu.Lock() n, ok := bs.source.Stream(tmp[:toRead]) + bs.srcMu.Unlock() if n > 0 { bs.mu.Lock() @@ -190,6 +199,20 @@ func (bs *BufferedStreamer) Flush() { bs.count = 0 } +// LockSource blocks the read-ahead goroutine from touching the +// underlying source, giving the caller exclusive access so it can +// safely Seek the non-thread-safe decoder. Every LockSource must +// be paired with an UnlockSource. +func (bs *BufferedStreamer) LockSource() { + bs.srcMu.Lock() +} + +// UnlockSource releases the exclusive source access acquired by +// LockSource, allowing the read-ahead goroutine to resume. +func (bs *BufferedStreamer) UnlockSource() { + bs.srcMu.Unlock() +} + // Close signals the read-ahead goroutine to stop. It is safe to // call multiple times. func (bs *BufferedStreamer) Close() { diff --git a/backend/player/player.go b/backend/player/player.go index d3243da..7ea5e8c 100644 --- a/backend/player/player.go +++ b/backend/player/player.go @@ -764,6 +764,17 @@ func (p *Player) seekLocked(targetSeconds int) error { return fmt.Errorf("cannot get track length: %w", err) } + // Block the read-ahead goroutine from reading the source while + // we seek it. The decoder (e.g. FLAC's bufseekio.ReadSeeker) is + // not safe for concurrent Read+Seek, and read-ahead runs on its + // own goroutine — without this it can panic with a slice-bounds + // error, especially right after load when the buffer is empty + // and read-ahead is filling at full speed. + if p.buffered != nil { + p.buffered.LockSource() + defer p.buffered.UnlockSource() + } + speaker.Lock() samples := int( @@ -863,7 +874,7 @@ func (p *Player) getCurrentTrackInfoLocked() TrackInfo { // Try to get metadata from database. if p.db != nil { - meta, err := p.db.Queries.GetTrackMetadataByPath( + meta, err := p.db.ReadQueries.GetTrackMetadataByPath( p.ctx, info.FilePath, ) if err == nil { @@ -1004,7 +1015,7 @@ func (p *Player) buildMediaMetadata( // full path; ResolveURLs converts it to relative HTTP paths // for the frontend, but MPRIS needs the actual file path. if p.db != nil && info.FilePath != "" { - dbMeta, err := p.db.Queries.GetTrackMetadataByPath( + dbMeta, err := p.db.ReadQueries.GetTrackMetadataByPath( p.ctx, info.FilePath, ) if err == nil && dbMeta.CoverArtPath != "" { @@ -1102,7 +1113,7 @@ func (p *Player) restoreStateLocked() { return } - state, err := p.db.Queries.GetPlayerState(p.db.Ctx) + state, err := p.db.ReadQueries.GetPlayerState(p.db.Ctx) if err != nil { p.logger.Error( "Failed to load player state", "err", err, diff --git a/backend/playlist/playlist.go b/backend/playlist/playlist.go index 49fa28d..7367fff 100644 --- a/backend/playlist/playlist.go +++ b/backend/playlist/playlist.go @@ -132,6 +132,11 @@ type Service struct { db *database.DB libraryDir LibraryDirProvider favoritesConf FavoritesConfigProvider + + // dataDirOverride, when non-empty, replaces the OS user data + // directory as the base for the playlists folder. Set by tests to + // keep M3U writes out of the real user data directory. + dataDirOverride string } // NewService creates a new playlist service. @@ -172,7 +177,7 @@ func (s *Service) SetContext(ctx context.Context) { // GetAllPlaylists returns all playlists ordered by most recently // updated. func (s *Service) GetAllPlaylists() ([]Summary, error) { - playlists, err := s.db.Queries.GetAllPlaylists(s.db.Ctx) + playlists, err := s.db.ReadQueries.GetAllPlaylists(s.db.Ctx) if err != nil { s.logger.Error( "Failed to get playlists", "err", err, @@ -204,7 +209,7 @@ func (s *Service) GetAllPlaylistsWithTracks() ( []WithTracks, error, ) { - playlists, err := s.db.Queries.GetAllPlaylists(s.db.Ctx) + playlists, err := s.db.ReadQueries.GetAllPlaylists(s.db.Ctx) if err != nil { s.logger.Error( "Failed to get playlists", "err", err, @@ -215,7 +220,7 @@ func (s *Service) GetAllPlaylistsWithTracks() ( ) } - rows, err := s.db.Queries.GetAllPlaylistTracksWithMetadata( + rows, err := s.db.ReadQueries.GetAllPlaylistTracksWithMetadata( s.db.Ctx, ) if err != nil { @@ -288,7 +293,7 @@ func (s *Service) GetAllPlaylistsWithTracks() ( func (s *Service) GetPlaylistTracks( playlistID int64, ) ([]Track, error) { - rows, err := s.db.Queries.GetPlaylistTracksWithMetadata( + rows, err := s.db.ReadQueries.GetPlaylistTracksWithMetadata( s.db.Ctx, playlistID, ) @@ -563,7 +568,7 @@ func (s *Service) FindDuplicateTracksInPlaylist( playlistID int64, filePaths []string, ) (DuplicateCheckResult, error) { - rows, err := s.db.Queries.GetPlaylistTracksWithMetadata( + rows, err := s.db.ReadQueries.GetPlaylistTracksWithMetadata( s.db.Ctx, playlistID, ) @@ -1229,11 +1234,17 @@ func (s *Service) addSingleTrack( // playlistsDir returns the path to the playlists directory, // creating it if needed. func (s *Service) playlistsDir() (string, error) { - dataDir, err := system.GetUserDataDirPath() - if err != nil { - return "", fmt.Errorf( - "could not get user data directory: %w", err, - ) + dataDir := s.dataDirOverride + + if dataDir == "" { + var err error + + dataDir, err = system.GetUserDataDirPath() + if err != nil { + return "", fmt.Errorf( + "could not get user data directory: %w", err, + ) + } } dir := filepath.Join(dataDir, playlistsDirName) @@ -1376,7 +1387,7 @@ func (s *Service) saveImportedPlaylistFile( func (s *Service) buildM3UEntries( playlistID int64, ) []m3uEntry { - rows, err := s.db.Queries.GetPlaylistTracksWithMetadata( + rows, err := s.db.ReadQueries.GetPlaylistTracksWithMetadata( s.db.Ctx, playlistID, ) @@ -1482,7 +1493,7 @@ func (s *Service) migrateExistingPlaylists() { } } - playlists, err := s.db.Queries.GetAllPlaylists(s.db.Ctx) + playlists, err := s.db.ReadQueries.GetAllPlaylists(s.db.Ctx) if err != nil { s.logger.Warn( "Could not get playlists for migration", @@ -1534,7 +1545,7 @@ func (s *Service) RepopulateFromM3U() { } // Get all playlists. - playlists, err := s.db.Queries.GetAllPlaylists(s.db.Ctx) + playlists, err := s.db.ReadQueries.GetAllPlaylists(s.db.Ctx) if err != nil { s.logger.Warn("could not get playlists for repopulation", "err", err) @@ -2677,11 +2688,139 @@ func (s *Service) UpdateSmartPlaylistRules( "playlistId", playlistID, ) + // Re-materialize the persisted snapshot so it reflects the new + // rules. RefreshSmartPlaylist emits PlaylistTracksChanged. + if err := s.RefreshSmartPlaylist(playlistID); err != nil { + return fmt.Errorf( + "failed to refresh smart playlist after rule update: %w", + err, + ) + } + + return nil +} + +// RefreshSmartPlaylist re-evaluates a smart playlist's rules against +// the current library and replaces its persisted membership in +// playlist_tracks with the result. This is the only path that +// re-evaluates a smart playlist — opening one otherwise reads the +// stored snapshot. Triggered on rule save and by the manual Refresh +// button. +func (s *Service) RefreshSmartPlaylist( + playlistID int64, +) error { + // EvaluateSmartPlaylist validates that the playlist exists and is + // smart, and returns the live rule-matched tracks. + tracks, err := s.EvaluateSmartPlaylist(playlistID) + if err != nil { + return err + } + + if err := s.db.Queries.ClearPlaylistTracks( + s.db.Ctx, playlistID, + ); err != nil { + return fmt.Errorf( + "failed to clear smart playlist tracks: %w", err, + ) + } + + for i, t := range tracks { + if strings.TrimSpace(t.FilePath) == "" { + continue + } + + if err := s.addSingleTrack( + playlistID, t.FilePath, int64(i), + ); err != nil { + // A track can vanish between evaluation and insertion + // (e.g. a concurrent rescan). Skip it rather than abort + // the whole refresh. + s.logger.Warn( + "Skipping smart playlist track during refresh", + "playlistId", playlistID, + "filePath", t.FilePath, + "err", err, + ) + + continue + } + } + + // SAFETY: Hand-crafted UPDATE for smart_snapshot_at column not + // yet in sqlc schema. Parameterized by playlist ID. + if _, err := s.db.ExecContext( + `UPDATE playlists + SET smart_snapshot_at = CURRENT_TIMESTAMP + WHERE id = ? AND is_smart = 1`, + playlistID, + ); err != nil { + return fmt.Errorf( + "failed to mark smart playlist snapshot: %w", err, + ) + } + + s.logger.Info( + "Smart playlist refreshed", + "playlistId", playlistID, + "trackCount", len(tracks), + ) + + s.savePlaylistFileByID(playlistID) s.emitEvent(events.PlaylistTracksChanged, playlistID) return nil } +// GetSmartPlaylistTracks returns the persisted snapshot of a smart +// playlist as regular playlist tracks (with resolved cover art and +// phantom entries), identical to a normal playlist. If the playlist +// has never been materialized — e.g. it predates snapshot support — +// it is evaluated and stored on first access. +func (s *Service) GetSmartPlaylistTracks( + playlistID int64, +) ([]Track, error) { + // SAFETY: Hand-crafted SELECT for smart_snapshot_at column not + // yet in sqlc schema. Parameterized by playlist ID. + rows, err := s.db.QueryContext( + `SELECT smart_snapshot_at FROM playlists + WHERE id = ? AND is_smart = 1`, + playlistID, + ) + if err != nil { + return nil, fmt.Errorf( + "failed to load smart playlist: %w", err, + ) + } + + if !rows.Next() { + _ = rows.Close() + + return nil, errNotSmartPlaylist + } + + var snapshotAt sql.NullString + + if err := rows.Scan(&snapshotAt); err != nil { + _ = rows.Close() + + return nil, fmt.Errorf( + "failed to read smart playlist snapshot state: %w", err, + ) + } + + // Close before RefreshSmartPlaylist / GetPlaylistTracks issue + // their own queries (MaxOpenConns=1 test DBs would deadlock). + _ = rows.Close() + + if !snapshotAt.Valid { + if err := s.RefreshSmartPlaylist(playlistID); err != nil { + return nil, err + } + } + + return s.GetPlaylistTracks(playlistID) +} + // EvaluateSmartPlaylist loads the rule set for a smart playlist // from the database and evaluates it against the track library, // returning the matching tracks. diff --git a/backend/playlist/smart_test.go b/backend/playlist/smart_test.go index cee6704..1ac4de2 100644 --- a/backend/playlist/smart_test.go +++ b/backend/playlist/smart_test.go @@ -170,8 +170,9 @@ func newTestService(t *testing.T, db *database.DB) *Service { t.Helper() return &Service{ - db: db, - logger: slog.Default(), + db: db, + logger: slog.Default(), + dataDirOverride: t.TempDir(), } } @@ -294,6 +295,141 @@ func TestSmartPlaylistUpdateRules(t *testing.T) { } } +// TestSmartPlaylistPersistedSnapshot verifies that a smart playlist's +// membership is materialized and read from a stored snapshot: it is +// backfilled on first access, does NOT re-evaluate when the rules +// change out from under it, and only re-materializes on an explicit +// refresh. +func TestSmartPlaylistPersistedSnapshot(t *testing.T) { + t.Parallel() + + db := database.NewTestDB(t) + seedSmartTestTracks(t, db) + + svc := newTestService(t, db) + + // Band A → tracks 1 and 3. + bandARules := makeRulesJSON(t, smartplaylist.RuleSet{ + Rules: []smartplaylist.Rule{ + {Field: "artist", Operator: "is", Value: "Band A"}, + }, + }) + + summary, err := svc.CreateSmartPlaylist("Snapshot Test", bandARules) + if err != nil { + t.Fatalf("CreateSmartPlaylist failed: %v", err) + } + + // First access backfills the snapshot (snapshot_at was NULL). + tracks, err := svc.GetSmartPlaylistTracks(summary.ID) + if err != nil { + t.Fatalf("GetSmartPlaylistTracks failed: %v", err) + } + + if len(tracks) != 2 { + t.Fatalf("initial snapshot: got %d tracks, want 2", len(tracks)) + } + + for _, tr := range tracks { + if tr.Artist != "Band A" { + t.Errorf("snapshot track %q has artist %q, want Band A", + tr.Title, tr.Artist) + } + } + + // Change the rules directly in the DB, bypassing + // UpdateSmartPlaylistRules so no refresh is triggered. The stored + // snapshot must be unaffected. + bandBRules := makeRulesJSON(t, smartplaylist.RuleSet{ + Rules: []smartplaylist.Rule{ + {Field: "artist", Operator: "is", Value: "Band B"}, + }, + }) + + if _, err := db.ExecContext( + "UPDATE playlists SET smart_rules = ? WHERE id = ?", + bandBRules, summary.ID, + ); err != nil { + t.Fatalf("failed to rewrite rules: %v", err) + } + + // Snapshot is served as-is: still Band A's two tracks, not Band B. + tracks, err = svc.GetSmartPlaylistTracks(summary.ID) + if err != nil { + t.Fatalf("GetSmartPlaylistTracks (post rule change) failed: %v", err) + } + + if len(tracks) != 2 { + t.Fatalf("snapshot re-read: got %d tracks, want 2 (must not re-evaluate)", + len(tracks)) + } + + // An explicit refresh re-materializes against the current rules. + if err := svc.RefreshSmartPlaylist(summary.ID); err != nil { + t.Fatalf("RefreshSmartPlaylist failed: %v", err) + } + + tracks, err = svc.GetSmartPlaylistTracks(summary.ID) + if err != nil { + t.Fatalf("GetSmartPlaylistTracks (post refresh) failed: %v", err) + } + + if len(tracks) != 1 { + t.Fatalf("post-refresh snapshot: got %d tracks, want 1", len(tracks)) + } + + if tracks[0].Artist != "Band B" { + t.Errorf("post-refresh artist = %q, want Band B", tracks[0].Artist) + } +} + +// TestSmartPlaylistSaveRulesRematerializes verifies that saving new +// rules through UpdateSmartPlaylistRules refreshes the stored snapshot. +func TestSmartPlaylistSaveRulesRematerializes(t *testing.T) { + t.Parallel() + + db := database.NewTestDB(t) + seedSmartTestTracks(t, db) + + svc := newTestService(t, db) + + bandARules := makeRulesJSON(t, smartplaylist.RuleSet{ + Rules: []smartplaylist.Rule{ + {Field: "artist", Operator: "is", Value: "Band A"}, + }, + }) + + summary, err := svc.CreateSmartPlaylist("Save Refresh", bandARules) + if err != nil { + t.Fatalf("CreateSmartPlaylist failed: %v", err) + } + + bandBRules := makeRulesJSON(t, smartplaylist.RuleSet{ + Rules: []smartplaylist.Rule{ + {Field: "artist", Operator: "is", Value: "Band B"}, + }, + }) + + if err := svc.UpdateSmartPlaylistRules(summary.ID, bandBRules); err != nil { + t.Fatalf("UpdateSmartPlaylistRules failed: %v", err) + } + + // The stored snapshot should already reflect the new rules without + // any manual refresh. + tracks, err := svc.GetSmartPlaylistTracks(summary.ID) + if err != nil { + t.Fatalf("GetSmartPlaylistTracks failed: %v", err) + } + + if len(tracks) != 1 { + t.Fatalf("got %d tracks, want 1", len(tracks)) + } + + if tracks[0].Artist != "Band B" { + t.Errorf("artist = %q, want Band B", tracks[0].Artist) + } +} + func TestSmartPlaylistCreateInvalidJSON(t *testing.T) { t.Parallel() diff --git a/backend/queue/persistence.go b/backend/queue/persistence.go index 046c776..92b2aee 100644 --- a/backend/queue/persistence.go +++ b/backend/queue/persistence.go @@ -237,7 +237,7 @@ func (q *Queue) lookupChunk( return } - rows, err := q.db.Queries.LookupTrackMetaByPaths(q.db.Ctx, paths) + rows, err := q.db.ReadQueries.LookupTrackMetaByPaths(q.db.Ctx, paths) if err != nil { q.logger.Error("Batch metadata lookup failed", "err", err) @@ -415,7 +415,7 @@ func (q *Queue) RestoreState() { defer q.mu.Unlock() // Restore queue metadata. - state, err := q.db.Queries.GetQueueState(q.db.Ctx) + state, err := q.db.ReadQueries.GetQueueState(q.db.Ctx) if err != nil { q.logger.Error("Failed to load queue state", "err", err) @@ -444,7 +444,7 @@ func (q *Queue) RestoreState() { } // Restore queue tracks. - rows, err := q.db.Queries.GetQueueTracks(q.db.Ctx) + rows, err := q.db.ReadQueries.GetQueueTracks(q.db.Ctx) if err != nil { q.logger.Error("Failed to load queue tracks", "err", err) diff --git a/backend/smartplaylist/smartplaylist.go b/backend/smartplaylist/smartplaylist.go index 4711472..577732b 100644 --- a/backend/smartplaylist/smartplaylist.go +++ b/backend/smartplaylist/smartplaylist.go @@ -13,6 +13,7 @@ import ( "strings" "time" + "yellowjacket/backend/coverart" "yellowjacket/backend/database" "yellowjacket/backend/library" ) @@ -54,6 +55,7 @@ var fieldMap = map[string]string{ "album": "album", "genre": "genre", "year": "year", + "release_year": "release_year", "composer": "composer", "file_type": "file_type", "duration": "length_milliseconds", @@ -72,6 +74,7 @@ var fieldMap = map[string]string{ // numericFields identifies fields that accept numeric operators. var numericFields = map[string]bool{ "year": true, + "release_year": true, "duration": true, "sample_rate": true, "bit_depth": true, @@ -552,7 +555,11 @@ const leanTrackQuery = `SELECT af.bitrate, af.file_size, af.play_count, - COALESCE(af.last_played, '') AS last_played + COALESCE(af.last_played, '') AS last_played, + af.cover_art_path, + af.artist_mbid, + af.release_group_mbid, + af.recording_mbid FROM ( SELECT af.id, @@ -564,7 +571,17 @@ FROM ( r.track_number, r.disc_number, COALESCE(rg.name, '') AS album, - COALESCE(r.year, 0) AS year, + -- Two year fields, matching the canonical track_metadata view: + -- year — the album's original (first-release) year, + -- the default users filter on. A 1977 album owned + -- as a 2010s reissue still filters as 1977. + -- release_year — the year of the specific release in the library + -- (the file/release-group tag), e.g. 2013 for that + -- reissue. + -- Both fall back through rg.year → r.year so a track without full + -- MusicBrainz data still gets a sensible year. + COALESCE(rg.original_year, rg.year, r.year, 0) AS year, + COALESCE(rg.year, r.year, 0) AS release_year, COALESCE(r.composer, '') AS composer, COALESCE(ft.extension, '') AS file_type, af.sample_rate, @@ -574,7 +591,15 @@ FROM ( af.file_size, af.library_id, af.play_count, - af.last_played + af.last_played, + COALESCE(ca.file_path, '') AS cover_art_path, + COALESCE((SELECT a.mbid + FROM artist_credit_artist aca + JOIN artists a ON a.id = aca.artist_id + WHERE aca.credit_id = ac.id + LIMIT 1), '') AS artist_mbid, + COALESCE(rg.mbid, '') AS release_group_mbid, + COALESCE(r.mbid, '') AS recording_mbid FROM audio_files af LEFT JOIN recordings r ON af.recording_id = r.id LEFT JOIN artist_credit ac ON r.artist_credit_id = ac.id @@ -585,6 +610,7 @@ FROM ( GROUP BY recording_id ) rgr ON r.id = rgr.recording_id LEFT JOIN release_groups rg ON rgr.release_group_id = rg.id + LEFT JOIN cover_art ca ON rg.cover_art_id = ca.id LEFT JOIN file_types ft ON af.file_type_id = ft.id ) af` @@ -752,6 +778,11 @@ func scanTracks(rows *sql.Rows) ([]library.Track, []int64, error) { fileSize int64 playCount int64 lastPlayed string + + coverArtPath string + artistMBID string + releaseGroupMBID string + recordingMBID string ) if err := rows.Scan( @@ -761,31 +792,46 @@ func scanTracks(rows *sql.Rows) ([]library.Track, []int64, error) { &sampleRate, &bitDepth, &channels, &bitrate, &fileSize, &playCount, &lastPlayed, + &coverArtPath, &artistMBID, + &releaseGroupMBID, &recordingMBID, ); err != nil { return nil, nil, fmt.Errorf( "could not scan smart playlist row: %w", err, ) } - tracks = append(tracks, library.Track{ - TrackName: title, - ArtistName: artistName, - TrackLength: strconv.FormatInt(lengthMs, 10), - FilePath: filePath, - TrackNumber: trackNumber.Int64, - DiscNumber: discNumber.Int64, - Album: album, - Year: year, - Composer: composer, - FileType: fileType, - SampleRate: sampleRate, - BitDepth: bitDepth, - Channels: channels, - Bitrate: bitrate, - FileSize: fileSize, - PlayCount: playCount, - LastPlayed: lastPlayed, - }) + track := library.Track{ + TrackName: title, + ArtistName: artistName, + TrackLength: strconv.FormatInt(lengthMs, 10), + FilePath: filePath, + TrackNumber: trackNumber.Int64, + DiscNumber: discNumber.Int64, + Album: album, + Year: year, + Composer: composer, + FileType: fileType, + SampleRate: sampleRate, + BitDepth: bitDepth, + Channels: channels, + Bitrate: bitrate, + FileSize: fileSize, + PlayCount: playCount, + LastPlayed: lastPlayed, + ArtistMBID: artistMBID, + ReleaseGroupMBID: releaseGroupMBID, + RecordingMBID: recordingMBID, + } + + if coverArtPath != "" { + urls := coverart.ResolveURLs(coverArtPath) + track.CoverArtPath = urls.Original + track.CoverArtSmall = urls.Small + track.CoverArtMedium = urls.Medium + track.CoverArtLarge = urls.Large + } + + tracks = append(tracks, track) recordingIDs = append(recordingIDs, recordingID.Int64) } diff --git a/backend/smartplaylist/smartplaylist_test.go b/backend/smartplaylist/smartplaylist_test.go index c0b2688..d515f45 100644 --- a/backend/smartplaylist/smartplaylist_test.go +++ b/backend/smartplaylist/smartplaylist_test.go @@ -1618,3 +1618,120 @@ func TestEvaluate_MultiGenreTrackGenreField(t *testing.T) { ) } } + +// TestEvaluate_YearUsesOriginalReleaseYear is a regression test for a +// bug where the smart-playlist year filter tested recordings.year (the +// file's ID3/reissue tag year) instead of the release group's original +// first-release year, the way the canonical track_metadata view and the +// UI do. That made a 1977 album owned as a 2010s reissue leak into a +// "2010s" year filter even though it displays as 1977. +func TestEvaluate_YearUsesOriginalReleaseYear(t *testing.T) { + t.Parallel() + + db := database.NewTestDB(t) + + // One track: a 1977 album the user owns as a 2013 reissue. The file + // tag / recording year is 2013, but the release group's original + // (first-release) year is 1977. + exec := func(query string, args ...any) { + t.Helper() + + if _, err := db.ExecContext(query, args...); err != nil { + t.Fatalf("exec %q: %v", query, err) + } + } + + exec("INSERT INTO artist_credit (id, text) VALUES (1, ?)", "The B-52's") + exec( + "INSERT INTO release_groups (id, name, year, original_year) "+ + "VALUES (1, ?, 2013, 1977)", + "Reissue Compilation", + ) + exec( + "INSERT INTO recordings (id, name, artist_credit_id, year) "+ + "VALUES (1, ?, 1, 2013)", + "Rock Lobster", + ) + exec( + "INSERT INTO audio_files (id, file_path, length_milliseconds, "+ + "file_type_id, recording_id) VALUES (1, ?, 300000, 1, 1)", + "/music/b52s/rock_lobster.mp3", + ) + exec( + "INSERT INTO release_group_recordings " + + "(release_group_id, recording_id) VALUES (1, 1)", + ) + + // A "2010s" filter must NOT match — the album is originally from 1977. + tracks, err := Evaluate(db, RuleSet{ + Rules: []Rule{ + {Field: "year", Operator: "between", Value: "2010,2019"}, + }, + }) + if err != nil { + t.Fatalf("Evaluate 2010s: %v", err) + } + + if len(tracks) != 0 { + t.Errorf( + "2010s filter matched %d tracks, want 0 "+ + "(reissue year leaked in)", len(tracks), + ) + } + + // A "1970s" filter must match — original_year is 1977. + tracks, err = Evaluate(db, RuleSet{ + Rules: []Rule{ + {Field: "year", Operator: "between", Value: "1970,1979"}, + }, + }) + if err != nil { + t.Fatalf("Evaluate 1970s: %v", err) + } + + if len(tracks) != 1 { + t.Fatalf( + "1970s filter matched %d tracks, want 1", len(tracks), + ) + } + + if tracks[0].Year != 1977 { + t.Errorf("Year = %d, want 1977", tracks[0].Year) + } + + // The release_year field, by contrast, tracks the specific release + // owned (the 2013 reissue), so a 2010s filter on it MUST match. + tracks, err = Evaluate(db, RuleSet{ + Rules: []Rule{ + {Field: "release_year", Operator: "between", Value: "2010,2019"}, + }, + }) + if err != nil { + t.Fatalf("Evaluate release_year 2010s: %v", err) + } + + if len(tracks) != 1 { + t.Fatalf( + "release_year 2010s filter matched %d tracks, want 1", + len(tracks), + ) + } + + // And a 1970s release_year filter must NOT match — the owned + // release is from 2013. + tracks, err = Evaluate(db, RuleSet{ + Rules: []Rule{ + {Field: "release_year", Operator: "between", Value: "1970,1979"}, + }, + }) + if err != nil { + t.Fatalf("Evaluate release_year 1970s: %v", err) + } + + if len(tracks) != 0 { + t.Errorf( + "release_year 1970s filter matched %d tracks, want 0", + len(tracks), + ) + } +} diff --git a/frontend/pnpm-workspace.yaml b/frontend/pnpm-workspace.yaml new file mode 100644 index 0000000..5ed0b5a --- /dev/null +++ b/frontend/pnpm-workspace.yaml @@ -0,0 +1,2 @@ +allowBuilds: + esbuild: true diff --git a/frontend/src/components/autotag-view/autotag-view.ts b/frontend/src/components/autotag-view/autotag-view.ts index a2a43f8..13c62a3 100644 --- a/frontend/src/components/autotag-view/autotag-view.ts +++ b/frontend/src/components/autotag-view/autotag-view.ts @@ -7,6 +7,7 @@ import { GetCandidates, GetCandidatesForPasteURL, GetCandidateCoverArt, + GetLocalCoverArt, GetPendingFolder, ListPendingFolders, ApplyAsync, @@ -14,19 +15,20 @@ import { LeaveAsIs, AckLibraryWarning, ClearCompletedEntries, + SearchCandidates, + SelectSearchCandidate, } from '@go/autotagservice/Service'; import type { autotagservice } from '@go/models'; import { EventsOn } from '@runtime/runtime'; import { Events } from '../../events'; -import { isMultiDisc, groupByDisc, discNumbers } from '../../utils/disc-grouping'; -import { inlineDiff } from '../../utils/text-diff'; +import { inlineDiff, normalizeStrict, isCosmeticDiff } from '../../utils/text-diff'; import { libraryStore } from '../../store/library-store'; type PendingItem = autotagservice.PendingItem; type ScoreView = autotagservice.ScoreView; type CandidateView = autotagservice.CandidateView; type AlignmentView = autotagservice.AlignmentView; -type LocalTrackView = autotagservice.LocalTrackView; +type SearchHitView = autotagservice.SearchHitView; interface ApplyJobState { state: 'running' | 'completed' | 'failed'; @@ -64,6 +66,28 @@ interface VersionCluster { bestIdx: number; // index into candidates[] of the highest-scoring one } +// Per-track detail rows behind the expandable match-detail lines. +// Each carries both sides so the dropdown can render an inline diff. +interface TitleDiffDetail { + pos: number; + local: string; + candidate: string; +} + +interface NumDiffDetail { + title: string; + local: number; + candidate: number; +} + +interface LengthDiffDetail { + pos: number; + title: string; + localMs: number; + candidateMs: number; + deltaMs: number; +} + /** autotag-view is the /autotag page: pick-apply-skip workflow for * pending tagging items. The left sidebar lists every folder * awaiting review; the main pane shows a beets-style single-view @@ -87,11 +111,11 @@ export class AutotagView extends LitElement { .root { display: grid; - grid-template-columns: 240px 1fr 220px; + grid-template-columns: 240px 1fr; grid-template-rows: auto 1fr; grid-template-areas: - "header header header" - "folders main versions"; + "header header" + "folders main"; gap: 0.75rem; height: 100%; box-sizing: border-box; @@ -178,13 +202,50 @@ export class AutotagView extends LitElement { } .folders-section-header { - padding: 0.55rem 0.75rem 0.3rem; - font-size: 0.72rem; - color: var(--yj-text-tertiary, #888); + display: flex; + align-items: center; + gap: 0.35rem; + width: 100%; + padding: 0.55rem 0.5rem 0.55rem 0.75rem; + font-size: 0.78rem; + font-family: inherit; + color: var(--yj-text-secondary, #b3b3b3); text-transform: uppercase; - letter-spacing: 0.5px; - margin-top: 0.4rem; - border-top: 1px solid var(--yj-bg-overlay, rgba(255, 255, 255, 0.06)); + letter-spacing: 0.4px; + background: transparent; + border: 0; + border-bottom: 1px solid var(--yj-bg-overlay, rgba(255, 255, 255, 0.08)); + cursor: pointer; + text-align: left; + } + + .folders-section-header:hover { + color: var(--yj-text-primary, #fff); + } + + /* Collapsible-section toggle used in the Pending header — + transparent button that inherits the header's type. */ + .section-toggle { + display: flex; + align-items: center; + gap: 0.35rem; + flex: 1; + min-width: 0; + padding: 0; + background: transparent; + border: 0; + font: inherit; + letter-spacing: inherit; + text-transform: inherit; + color: inherit; + cursor: pointer; + text-align: left; + } + + .section-chevron { + font-size: 0.85rem; + flex-shrink: 0; + color: var(--yj-text-tertiary, #888); } .folders-menu-trigger { @@ -382,6 +443,63 @@ export class AutotagView extends LitElement { .album-artist { font-size: 1rem; color: var(--yj-text-secondary, #b3b3b3); } .album-line { font-size: 0.8rem; color: var(--yj-text-secondary, #b3b3b3); } + /* Skeleton placeholders — the layout (header + panes) paints + immediately and these shimmer blocks stand in for data + that's still loading (folder list, candidate scoring), + instead of a blank full-screen "Loading…". */ + .skeleton { + position: relative; + overflow: hidden; + background: var(--yj-bg-elevated, #343a40); + border-radius: 4px; + } + + .skeleton::after { + content: ''; + position: absolute; + inset: 0; + transform: translateX(-100%); + background: linear-gradient( + 90deg, + transparent, + rgba(255, 255, 255, 0.06), + transparent + ); + animation: yj-shimmer 1.2s ease-in-out infinite; + } + + @media (prefers-reduced-motion: reduce) { + .skeleton::after { animation: none; } + } + + @keyframes yj-shimmer { + 100% { transform: translateX(100%); } + } + + .sk-line { height: 0.85rem; } + .sk-title { height: 1.4rem; width: 65%; } + .sk-artist { height: 1rem; width: 45%; } + .sk-cover { width: 180px; height: 180px; border-radius: 4px; } + .sk-pill { height: 1rem; width: 3.5rem; border-radius: 999px; } + + .sk-folder-row { + display: grid; + grid-template-columns: 22px 1fr; + column-gap: 0.55rem; + row-gap: 0.35rem; + padding: 0.55rem 0.75rem 1.1rem; + border-bottom: 1px solid var(--yj-bg-overlay, rgba(255, 255, 255, 0.04)); + } + + .sk-folder-row .sk-icon { + grid-column: 1; + grid-row: 1 / span 2; + width: 18px; + height: 18px; + border-radius: 50%; + align-self: center; + } + .banner { background: rgba(255, 200, 90, 0.12); border: 1px solid rgba(255, 200, 90, 0.4); @@ -416,35 +534,121 @@ export class AutotagView extends LitElement { } .md-items { - list-style: none; margin: 0; padding: 0; display: flex; flex-direction: column; - gap: 0.2rem; + gap: 0.1rem; } - .md-items li { - display: flex; - align-items: baseline; - gap: 0.45rem; + .md-item { line-height: 1.35; } - .md-items li::before { - content: ''; - display: inline-block; - width: 0.65em; - flex-shrink: 0; + .md-item > summary, + .md-item:not(.expandable) { + display: flex; + align-items: baseline; + gap: 0.45rem; + padding: 0.15rem 0; } - .md-items li.ok::before { content: '✓'; color: #9be09b; } - .md-items li.warn::before { content: '⚠'; color: #ffd089; } - .md-items li.info::before { content: '○'; color: var(--yj-text-tertiary, #888); } + .md-item.expandable > summary { + cursor: pointer; + list-style: none; + border-radius: 3px; + } - .md-items li.ok { color: var(--yj-text-secondary, #b3b3b3); } - .md-items li.warn { color: var(--yj-text-primary, #fff); } - .md-items li.info { color: var(--yj-text-secondary, #b3b3b3); } + .md-item.expandable > summary::-webkit-details-marker { display: none; } + .md-item.expandable > summary:hover { + background: var(--yj-bg-elevated, #343a40); + } + + .md-mark { + display: inline-block; + width: 0.9em; + flex-shrink: 0; + text-align: center; + } + + .md-mark-ok::before { content: '✓'; color: #9be09b; } + .md-mark-warn::before { content: '⚠'; color: #ffd089; } + .md-mark-info::before { content: '○'; color: var(--yj-text-tertiary, #888); } + + .md-text { flex: 1; min-width: 0; } + + .md-count { + font-variant-numeric: tabular-nums; + color: #9be09b; + font-weight: 500; + } + + .md-count.bad { color: #ffd089; } + + .md-item.ok { color: var(--yj-text-secondary, #b3b3b3); } + .md-item.warn { color: var(--yj-text-primary, #fff); } + .md-item.info { color: var(--yj-text-secondary, #b3b3b3); } + + .md-chevron { + flex-shrink: 0; + font-size: 0.8rem; + color: var(--yj-text-tertiary, #888); + transition: transform 0.15s ease; + } + + @media (prefers-reduced-motion: reduce) { + .md-chevron { transition: none; } + } + + .md-item.expandable[open] > summary .md-chevron { + transform: rotate(90deg); + } + + .md-body { + margin: 0.15rem 0 0.35rem 1.35rem; + padding: 0.35rem 0.5rem; + border-left: 2px solid var(--yj-bg-overlay, rgba(255, 255, 255, 0.12)); + background: var(--yj-bg-base, rgba(0, 0, 0, 0.18)); + border-radius: 0 4px 4px 0; + display: flex; + flex-direction: column; + gap: 0.2rem; + font-size: 0.82rem; + } + + .md-diff-row { + display: flex; + align-items: baseline; + gap: 0.5rem; + line-height: 1.3; + } + + .md-diff-pos { + flex-shrink: 0; + min-width: 1.4rem; + text-align: right; + color: var(--yj-text-tertiary, #888); + font-variant-numeric: tabular-nums; + font-size: 0.76rem; + } + + .md-diff-text { + flex: 1; + min-width: 0; + color: var(--yj-text-primary, #fff); + } + + .md-diff-nums { + flex-shrink: 0; + display: inline-flex; + align-items: baseline; + gap: 0.3rem; + font-variant-numeric: tabular-nums; + } + + .md-arrow, .md-delta { + color: var(--yj-text-tertiary, #888); + } .breakdown-line { margin-top: 0.45rem; @@ -549,25 +753,194 @@ export class AutotagView extends LitElement { color: #ffd089; } - /* ── Versions sidebar ── */ + /* Cosmetic-only difference (case / punctuation): the + * normalized strings match, so the score is unaffected. + * Render muted + dotted rather than the alarming + * red-strike / green so it reads as "formatting, not a + * real change". */ + .diff-cosmetic-old { + color: var(--yj-text-tertiary, #888); + text-decoration: line-through dotted; + text-decoration-thickness: 1px; + opacity: 0.7; + } - .versions { - grid-area: versions; + .diff-cosmetic-new { + color: var(--yj-text-secondary, #b3b3b3); + border-bottom: 1px dotted var(--yj-text-tertiary, #888); + } + + /* ── Two-column folder-vs-candidate comparison ── */ + + .compare { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 0.6rem; + align-items: start; + } + + .compare-col { background: var(--yj-bg-surface, #222); border: 1px solid var(--yj-bg-overlay, rgba(255, 255, 255, 0.1)); border-radius: 6px; - overflow: auto; - display: flex; - flex-direction: column; + padding: 0.5rem 0; + min-width: 0; } - .versions-header { - padding: 0.55rem 0.75rem; + .compare-col-header { + display: flex; + align-items: baseline; + justify-content: space-between; + padding: 0.2rem 0.9rem 0.45rem; border-bottom: 1px solid var(--yj-bg-overlay, rgba(255, 255, 255, 0.08)); + margin-bottom: 0.3rem; + } + + .compare-col-header .title { font-size: 0.78rem; - color: var(--yj-text-secondary, #b3b3b3); text-transform: uppercase; letter-spacing: 0.4px; + color: var(--yj-text-secondary, #b3b3b3); + } + + .compare-col-header .count { + font-size: 0.72rem; + color: var(--yj-text-tertiary, #888); + font-variant-numeric: tabular-nums; + } + + /* Per-column album header — each comparison column shows + * its own artwork + album/artist so the local (embedded) + * and candidate (fetched) identities sit side by side + * without any diff coloring. */ + .cc-album { + display: grid; + grid-template-columns: 84px 1fr; + gap: 0.75rem; + padding: 0.6rem 0.9rem 0.7rem; + border-bottom: 1px solid var(--yj-bg-overlay, rgba(255, 255, 255, 0.08)); + margin-bottom: 0.3rem; + align-items: center; + } + + .cc-cover { + width: 84px; + height: 84px; + border-radius: 4px; + object-fit: cover; + display: block; + background: var(--yj-bg-elevated, #343a40); + } + + .cc-cover.placeholder { + display: flex; + align-items: center; + justify-content: center; + color: var(--yj-text-tertiary, #888); + font-size: 0.68rem; + text-align: center; + padding: 0.2rem; + box-sizing: border-box; + } + + .cc-meta { + display: flex; + flex-direction: column; + gap: 0.15rem; + min-width: 0; + } + + .cc-kicker { + font-size: 0.68rem; + text-transform: uppercase; + letter-spacing: 0.4px; + color: var(--yj-text-tertiary, #888); + } + + .cc-title { + font-size: 1rem; + font-weight: 600; + line-height: 1.2; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } + + .cc-artist { + font-size: 0.82rem; + color: var(--yj-text-secondary, #b3b3b3); + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } + + .cc-line { + font-size: 0.74rem; + color: var(--yj-text-tertiary, #888); + display: flex; + align-items: center; + gap: 0.4rem; + flex-wrap: wrap; + margin-top: 0.1rem; + } + + /* A row whose partner (same data-pair) is being hovered. */ + .track-row.pair-hi { background: rgba(120, 170, 255, 0.16); } + + /* Folder-side track with no candidate partner (extra), and + * candidate-side track with no folder partner (missing) — + * both are gaps, both amber. */ + .track-row.extra { background: rgba(255, 200, 90, 0.10); } + .track-row.gap-spacer { visibility: hidden; } + + /* ── Candidate picker (ranked alternatives) ── */ + + .cand-picker { + display: flex; + gap: 0.4rem; + overflow-x: auto; + padding: 0.1rem; + } + + .cand-chip { + flex: 0 0 auto; + display: flex; + align-items: center; + gap: 0.4rem; + padding: 0.3rem 0.55rem; + border-radius: 6px; + cursor: pointer; + background: var(--yj-bg-surface, #222); + border: 1px solid var(--yj-bg-overlay, rgba(255, 255, 255, 0.1)); + font-size: 0.8rem; + white-space: nowrap; + max-width: 22rem; + } + + .cand-chip:hover { background: var(--yj-bg-elevated, #343a40); } + + .cand-chip.selected { + border-color: var(--yj-accent, #ffd43b); + background: var(--yj-bg-elevated, #343a40); + } + + .cand-chip .label { + overflow: hidden; + text-overflow: ellipsis; + max-width: 15rem; + } + + /* ── Versions dropdown (editions within the active album) ── */ + + .versions-select { + font: inherit; + font-size: 0.8rem; + background: var(--yj-bg-elevated, #343a40); + color: var(--yj-text-primary, #fff); + border: 1px solid var(--yj-bg-overlay, rgba(255, 255, 255, 0.15)); + border-radius: 4px; + padding: 0.15rem 0.3rem; + max-width: 100%; } .version-row { @@ -618,24 +991,6 @@ export class AutotagView extends LitElement { color: var(--yj-bg-base, #000); } - .provenance-badge { - display: inline-block; - padding: 0 0.3rem; - border-radius: 8px; - font-size: 0.62rem; - text-transform: uppercase; - letter-spacing: 0.3px; - color: var(--yj-bg-base, #000); - } - - .provenance-local { background: #9c7; } - .provenance-strict { background: #7bf; } - .provenance-no-track-count, - .provenance-title-only { background: #fb7; } - .provenance-fuzzy-title { background: #e9d; } - .provenance-paste { background: var(--yj-accent, #ffd43b); } - .provenance-unknown { background: #777; color: #fff; } - /* ── Generic states ── */ .empty { @@ -716,6 +1071,74 @@ export class AutotagView extends LitElement { justify-content: flex-end; margin-top: 1rem; } + + /* ── In-app search dialog ── */ + + .search-dialog { min-width: 480px; } + + .search-kind { + display: flex; + gap: 1rem; + margin-bottom: 0.6rem; + font-size: 0.85rem; + } + + .search-kind label { + display: flex; + align-items: center; + gap: 0.3rem; + cursor: pointer; + } + + .search-input { + width: 100%; + padding: 0.4rem; + font: inherit; + margin-bottom: 0.5rem; + background: var(--yj-bg-elevated, #343a40); + color: var(--yj-text-primary, #fff); + border: 1px solid var(--yj-bg-overlay, rgba(255, 255, 255, 0.15)); + border-radius: 4px; + box-sizing: border-box; + } + + .search-results { + margin-top: 0.75rem; + max-height: 320px; + overflow: auto; + border-top: 1px solid var(--yj-bg-overlay, rgba(255, 255, 255, 0.08)); + } + + .search-result { + padding: 0.45rem 0.5rem; + cursor: pointer; + border-bottom: 1px solid var(--yj-bg-overlay, rgba(255, 255, 255, 0.05)); + border-radius: 4px; + } + + .search-result:hover { background: var(--yj-bg-elevated, #343a40); } + + .sr-title { + font-weight: 500; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } + + .sr-sub { + display: flex; + gap: 0.5rem; + font-size: 0.78rem; + color: var(--yj-text-secondary, #b3b3b3); + } + + .sr-detail { color: var(--yj-text-tertiary, #888); } + + .search-empty { + margin-top: 0.75rem; + font-size: 0.85rem; + color: var(--yj-text-secondary, #b3b3b3); + } `, ]; @@ -723,11 +1146,32 @@ export class AutotagView extends LitElement { @state() private current: PendingItem | null = null; @state() private score: ScoreView | null = null; @state() private selectedCandidateIdx = 0; + // Data-URI for artwork embedded in the current folder's local + // files, fetched lazily per folder so the local comparison + // column can show "what the folder already looks like". + @state() private localCoverUrl = ''; + // loading tracks candidate scoring for the selected folder; + // foldersLoading tracks the folder-list fetch. They're separate + // so the sidebar can paint as soon as the list arrives while the + // main pane still shows a candidate skeleton. @state() private loading = false; + @state() private foldersLoading = false; @state() private errorMessage = ''; - @state() private dialog: 'none' | 'paste' | 'warning' | 'leave' = 'none'; + @state() private dialog: 'none' | 'paste' | 'warning' | 'leave' | 'search' = 'none'; @state() private pasteURL = ''; + + // In-app MusicBrainz search ("suggest a candidate") dialog state. + @state() private searchKind: 'releasegroup' | 'recording' = 'releasegroup'; + @state() private searchQuery = ''; + @state() private searchArtist = ''; + @state() private searchResults: SearchHitView[] = []; + @state() private searchLoading = false; + @state() private searchError = ''; @state() private queueMenuOpen = false; + // Collapsible sidebar sections. Pending stays expanded so the + // actionable queue is always visible; the non-actionable Skipped + // and Completed sections start collapsed to keep the list short. + @state() private collapsedSections = new Set(['skipped', 'completed']); // Per-folder apply-job state, keyed by groupKey. Updated from // AutotagApply{Started,Progress,Finished} events emitted by the // backend. Drives the sidebar status icons (running ring, @@ -809,7 +1253,7 @@ export class AutotagView extends LitElement { if (this.prefetchRefreshTimer !== undefined) return; this.prefetchRefreshTimer = window.setTimeout(() => { this.prefetchRefreshTimer = undefined; - void this.loadFolders(); + void this.loadFolders().then(() => this.reconcileSelection()); }, 500); } @@ -896,6 +1340,52 @@ export class AutotagView extends LitElement { await this.refreshAfterAction(); }; + /** Open the in-app MB search dialog, seeding the query fields from + * the current folder's album/artist so the common case is one + * keystroke away. */ + private openSearch(): void { + this.searchKind = 'releasegroup'; + this.searchQuery = this.current?.albumName ?? ''; + this.searchArtist = this.current?.albumArtist ?? ''; + this.searchResults = []; + this.searchError = ''; + this.dialog = 'search'; + } + + private onSearchCancel = () => { this.dialog = 'none'; }; + + private async runSearch(): Promise { + const query = this.searchQuery.trim(); + if (!query) return; + this.searchLoading = true; + this.searchError = ''; + try { + this.searchResults = await SearchCandidates( + this.searchKind, query, this.searchArtist.trim(), + ); + } catch (err) { + this.searchError = `Search failed: ${(err as Error).message}`; + this.searchResults = []; + } finally { + this.searchLoading = false; + } + } + + private async pickSearchResult(hit: SearchHitView): Promise { + if (!this.current) return; + const groupKey = this.current.groupKey; + this.dialog = 'none'; + this.loading = true; + try { + this.score = await SelectSearchCandidate(groupKey, hit.kind, hit.mbid); + this.selectedCandidateIdx = 0; + } catch (err) { + this.errorMessage = `Failed to load candidate: ${(err as Error).message}`; + } finally { + this.loading = false; + } + } + private toggleQueueMenu = (e: Event) => { e.stopPropagation(); this.queueMenuOpen = !this.queueMenuOpen; @@ -907,6 +1397,7 @@ export class AutotagView extends LitElement { try { await ClearCompletedEntries(this.libraryFilterID()); await this.loadFolders(); + await this.reconcileSelection(); } catch (err) { this.errorMessage = `Clear completed failed: ${(err as Error).message}`; } @@ -928,8 +1419,25 @@ export class AutotagView extends LitElement { /* ── Queue ops ── */ + // Incremented on every startQueue so an older in-flight run + // (e.g. the initial load racing a library-filter change) can + // detect it's stale and stop before clobbering the newer + // run's selection. + private queueGeneration = 0; + + /** First actionable folder, matching the sidebar's visual + * order: the Pending section renders first and Skipped / + * Completed start collapsed, so raw folders[0] (sorted by + * score across all statuses) may be hidden from view. */ + private firstPendingFolder(): PendingItem | undefined { + return this.folders.find( + (f) => f.status !== 'confirmed' && f.status !== 'skipped', + ); + } + private async startQueue(): Promise { - this.loading = true; + const generation = ++this.queueGeneration; + this.foldersLoading = true; // Reset state so a library-filter change doesn't leave a // stale folder selected from the previous library. this.current = null; @@ -937,15 +1445,23 @@ export class AutotagView extends LitElement { this.selectedCandidateIdx = 0; try { await StartAutotagQueue(this.libraryFilterID()); + if (generation !== this.queueGeneration) return; await this.loadFolders(); - // Auto-select the first folder so the user lands on - // something to review. - const first = this.folders[0]; + if (generation !== this.queueGeneration) return; + // Folder list is in — let the sidebar paint now, before we + // block on scoring the first folder's candidates. + this.foldersLoading = false; + // Auto-select the first pending folder so the user lands + // on something actionable (selectFolder drives its own + // candidate-loading skeleton via this.loading). + const first = this.firstPendingFolder(); if (first) { await this.selectFolder(first.groupKey); } } finally { - this.loading = false; + if (generation === this.queueGeneration) { + this.foldersLoading = false; + } } } @@ -958,10 +1474,32 @@ export class AutotagView extends LitElement { } } + /** Called after a background folder-list reload (prefetch + * refresh, clear-completed). If the selected folder is no + * longer in the list, fall back to the first pending row so + * the main pane never shows a folder the sidebar doesn't. */ + private async reconcileSelection(): Promise { + const key = this.current?.groupKey; + if (key !== undefined && this.folders.some((f) => f.groupKey === key)) { + return; + } + + const first = this.firstPendingFolder(); + if (first) { + await this.selectFolder(first.groupKey); + } else if (this.current) { + this.current = null; + this.score = null; + } + } + private async selectFolder(groupKey: string): Promise { + const generation = this.queueGeneration; this.errorMessage = ''; this.score = null; this.selectedCandidateIdx = 0; + this.localCoverUrl = ''; + void this.loadLocalCover(groupKey); // Show what we already know about this folder while // candidates load — keeps the header from blanking. const inList = this.folders.find((f) => f.groupKey === groupKey); @@ -969,6 +1507,10 @@ export class AutotagView extends LitElement { try { if (!inList) { const fetched = await GetPendingFolder(groupKey); + // A library-filter change restarted the queue while + // this fetch was in flight — don't resurrect a + // folder from the previous library. + if (generation !== this.queueGeneration) return; this.current = fetched ?? null; } if (this.current) { @@ -979,6 +1521,20 @@ export class AutotagView extends LitElement { } } + /** Fetch the folder's embedded local artwork (best-effort). + * Guarded on groupKey so a fast folder switch doesn't paint the + * previous folder's cover against the new one. */ + private async loadLocalCover(groupKey: string): Promise { + try { + const url = await GetLocalCoverArt(groupKey); + if (this.current?.groupKey === groupKey) { + this.localCoverUrl = url ?? ''; + } + } catch { + // No local art is a normal case; leave the placeholder. + } + } + private async loadCandidates(groupKey: string): Promise { this.loading = true; try { @@ -1178,6 +1734,7 @@ export class AutotagView extends LitElement { case 's': e.preventDefault(); void this.onSkip(); break; case 'l': e.preventDefault(); void this.onLeave(); break; case 'u': e.preventDefault(); this.dialog = 'paste'; break; + case 'f': e.preventDefault(); this.openSearch(); break; case 'arrowdown': e.preventDefault(); void this.navigateFolder(1); @@ -1291,22 +1848,23 @@ export class AutotagView extends LitElement { * instead of the whole title. */ private diffText(local: string, candidate: string): TemplateResult | string { if (local === candidate || candidate === '') return candidate || local || ''; + + // Cosmetic-only difference (case / punctuation / the spacing + // punctuation induces): render it muted rather than alarming + // red-green so the user can see the formatting change without + // reading it as a real conflict. + const cosmetic = isCosmeticDiff(local, candidate); + const removeCls = cosmetic ? 'diff-cosmetic-old' : 'diff-old'; + const addCls = cosmetic ? 'diff-cosmetic-new' : 'diff-new'; + const segments = inlineDiff(local, candidate); return html`${segments.map((seg) => { if (seg.type === 'equal') return seg.text; - if (seg.type === 'remove') return html`${seg.text}`; - return html`${seg.text}`; + if (seg.type === 'remove') return html`${seg.text}`; + return html`${seg.text}`; })}`; } - /** Render an inline diff for a number-valued field (track #). - * Renders nothing for matched values. */ - private diffNumber(local: number, candidate: number): TemplateResult | string { - if (local === candidate || candidate === 0) return String(candidate || local || ''); - if (!local) return html`${candidate}`; - return html`${local}${candidate}`; - } - /* ── Render ── */ private renderHeader() { @@ -1339,6 +1897,26 @@ export class AutotagView extends LitElement { `; } + /** Toggle a sidebar section's collapsed state. Reassigns a new + * Set so Lit sees the change (in-place mutation wouldn't). */ + private toggleSection(key: string): void { + const next = new Set(this.collapsedSections); + if (next.has(key)) { + next.delete(key); + } else { + next.add(key); + } + this.collapsedSections = next; + } + + /** Disclosure chevron for a collapsible section header. */ + private sectionChevron(key: string): TemplateResult { + const collapsed = this.collapsedSections.has(key); + return html``; + } + private renderFolderSidebar() { // Group folders by review state. The backend now returns // pending + skipped + confirmed in a single list (sorted by @@ -1366,7 +1944,11 @@ export class AutotagView extends LitElement { return html`
- Pending (${pending.length}) + ${completed.length > 0 ? html` + ${this.collapsedSections.has('skipped') + ? nothing + : skipped.map((f) => this.renderFolderRow(f))} ` : nothing} ${completed.length > 0 ? html` -
- Completed (${completed.length}) -
- ${completed.map((f) => this.renderFolderRow(f))} + + ${this.collapsedSections.has('completed') + ? nothing + : completed.map((f) => this.renderFolderRow(f))} ` : nothing} `}
@@ -1483,53 +2077,47 @@ export class AutotagView extends LitElement { `; } - private renderAlbumCard(cand: CandidateView) { - const localAlbum = this.current?.albumName ?? ''; - const localArtist = this.current?.albumArtist ?? ''; - const releaseYear = (cand.date ?? '').slice(0, 4); - const originalYear = (cand.originalDate ?? '').slice(0, 4); - const subParts: string[] = []; - if (originalYear && releaseYear && originalYear !== releaseYear) { - // Remaster/reissue: show the original year prominently, - // mark the technical re-release year as such. - subParts.push(`${originalYear} (${releaseYear} reissue)`); - } else if (releaseYear) { - subParts.push(releaseYear); - } - if (cand.country) subParts.push(cand.country); - if (cand.status) subParts.push(cand.status); - if (cand.trackCount) subParts.push(`${cand.trackCount} tracks`); + /** + * Versions dropdown: the editions *within the active cluster* + * (remaster / country / reissue of the same album), so the user + * picks a release edition here without leaving the current album. + * Picking a different album entirely is the candidate picker's job. + * Hidden when the active cluster has only one edition. + */ + private renderVersionsDropdown(cand: CandidateView, clusters: VersionCluster[]) { + const cluster = clusters.find((c) => c.candidates.includes(cand)); + if (!cluster || cluster.candidates.length <= 1) return nothing; + + const editions = cluster.candidates + .map((c) => ({ cand: c, idx: this.score?.candidates.indexOf(c) ?? -1 })) + .filter((e) => e.idx >= 0); return html` -
- ${cand.coverArtUrl - ? html`cover art` - : html`
No cover
`} -
-
${this.diffText(localAlbum, cand.title)}
-
${this.diffText(localArtist, cand.artistCredit)}
-
${subParts.join(' · ') || '\u00a0'}
-
- - ${(cand.score * 100).toFixed(0)}% match - - ${this.renderProvenanceBadge(cand)} -
-
-
+ `; } - private renderProvenanceBadge(cand: CandidateView) { - const prov = cand.provenance || 'unknown'; - const label = prov === 'local' ? 'local' - : prov === 'paste' ? 'paste' - : prov === 'strict' ? 'exact' - : prov === 'no-track-count' ? 'any-tracks' - : prov === 'title-only' ? 'title-only' - : prov === 'fuzzy-title' ? 'fuzzy' - : prov; - return html`${label}`; + /** Compact one-line label for a release edition in the versions + * dropdown: year, country, status, falling back to the title. */ + private versionOptionLabel(c: CandidateView): string { + const parts: string[] = []; + const year = (c.date ?? '').slice(0, 4); + if (year) parts.push(year); + if (c.country) parts.push(c.country); + if (c.status) parts.push(c.status); + if (c.trackCount) parts.push(`${c.trackCount} trk`); + return parts.join(' · ') || c.title || 'release'; } /** @@ -1537,21 +2125,33 @@ export class AutotagView extends LitElement { * given candidate. This pairs with renderMatchDetails to give * the user a concrete answer to "why isn't this 100%?" — the * tracklist already shows per-track title/length/# diffs, but - * subtle length drift (<2s) is suppressed there and album- + * subtle length drift (<5s) is suppressed there and album- * level metadata factors aren't visible at all. */ private computeMatchSummary(cand: CandidateView) { let paired = 0; - let missingFromFolder = 0; - let unmatchedInFolder = 0; - let titleDiffs = 0; - let trackNumDiffs = 0; - let visibleLengthDiffs = 0; - let subtleLengthDiffs = 0; + // Paired tracks where both sides carry a usable track number, + // so "Track Numbers: N/M" only counts tracks it can judge. + let numberedPaired = 0; let driftSum = 0; let driftCount = 0; - const SUBTLE_LENGTH_MAX_MS = 2000; + // Title diffs split by significance: a cosmetic diff (case / + // punctuation) has titleScore 1.0 — it does not move the score, + // so it shouldn't read as a warning. A significant diff has + // titleScore < 1. Each detail row keeps both sides so the + // dropdown can render an inline diff of exactly what changed. + const significantTitleDiffs: TitleDiffDetail[] = []; + const cosmeticTitleDiffs: TitleDiffDetail[] = []; + const trackNumDiffs: NumDiffDetail[] = []; + const visibleLengthDiffs: LengthDiffDetail[] = []; + const subtleLengthDiffs: LengthDiffDetail[] = []; + const missingTitles: string[] = []; + const extraTitles: string[] = []; + + // Mirrors the scorer's lengthExactMs grace band — anything + // the score forgives, the UI mutes. + const SUBTLE_LENGTH_MAX_MS = 5000; for (const a of cand.alignments) { if (a.status === 'matched' || a.status === 'mismatched') { @@ -1560,26 +2160,60 @@ export class AutotagView extends LitElement { ? this.score?.localTracks[a.localIndex] ?? null : null; if (local) { - if (local.title !== a.candidateTitle) titleDiffs++; - if ( - a.candidatePosition > 0 - && local.trackNumber > 0 - && local.trackNumber !== a.candidatePosition - ) { - trackNumDiffs++; + if (local.title !== a.candidateTitle) { + const detail: TitleDiffDetail = { + pos: a.candidatePosition, + local: local.title, + candidate: a.candidateTitle, + }; + // Cosmetic when the backend already scored it a + // perfect title match, OR when the only + // difference is case/punctuation/spacing that the + // backend's punctuation-deletion left as a stray + // space (so titleScore dipped just under 1). + if (a.titleScore >= 1 || isCosmeticDiff(local.title, a.candidateTitle)) { + cosmeticTitleDiffs.push(detail); + } else { + significantTitleDiffs.push(detail); + } + } + if (a.candidatePosition > 0 && local.trackNumber > 0) { + numberedPaired++; + if (local.trackNumber !== a.candidatePosition) { + trackNumDiffs.push({ + title: a.candidateTitle || local.title || '(untitled)', + local: local.trackNumber, + candidate: a.candidatePosition, + }); + } } } if (a.lengthDeltaMs > SUBTLE_LENGTH_MAX_MS) { - visibleLengthDiffs++; + visibleLengthDiffs.push({ + pos: a.candidatePosition, + title: a.candidateTitle || local?.title || '(untitled)', + localMs: a.localLengthMillis || local?.lengthMillis || 0, + candidateMs: a.candidateLength, + deltaMs: a.lengthDeltaMs, + }); } else if (a.lengthDeltaMs > 0) { - subtleLengthDiffs++; + subtleLengthDiffs.push({ + pos: a.candidatePosition, + title: a.candidateTitle || local?.title || '(untitled)', + localMs: a.localLengthMillis || local?.lengthMillis || 0, + candidateMs: a.candidateLength, + deltaMs: a.lengthDeltaMs, + }); driftSum += a.lengthDeltaMs; driftCount++; } } else if (a.status === 'missing') { - missingFromFolder++; + missingTitles.push(a.candidateTitle || '(untitled)'); } else if (a.status === 'unmatched') { - unmatchedInFolder++; + const local = a.localIndex >= 0 + ? this.score?.localTracks[a.localIndex] ?? null + : null; + extraTitles.push(a.localTitle || local?.title || '(untitled)'); } } @@ -1590,102 +2224,233 @@ export class AutotagView extends LitElement { return { paired, - missingFromFolder, - unmatchedInFolder, - titleDiffs, + numberedPaired, + missingFromFolder: missingTitles.length, + unmatchedInFolder: extraTitles.length, + missingTitles, + extraTitles, + significantTitleDiffs, + cosmeticTitleDiffs, trackNumDiffs, visibleLengthDiffs, subtleLengthDiffs, avgDriftMs: driftCount > 0 ? driftSum / driftCount : 0, - albumTitleMatches: localAlbum === '' || localAlbum === candAlbum, - albumArtistMatches: localArtist === '' || localArtist === candArtist, + localAlbum, + candAlbum, + localArtist, + candArtist, + albumTitleMatches: localAlbum === '' || normalizeStrict(localAlbum) === normalizeStrict(candAlbum), + albumArtistMatches: localArtist === '' || normalizeStrict(localArtist) === normalizeStrict(candArtist), }; } + /** One match-detail line. When `body` is provided the line is a + * native
disclosure so the user can expand it to see + * exactly what differs; otherwise it's a plain, non-expandable + * row. `cls` drives the leading marker (ok / warn / info). */ + private mdItem( + cls: 'ok' | 'warn' | 'info', + summary: TemplateResult | string, + body: TemplateResult | null = null, + ): TemplateResult { + const mark = html``; + if (!body) { + return html` +
+ ${mark}${summary} +
+ `; + } + return html` + + `; + } + + /** Inline before/after diff rows for a set of title changes. */ + private renderTitleDiffBody(details: TitleDiffDetail[]): TemplateResult { + return html`${details.map((d) => html` +
+ ${d.pos > 0 ? html`${d.pos}` : nothing} + ${this.diffText(d.local, d.candidate)} +
+ `)}`; + } + + /** Track-number change rows: local # → candidate #. */ + private renderNumDiffBody(details: NumDiffDetail[]): TemplateResult { + return html`${details.map((d) => html` +
+ ${d.title || '(untitled)'} + + #${d.local} + → + #${d.candidate} + +
+ `)}`; + } + + /** Length-drift rows: local m:ss → candidate m:ss (±Xs). + * When `subtle` (drift under 5s, treated as a successful match) + * the values render muted rather than red-strike/green so the + * row doesn't read as a real mismatch. */ + private renderLengthDiffBody( + details: LengthDiffDetail[], subtle = false, + ): TemplateResult { + const oldCls = subtle ? 'diff-cosmetic-old' : 'diff-old'; + const newCls = subtle ? 'diff-cosmetic-new' : 'diff-new'; + return html`${details.map((d) => { + const delta = (d.deltaMs / 1000).toFixed(1); + return html` +
+ ${d.pos > 0 ? html`${d.pos}` : nothing} + ${d.title || '(untitled)'} + + ${this.formatLength(d.localMs) || '—'} + → + ${this.formatLength(d.candidateMs) || '—'} + (±${delta}s) + +
+ `; + })}`; + } + + /** Plain title list body for missing / extra tracks. */ + private renderTitleListBody( + titles: string[], side: 'candidate' | 'local', + ): TemplateResult { + const cls = side === 'candidate' ? 'diff-new' : 'diff-old'; + return html`${titles.map((t) => html` +
+ ${t || '(untitled)'} +
+ `)}`; + } + + /** A counted match category rendered as "Label: matched/total". + * When every item matches (matched >= total) the line is a plain + * green-check row with no dropdown; only a real conflict gets the + * expandable detail body. */ + private mdCategory( + label: string, + matched: number, + total: number, + body: TemplateResult | null = null, + ): TemplateResult { + const ok = matched >= total; + const summary = html`${label}: + ${matched}/${total}`; + return this.mdItem(ok ? 'ok' : 'warn', summary, ok ? null : body); + } + + /** A single-count match line rendered as "Label: N". `cls` + * drives the marker/colour; a body makes it an expandable + * disclosure (used for the missing/extra track lists). */ + private mdCount( + cls: 'ok' | 'warn' | 'info', + label: string, + count: number, + body: TemplateResult | null = null, + ): TemplateResult { + const summary = html`${label}: + ${count}`; + return this.mdItem(cls, summary, body); + } + private renderMatchDetails(cand: CandidateView) { const s = this.computeMatchSummary(cand); - const totalCand = s.paired + s.missingFromFolder; const items: TemplateResult[] = []; - // ── Track pairing ────────────────────────────────────── - if (s.missingFromFolder === 0 && s.unmatchedInFolder === 0) { - items.push(html` -
  • - All ${s.paired} ${s.paired === 1 ? 'track' : 'tracks'} paired -
    • - `); - } else { - if (s.missingFromFolder > 0) { - items.push(html` -
  • - ${s.missingFromFolder} ${s.missingFromFolder === 1 ? 'track' : 'tracks'} - on candidate but not in folder -
    • - `); - } - if (s.unmatchedInFolder > 0) { - items.push(html` -
  • - ${s.unmatchedInFolder} ${s.unmatchedInFolder === 1 ? 'track' : 'tracks'} - in folder but not on candidate -
    • - `); - } + // ── Track pairing — matched / missing / extra as separate + // single-count lines, so there's no ambiguous "total". + // Matched is always shown; missing/extra only when non-zero + // (they're the conflicts, and carry the expandable lists). + items.push(this.mdCount('ok', 'Matched Tracks', s.paired)); + if (s.missingFromFolder > 0) { + items.push(this.mdCount( + 'warn', + 'Missing Tracks', + s.missingFromFolder, + this.renderTitleListBody(s.missingTitles, 'candidate'), + )); + } + if (s.unmatchedInFolder > 0) { + items.push(this.mdCount( + 'warn', + 'Extra Tracks', + s.unmatchedInFolder, + this.renderTitleListBody(s.extraTitles, 'local'), + )); } - // ── Title diffs (pointer to tracklist) ───────────────── - if (s.titleDiffs > 0) { - items.push(html` -
  • - ${s.titleDiffs} of ${s.paired} ${s.titleDiffs === 1 ? 'title differs' : 'titles differ'} - — see tracklist -
    • - `); - } else if (s.paired > 0) { - items.push(html`
  • All track titles match
    • `); + // ── Track titles: cosmetic (case/punctuation) diffs count as + // a match since they don't move the score. Only significant + // diffs are conflicts — but when there is one, the dropdown + // shows every title change (significant + cosmetic) for context. + if (s.paired > 0) { + const allTitleDiffs = [...s.significantTitleDiffs, ...s.cosmeticTitleDiffs] + .sort((a, b) => a.pos - b.pos); + items.push(this.mdCategory( + 'Track Titles', + s.paired - s.significantTitleDiffs.length, + s.paired, + this.renderTitleDiffBody(allTitleDiffs), + )); } - // ── Track-number diffs ───────────────────────────────── - if (s.trackNumDiffs > 0) { - items.push(html` -
  • - ${s.trackNumDiffs} track ${s.trackNumDiffs === 1 ? 'number' : 'numbers'} would change -
    • - `); + // ── Track lengths: drift under 5s counts as a match, so only + // the >5s differences show up (and only they get a dropdown). + if (s.paired > 0) { + items.push(this.mdCategory( + 'Track Lengths', + s.paired - s.visibleLengthDiffs.length, + s.paired, + this.renderLengthDiffBody(s.visibleLengthDiffs), + )); } - // ── Length diffs: visible (>2s) vs subtle (<=2s) ─────── - if (s.visibleLengthDiffs > 0) { - items.push(html` -
  • - ${s.visibleLengthDiffs} - ${s.visibleLengthDiffs === 1 ? 'track length differs' : 'track lengths differ'} - by more than 2s — see tracklist -
    • - `); - } - if (s.subtleLengthDiffs > 0) { - const avg = (s.avgDriftMs / 1000).toFixed(1); - items.push(html` -
  • - ${s.subtleLengthDiffs} - ${s.subtleLengthDiffs === 1 ? 'track' : 'tracks'} drift by ~${avg}s - (under 2s, suppressed in tracklist) -
    • - `); + // ── Track numbers: only counts tracks that carry a number on + // both sides, so an untagged folder doesn't read as all-wrong. + if (s.numberedPaired > 0) { + items.push(this.mdCategory( + 'Track Numbers', + s.numberedPaired - s.trackNumDiffs.length, + s.numberedPaired, + this.renderNumDiffBody(s.trackNumDiffs), + )); } - // ── Album header diffs (mirror what the header card shows) ── + // ── Album header — boolean fields, shown only when they'd + // change (a conflict); the dropdown shows the exact diff. if (!s.albumTitleMatches) { - items.push(html`
  • Album name would change
    • `); + items.push(this.mdItem( + 'warn', + 'Album name would change', + html`
    + ${this.diffText(s.localAlbum, s.candAlbum)} +
    `, + )); } if (!s.albumArtistMatches) { - items.push(html`
  • Album artist would change
    • `); + items.push(this.mdItem( + 'warn', + 'Album artist would change', + html`
    + ${this.diffText(s.localArtist, s.candArtist)} +
    `, + )); } // ── Release-info line: not a "diff" per se but a useful // signal since older / non-Official releases score lower - // even when every track lines up. + // even when every track lines up. No dropdown — informational. const releaseParts: string[] = []; const releaseYear = (cand.date ?? '').slice(0, 4); const originalYear = (cand.originalDate ?? '').slice(0, 4); @@ -1694,20 +2459,11 @@ export class AutotagView extends LitElement { } else if (releaseYear) { releaseParts.push(releaseYear); } + if (cand.primaryType) releaseParts.push(cand.primaryType); if (cand.country) releaseParts.push(cand.country); if (cand.status) releaseParts.push(cand.status); if (releaseParts.length) { - items.push(html` -
  • Release: ${releaseParts.join(' · ')}
    • - `); - } - - if (totalCand > 0 && cand.trackCount > 0 && cand.trackCount !== totalCand) { - items.push(html` -
  • - Candidate has ${cand.trackCount} tracks total -
    • - `); + items.push(this.mdItem('info', html`Release: ${releaseParts.join(' · ')}`)); } const b = cand.breakdown; @@ -1718,14 +2474,21 @@ export class AutotagView extends LitElement {
    Match details
    -
      ${items}
    +
    ${items}
    ${b ? html`
    Match ${pct(cand.score)} Title ${pct(b.titleAvg)} Length ${pct(b.lengthAvg)} + Artist ${pct(b.artistFit)} + ${b.albumFit < 1 + ? html`Album ${pct(b.albumFit)}` + : nothing} Tracks ${pct(b.trackCountFit)} Release ${pct(b.releaseMeta)} + ${b.evidence < 1 + ? html`Evidence ${pct(b.evidence)}` + : nothing}
    ` : nothing}
    @@ -1757,161 +2520,256 @@ export class AutotagView extends LitElement { * group at the same level so the user can see what's expected * but absent on disk. */ - private renderTracklist(cand: CandidateView) { - const locals = new Map(); - this.score?.localTracks.forEach((t, i) => locals.set(i, t)); - - // Bucket alignments: paired (matched/mismatched) and - // candidate-side missing get sorted into the candidate - // tracklist by candidate position; folder-side unmatched - // is rendered separately. - const paired: AlignmentView[] = []; - const missing: AlignmentView[] = []; - const unmatched: AlignmentView[] = []; + /** + * The two-column comparison: the folder's files on the left (in + * folder order) and the candidate release on the right (in + * candidate order). Matched rows share a data-pair id so hovering + * either side highlights its partner; folder tracks with no + * candidate partner (extra) and candidate tracks with no folder + * partner (missing) both surface as amber gaps, which is what makes + * "do I have extra or missing tracks?" answerable at a glance. + * Field-level title/length conflicts are summarised in + * renderMatchDetails, so each column shows its own values plainly. + */ + private renderComparison(cand: CandidateView, clusters: VersionCluster[] = []) { + const locals = this.score?.localTracks ?? []; + // localIndex -> its alignment, so a folder row knows whether it + // paired and (if so) how confidently. + const alignByLocal = new Map(); for (const a of cand.alignments) { - switch (a.status) { - case 'matched': - case 'mismatched': - paired.push(a); - break; - case 'missing': - missing.push(a); - break; - case 'unmatched': - unmatched.push(a); - break; - default: - paired.push(a); - } + if (a.localIndex >= 0) alignByLocal.set(a.localIndex, a); } - const candTracks = [...paired, ...missing].map((a) => ({ - ...a, - discNumber: a.candidateDiscNumber || 1, - position: a.candidatePosition || 0, - })); + // Candidate side: every alignment that has a candidate track + // (paired or missing-from-folder), in candidate order. + const candRows = cand.alignments + .filter((a) => a.status !== 'unmatched' && a.candidatePosition > 0) + .slice() + .sort((x, y) => + (x.candidateDiscNumber - y.candidateDiscNumber) + || (x.candidatePosition - y.candidatePosition)); - const multi = isMultiDisc(candTracks); - const grouped = groupByDisc(candTracks); - const discs = discNumbers(grouped); + const folderRows = locals.map((t, i) => { + const a = alignByLocal.get(i); + const paired = !!a && (a.status === 'matched' || a.status === 'mismatched'); + const cls = !paired ? 'extra' : a!.status === 'mismatched' ? 'mismatched' : 'matched'; + const pair = paired ? `p${i}` : ''; + return this.renderCompareRow(cls, pair, t.trackNumber, t.title, t.lengthMillis); + }); + + const candItems = candRows.map((a) => { + const paired = a.status === 'matched' || a.status === 'mismatched'; + const cls = !paired ? 'extra' : a.status === 'mismatched' ? 'mismatched' : 'matched'; + const pair = paired ? `p${a.localIndex}` : ''; + return this.renderCompareRow( + cls, pair, a.candidatePosition, a.candidateTitle, a.candidateLength, + ); + }); return html` -
    -
    Tracks
    - ${discs.map((discNum) => { - const rows = grouped.get(discNum) ?? []; +
    +
    + ${this.renderLocalAlbumHeader(folderRows.length)} + ${folderRows.length > 0 + ? folderRows + : html`
    No local tracks.
    `} +
    +
    + ${this.renderCandidateAlbumHeader(cand, candItems.length, clusters)} + ${candItems.length > 0 + ? candItems + : html`
    No candidate tracks.
    `} +
    +
    + `; + } + + /** Local column header: the folder's own artwork + album/artist, + * shown plainly (no diff coloring) so the user sees what the + * files currently look like. */ + private renderLocalAlbumHeader(trackCount: number) { + const album = this.current?.albumName || '(no album)'; + const artist = this.current?.albumArtist || 'Unknown artist'; + return html` +
    + ${this.localCoverUrl + ? html`local cover art` + : html`
    No cover
    `} +
    +
    Your folder
    +
    ${album}
    +
    ${artist}
    +
    ${trackCount} ${trackCount === 1 ? 'track' : 'tracks'}
    +
    +
    + `; + } + + /** Candidate column header: the fetched release's artwork + + * title/artist, its release-info line, the match percentage and + * — when the cluster has multiple editions — the versions picker. + * Shown plainly (no diff coloring). */ + private renderCandidateAlbumHeader( + cand: CandidateView, trackCount: number, clusters: VersionCluster[], + ) { + const title = cand.title || '(untitled)'; + const artist = cand.artistCredit || 'Unknown artist'; + const releaseYear = (cand.date ?? '').slice(0, 4); + const originalYear = (cand.originalDate ?? '').slice(0, 4); + const subParts: string[] = []; + if (originalYear && releaseYear && originalYear !== releaseYear) { + subParts.push(`${originalYear} (${releaseYear} reissue)`); + } else if (releaseYear) { + subParts.push(releaseYear); + } + if (cand.country) subParts.push(cand.country); + if (cand.status) subParts.push(cand.status); + subParts.push(`${trackCount} ${trackCount === 1 ? 'track' : 'tracks'}`); + + const versions = this.renderVersionsDropdown(cand, clusters); + + return html` +
    + ${cand.coverArtUrl + ? html`candidate cover art` + : html`
    No cover
    `} +
    +
    Candidate
    +
    ${title}
    +
    ${artist}
    +
    + + ${(cand.score * 100).toFixed(0)}% match + + ${subParts.join(' · ')} +
    + ${versions !== nothing + ? html`
    ${versions}
    ` + : nothing} +
    +
    + `; + } + + private renderCompareRow( + cls: string, pair: string, pos: number, title: string, lengthMs: number, + ): TemplateResult { + const hover = pair + ? { + enter: () => this.highlightPair(pair, true), + leave: () => this.highlightPair(pair, false), + } + : { enter: () => {}, leave: () => {} }; + return html` +
    + ${pos || '-'} + ${title || '(untitled)'} + ${this.formatLength(lengthMs)} +
    + `; + } + + /** Toggle the partner-highlight class on every row that shares the + * given data-pair id (the local row and its candidate row). */ + private highlightPair(pair: string, on: boolean): void { + const rows = this.renderRoot.querySelectorAll(`[data-pair="${pair}"]`); + rows.forEach((r) => r.classList.toggle('pair-hi', on)); + } + + /** + * Ranked candidate picker: one chip per version-cluster (its best + * edition), sorted by score, so the user can jump to a lower-scored + * alternative — the beets-style "here are the other matches" list. + * Editions *within* the selected cluster are chosen via the + * versions dropdown in the album card, not here. + */ + private renderCandidatePicker(clusters: VersionCluster[], activeCand: CandidateView) { + if (clusters.length <= 1) return nothing; + + return html` +
    + ${clusters.map((cluster) => { + const best = cluster.candidates[cluster.bestIdx]!; + const active = cluster.candidates.includes(activeCand); + const idx = this.score?.candidates.indexOf(best) ?? -1; return html` - ${multi - ? html`
    Disc ${discNum}
    ` - : nothing} - ${rows.map((a) => this.renderTrackRow(a, locals))} +
    { void this.selectCandidateByIdx(idx); }}> + ${cluster.label} + + ${(best.score * 100).toFixed(0)}% + +
    `; })} - ${unmatched.length > 0 ? html` -
    - Unmatched (in folder, not in candidate) -
    - ${unmatched.map((a) => this.renderUnmatchedRow(a, locals))} - ` : nothing}
    `; } - private renderTrackRow(a: AlignmentView, locals: Map) { - const local = a.localIndex >= 0 ? locals.get(a.localIndex) ?? null : null; - const cls = `track-row ${a.status}`; - - if (a.status === 'missing') { - // Candidate has this track, folder doesn't. - return html` -
    - ${a.candidatePosition} - - ${a.candidateTitle} - - ${this.formatLength(a.candidateLength)} + /** Shimmer placeholder rows for the folder sidebar while the + * pending-folder list is still loading. */ + private renderFolderSkeleton(): TemplateResult { + const rows = 8; + return html` + ${Array.from({ length: rows }, () => html` +
    +
    +
    +
    - `; - } - - // Paired (matched / mismatched). Show track-#, title, - // length — each as a diff against the local file. - const localTitle = local?.title ?? ''; - const localLen = local?.lengthMillis ?? 0; - const localPos = local?.trackNumber ?? 0; - - return html` -
    - - ${this.diffNumber(localPos, a.candidatePosition)} - - - ${this.diffText(localTitle, a.candidateTitle)} - - - ${this.renderLengthDiff(localLen, a.candidateLength)} - -
    + `)} `; } - private renderLengthDiff(localMs: number, candMs: number): TemplateResult | string { - const candStr = this.formatLength(candMs); - const localStr = this.formatLength(localMs); - if (!candStr) return localStr; - if (!localStr || localStr === candStr) return candStr; - // Show only the candidate length when the difference is - // tiny (<= 2s) — disk seek noise, not a real diff. - if (Math.abs(localMs - candMs) <= 2000) return candStr; - return html`${localStr}${candStr}`; - } - - private renderUnmatchedRow(a: AlignmentView, locals: Map) { - const local = a.localIndex >= 0 ? locals.get(a.localIndex) ?? null : null; - const title = a.localTitle || local?.title || '(untitled)'; - const len = a.localLengthMillis || local?.lengthMillis || 0; - const pos = local?.trackNumber ?? 0; + /** Shimmer placeholder for the main pane — mirrors the album card + * (cover + meta) and a few tracklist rows so the layout doesn't + * jump when the real candidate data arrives. */ + private renderMainSkeleton(): TemplateResult { return html` -
    - ${pos || '-'} - ${title} - ${this.formatLength(len)} -
    - `; - } - - private renderVersionsSidebar(clusters: VersionCluster[]) { - const selectedCand = this.currentCandidate(); - return html` -
    -
    Versions (${clusters.length})
    - ${clusters.length === 0 - ? html`
    No candidates.
    Hit U to paste a URL.
    ` - : clusters.map((cluster) => { - const best = cluster.candidates[cluster.bestIdx]!; - const isSelected = best === selectedCand; - const idx = this.score?.candidates.indexOf(best) ?? -1; - return html` -
    { void this.selectCandidateByIdx(idx); }}> -
    ${cluster.label}
    -
    - ${cluster.sublabel || '\u00a0'} - - ${(best.score * 100).toFixed(0)}% - -
    -
    - `; - })} +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    + ${Array.from({ length: 6 }, () => html` +
    + + + +
    + `)} +
    `; } private renderMain() { - if (this.loading && !this.score) { - return html`
    Loading candidates\u2026
    `; + // A scoring error on the selected folder surfaces as an error, + // not an endless skeleton. + if (this.current && !this.score && this.errorMessage) { + return html` +
    +
    + ${this.errorMessage} + +
    +
    + `; + } + + // Folder list still arriving, or the selected folder's + // candidates are still being scored \u2192 skeleton, not text. + if (this.foldersLoading || (this.current && (this.loading || !this.score))) { + return this.renderMainSkeleton(); } if (!this.current) { @@ -1933,7 +2791,7 @@ export class AutotagView extends LitElement {
    No candidates found for this folder.
    - Hit U to paste a MusicBrainz URL. + Hit F to search MusicBrainz or U to paste a URL.
    `; @@ -1955,9 +2813,9 @@ export class AutotagView extends LitElement {
    ` : nothing} ${this.renderLowConfidenceBanner(clusters)} - ${this.renderAlbumCard(cand)} + ${this.renderCandidatePicker(clusters, cand)} ${this.renderMatchDetails(cand)} - ${this.renderTracklist(cand)} + ${this.renderComparison(cand, clusters)}
    `; } @@ -2061,30 +2919,92 @@ export class AutotagView extends LitElement { `; } + private renderSearchDialog() { + return html` +
    { + if (e.target === e.currentTarget) this.onSearchCancel(); + }}> +
    +
    +

    Search MusicBrainz

    + +
    +
    + + +
    + { this.searchQuery = (e.target as HTMLInputElement).value; }} + @keydown=${(e: KeyboardEvent) => { if (e.key === 'Enter') void this.runSearch(); }} + autofocus> + { this.searchArtist = (e.target as HTMLInputElement).value; }} + @keydown=${(e: KeyboardEvent) => { if (e.key === 'Enter') void this.runSearch(); }}> +
    + + +
    + ${this.searchError + ? html`
    ${this.searchError}
    ` + : nothing} + ${this.searchResults.length > 0 ? html` +
    + ${this.searchResults.map((hit) => html` +
    { void this.pickSearchResult(hit); }}> +
    ${hit.title}
    +
    + ${hit.artist || '—'} + ${hit.detail ? html`${hit.detail}` : nothing} +
    +
    + `)} +
    + ` : this.searchLoading || this.searchError || this.searchQuery.trim() === '' + ? nothing + : html`
    No results — try dropping the artist or switching Album/Track.
    `} +
    +
    + `; + } + private renderDialog() { switch (this.dialog) { case 'paste': return this.renderPasteDialog(); case 'warning': return this.renderWarningDialog(); case 'leave': return this.renderLeaveDialog(); + case 'search': return this.renderSearchDialog(); default: return nothing; } } override render() { - if (this.loading && !this.current && this.folders.length === 0) { - return html`
    Loading pending folders\u2026
    `; - } - - const clusters = this.score - ? this.clusterVersions(this.score.candidates) - : []; - + // The layout chrome always renders immediately; each pane owns + // its own skeleton while its data resolves, so the user never + // sees a blank full-screen "Loading\u2026". return html`
    ${this.renderHeader()} ${this.renderFolderSidebar()} ${this.renderMain()} - ${this.renderVersionsSidebar(clusters)}
    ${this.renderDialog()} `; diff --git a/frontend/src/components/config-page/config-page.ts b/frontend/src/components/config-page/config-page.ts index 8d010b8..ec737c7 100644 --- a/frontend/src/components/config-page/config-page.ts +++ b/frontend/src/components/config-page/config-page.ts @@ -1945,7 +1945,7 @@ export class ConfigPage extends LitElement { return html`
    diff --git a/frontend/src/components/cover-grid/cover-grid.ts b/frontend/src/components/cover-grid/cover-grid.ts index c9560d9..b401352 100644 --- a/frontend/src/components/cover-grid/cover-grid.ts +++ b/frontend/src/components/cover-grid/cover-grid.ts @@ -46,6 +46,7 @@ import type { DragPayload } from '@utils/drag-controller'; import { ContextMenuController } from '@utils/context-menu-controller.js'; import type { ContextMenuHost } from '@utils/context-menu-controller.js'; import { FavoritesController } from '@store/controllers/favorites-controller'; +import { artistLink, exploreLinkStyles } from '../../utils/explore-link'; import { createAlbumArtDragImage, createDragImage, @@ -264,7 +265,7 @@ export class CoverGrid private splitEntriesCacheKey: GridEntry[] | null = null; private splitEntriesCacheIndex = -1; - static override styles = coverGridStyles; + static override styles = [coverGridStyles, exploreLinkStyles]; /* ==================================================================== * Reactive state @@ -1832,7 +1833,7 @@ export class CoverGrid class="artist-name" title="${album.ArtistName}" > - ${album.ArtistName} + ${artistLink(album.ArtistName, album.ArtistMBID ?? '')}
    diff --git a/frontend/src/components/explore-album-details/explore-album-details.ts b/frontend/src/components/explore-album-details/explore-album-details.ts index 80f1497..b6cede1 100644 --- a/frontend/src/components/explore-album-details/explore-album-details.ts +++ b/frontend/src/components/explore-album-details/explore-album-details.ts @@ -15,6 +15,9 @@ type MBTrack = explore.MBTrack; import { exploreCache } from '../../store/explore-cache'; import { exploreSettings } from '../../store/explore-settings'; import { libraryStore } from '../../store/library-store'; +import { artistLink, exploreLinkStyles } from '../../utils/explore-link'; +import { EventsOn } from '@runtime/runtime'; +import { Events } from '../../events'; import '@awesome.me/webawesome/dist/components/icon/icon.js'; import '../library-status-indicator/library-status-indicator.js'; @@ -113,6 +116,7 @@ export class ExploreAlbumDetails extends LitElement { static override styles = [ designTokens, + exploreLinkStyles, css` :host { display: flex; @@ -451,6 +455,13 @@ export class ExploreAlbumDetails extends LitElement { /* ── Lifecycle ── */ private unsubSettings?: () => void; + private unsubReleasesReady?: () => void; + /** Release-group MBIDs whose AlbumReleasesReady event we've handled, + * so a background BrowseReleases fetch re-hydrates versions once. */ + private releasesReloaded = new Set(); + /** Fallback timer that stops the versions spinner if the background + * BrowseReleases fetch never signals readiness. */ + private releasesFallbackTimer?: number; override connectedCallback() { super.connectedCallback(); @@ -465,11 +476,45 @@ export class ExploreAlbumDetails extends LitElement { void this.loadAllData(); } }); + + // A background BrowseReleases fetch (cold album, versions + + // tracklist not cached yet) finished — re-fetch the versions once + // per release group so they fill in without the initial request + // having blocked on a live MusicBrainz browse. + this.unsubReleasesReady = EventsOn( + Events.AlbumReleasesReady, + (mbid: string) => { + if (mbid !== this.releaseGroupMBID) return; + if (this.releasesReloaded.has(mbid)) return; + + if (this.releasesFallbackTimer) clearTimeout(this.releasesFallbackTimer); + this.releasesReloaded.add(mbid); + void this.fetchReleases(mbid); + }, + ); } override disconnectedCallback() { super.disconnectedCallback(); this.unsubSettings?.(); + this.unsubReleasesReady?.(); + if (this.releasesFallbackTimer) clearTimeout(this.releasesFallbackTimer); + } + + /** + * Arm a one-shot fallback that stops the versions spinner if + * AlbumReleasesReady never arrives (e.g. the background browse stalled + * or the release group genuinely has no releases). + */ + private armReleasesFallback(mbid: string) { + if (this.releasesFallbackTimer) clearTimeout(this.releasesFallbackTimer); + + this.releasesFallbackTimer = window.setTimeout(() => { + if (this.releasesReloaded.has(mbid)) return; + + this.releasesReloaded.add(mbid); + if (this.releases.length === 0) this.loadingReleases = false; + }, 12000); } /** Whether we've already scrolled to the highlight target. */ @@ -595,7 +640,11 @@ export class ExploreAlbumDetails extends LitElement { } // Phase 2: fire API calls independently so each section - // renders as its data arrives. + // renders as its data arrives. Allow the versions section one + // background-fetch re-fetch and arm a fallback so it can't spin + // forever if AlbumReleasesReady never arrives. + this.releasesReloaded.delete(mbid); + this.armReleasesFallback(mbid); void this.fetchReleaseGroup(mbid); void this.fetchReleases(mbid); @@ -828,13 +877,28 @@ export class ExploreAlbumDetails extends LitElement { private async fetchReleases(mbid: string) { try { const releases = await BrowseReleases(mbid); - this.releases = releases ?? []; - this.buildClusters(); + + if (releases && releases.length > 0) { + // Warm cache hit (or the background re-fetch landed): + // authoritative MB versions replace any local placeholder. + this.releases = releases; + this.buildClusters(); + this.loadingReleases = false; + return; + } + + // Cold miss: BrowseReleases is cache-first + async and the + // versions/tracklist are still being fetched in the background. + // Don't clobber a tracklist already hydrated from the library — + // keep showing it. Hold the spinner only when there's nothing + // on screen yet; AlbumReleasesReady (or the fallback) resolves it. + if (this.releases.length > 0 || this.releasesReloaded.has(mbid)) { + this.loadingReleases = false; + } } catch (err) { const msg = err instanceof Error ? err.message : String(err); this.errorReleases = msg; console.error(`[explore-album] BrowseReleases error: ${msg}`); - } finally { this.loadingReleases = false; } } @@ -1437,6 +1501,7 @@ export class ExploreAlbumDetails extends LitElement { const rg = this.releaseGroup; const artist = rg.artistCredit || ''; + const artistMbid = rg.artistMbid ?? ''; const year = extractYear(rg.firstReleaseDate); const type = rg.primaryType || ''; @@ -1446,7 +1511,9 @@ export class ExploreAlbumDetails extends LitElement { return html` ${artist - ? html`
    ${artist}
    ` + ? html`
    + ${artistLink(artist, artistMbid)} +
    ` : nothing} ${metaParts.length > 0 ? html` diff --git a/frontend/src/components/explore-artist-details/explore-artist-details.ts b/frontend/src/components/explore-artist-details/explore-artist-details.ts index 5efc5f6..63649a4 100644 --- a/frontend/src/components/explore-artist-details/explore-artist-details.ts +++ b/frontend/src/components/explore-artist-details/explore-artist-details.ts @@ -15,6 +15,7 @@ import { GetTrackThumbnail, GetTrackThumbnails, ResolveReleaseGroupMBIDs, + PrefetchReleases, } from '@go/explore/Service'; import type { explore } from '@go/models'; type MBArtist = explore.MBArtist; @@ -25,7 +26,10 @@ type LBSimilarArtist = explore.LBSimilarArtist; import { exploreCache } from '../../store/explore-cache'; import { exploreSettings } from '../../store/explore-settings'; import { libraryStore } from '../../store/library-store'; +import { trackLink, exploreLinkStyles } from '../../utils/explore-link'; import { GetAlbumsByArtist } from '@go/library/Library'; +import { EventsOn } from '@runtime/runtime'; +import { Events } from '../../events'; import '@awesome.me/webawesome/dist/components/icon/icon.js'; import '../library-status-indicator/library-status-indicator.js'; @@ -123,6 +127,7 @@ export class ExploreArtistDetails extends LitElement { static override styles = [ designTokens, + exploreLinkStyles, css` :host { display: flex; @@ -826,6 +831,17 @@ export class ExploreArtistDetails extends LitElement { /* ── Lifecycle ── */ private unsubSettings?: () => void; + private unsubDiscogReady?: () => void; + private unsubSimilarReady?: () => void; + /** MBIDs whose ArtistSimilarReady event we've already handled, so a + * background similar-artists fetch re-hydrates that section once. */ + private similarReloaded = new Set(); + /** MBIDs whose ArtistDiscographyReady event we've already handled, + * so an artist with no discography can't trigger a re-fetch loop. */ + private discogReloaded = new Set(); + /** Fallback timer that stops the top-section spinners if the + * background discography fetch never signals readiness. */ + private discogFallbackTimer?: number; override connectedCallback() { super.connectedCallback(); @@ -840,15 +856,72 @@ export class ExploreArtistDetails extends LitElement { void this.loadAllData(); } }); + + // A background discography fetch (top tracks / top releases for an + // artist that wasn't indexed yet) finished — re-fetch those two + // sections, once per artist, so they fill in without the initial + // request having blocked. + this.unsubDiscogReady = EventsOn( + Events.ArtistDiscographyReady, + (mbid: string) => { + if (mbid !== this.artistMBID) return; + if (this.discogReloaded.has(mbid)) return; + + if (this.discogFallbackTimer) clearTimeout(this.discogFallbackTimer); + this.discogReloaded.add(mbid); + void this.fetchTopTracks(mbid); + void this.fetchTopReleaseGroups(mbid); + // Full discography section is also index-first + async now, + // so re-read it from the freshly-populated index too. + void this.fetchReleaseGroups(mbid); + }, + ); + + // A background similar-artists fetch (LB labs, first view of an + // artist) finished — re-fetch that section once per artist. + this.unsubSimilarReady = EventsOn( + Events.ArtistSimilarReady, + (mbid: string) => { + if (mbid !== this.artistMBID) return; + if (this.similarReloaded.has(mbid)) return; + + this.similarReloaded.add(mbid); + void this.fetchSimilarArtists(mbid); + }, + ); } override disconnectedCallback() { super.disconnectedCallback(); this.unsubSettings?.(); + this.unsubDiscogReady?.(); + this.unsubSimilarReady?.(); + if (this.discogFallbackTimer) clearTimeout(this.discogFallbackTimer); this.topSectionObserver?.disconnect(); this.discoObserver?.disconnect(); } + /** + * Arm a one-shot fallback that clears the top-section loading state + * if ArtistDiscographyReady never arrives (e.g. the artist genuinely + * has no discography, or the background fetch stalled). Treated as a + * "reload happened" so the finally blocks resolve to empty state. + */ + private armDiscogFallback(mbid: string) { + if (this.discogFallbackTimer) clearTimeout(this.discogFallbackTimer); + + this.discogFallbackTimer = window.setTimeout(() => { + if (this.discogReloaded.has(mbid)) return; + + this.discogReloaded.add(mbid); + this.similarReloaded.add(mbid); + if (this.topTracks.length === 0) this.loadingTracks = false; + if (this.topReleaseGroups.length === 0) this.loadingTopReleases = false; + if (this.releaseGroups.length === 0) this.loadingReleases = false; + if (this.similarArtists.length === 0) this.loadingSimilar = false; + }, 12000); + } + protected override firstUpdated() { this.observeTopSectionWidth(); this.observeDiscoWidth(); @@ -1038,6 +1111,13 @@ export class ExploreArtistDetails extends LitElement { return; } + // Fresh load for this artist: allow the top sections one + // background-fetch re-fetch, and arm a fallback so they can't spin + // forever if ArtistDiscographyReady never arrives. + this.discogReloaded.delete(mbid); + this.similarReloaded.delete(mbid); + this.armDiscogFallback(mbid); + // Phase 1: fire all API requests independently so the UI // renders each section as its data arrives, rather than // waiting for the slowest call to finish. @@ -1290,7 +1370,13 @@ export class ExploreArtistDetails extends LitElement { `[explore-artist] TopRecordingsForArtist error: ${msg}`, ); } finally { - this.loadingTracks = false; + // An empty first pass may mean a background discography fetch + // is still in flight (the artist wasn't indexed yet). Keep + // the loading state up until the ArtistDiscographyReady + // re-fetch runs, so the section doesn't flash empty. + if (this.topTracks.length > 0 || this.discogReloaded.has(mbid)) { + this.loadingTracks = false; + } } } @@ -1367,6 +1453,10 @@ export class ExploreArtistDetails extends LitElement { rgs?.map((r) => ({ mbid: r.releaseGroupMbid, albumName: r.title, artistName: r.artistName })) ?? [], ); + + // Warm the release/tracklist cache for the top albums — these + // are the most likely to be clicked from the artist page. + this.prefetchReleases(rgs?.map((r) => r.releaseGroupMbid) ?? []); } catch (err) { const msg = err instanceof Error ? err.message : String(err); console.error( @@ -1374,7 +1464,11 @@ export class ExploreArtistDetails extends LitElement { ); this.topReleaseGroups = []; } finally { - this.loadingTopReleases = false; + // See fetchTopTracks: hold the spinner while a background + // discography fetch may still populate this section. + if (this.topReleaseGroups.length > 0 || this.discogReloaded.has(mbid)) { + this.loadingTopReleases = false; + } } } @@ -1392,6 +1486,10 @@ export class ExploreArtistDetails extends LitElement { rgs?.map((r) => ({ mbid: r.mbid, albumName: r.title, artistName: r.artistCredit })) ?? [], ); + + // Warm the release/tracklist cache for these albums so opening + // one from here is instant instead of a cold MB browse. + this.prefetchReleases(rgs?.map((r) => r.mbid) ?? []); } catch (err) { const msg = err instanceof Error ? err.message : String(err); this.errorReleases = msg; @@ -1399,10 +1497,29 @@ export class ExploreArtistDetails extends LitElement { `[explore-artist] BrowseReleaseGroups error: ${msg}`, ); } finally { - this.loadingReleases = false; + // BrowseReleaseGroups is index-first + async: an empty result on + // a cold artist means the discography is still being fetched in + // the background. Hold the spinner until it arrives (via + // ArtistDiscographyReady) or the fallback fires. + if (this.releaseGroups.length > 0 || this.discogReloaded.has(mbid)) { + this.loadingReleases = false; + } } } + /** + * Warm the backend's release/tracklist cache for a set of release + * groups so opening an album from this page is instant. Fire-and-forget. + */ + private prefetchReleases(mbids: string[]) { + const filtered = mbids.filter((m) => m); + if (filtered.length === 0) return; + + void PrefetchReleases(filtered).catch(() => { + /* best-effort cache warming — ignore failures */ + }); + } + private async fetchSimilarArtists(mbid: string) { try { const artists = await SimilarArtists(mbid); @@ -1415,7 +1532,13 @@ export class ExploreArtistDetails extends LitElement { ); this.similarArtists = []; } finally { - this.loadingSimilar = false; + // SimilarArtists is DB-first + async: an empty result on the + // first view means the LB labs fetch is still running. Hold the + // spinner until ArtistSimilarReady re-fetches (or the fallback + // fires); once reloaded, an empty list is genuinely "none". + if (this.similarArtists.length > 0 || this.similarReloaded.has(mbid)) { + this.loadingSimilar = false; + } } // Fire-and-forget: resolve images for similar artists in parallel. @@ -1973,7 +2096,7 @@ export class ExploreArtistDetails extends LitElement { })()}
    -
    ${t.trackName}
    +
    ${trackLink(t.trackName, t.releaseName, t.releaseGroupMbid ?? '', t.recordingMbid)}
    ${t.artistName}
    diff --git a/frontend/src/components/explore-view/explore-view.ts b/frontend/src/components/explore-view/explore-view.ts index 0717865..3ff2ac0 100644 --- a/frontend/src/components/explore-view/explore-view.ts +++ b/frontend/src/components/explore-view/explore-view.ts @@ -1,74 +1,28 @@ import { LitElement, html, css, nothing } from 'lit'; import { customElement, state, query as litQuery } from 'lit/decorators.js'; import { designTokens } from '../../styles/tokens.css'; -import { Search, GetThumbnail, GetThumbnails, GetArtistImageURL, GetPopularityBatch, RecordSearchClick } from '@go/explore/Service'; +import { SearchLocal, SearchLyrics, GetThumbnail, GetThumbnails, GetArtistImageURL, RecordSearchClick } from '@go/explore/Service'; import { libraryStore } from '../../store/library-store'; import { exploreCache } from '../../store/explore-cache'; -import { exploreSettings } from '../../store/explore-settings'; +import { queueStore } from '../../store/queue-store'; +import { artistLink, trackLink, exploreLinkStyles } from '../../utils/explore-link'; import '@awesome.me/webawesome/dist/components/icon/icon.js'; import '../library-status-indicator/library-status-indicator.js'; import '../top-results-row/top-results-row.js'; -import type { explore } from '@go/models'; +import { explore } from '@go/models'; type ThumbnailRequest = explore.ThumbnailRequest; type MBSearchResult = explore.MBSearchResult; +type LyricsResult = explore.LyricsResult; type MBArtist = explore.MBArtist; type MBReleaseGroup = explore.MBReleaseGroup; type MBRecording = explore.MBRecording; /* ── Constants ── */ -const DEBOUNCE_MS = 300; const MIN_QUERY_LENGTH = 2; -const FUZZY_MAX_DISTANCE = 2; +// Debounce window for live search-as-you-type. The index query is +// local (no network), so this only coalesces rapid keystrokes. +const SEARCH_DEBOUNCE_MS = 180; -/* ── Fuzzy matching ── */ - -/** Levenshtein edit distance between two strings. */ -function editDistance(a: string, b: string): number { - if (a.length === 0) return b.length; - if (b.length === 0) return a.length; - - const matrix: number[][] = []; - - for (let i = 0; i <= a.length; i++) matrix[i] = [i]; - for (let j = 0; j <= b.length; j++) matrix[0]![j] = j; - - for (let i = 1; i <= a.length; i++) { - for (let j = 1; j <= b.length; j++) { - const cost = a[i - 1] === b[j - 1] ? 0 : 1; - matrix[i]![j] = Math.min( - matrix[i - 1]![j]! + 1, - matrix[i]![j - 1]! + 1, - matrix[i - 1]![j - 1]! + cost, - ); - } - } - - return matrix[a.length]![b.length]!; -} - -/** - * Check if a name fuzzy-matches a query. Returns true if: - * - the name contains the query as a substring (exact), OR - * - any word-aligned segment of the name is within edit distance - * FUZZY_MAX_DISTANCE of the query - */ -function fuzzyMatch(query: string, name: string): boolean { - if (name.includes(query)) return true; - - // Split both into words and check if all query words match - // a name word within edit distance (handles per-word typos). - const qWords = query.split(/\s+/); - const nWords = name.split(/\s+/); - - return qWords.every((qw) => - nWords.some( - (nw) => - nw.includes(qw) || - (nw.length >= 3 && qw.includes(nw)) || - (qw.length >= 4 && nw.length >= 4 && editDistance(qw, nw) <= FUZZY_MAX_DISTANCE), - ), - ); -} const MAX_SECTION_RESULTS = 10; @@ -98,27 +52,6 @@ function formatPopularity(count: number): string { return `${count} plays`; } -/** - * Check if `text` contains `word` as a whole word, bounded by - * spaces, hyphens, or string boundaries. Both args must be - * pre-lowercased. - */ -function containsWord(text: string, word: string): boolean { - const re = new RegExp(`(?:^|[\\s\\-])${escapeRegExp(word)}(?:$|[\\s\\-])`, 'i'); - return re.test(text); -} - -function escapeRegExp(s: string): string { - return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); -} - -/** Parse a TrackLength string (milliseconds as string) to number. */ -function parseDuration(s: string): number { - if (!s) return 0; - const n = Number(s); - return isNaN(n) ? 0 : n; -} - /** Extract the year from a date string like "2005-03-29" or "2005". */ function extractYear(dateStr: string): string { if (!dateStr) return ''; @@ -154,10 +87,15 @@ export class ExploreView extends LitElement { @state() private loading = false; @state() private error = ''; @state() private queryTooShort = false; + /** Which search surface is active: catalog (index) or lyrics. */ + @state() private searchMode: 'catalog' | 'lyrics' = 'catalog'; + /** Lyric-search hits (library tracks matched by lyric fragment). */ + @state() private lyricsResults: LyricsResult[] | null = null; /** Monotonic counter to discard stale responses. */ private searchVersion = 0; - private debounceTimer: ReturnType | null = null; + /** Debounce timer for live search-as-you-type. */ + private searchDebounceTimer?: ReturnType; private thumbnailCache = new Map(); private artistImageCache = new Map(); private libraryMBIDs = new Set(); @@ -168,6 +106,7 @@ export class ExploreView extends LitElement { static override styles = [ designTokens, + exploreLinkStyles, css` :host { display: block; @@ -177,6 +116,37 @@ export class ExploreView extends LitElement { box-sizing: border-box; } + /* ── Search mode tabs ── */ + .search-mode-tabs { + display: flex; + gap: 4px; + margin-bottom: 10px; + } + + .search-mode-tab { + display: inline-flex; + align-items: center; + gap: 6px; + background: none; + border: 1px solid transparent; + border-radius: 6px; + color: var(--yj-text-tertiary, #888); + cursor: pointer; + padding: 5px 12px; + font-size: var(--yj-text-sm); + font-family: inherit; + transition: color 0.15s ease, background 0.15s ease; + } + + .search-mode-tab:hover { + color: var(--yj-text-primary, #fff); + } + + .search-mode-tab.active { + color: var(--yj-bg-base, #1a1a1a); + background: var(--yj-accent, #ffd43b); + } + /* ── Search input ── */ .search-container { display: flex; @@ -191,6 +161,66 @@ export class ExploreView extends LitElement { transition: border-color 0.15s ease; } + /* ── Lyrics results ── */ + .lyrics-results { + margin-top: 20px; + display: flex; + flex-direction: column; + gap: 2px; + max-width: 640px; + } + + .lyrics-hit { + display: flex; + align-items: center; + gap: 12px; + width: 100%; + text-align: left; + background: none; + border: none; + border-radius: 6px; + color: var(--yj-text-primary, #fff); + cursor: pointer; + padding: 8px 10px; + font-family: inherit; + transition: background 0.12s ease; + } + + .lyrics-hit:hover { + background: var(--yj-bg-surface, #212529); + } + + .lyrics-hit-play { + color: var(--yj-text-tertiary, #888); + font-size: var(--yj-icon-sm); + flex-shrink: 0; + } + + .lyrics-hit:hover .lyrics-hit-play { + color: var(--yj-accent, #ffd43b); + } + + .lyrics-hit-main { + display: flex; + flex-direction: column; + min-width: 0; + } + + .lyrics-hit-title { + font-size: var(--yj-text-md); + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } + + .lyrics-hit-meta { + font-size: var(--yj-text-sm); + color: var(--yj-text-secondary, #b3b3b3); + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } + .search-container:focus-within { border-color: var(--yj-accent, #ffd43b); } @@ -560,26 +590,10 @@ export class ExploreView extends LitElement { /* ── Lifecycle ── */ - private unsubSettings?: () => void; - - override connectedCallback() { - super.connectedCallback(); - // Re-render and re-search when library-only mode toggles. - this.unsubSettings = exploreSettings.subscribe(() => { - this.requestUpdate(); - // Re-run the current search with the new mode. - if (this.searchQuery.trim().length >= MIN_QUERY_LENGTH) { - void this.executeSearch(); - } - }); - } - override disconnectedCallback() { super.disconnectedCallback(); - this.unsubSettings?.(); - if (this.debounceTimer !== null) { - clearTimeout(this.debounceTimer); - this.debounceTimer = null; + if (this.searchDebounceTimer) { + clearTimeout(this.searchDebounceTimer); } } @@ -589,14 +603,10 @@ export class ExploreView extends LitElement { const input = e.target as HTMLInputElement; this.searchQuery = input.value; - if (this.debounceTimer !== null) { - clearTimeout(this.debounceTimer); - this.debounceTimer = null; - } - const trimmed = this.searchQuery.trim(); if (!trimmed) { + this.cancelPendingSearch(); this.results = null; this.error = ''; this.loading = false; @@ -605,6 +615,7 @@ export class ExploreView extends LitElement { } if (trimmed.length < MIN_QUERY_LENGTH) { + this.cancelPendingSearch(); this.results = null; this.error = ''; this.loading = false; @@ -614,22 +625,57 @@ export class ExploreView extends LitElement { this.queryTooShort = false; - this.debounceTimer = setTimeout(() => { - this.debounceTimer = null; + // Both modes debounce straight to their backend search — catalog + // to the offline index (SearchLocal), lyrics to the FTS lyric + // search. No owned-library seed: the index is the sole source of + // catalog results, so we never paint temporary library matches. + this.scheduleSearch(); + } + + /** Switch between catalog and lyric search, resetting results. */ + private setSearchMode(mode: 'catalog' | 'lyrics') { + if (this.searchMode === mode) return; + + this.cancelPendingSearch(); + this.searchMode = mode; + this.results = null; + this.lyricsResults = null; + this.error = ''; + this.loading = false; + + if (this.searchQuery.trim().length >= MIN_QUERY_LENGTH) { void this.executeSearch(); - }, DEBOUNCE_MS); + } + + this.inputEl?.focus(); + } + + /** Debounce a live index search after the latest keystroke. */ + private scheduleSearch() { + this.cancelPendingSearch(); + this.searchDebounceTimer = setTimeout(() => { + this.searchDebounceTimer = undefined; + if (this.searchQuery.trim().length >= MIN_QUERY_LENGTH) { + void this.executeSearch(); + } + }, SEARCH_DEBOUNCE_MS); + } + + private cancelPendingSearch() { + if (this.searchDebounceTimer) { + clearTimeout(this.searchDebounceTimer); + this.searchDebounceTimer = undefined; + } } private handleClear() { + this.cancelPendingSearch(); this.searchQuery = ''; this.results = null; + this.lyricsResults = null; this.error = ''; this.loading = false; this.queryTooShort = false; - if (this.debounceTimer !== null) { - clearTimeout(this.debounceTimer); - this.debounceTimer = null; - } if (this.inputEl) { this.inputEl.value = ''; this.inputEl.focus(); @@ -639,6 +685,17 @@ export class ExploreView extends LitElement { private handleKeydown(e: KeyboardEvent) { if (e.key === 'Escape') { this.handleClear(); + return; + } + + // Enter is optional now — search runs live as you type — but it + // still fires an immediate search, skipping the debounce wait. + if (e.key === 'Enter') { + e.preventDefault(); + if (this.searchQuery.trim().length >= MIN_QUERY_LENGTH) { + this.cancelPendingSearch(); + void this.executeSearch(); + } } } @@ -651,287 +708,18 @@ export class ExploreView extends LitElement { this.error = ''; const startTime = performance.now(); - console.log(`[explore] search started: "${query}"`); + console.log(`[explore] search started: "${query}" (${this.searchMode})`); - // Phase 1: instant library search — pure frontend, no Go calls. - const localResults = this.searchLibraryCache(query); - if (localResults && (localResults.artists?.length || localResults.releaseGroups?.length)) { - this.results = localResults; - exploreCache.populateFromSearch( - localResults.artists || [], - localResults.releaseGroups || [], - ); - - // Seed artist image cache from library data. - for (const a of localResults.artists || []) { - const img = (a as any)._imageMedium || (a as any)._imageSmall; - if (img && a.mbid) { - this.artistImageCache.set(a.mbid, img); - } - } - - // Fallback: album art for artists without images. - for (const a of localResults.artists || []) { - if (a.mbid && !this.artistImageCache.get(a.mbid)) { - const albumArt = getArtistAlbumArt(a.name); - if (albumArt) { - this.artistImageCache.set(a.mbid, albumArt); - } - } - } - - // In library-only mode, local results already have cover art - // and artist images from the library store — seed both caches - // from library data without making any API calls. - if (exploreSettings.libraryOnly) { - this.seedThumbnailsFromLibrary(); - this.seedArtistImagesFromLibrary(); - } else { - this.loadThumbnails(); - this.loadArtistImages(); - } - const elapsed = (performance.now() - startTime).toFixed(0); - console.log( - `[explore] library results: "${query}" in ${elapsed}ms — ` + - `artists=${localResults.artists?.length ?? 0}, ` + - `albums=${localResults.releaseGroups?.length ?? 0}`, - ); + // Lyrics mode: a single FTS lyric search over the library. + if (this.searchMode === 'lyrics') { + void this.executeLyricsSearch(version, query, startTime); + return; } - // Phase 2: full pipeline (MB + LB + reranking) via Wails RPC. - // Skip entirely in library-only mode — local results are final. - if (!exploreSettings.libraryOnly) { - void this.executeFullSearch(version, query, startTime); - } else { - // Rerank with popularity from the explore index, then finalize. - void this.rerankWithPopularity(localResults).then(() => { - this.loading = false; - }); - } - } - - /** - * Search the frontend library cache for matching artists and albums. - * Pure JS — no Go calls, guaranteed instant. Returns results with - * MBIDs and local cover art so they can navigate to explore pages. - */ - private searchLibraryCache(query: string): MBSearchResult | null { - const q = query.toLowerCase(); - - // Collect all matching artists with match-quality scores. - const artistMatches: Array<{ artist: any; score: number }> = []; - const cachedArtists = libraryStore.cachedArtists; - if (cachedArtists) { - for (const a of cachedArtists) { - const name = a.Name.toLowerCase(); - if (!fuzzyMatch(q, name)) continue; - - // Score by match quality. - let score: number; - if (name === q) { - score = 100; // exact - } else if (name.startsWith(q)) { - score = 90; // starts with - } else if (containsWord(name, q)) { - score = 75; // contains word - } else if (name.includes(q)) { - score = 60; // substring - } else { - score = 40; // fuzzy/word match - } - - artistMatches.push({ artist: a, score }); - } - } - - // Sort by score descending, then alphabetically. - artistMatches.sort((a, b) => b.score - a.score || a.artist.Name.localeCompare(b.artist.Name)); - - const artists: MBArtist[] = artistMatches.slice(0, 10).map((m) => ({ - mbid: m.artist.MBID || '', - name: m.artist.Name, - sortName: '', - type: '', - country: '', - disambiguation: '', - score: m.score, - inLibrary: true, - localId: m.artist.ID, - _imageSmall: m.artist.ImageSmall || '', - _imageMedium: m.artist.ImageMedium || '', - _inLibrary: true, - } as MBArtist & { _imageSmall: string; _imageMedium: string; _inLibrary: boolean })); - - // Collect all matching albums with match-quality scores. - const albumMatches: Array<{ album: any; score: number }> = []; - const cachedAlbums = libraryStore.cachedAlbums; - if (cachedAlbums) { - for (const a of cachedAlbums) { - const name = a.Name.toLowerCase(); - const artist = a.ArtistName.toLowerCase(); - const matchesName = fuzzyMatch(q, name); - const matchesArtist = fuzzyMatch(q, artist); - if (!matchesName && !matchesArtist) continue; - - let score: number; - if (artist === q) { - score = 100; - } else if (name === q) { - score = 95; - } else if (artist.startsWith(q)) { - score = 88; - } else if (name.startsWith(q)) { - score = 85; - } else if (containsWord(artist, q)) { - score = 78; - } else if (containsWord(name, q)) { - score = 75; - } else if (artist.includes(q)) { - score = 65; - } else if (name.includes(q)) { - score = 60; - } else { - score = 40; - } - - albumMatches.push({ album: a, score }); - } - } - - albumMatches.sort((a, b) => b.score - a.score || a.album.Name.localeCompare(b.album.Name)); - - const releaseGroups: MBReleaseGroup[] = albumMatches.slice(0, 10).map((m) => ({ - mbid: m.album.MBID || '', - title: m.album.Name, - primaryType: 'Album', - artistCredit: m.album.ArtistName, - firstReleaseDate: m.album.Year ? String(m.album.Year) : '', - _coverArt: m.album.CoverArtMedium || m.album.CoverArtSmall || '', - _inLibrary: true, - } as MBReleaseGroup & { _coverArt: string; _inLibrary: boolean })); - - // Collect matching tracks by title or artist name. - const trackMatches: Array<{ track: any; score: number }> = []; - const cachedTracks = libraryStore.getCachedTracks(); - if (cachedTracks) { - for (const t of cachedTracks) { - const title = t.TrackName.toLowerCase(); - const artist = t.ArtistName.toLowerCase(); - const matchesTitle = fuzzyMatch(q, title); - const matchesArtist = fuzzyMatch(q, artist); - if (!matchesTitle && !matchesArtist) continue; - - let score: number; - if (title === q) { - score = 100; - } else if (artist === q) { - score = 95; - } else if (title.startsWith(q)) { - score = 88; - } else if (artist.startsWith(q)) { - score = 85; - } else if (containsWord(title, q)) { - score = 78; - } else if (containsWord(artist, q)) { - score = 75; - } else if (title.includes(q)) { - score = 65; - } else if (artist.includes(q)) { - score = 60; - } else { - score = 40; - } - - trackMatches.push({ track: t, score }); - } - } - - trackMatches.sort((a, b) => b.score - a.score || a.track.TrackName.localeCompare(b.track.TrackName)); - - // Deduplicate by recording MBID (keep highest score). - const seenRecMBIDs = new Set(); - const recordings: MBRecording[] = []; - for (const m of trackMatches) { - if (recordings.length >= 15) break; - const mbid = m.track.RecordingMBID || ''; - if (mbid && seenRecMBIDs.has(mbid)) continue; - if (mbid) seenRecMBIDs.add(mbid); - - recordings.push({ - mbid, - title: m.track.TrackName, - length: parseDuration(m.track.TrackLength), - artistCredit: m.track.ArtistName, - score: m.score, - } as MBRecording); - } - - if (artists.length === 0 && releaseGroups.length === 0 && recordings.length === 0) { - return null; - } - - return { artists, releaseGroups, recordings } as MBSearchResult; - } - - /** - * Fetch LB popularity for all MBIDs in the result and re-sort - * each category using a blended score: match quality + log-scaled - * popularity. Same approach as the backend reranker. - */ - private async rerankWithPopularity(result: MBSearchResult | null): Promise { - if (!result) return; - - // Collect all non-empty MBIDs. - const mbids: string[] = []; - for (const a of result.artists ?? []) if (a.mbid) mbids.push(a.mbid); - for (const rg of result.releaseGroups ?? []) if (rg.mbid) mbids.push(rg.mbid); - for (const r of result.recordings ?? []) if (r.mbid) mbids.push(r.mbid); - - if (mbids.length === 0) return; - - let batch: Record; - try { - batch = await GetPopularityBatch(mbids); - } catch { - return; // degrade gracefully — keep match-quality order - } - - if (!batch || Object.keys(batch).length === 0) return; - - // Weights matching backend: 0.35 relevance + 0.50 popularity + 0.15 personalization - const maxPop = Math.max(1, ...Object.values(batch).map(b => b.popularity)); - const logMax = Math.log10(maxPop + 1); - const maxSim = Math.max(1, ...Object.values(batch).map(b => b.similarityScore || 0)); - - const blendedScore = (mbid: string, matchScore: number): number => { - const b = batch[mbid]; - const relevance = matchScore / 100; - const logPop = b ? Math.log10(b.popularity + 1) / logMax : 0; - let personal = 0; - if (b?.inLibrary) { - personal = 1.0; - } else if (b?.similarityScore && maxSim > 0) { - personal = 0.5 * (b.similarityScore / maxSim); - } - return 0.35 * relevance + 0.50 * logPop + 0.15 * personal; - }; - - // Re-sort each category. Backend stamps a `score` field - // onto entries before returning them, but the Wails- - // generated MB types don't model it — cast through any to - // read it on the way to the comparator. - const cmp = (a: { mbid: string }, b: { mbid: string }): number => - blendedScore(b.mbid, (b as any).score ?? 0) - blendedScore(a.mbid, (a as any).score ?? 0); - (result.artists ?? []).sort(cmp); - (result.releaseGroups ?? []).sort(cmp); - (result.recordings ?? []).sort(cmp); - - // Trigger re-render. MBSearchResult is a Wails-generated - // class with bound methods (convertValues), so request an - // update directly rather than spreading the object — that - // would drop the methods. - this.results = result; - this.requestUpdate(); + // Offline search over the local popularity index via Wails RPC. + // No network, and no owned-library seed — the index is the sole + // source of catalog results. + void this.executeIndexSearch(version, query, startTime); } /** @@ -987,48 +775,19 @@ export class ExploreView extends LitElement { return result; } - /** - * Merge library-only results from this.results into the full - * search result. Adds local artists/albums that the MB search - * didn't find (by name dedup) so they aren't lost. - */ - private mergeLocalIntoFull(full: MBSearchResult) { - const prev = this.results; - if (!prev) return; - - // Dedup artists by name (case-insensitive). - if (prev.artists?.length) { - const existing = new Set( - (full.artists || []).map((a) => a.name.toLowerCase()), - ); - for (const a of prev.artists) { - if (!existing.has(a.name.toLowerCase())) { - full.artists = full.artists || []; - full.artists.push(a); - } - } - } - - // Dedup albums by title + artist (case-insensitive). - if (prev.releaseGroups?.length) { - const existing = new Set( - (full.releaseGroups || []).map( - (rg: MBReleaseGroup) => `${rg.title}|${rg.artistCredit}`.toLowerCase(), - ), - ); - for (const rg of prev.releaseGroups) { - const key = `${rg.title}|${rg.artistCredit}`.toLowerCase(); - if (!existing.has(key)) { - full.releaseGroups = full.releaseGroups || []; - full.releaseGroups.push(rg); - } - } - } - } - - private async executeFullSearch(version: number, query: string, startTime: number) { + private async executeIndexSearch(version: number, query: string, startTime: number) { try { - const result = await Search(query); + // Local FTS index only — no network. Returns null when the + // index has no hits, in which case we keep the owned-library + // matches already displayed. + const result = + (await SearchLocal(query)) ?? + explore.MBSearchResult.createFrom({ + artists: [], + releaseGroups: [], + recordings: [], + topResults: [], + }); // Discard stale response if (version !== this.searchVersion) { @@ -1038,32 +797,10 @@ export class ExploreView extends LitElement { return; } - const merged = this.mergeWithLibrary(result); - - // If the full search returned results, use them. - // If it returned nothing but we had local results, keep those. - const hasFullResults = - (merged.artists?.length ?? 0) > 0 || - (merged.releaseGroups?.length ?? 0) > 0 || - (merged.recordings?.length ?? 0) > 0; - const hadLocalResults = this.results && - ((this.results.artists?.length ?? 0) > 0 || - (this.results.releaseGroups?.length ?? 0) > 0 || - (this.results.recordings?.length ?? 0) > 0); - - if (hasFullResults) { - // Preserve any library-only artists/albums that the MB - // search didn't find (no MBID, or MB didn't match). - if (hadLocalResults) { - this.mergeLocalIntoFull(merged); - } - - this.results = merged; - } else if (!hadLocalResults) { - // Both local and full are empty — show empty state. - this.results = merged; - } - // else: keep existing local results as-is. + // Enrich index results with local cover art / "In Library" + // badges, but never inject library-only entries — the index + // is the sole source of results shown. + this.results = this.mergeWithLibrary(result); exploreCache.populateFromSearch( this.results?.artists || [], @@ -1092,6 +829,37 @@ export class ExploreView extends LitElement { } } + /* ── Lyrics Search ── */ + + private async executeLyricsSearch(version: number, query: string, startTime: number) { + try { + const hits = await SearchLyrics(query); + if (version !== this.searchVersion) return; + + this.lyricsResults = hits ?? []; + + const elapsed = (performance.now() - startTime).toFixed(0); + console.log( + `[explore] lyrics search: "${query}" in ${elapsed}ms — ` + + `hits=${this.lyricsResults.length}`, + ); + } catch (err) { + if (version !== this.searchVersion) return; + this.error = err instanceof Error ? err.message : String(err); + console.error(`[explore] lyrics search error: "${query}" — ${this.error}`); + } finally { + if (version === this.searchVersion) { + this.loading = false; + } + } + } + + /** Play a lyric-search hit immediately (replaces the queue). */ + private playLyricHit(hit: LyricsResult) { + if (!hit.filePath) return; + queueStore.setQueue([hit.filePath], 0); + } + /* ── Thumbnail Loading ── */ private thumbnailBatchPending = false; @@ -1446,9 +1214,26 @@ export class ExploreView extends LitElement { ); break; case 'recording': - // Navigate to the album page if we can resolve it, - // otherwise navigate to the artist. - if (r.artistCredit) { + // Standard track-click behaviour (matches the library list, + // queue, playlists, etc.): open the track's album page with + // the track highlighted. The backend resolves the parent + // release group from the local index, so this is an instant, + // index-backed load rather than a name-only artist lookup. + if (r.releaseGroupMbid) { + this.dispatchEvent( + new CustomEvent('navigate', { + bubbles: true, + composed: true, + detail: { + view: 'explore-album-details', + releaseGroupMBID: r.releaseGroupMbid, + albumName: r.releaseName || '', + highlightTrackMBID: r.mbid, + }, + }), + ); + } else if (r.artistCredit) { + // Fallback only when the album can't be resolved locally. this.dispatchEvent( new CustomEvent('navigate', { bubbles: true, @@ -1496,12 +1281,33 @@ export class ExploreView extends LitElement { } private renderSearchInput() { + const placeholder = + this.searchMode === 'lyrics' + ? 'Search by a lyric\u2026' + : 'Search artists, albums, and tracks\u2026'; + return html` +
    + + +
    Keep typing…
    `; + } + + if (!this.searchQuery.trim() && !this.lyricsResults) { + return html`
    + Type a line of lyrics to find the track in your library. +
    `; + } + + if (this.loading && !this.lyricsResults) return nothing; + + if (this.lyricsResults && this.lyricsResults.length === 0) { + return html`
    + No tracks with lyrics matching “${this.searchQuery}”. +
    `; + } + + if (!this.lyricsResults) return nothing; + + return html` +
    + ${this.lyricsResults.map( + (hit) => html` + + `, + )} +
    + `; + } + private renderBody() { + if (this.searchMode === 'lyrics') { + return this.renderLyricsBody(); + } + // Query too short if (this.queryTooShort) { return html`
    @@ -1531,7 +1387,7 @@ export class ExploreView extends LitElement { // No query entered yet if (!this.searchQuery.trim() && !this.results) { return html`
    - Search MusicBrainz to discover artists, albums, and tracks. + Search to discover artists, albums, and tracks.
    `; } @@ -1673,7 +1529,7 @@ export class ExploreView extends LitElement {
    ${rg.title}
    -
    ${rg.artistCredit}
    +
    ${artistLink(rg.artistCredit, rg.artistMbid ?? '')}
    ${rg.primaryType @@ -1706,9 +1562,11 @@ export class ExploreView extends LitElement { (r) => html`
    -
    ${r.title}
    +
    + ${trackLink(r.title, r.releaseName ?? '', r.releaseGroupMbid ?? '', r.mbid)} +
    - ${r.artistCredit} + ${artistLink(r.artistCredit, r.artistMbid ?? '')}
    diff --git a/frontend/src/components/smart-playlist-details/smart-playlist-details.ts b/frontend/src/components/smart-playlist-details/smart-playlist-details.ts index 5bf5e1c..9590be1 100644 --- a/frontend/src/components/smart-playlist-details/smart-playlist-details.ts +++ b/frontend/src/components/smart-playlist-details/smart-playlist-details.ts @@ -3,18 +3,53 @@ import { customElement, property, state, + query, } from 'lit/decorators.js'; -import { library } from '@go/models'; +import type { playlist, library } from '@go/models'; import { - EvaluateSmartPlaylist, + GetSmartPlaylistTracks, + RefreshSmartPlaylist, GetSmartPlaylistRules, UpdateSmartPlaylistRules, } from '@go/playlist/Service'; import { EventsOn } from '@runtime/runtime'; import { Events } from '../../events'; import { queueStore } from '@store/queue-store'; +import { PlayerController } from '@store/controllers/player-controller'; +import { SearchController } from '@store/controllers/search-controller'; +import { SelectionController } from '@utils/selection-controller'; +import type { SelectionHost } from '@utils/selection-controller'; +import { + ContextMenuController, + contextMenuStyles, +} from '@utils/context-menu-controller.js'; +import type { ContextMenuHost } from '@utils/context-menu-controller.js'; +import { FavoritesController } from '@store/controllers/favorites-controller'; +import { + setDragPayload, + emitDragActive, +} from '@utils/drag-controller'; +import { + createDragImage, + createTrackCardDragImage, + removeDragImage, +} from '@utils/drag-image'; import '@awesome.me/webawesome/dist/components/icon/icon.js'; -import '@components/track-list/track-list.js'; +import '@awesome.me/webawesome/dist/components/popup/popup.js'; +import type WaPopup from '@awesome.me/webawesome/dist/components/popup/popup.js'; +import '@awesome.me/webawesome/dist/components/dropdown-item/dropdown-item.js'; +import '@components/playlist-picker/playlist-picker.js'; +import '@components/track-details/track-details.js'; +import type { TrackDetails } from '@components/track-details/track-details.js'; +import type { CoverArtUrls } from '@components/track-details/track-details.js'; +import { libraryStore } from '@store/library-store'; +import { formatMilliseconds } from '@utils/time'; +import { + artistLink, + albumLink, + trackLink, + exploreLinkStyles, +} from '@utils/explore-link'; import '@components/smart-playlist-editor/smart-playlist-editor.js'; import { designTokens } from '../../styles/tokens.css'; @@ -44,7 +79,10 @@ function formatTotalDuration(totalMs: number): string { } @customElement('smart-playlist-details') -export class SmartPlaylistDetails extends LitElement { +export class SmartPlaylistDetails + extends LitElement + implements SelectionHost, ContextMenuHost +{ @property({ type: Number, attribute: 'playlist-id' }) playlistId = 0; @@ -55,7 +93,7 @@ export class SmartPlaylistDetails extends LitElement { autoEdit = false; @state() - private tracks: library.Track[] = []; + private tracks: playlist.Track[] = []; @state() private loading = true; @@ -72,14 +110,74 @@ export class SmartPlaylistDetails extends LitElement { @state() private saving = false; + @state() + private refreshing = false; + + private player = new PlayerController(this); + private searchCtrl = new SearchController(this); + private selection = new SelectionController(this); + private ctxMenu = new ContextMenuController(this); + private favCtrl = new FavoritesController(this); + private playlistDeletedCleanup: (() => void) | null = null; private playlistRenamedCleanup: (() => void) | null = null; + private dragImageEl: HTMLElement | null = null; + + @query('#context-menu') + private contextMenuPopup!: WaPopup; + + @query('#playlist-submenu') + private playlistSubmenuPopup!: WaPopup; + + @query('track-details') + private trackDetailsDialog!: TrackDetails; + + // ================================================================= + // ContextMenuHost interface + // ================================================================= + + getContextMenuPopup(): WaPopup | undefined { + return this.contextMenuPopup; + } + + getPlaylistSubmenuPopup(): WaPopup | undefined { + return this.playlistSubmenuPopup; + } + + onContextMenuClose(): void { + // No-op. + } + + // ================================================================= + // SelectionHost interface + // ================================================================= + + getItemKey(index: number): string | undefined { + if (index < 0 || index >= this.tracks.length) { + return undefined; + } + + return String(index); + } + + getItemCount(): number { + return this.tracks.length; + } + + onSelectionChanged(): void { + this.requestUpdate(); + } + // ================================================================= // Styles // ================================================================= - static override styles = [designTokens, css` + static override styles = [ + designTokens, + contextMenuStyles, + exploreLinkStyles, + css` :host { display: flex; flex-direction: column; @@ -236,12 +334,162 @@ export class SmartPlaylistDetails extends LitElement { .content { flex: 1; - overflow: hidden; + overflow-y: auto; + padding: 0 16px 12px 16px; } - track-list { + .search-bar-row { + position: relative; + display: flex; + align-items: center; + justify-content: center; + min-height: 30px; + border-bottom: 1px solid + var(--yj-border-subtle, #333); + flex-shrink: 0; + user-select: none; + } + + .search-indicator { + position: absolute; + left: 50%; + transform: translateX(-50%); + pointer-events: none; + background: var( + --yj-bg-overlay, + #495057 + ); + color: var( + --yj-text-secondary, + #b3b3b3 + ); + font-size: 12px; + padding: 2px 14px; + border-radius: 12px; + border: 1px solid + var(--yj-border-subtle, #555); + white-space: nowrap; + opacity: 0.92; + } + + /* Column grid layout */ + .track-header, + .track-item { + display: grid; + grid-template-columns: 40px 36px 1fr 1fr 1fr 80px; + align-items: center; + gap: 0; + } + + .track-header { + padding: 6px 8px; + font-size: 11px; + font-weight: 600; + color: var(--yj-text-secondary, #b3b3b3); + text-transform: uppercase; + letter-spacing: 0.03em; + border-bottom: 1px solid var(--yj-text-tertiary, #666); + user-select: none; + } + + .track-art { + width: 32px; + height: 32px; + border-radius: 4px; + overflow: hidden; + flex-shrink: 0; + background: var(--yj-bg-overlay, rgba(255, 255, 255, 0.06)); + } + + .track-art img { width: 100%; height: 100%; + object-fit: cover; + display: block; + } + + .header-cell, + .cell { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + min-width: 0; + padding: 0 4px; + } + + .col-number { + text-align: center; + color: var(--yj-text-tertiary, #888); + font-variant-numeric: tabular-nums; + } + + .col-duration { + text-align: right; + color: var(--yj-text-tertiary, #888); + font-variant-numeric: tabular-nums; + } + + .track-item { + padding: 6px 8px; + border-bottom: 1px solid + rgba(255, 255, 255, 0.03); + cursor: default; + user-select: none; + } + + .track-item:hover { + background-color: var(--yj-hover-overlay, rgba(255, 255, 255, 0.05)); + } + + .track-item.selected { + background-color: var(--yj-selection-bg, rgba(100, 160, 255, 0.15)); + } + + .track-item.active { + background-color: var(--yj-accent-bg, rgba(255, 212, 59, 0.1)); + color: var(--yj-accent, #ffd43b); + } + + .track-item.selected.active { + background-color: var(--yj-selection-bg, rgba(100, 160, 255, 0.15)); + } + + /* Phantom rows span the full grid */ + .track-item.phantom { + display: grid; + grid-template-columns: 40px 36px 1fr 1fr 1fr 80px; + cursor: default; + } + + .phantom-row { + grid-column: 1 / -1; + display: flex; + align-items: center; + gap: 8px; + min-width: 0; + width: 100%; + } + + .phantom-caution { + flex-shrink: 0; + font-size: 14px; + color: var(--yj-warning, #e67700); + } + + .phantom-path { + flex: 1; + min-width: 0; + font-size: 12px; + color: var(--yj-text-tertiary, #888); + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } + + .tracks-empty { + padding: 12px 0; + color: var(--yj-text-tertiary, #666); + font-size: 12px; } .loading { @@ -273,6 +521,24 @@ export class SmartPlaylistDetails extends LitElement { // Lifecycle // ================================================================= + private handleSelectAll = (): void => { + this.selection.selectAll(); + }; + + private clearSelectionHandler = (e: MouseEvent) => { + const path = e.composedPath(); + const isTrackClick = path.some( + (el) => + el instanceof HTMLElement && + el.classList.contains('track-item') && + this.shadowRoot?.contains(el), + ); + + if (!isTrackClick) { + this.selection.clear(); + } + }; + override connectedCallback() { super.connectedCallback(); @@ -303,6 +569,15 @@ export class SmartPlaylistDetails extends LitElement { } }, ); + + document.addEventListener( + 'click', + this.clearSelectionHandler, + ); + document.addEventListener( + 'shortcut:select-all', + this.handleSelectAll, + ); } override disconnectedCallback() { @@ -317,6 +592,15 @@ export class SmartPlaylistDetails extends LitElement { this.playlistRenamedCleanup(); this.playlistRenamedCleanup = null; } + + document.removeEventListener( + 'click', + this.clearSelectionHandler, + ); + document.removeEventListener( + 'shortcut:select-all', + this.handleSelectAll, + ); } // ================================================================= @@ -329,12 +613,15 @@ export class SmartPlaylistDetails extends LitElement { this.loading = true; try { - const result = await EvaluateSmartPlaylist(this.playlistId); + const result = await GetSmartPlaylistTracks( + this.playlistId, + ); this.tracks = result ?? []; + this.selection.clear(); } catch (error) { console.error( - 'Failed to evaluate smart playlist:', + 'Failed to load smart playlist tracks:', error, ); this.tracks = []; @@ -343,6 +630,21 @@ export class SmartPlaylistDetails extends LitElement { } } + private async refreshTracks() { + if (!this.playlistId) return; + + try { + this.tracks = await GetSmartPlaylistTracks( + this.playlistId, + ); + } catch (error) { + console.error( + 'Failed to refresh smart playlist tracks:', + error, + ); + } + } + // ================================================================= // Navigation // ================================================================= @@ -363,7 +665,7 @@ export class SmartPlaylistDetails extends LitElement { private handlePlay() { const filePaths = this.tracks - .filter((t) => t.FilePath) + .filter((t) => !t.Phantom) .map((t) => t.FilePath); if (filePaths.length === 0) return; @@ -373,7 +675,7 @@ export class SmartPlaylistDetails extends LitElement { private handleShuffle() { const filePaths = this.tracks - .filter((t) => t.FilePath) + .filter((t) => !t.Phantom) .map((t) => t.FilePath); if (filePaths.length === 0) return; @@ -381,8 +683,20 @@ export class SmartPlaylistDetails extends LitElement { queueStore.setQueue(filePaths, 0, true); } - private handleRefresh() { - void this.loadTracks(); + private async handleRefresh() { + this.refreshing = true; + + try { + await RefreshSmartPlaylist(this.playlistId); + await this.refreshTracks(); + } catch (error) { + console.error( + 'Failed to refresh smart playlist:', + error, + ); + } finally { + this.refreshing = false; + } } private async handleEditRules() { @@ -424,13 +738,278 @@ export class SmartPlaylistDetails extends LitElement { this.pendingRulesJSON = e.detail.json; } + // ================================================================= + // Track interactions + // ================================================================= + + private handleTrackClick( + e: MouseEvent, + trackIndex: number, + ) { + this.selection.handleItemClick( + e, + String(trackIndex), + trackIndex, + ); + } + + private handleTrackDblClick(trackIndex: number) { + this.selection.clear(); + + const filePaths = this.tracks.map( + (t) => t.FilePath, + ); + + queueStore.setQueue(filePaths, trackIndex); + } + + private handleTrackContextMenu( + e: MouseEvent, + trackIndex: number, + ) { + e.preventDefault(); + e.stopPropagation(); + + this.selection.handleContextMenu( + String(trackIndex), + ); + this.ctxMenu.openAt(e.clientX, e.clientY); + } + + private isActiveTrack( + track: playlist.Track, + ): boolean { + const currentTrack = this.player.currentTrack; + + if (!currentTrack) return false; + + return currentTrack.filePath === track.FilePath; + } + + // ================================================================= + // Selection helpers + // ================================================================= + + private getSelectedFilePaths(): string[] { + return this.selection + .getSelectedIndices() + .map((i) => this.tracks[i]!.FilePath); + } + + // ================================================================= + // Context menu actions + // ================================================================= + + private onContextMenuAction(action: string) { + const filePaths = this.getSelectedFilePaths(); + + if (filePaths.length === 0) return; + + switch (action) { + case 'play': + queueStore.setQueue(filePaths, 0, true); + break; + case 'add-to-queue': + queueStore.addTracksToQueue(filePaths); + break; + case 'play-next': + queueStore.playTracksNext(filePaths); + break; + case 'track-details': + if (filePaths.length === 1) { + this.openTrackDetails(filePaths[0]!); + } else { + this.openBatchTrackDetails(filePaths); + } + break; + } + + this.selection.clear(); + this.ctxMenu.close(); + } + + private onContextMenuFavoriteToggle() { + const filePaths = this.getSelectedFilePaths(); + + if (filePaths.length === 0) return; + + if (this.favCtrl.allFavorited(filePaths)) { + void this.favCtrl.removeFromFavorites( + filePaths, + ); + } else { + void this.favCtrl.addToFavorites( + filePaths, + ); + } + + this.selection.clear(); + this.ctxMenu.close(); + } + + private openTrackDetails(filePath: string) { + const tracks = libraryStore.getCachedTracks(); + const track = tracks?.find( + (t) => t.FilePath === filePath, + ); + + if (!track) return; + + const coverArt = track.CoverArtPath + ? { + coverArtPath: track.CoverArtPath, + coverArtSmall: track.CoverArtSmall, + coverArtMedium: track.CoverArtMedium, + coverArtLarge: track.CoverArtLarge, + } + : undefined; + + this.trackDetailsDialog?.show( + track, + coverArt, + ); + } + + private openBatchTrackDetails( + filePaths: string[], + ) { + const cachedTracks = + libraryStore.getCachedTracks(); + + if (!cachedTracks) return; + + const tracks = filePaths + .map((fp) => + cachedTracks.find( + (t) => t.FilePath === fp, + ), + ) + .filter( + (t): t is library.Track => t != null, + ); + + if (tracks.length === 0) return; + + const first = tracks[0]!; + const albumNames = new Set(tracks.map((t) => t.Album)); + let coverArt: CoverArtUrls | null = null; + let coverArtMixed = false; + + if (albumNames.size === 1 && first.CoverArtPath) { + coverArt = { + coverArtPath: first.CoverArtPath, + coverArtSmall: first.CoverArtSmall, + coverArtMedium: first.CoverArtMedium, + coverArtLarge: first.CoverArtLarge, + }; + } else if (albumNames.size > 1) { + coverArtMixed = true; + } + + this.trackDetailsDialog?.showBatch( + tracks, + coverArt, + coverArtMixed, + ); + } + + // ================================================================= + // Drag source (smart-playlist tracks -> queue or playlist) + // ================================================================= + + private onTrackDragStart = ( + e: DragEvent, + track: playlist.Track, + trackIndex: number, + ) => { + let filePaths: string[]; + + if ( + this.selection.isSelected( + String(trackIndex), + ) + ) { + filePaths = this.getSelectedFilePaths(); + } else { + filePaths = [track.FilePath]; + } + + if (filePaths.length === 0) return; + + setDragPayload(e, { + filePaths, + source: 'track-list', + }); + + this.dragImageEl = + filePaths.length === 1 + ? createTrackCardDragImage( + track.Title, + track.Artist, + track.FilePath, + ) + : createDragImage(filePaths.length); + e.dataTransfer?.setDragImage( + this.dragImageEl, + 0, + 0, + ); + + emitDragActive(true); + }; + + private onTrackDragEnd = () => { + if (this.dragImageEl) { + removeDragImage(this.dragImageEl); + this.dragImageEl = null; + } + + emitDragActive(false); + }; + + // ================================================================= + // Search filtering + // ================================================================= + + private getVisibleTracks(): { + track: playlist.Track; + trackIndex: number; + }[] { + const term = + this.searchCtrl.term.toLowerCase(); + + if (!term) { + return this.tracks.map( + (track, trackIndex) => ({ + track, + trackIndex, + }), + ); + } + + return this.tracks + .map((track, trackIndex) => ({ + track, + trackIndex, + })) + .filter( + ({ track }) => + track.Title.toLowerCase().includes( + term, + ) || + track.Artist.toLowerCase().includes( + term, + ), + ); + } + // ================================================================= // Helpers // ================================================================= private getTotalDuration(): string { const totalMs = this.tracks.reduce( - (sum, t) => sum + Number(t.TrackLength || 0), + (sum, t) => sum + Number(t.Duration || 0), 0, ); @@ -444,7 +1023,16 @@ export class SmartPlaylistDetails extends LitElement { override render() { const trackCount = this.tracks.length; const trackLabel = trackCount === 1 ? 'track' : 'tracks'; - const hasPlayableTracks = this.tracks.some((t) => t.FilePath); + const hasPlayableTracks = this.tracks.some((t) => !t.Phantom); + + const searchBar = this.searchCtrl.term + ? html`
    +
    + Showing results for + “${this.searchCtrl.term}” +
    +
    ` + : nothing; return html`
    @@ -480,7 +1068,7 @@ export class SmartPlaylistDetails extends LitElement { ${this.loading ? html`
    - Evaluating smart playlist… + Loading tracks…
    ` : html`
    @@ -527,10 +1115,11 @@ export class SmartPlaylistDetails extends LitElement {
    `} `} + + ${this.renderContextMenu()} + + + `; + } + + private renderTrackList() { + const visibleTracks = this.getVisibleTracks(); + + return html` +
    +
    #
    +
    +
    Title
    +
    Artist
    +
    Album
    +
    Duration
    +
    + ${visibleTracks.map( + ({ track, trackIndex }) => { + const isPhantom = track.Phantom; + const active = + !isPhantom && + this.isActiveTrack(track); + const selected = + this.selection.isSelected( + String(trackIndex), + ); + + const classes = [ + 'track-item', + active ? 'active' : '', + selected ? 'selected' : '', + isPhantom ? 'phantom' : '', + ] + .filter(Boolean) + .join(' '); + + return html` +
    + this.handleTrackClick( + e, + trackIndex, + )} + @dblclick=${isPhantom + ? nothing + : () => + this.handleTrackDblClick( + trackIndex, + )} + @contextmenu=${isPhantom + ? nothing + : (e: MouseEvent) => + this.handleTrackContextMenu( + e, + trackIndex, + )} + @dragstart=${isPhantom + ? nothing + : (e: DragEvent) => + this.onTrackDragStart( + e, + track, + trackIndex, + )} + @dragend=${isPhantom + ? nothing + : this.onTrackDragEnd} + > + ${isPhantom + ? html`
    + + ${track.FilePath} +
    ` + : html`${trackIndex + 1} +
    + ${track.CoverArtSmall || track.CoverArtMedium + ? html`` + : nothing} +
    + ${trackLink(track.Title, track.Album, track.ReleaseGroupMBID, track.RecordingMBID) || track.FilePath} + ${artistLink(track.Artist, track.ArtistMBID)} + ${albumLink(track.Album, track.ReleaseGroupMBID)} + ${formatMilliseconds(track.Duration)}`} +
    + `; + }, + )} + `; + } + + private renderContextMenu() { + return html` + + ${this.ctxMenu.contextMenuOpen + ? html` +
    + + this.onContextMenuAction( + 'play', + )} + @mouseenter=${() => + this.ctxMenu.closePlaylistSubmenu()} + > + + Play + + + this.onContextMenuAction( + 'add-to-queue', + )} + @mouseenter=${() => + this.ctxMenu.closePlaylistSubmenu()} + > + + Add to Queue + + + this.onContextMenuAction( + 'play-next', + )} + @mouseenter=${() => + this.ctxMenu.closePlaylistSubmenu()} + > + + Play Next + + { + this.ctxMenu.clearSubmenuCloseTimer(); + void this.ctxMenu.showPlaylistSubmenu( + this.getSelectedFilePaths(), + ); + }} + @mouseleave=${this + .ctxMenu + .scheduleSubmenuClose} + @click=${( + e: Event, + ) => { + e.stopPropagation(); + void this.ctxMenu.showPlaylistSubmenu( + this.getSelectedFilePaths(), + ); + }} + > + + Add to Playlist + + ▶ + + + + this.onContextMenuFavoriteToggle()} + @mouseenter=${() => + this.ctxMenu.closePlaylistSubmenu()} + > + + ${this.favCtrl.allFavorited( + this.getSelectedFilePaths(), + ) + ? `Remove from ${this.favCtrl.playlistName}` + : `Add to ${this.favCtrl.playlistName}`} + + + this.onContextMenuAction( + 'track-details', + )} + @mouseenter=${() => + this.ctxMenu.closePlaylistSubmenu()} + > + + Track Details + +
    + ` + : nothing} +     + + + ${this.ctxMenu.playlistSubmenuOpen && + this.selection.hasSelection + ? html` +
    + this.ctxMenu.clearSubmenuCloseTimer()} + @mouseleave=${this + .ctxMenu + .scheduleSubmenuClose} + > + + e.stopPropagation()} + > +
    + ` + : nothing} +     `; } } + +declare global { + interface HTMLElementTagNameMap { + 'smart-playlist-details': SmartPlaylistDetails; + } +} diff --git a/frontend/src/components/smart-playlist-editor/smart-playlist-editor.ts b/frontend/src/components/smart-playlist-editor/smart-playlist-editor.ts index ea560a1..c253a0c 100644 --- a/frontend/src/components/smart-playlist-editor/smart-playlist-editor.ts +++ b/frontend/src/components/smart-playlist-editor/smart-playlist-editor.ts @@ -8,13 +8,14 @@ import '@components/combobox/combobox.ts'; // ── Field / Operator constants ────────────────────────────────────── -/** All 16 fields matching the backend `fieldMap` keys. */ +/** All fields matching the backend `fieldMap` keys. */ const FIELDS: string[] = [ 'title', 'artist', 'album', 'genre', 'year', + 'release_year', 'composer', 'file_type', 'duration', @@ -32,6 +33,7 @@ const FIELDS: string[] = [ const NUMERIC_FIELDS = new Set([ 'year', + 'release_year', 'duration', 'sample_rate', 'bit_depth', @@ -63,7 +65,16 @@ const NUMERIC_OPERATORS = [ 'between', ]; -const SORT_FIELDS = ['title', 'artist', 'album', 'year', 'duration', 'play_count', 'random']; +const SORT_FIELDS = [ + 'title', + 'artist', + 'album', + 'year', + 'release_year', + 'duration', + 'play_count', + 'random', +]; // ── Helpers ───────────────────────────────────────────────────────── @@ -103,7 +114,12 @@ function getAutocompleteOptions(field: string): string[] { if (!tracks) return []; return [...new Set(tracks.map((t) => t.FileType).filter(Boolean))]; } - case 'year': { + case 'year': + case 'release_year': { + // Both year fields draw suggestions from the set of years + // present in the library. The cached Track only carries the + // display (original) year, so it seeds both datalists — the + // list is just a hint, and the real filter runs server-side. const tracks = libraryStore.getCachedTracks(); if (!tracks) return []; return [ @@ -120,8 +136,21 @@ function getAutocompleteOptions(field: string): string[] { } } +/** + * Overrides for fields whose title-cased name would be ambiguous. The + * two year fields in particular need to disambiguate the album's + * original release from the specific (possibly reissue) release owned. + */ +const FIELD_LABEL_OVERRIDES: Record = { + year: 'Year (Original Release)', + release_year: 'Year (This Release)', +}; + /** Format a field name for display: `file_type` → "File Type". */ function formatFieldLabel(field: string): string { + const override = FIELD_LABEL_OVERRIDES[field]; + if (override) return override; + return field .split('_') .map((w) => w.charAt(0).toUpperCase() + w.slice(1)) diff --git a/frontend/src/components/top-results-row/top-results-row.ts b/frontend/src/components/top-results-row/top-results-row.ts index 7e0e32d..22c9b54 100644 --- a/frontend/src/components/top-results-row/top-results-row.ts +++ b/frontend/src/components/top-results-row/top-results-row.ts @@ -9,6 +9,7 @@ import { } from '@go/explore/Service'; import '../library-status-indicator/library-status-indicator.js'; import type { LibraryStatus } from '../library-status-indicator/library-status-indicator.js'; +import { artistLink, exploreLinkStyles } from '../../utils/explore-link'; /** Format milliseconds as m:ss. */ function formatDuration(ms: number | undefined): string { @@ -51,6 +52,7 @@ export class TopResultsRow extends LitElement { static override styles = [ designTokens, + exploreLinkStyles, css` :host { display: block; @@ -242,13 +244,16 @@ export class TopResultsRow extends LitElement { const imgUrl = this.images.get(r.mbid); const isArtist = r.entityType === 'artist'; - const subtitle = isArtist - ? [r.artistType, r.country].filter(Boolean).join(' · ') || '' + // The artist portion of the subtitle links to the artist page; + // the remaining metadata (type/country, year, duration) is plain + // text. Artist cards have no artist credit — their whole subtitle + // is metadata. + const artistPart = isArtist ? '' : r.artistCredit || ''; + const metaPart = isArtist + ? [r.artistType, r.country].filter(Boolean).join(' · ') : r.entityType === 'release_group' - ? [r.artistCredit, r.year].filter(Boolean).join(' · ') - : [r.artistCredit, formatDuration(r.length)] - .filter(Boolean) - .join(' · '); + ? r.year || '' + : formatDuration(r.length) || ''; const status: LibraryStatus = r.inLibrary ? 'in-library' : 'not-in-library'; const entityType: 'artist' | 'album' | 'track' = @@ -280,9 +285,13 @@ export class TopResultsRow extends LitElement {
    `}
    ${r.name} - ${subtitle + ${artistPart || metaPart ? html`${subtitle}${artistPart + ? artistLink(artistPart, r.artistMbid ?? '') + : nothing}${artistPart && metaPart + ? ' · ' + : ''}${metaPart}` : nothing}
    diff --git a/frontend/src/events.ts b/frontend/src/events.ts index b82a2d1..448898d 100644 --- a/frontend/src/events.ts +++ b/frontend/src/events.ts @@ -63,6 +63,9 @@ export const Events = { // Explore / search index events IndexStatusChanged: "IndexStatusChanged", + ArtistDiscographyReady: "ArtistDiscographyReady", + ArtistSimilarReady: "ArtistSimilarReady", + AlbumReleasesReady: "AlbumReleasesReady", } as const; export type EventName = (typeof Events)[keyof typeof Events]; diff --git a/frontend/src/utils/text-diff.ts b/frontend/src/utils/text-diff.ts index 9f84516..ff0290d 100644 --- a/frontend/src/utils/text-diff.ts +++ b/frontend/src/utils/text-diff.ts @@ -19,6 +19,47 @@ function tokenize(s: string): string[] { return s.match(tokenRe) ?? []; } +/** + * Loose comparison-only normalization: lowercase, drop punctuation + * (keep letters/digits/spaces), collapse whitespace, trim. Mirrors + * the significant part of the backend's autotag.Normalize() so the + * UI can tell a cosmetic-only difference (case / punctuation / + * whitespace — normalized-equal, score unaffected) from a real one. + * Not exhaustive (no qualifier stripping); it only needs to agree on + * "is this difference purely formatting?". + */ +export function normalizeLoose(s: string): string { + return s + .toLowerCase() + .replace(/[^\p{L}\p{N}\s]/gu, '') + .replace(/\s+/g, ' ') + .trim(); +} + +/** + * Strict compare-only normalization: like normalizeLoose but also + * drops *all* whitespace, so punctuation that merely changes spacing + * doesn't register as a difference. This is what closes the + * "Rock&Roll" vs "Rock & Roll" gap: the backend's Normalize() deletes + * punctuation without collapsing the surrounding spaces, leaving a + * stray space that scores the pair below 1.0 even though the only + * real difference is punctuation/case. + */ +export function normalizeStrict(s: string): string { + return s.toLowerCase().replace(/[^\p{L}\p{N}]/gu, ''); +} + +/** + * True when `a` and `b` differ only cosmetically — i.e. by + * capitalization, punctuation, or the spacing punctuation induces. + * Used to decide whether a title change is a real conflict or just + * formatting. Empty `a` (no local value) is never cosmetic. + */ +export function isCosmeticDiff(a: string, b: string): boolean { + if (a === '' || a === b) return false; + return normalizeStrict(a) === normalizeStrict(b); +} + /** * Compute an inline word/punct-level diff between `a` (old) and * `b` (new), returning a list of segments suitable for inline diff --git a/frontend/wailsjs/go/autotagservice/Service.d.ts b/frontend/wailsjs/go/autotagservice/Service.d.ts index deac048..10a7c01 100755 --- a/frontend/wailsjs/go/autotagservice/Service.d.ts +++ b/frontend/wailsjs/go/autotagservice/Service.d.ts @@ -17,6 +17,8 @@ export function GetCandidates(arg1:string):Promise; export function GetCandidatesForPasteURL(arg1:string,arg2:string):Promise; +export function GetLocalCoverArt(arg1:string):Promise; + export function GetNextPending():Promise; export function GetPendingFolder(arg1:string):Promise; @@ -27,6 +29,10 @@ export function ListPendingFolders(arg1:number):Promise; +export function SearchCandidates(arg1:string,arg2:string,arg3:string):Promise>; + +export function SelectSearchCandidate(arg1:string,arg2:string,arg3:string):Promise; + export function SetContext(arg1:context.Context):Promise; export function Skip(arg1:string):Promise; diff --git a/frontend/wailsjs/go/autotagservice/Service.js b/frontend/wailsjs/go/autotagservice/Service.js index 43bb339..342a2d0 100755 --- a/frontend/wailsjs/go/autotagservice/Service.js +++ b/frontend/wailsjs/go/autotagservice/Service.js @@ -30,6 +30,10 @@ export function GetCandidatesForPasteURL(arg1, arg2) { return window['go']['autotagservice']['Service']['GetCandidatesForPasteURL'](arg1, arg2); } +export function GetLocalCoverArt(arg1) { + return window['go']['autotagservice']['Service']['GetLocalCoverArt'](arg1); +} + export function GetNextPending() { return window['go']['autotagservice']['Service']['GetNextPending'](); } @@ -50,6 +54,14 @@ export function RetagGroup(arg1) { return window['go']['autotagservice']['Service']['RetagGroup'](arg1); } +export function SearchCandidates(arg1, arg2, arg3) { + return window['go']['autotagservice']['Service']['SearchCandidates'](arg1, arg2, arg3); +} + +export function SelectSearchCandidate(arg1, arg2, arg3) { + return window['go']['autotagservice']['Service']['SelectSearchCandidate'](arg1, arg2, arg3); +} + export function SetContext(arg1) { return window['go']['autotagservice']['Service']['SetContext'](arg1); } diff --git a/frontend/wailsjs/go/explore/Service.d.ts b/frontend/wailsjs/go/explore/Service.d.ts index 4700610..ae93992 100755 --- a/frontend/wailsjs/go/explore/Service.d.ts +++ b/frontend/wailsjs/go/explore/Service.d.ts @@ -3,6 +3,8 @@ import {explore} from '../models'; import {context} from '../models'; +export function BackfillLibraryLyrics():Promise; + export function BrowseReleaseGroups(arg1:string):Promise>; export function BrowseReleases(arg1:string):Promise>; @@ -39,14 +41,16 @@ export function GetThumbnail(arg1:string,arg2:string,arg3:string):Promise):Promise>; +export function GetTrackLyrics(arg1:number):Promise; + export function GetTrackThumbnail(arg1:string,arg2:string,arg3:string,arg4:string):Promise; export function GetTrackThumbnails(arg1:Array):Promise>; -export function IndexNewArtists():Promise; - export function InvalidateIndexDiscographies():Promise; +export function InvalidateLibrarySync():Promise; + export function IsIndexReady():Promise; export function LookupArtist(arg1:string):Promise; @@ -57,19 +61,23 @@ export function MusicBrainz():Promise; export function PopulateLocalCrossReferences():Promise; +export function PopulateLocalCrossReferencesIfNeeded():Promise; + +export function PrefetchReleases(arg1:Array):Promise; + +export function RebuildLyricsIndex():Promise; + +export function RebuildLyricsIndexIfNeeded():Promise; + export function RecordSearchClick(arg1:string,arg2:string,arg3:string):Promise; +export function RefreshListenCounts():Promise; + export function ResolveReleaseGroupMBIDs(arg1:Array):Promise>; -export function Search(arg1:string):Promise; - -export function SearchArtists(arg1:string):Promise>; - export function SearchLocal(arg1:string):Promise; -export function SearchRecordings(arg1:string):Promise>; - -export function SearchReleaseGroups(arg1:string):Promise>; +export function SearchLyrics(arg1:string):Promise>; export function SetContext(arg1:context.Context):Promise; diff --git a/frontend/wailsjs/go/explore/Service.js b/frontend/wailsjs/go/explore/Service.js index 9faf63d..5c52560 100755 --- a/frontend/wailsjs/go/explore/Service.js +++ b/frontend/wailsjs/go/explore/Service.js @@ -2,6 +2,10 @@ // Cynhyrchwyd y ffeil hon yn awtomatig. PEIDIWCH Â MODIWL // This file is automatically generated. DO NOT EDIT +export function BackfillLibraryLyrics() { + return window['go']['explore']['Service']['BackfillLibraryLyrics'](); +} + export function BrowseReleaseGroups(arg1) { return window['go']['explore']['Service']['BrowseReleaseGroups'](arg1); } @@ -74,6 +78,10 @@ export function GetThumbnails(arg1) { return window['go']['explore']['Service']['GetThumbnails'](arg1); } +export function GetTrackLyrics(arg1) { + return window['go']['explore']['Service']['GetTrackLyrics'](arg1); +} + export function GetTrackThumbnail(arg1, arg2, arg3, arg4) { return window['go']['explore']['Service']['GetTrackThumbnail'](arg1, arg2, arg3, arg4); } @@ -82,14 +90,14 @@ export function GetTrackThumbnails(arg1) { return window['go']['explore']['Service']['GetTrackThumbnails'](arg1); } -export function IndexNewArtists() { - return window['go']['explore']['Service']['IndexNewArtists'](); -} - export function InvalidateIndexDiscographies() { return window['go']['explore']['Service']['InvalidateIndexDiscographies'](); } +export function InvalidateLibrarySync() { + return window['go']['explore']['Service']['InvalidateLibrarySync'](); +} + export function IsIndexReady() { return window['go']['explore']['Service']['IsIndexReady'](); } @@ -110,32 +118,40 @@ export function PopulateLocalCrossReferences() { return window['go']['explore']['Service']['PopulateLocalCrossReferences'](); } +export function PopulateLocalCrossReferencesIfNeeded() { + return window['go']['explore']['Service']['PopulateLocalCrossReferencesIfNeeded'](); +} + +export function PrefetchReleases(arg1) { + return window['go']['explore']['Service']['PrefetchReleases'](arg1); +} + +export function RebuildLyricsIndex() { + return window['go']['explore']['Service']['RebuildLyricsIndex'](); +} + +export function RebuildLyricsIndexIfNeeded() { + return window['go']['explore']['Service']['RebuildLyricsIndexIfNeeded'](); +} + export function RecordSearchClick(arg1, arg2, arg3) { return window['go']['explore']['Service']['RecordSearchClick'](arg1, arg2, arg3); } +export function RefreshListenCounts() { + return window['go']['explore']['Service']['RefreshListenCounts'](); +} + export function ResolveReleaseGroupMBIDs(arg1) { return window['go']['explore']['Service']['ResolveReleaseGroupMBIDs'](arg1); } -export function Search(arg1) { - return window['go']['explore']['Service']['Search'](arg1); -} - -export function SearchArtists(arg1) { - return window['go']['explore']['Service']['SearchArtists'](arg1); -} - export function SearchLocal(arg1) { return window['go']['explore']['Service']['SearchLocal'](arg1); } -export function SearchRecordings(arg1) { - return window['go']['explore']['Service']['SearchRecordings'](arg1); -} - -export function SearchReleaseGroups(arg1) { - return window['go']['explore']['Service']['SearchReleaseGroups'](arg1); +export function SearchLyrics(arg1) { + return window['go']['explore']['Service']['SearchLyrics'](arg1); } export function SetContext(arg1) { diff --git a/frontend/wailsjs/go/models.ts b/frontend/wailsjs/go/models.ts index dfde226..a16b5e4 100755 --- a/frontend/wailsjs/go/models.ts +++ b/frontend/wailsjs/go/models.ts @@ -87,8 +87,11 @@ export namespace autotagservice { export class ScoreBreakdownView { titleAvg: number; lengthAvg: number; + artistFit: number; + albumFit: number; trackCountFit: number; releaseMeta: number; + evidence: number; static createFrom(source: any = {}) { return new ScoreBreakdownView(source); @@ -98,8 +101,11 @@ export namespace autotagservice { if ('string' === typeof source) source = JSON.parse(source); this.titleAvg = source["titleAvg"]; this.lengthAvg = source["lengthAvg"]; + this.artistFit = source["artistFit"]; + this.albumFit = source["albumFit"]; this.trackCountFit = source["trackCountFit"]; this.releaseMeta = source["releaseMeta"]; + this.evidence = source["evidence"]; } } export class CandidateView { @@ -111,6 +117,7 @@ export namespace autotagservice { originalDate: string; country: string; status: string; + primaryType: string; trackCount: number; score: number; breakdown: ScoreBreakdownView; @@ -133,6 +140,7 @@ export namespace autotagservice { this.originalDate = source["originalDate"]; this.country = source["country"]; this.status = source["status"]; + this.primaryType = source["primaryType"]; this.trackCount = source["trackCount"]; this.score = source["score"]; this.breakdown = this.convertValues(source["breakdown"], ScoreBreakdownView); @@ -224,6 +232,7 @@ export namespace autotagservice { groupKey: string; localTracks: LocalTrackView[]; candidates: CandidateView[]; + recommendation: string; static createFrom(source: any = {}) { return new ScoreView(source); @@ -234,6 +243,7 @@ export namespace autotagservice { this.groupKey = source["groupKey"]; this.localTracks = this.convertValues(source["localTracks"], LocalTrackView); this.candidates = this.convertValues(source["candidates"], CandidateView); + this.recommendation = source["recommendation"]; } convertValues(a: any, classs: any, asMap: boolean = false): any { @@ -254,6 +264,26 @@ export namespace autotagservice { return a; } } + export class SearchHitView { + mbid: string; + kind: string; + title: string; + artist: string; + detail: string; + + static createFrom(source: any = {}) { + return new SearchHitView(source); + } + + constructor(source: any = {}) { + if ('string' === typeof source) source = JSON.parse(source); + this.mbid = source["mbid"]; + this.kind = source["kind"]; + this.title = source["title"]; + this.artist = source["artist"]; + this.detail = source["detail"]; + } + } } @@ -345,6 +375,7 @@ export namespace explore { trackName: string; totalListenCount: number; caaReleaseMbid: string; + releaseGroupMbid?: string; releaseName: string; length: number; inLibrary: boolean; @@ -361,6 +392,7 @@ export namespace explore { this.trackName = source["trackName"]; this.totalListenCount = source["totalListenCount"]; this.caaReleaseMbid = source["caaReleaseMbid"]; + this.releaseGroupMbid = source["releaseGroupMbid"]; this.releaseName = source["releaseName"]; this.length = source["length"]; this.inLibrary = source["inLibrary"]; @@ -395,6 +427,28 @@ export namespace explore { this.localId = source["localId"]; } } + export class LyricsResult { + recordingId: number; + filePath: string; + lengthMs: number; + title: string; + artist: string; + album: string; + + static createFrom(source: any = {}) { + return new LyricsResult(source); + } + + constructor(source: any = {}) { + if ('string' === typeof source) source = JSON.parse(source); + this.recordingId = source["recordingId"]; + this.filePath = source["filePath"]; + this.lengthMs = source["lengthMs"]; + this.title = source["title"]; + this.artist = source["artist"]; + this.album = source["album"]; + } + } export class MBArtist { mbid: string; name: string; @@ -434,9 +488,13 @@ export namespace explore { title: string; length: number; artistCredit: string; + artistMbid?: string; score: number; popularity: number; listenerCount: number; + caaReleaseMbid?: string; + releaseGroupMbid?: string; + releaseName?: string; inLibrary: boolean; localId?: number; @@ -450,9 +508,13 @@ export namespace explore { this.title = source["title"]; this.length = source["length"]; this.artistCredit = source["artistCredit"]; + this.artistMbid = source["artistMbid"]; this.score = source["score"]; this.popularity = source["popularity"]; this.listenerCount = source["listenerCount"]; + this.caaReleaseMbid = source["caaReleaseMbid"]; + this.releaseGroupMbid = source["releaseGroupMbid"]; + this.releaseName = source["releaseName"]; this.inLibrary = source["inLibrary"]; this.localId = source["localId"]; } @@ -530,6 +592,7 @@ export namespace explore { secondaryTypes?: string[]; firstReleaseDate: string; artistCredit: string; + artistMbid?: string; popularity: number; listenerCount: number; inLibrary: boolean; @@ -547,6 +610,7 @@ export namespace explore { this.secondaryTypes = source["secondaryTypes"]; this.firstReleaseDate = source["firstReleaseDate"]; this.artistCredit = source["artistCredit"]; + this.artistMbid = source["artistMbid"]; this.popularity = source["popularity"]; this.listenerCount = source["listenerCount"]; this.inLibrary = source["inLibrary"]; @@ -558,12 +622,16 @@ export namespace explore { mbid: string; name: string; artistCredit?: string; + artistMbid?: string; intentScore: number; artistType?: string; country?: string; primaryType?: string; year?: string; length?: number; + caaReleaseMbid?: string; + releaseGroupMbid?: string; + releaseName?: string; inLibrary: boolean; static createFrom(source: any = {}) { @@ -576,12 +644,16 @@ export namespace explore { this.mbid = source["mbid"]; this.name = source["name"]; this.artistCredit = source["artistCredit"]; + this.artistMbid = source["artistMbid"]; this.intentScore = source["intentScore"]; this.artistType = source["artistType"]; this.country = source["country"]; this.primaryType = source["primaryType"]; this.year = source["year"]; this.length = source["length"]; + this.caaReleaseMbid = source["caaReleaseMbid"]; + this.releaseGroupMbid = source["releaseGroupMbid"]; + this.releaseName = source["releaseName"]; this.inLibrary = source["inLibrary"]; } } @@ -664,6 +736,24 @@ export namespace explore { } + export class TrackLyrics { + plain: string; + synced: string; + instrumental: boolean; + source: string; + + static createFrom(source: any = {}) { + return new TrackLyrics(source); + } + + constructor(source: any = {}) { + if ('string' === typeof source) source = JSON.parse(source); + this.plain = source["plain"]; + this.synced = source["synced"]; + this.instrumental = source["instrumental"]; + this.source = source["source"]; + } + } export class TrackThumbnailRequest { key: string; releaseMbid: string; @@ -693,6 +783,7 @@ export namespace library { ID: number; Name: string; ArtistName: string; + ArtistMBID: string; MBID: string; CoverArtPath: string; CoverArtSmall: string; @@ -710,6 +801,7 @@ export namespace library { this.ID = source["ID"]; this.Name = source["Name"]; this.ArtistName = source["ArtistName"]; + this.ArtistMBID = source["ArtistMBID"]; this.MBID = source["MBID"]; this.CoverArtPath = source["CoverArtPath"]; this.CoverArtSmall = source["CoverArtSmall"]; diff --git a/frontend/wailsjs/go/playlist/Service.d.ts b/frontend/wailsjs/go/playlist/Service.d.ts index ab3d2e0..9e0ddb4 100755 --- a/frontend/wailsjs/go/playlist/Service.d.ts +++ b/frontend/wailsjs/go/playlist/Service.d.ts @@ -38,12 +38,16 @@ export function GetPlaylistTracks(arg1:number):Promise>; export function GetSmartPlaylistRules(arg1:number):Promise; +export function GetSmartPlaylistTracks(arg1:number):Promise>; + export function ImportPlaylist(arg1:string):Promise; export function ImportPlaylists(arg1:Array):Promise>; export function PreviewSmartPlaylist(arg1:string):Promise>; +export function RefreshSmartPlaylist(arg1:number):Promise; + export function RemoveFromDefaultPlaylist(arg1:Array):Promise; export function RemovePhantomTracks(arg1:number,arg2:Array):Promise; diff --git a/frontend/wailsjs/go/playlist/Service.js b/frontend/wailsjs/go/playlist/Service.js index 3489c6d..474100b 100755 --- a/frontend/wailsjs/go/playlist/Service.js +++ b/frontend/wailsjs/go/playlist/Service.js @@ -70,6 +70,10 @@ export function GetSmartPlaylistRules(arg1) { return window['go']['playlist']['Service']['GetSmartPlaylistRules'](arg1); } +export function GetSmartPlaylistTracks(arg1) { + return window['go']['playlist']['Service']['GetSmartPlaylistTracks'](arg1); +} + export function ImportPlaylist(arg1) { return window['go']['playlist']['Service']['ImportPlaylist'](arg1); } @@ -82,6 +86,10 @@ export function PreviewSmartPlaylist(arg1) { return window['go']['playlist']['Service']['PreviewSmartPlaylist'](arg1); } +export function RefreshSmartPlaylist(arg1) { + return window['go']['playlist']['Service']['RefreshSmartPlaylist'](arg1); +} + export function RemoveFromDefaultPlaylist(arg1) { return window['go']['playlist']['Service']['RemoveFromDefaultPlaylist'](arg1); } diff --git a/go.mod b/go.mod index 781031b..f602aba 100644 --- a/go.mod +++ b/go.mod @@ -13,11 +13,14 @@ require ( github.com/godbus/dbus/v5 v5.1.0 github.com/golang-cz/devslog v0.0.15 github.com/gopxl/beep/v2 v2.1.1 + github.com/klauspost/compress v1.17.9 + github.com/parquet-go/parquet-go v0.30.1 github.com/wailsapp/wails/v2 v2.10.2 go.uploadedlobster.com/mbtypes v0.4.0 go.uploadedlobster.com/musicbrainzws2 v0.18.0 golang.org/x/image v0.12.0 golang.org/x/sync v0.19.0 + golang.org/x/sys v0.41.0 golang.org/x/text v0.34.0 golang.org/x/time v0.15.0 modernc.org/sqlite v1.46.1 @@ -62,7 +65,7 @@ require ( github.com/alfatraining/structtag v1.0.0 // indirect github.com/alingse/asasalint v0.0.11 // indirect github.com/alingse/nilnesserr v0.2.0 // indirect - github.com/andybalholm/brotli v1.1.0 // indirect + github.com/andybalholm/brotli v1.1.1 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/ashanbrown/forbidigo/v2 v2.3.0 // indirect github.com/ashanbrown/makezero/v2 v2.1.0 // indirect @@ -254,9 +257,12 @@ require ( github.com/nishanths/exhaustive v0.12.0 // indirect github.com/nishanths/predeclared v0.2.2 // indirect github.com/nunnatsa/ginkgolinter v0.23.0 // indirect + github.com/parquet-go/bitpack v1.0.0 // indirect + github.com/parquet-go/jsonlite v1.0.0 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pganalyze/pg_query_go/v6 v6.1.0 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pingcap/errors v0.11.5-0.20240311024730-e056997136bb // indirect github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 // indirect github.com/pingcap/log v1.1.0 // indirect @@ -321,6 +327,7 @@ require ( github.com/tkrajina/go-reflector v0.5.8 // indirect github.com/tomarrell/wrapcheck/v2 v2.12.0 // indirect github.com/tommy-muehle/go-mnd/v2 v2.5.1 // indirect + github.com/twpayne/go-geom v1.6.1 // indirect github.com/ultraware/funlen v0.2.0 // indirect github.com/ultraware/whitespace v0.2.0 // indirect github.com/uudashr/gocognit v1.2.0 // indirect @@ -355,7 +362,6 @@ require ( golang.org/x/exp/typeparams v0.0.0-20260209203927-2842357ff358 // indirect golang.org/x/mod v0.33.0 // indirect golang.org/x/net v0.50.0 // indirect - golang.org/x/sys v0.41.0 // indirect golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4 // indirect golang.org/x/term v0.40.0 // indirect golang.org/x/tools v0.42.0 // indirect diff --git a/go.sum b/go.sum index fa92231..dbd1380 100644 --- a/go.sum +++ b/go.sum @@ -75,6 +75,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= +github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/Djarvur/go-err113 v0.1.1 h1:eHfopDqXRwAi+YmCUas75ZE0+hoBHJ2GQNLYRSxao4g= github.com/Djarvur/go-err113 v0.1.1/go.mod h1:IaWJdYFLg76t2ihfflPZnM1LIQszWOsFDh2hhhAVF6k= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= @@ -134,8 +136,8 @@ github.com/alingse/asasalint v0.0.11 h1:SFwnQXJ49Kx/1GghOFz1XGqHYKp21Kq1nHad/0WQ github.com/alingse/asasalint v0.0.11/go.mod h1:nCaoMhw7a9kSJObvQyVzNTPBDbNpdocqrSP7t/cW5+I= github.com/alingse/nilnesserr v0.2.0 h1:raLem5KG7EFVb4UIDAXgrv3N2JIaffeKNtcEXkEWd/w= github.com/alingse/nilnesserr v0.2.0/go.mod h1:1xJPrXonEtX7wyTq8Dytns5P2hNzoWymVUIaKm4HNFg= -github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= -github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= @@ -570,6 +572,8 @@ github.com/kisielk/errcheck v1.9.0/go.mod h1:kQxWMMVZgIkDq7U8xtG/n2juOjbLgZtedi0 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkHAIKE/contextcheck v1.1.6 h1:7HIyRcnyzxL9Lz06NGhiKvenXq7Zw6Q0UQu/ttjfJCE= github.com/kkHAIKE/contextcheck v1.1.6/go.mod h1:3dDbMRNBFaq8HFXWC1JyvDSPm43CmE6IuHam8Wr0rkg= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.10/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= @@ -728,12 +732,20 @@ github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJ github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= +github.com/parquet-go/bitpack v1.0.0 h1:AUqzlKzPPXf2bCdjfj4sTeacrUwsT7NlcYDMUQxPcQA= +github.com/parquet-go/bitpack v1.0.0/go.mod h1:XnVk9TH+O40eOOmvpAVZ7K2ocQFrQwysLMnc6M/8lgs= +github.com/parquet-go/jsonlite v1.0.0 h1:87QNdi56wOfsE5bdgas0vRzHPxfJgzrXGml1zZdd7VU= +github.com/parquet-go/jsonlite v1.0.0/go.mod h1:nDjpkpL4EOtqs6NQugUsi0Rleq9sW/OtC1NnZEnxzF0= +github.com/parquet-go/parquet-go v0.30.1 h1:Oy6ganNrAdFiVwy7wNmWagfPTWA2X9Z3tVHBc7JtuX8= +github.com/parquet-go/parquet-go v0.30.1/go.mod h1:navtkAYr2LGoJVp141oXPlO/sxLvaOe3la2JEoD8+rg= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pganalyze/pg_query_go/v6 v6.1.0 h1:jG5ZLhcVgL1FAw4C/0VNQaVmX1SUJx71wBGdtTtBvls= github.com/pganalyze/pg_query_go/v6 v6.1.0/go.mod h1:nvTHIuoud6e1SfrUaFwHqT0i4b5Nr+1rPWVds3B5+50= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20240311024730-e056997136bb h1:3pSi4EDG6hg0orE1ndHkXvX6Qdq2cZn8gAPir8ymKZk= github.com/pingcap/errors v0.11.5-0.20240311024730-e056997136bb/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= @@ -914,6 +926,8 @@ github.com/tomarrell/wrapcheck/v2 v2.12.0 h1:H/qQ1aNWz/eeIhxKAFvkfIA+N7YDvq6TWVF github.com/tomarrell/wrapcheck/v2 v2.12.0/go.mod h1:AQhQuZd0p7b6rfW+vUwHm5OMCGgp63moQ9Qr/0BpIWo= github.com/tommy-muehle/go-mnd/v2 v2.5.1 h1:NowYhSdyE/1zwK9QCLeRb6USWdoif80Ie+v+yU8u1Zw= github.com/tommy-muehle/go-mnd/v2 v2.5.1/go.mod h1:WsUAkMJMYww6l/ufffCD3m+P7LEvr8TnZn9lwVDlgzw= +github.com/twpayne/go-geom v1.6.1 h1:iLE+Opv0Ihm/ABIcvQFGIiFBXd76oBIar9drAwHFhR4= +github.com/twpayne/go-geom v1.6.1/go.mod h1:Kr+Nly6BswFsKM5sd31YaoWS5PeDDH2NftJTK7Gd028= github.com/ultraware/funlen v0.2.0 h1:gCHmCn+d2/1SemTdYMiKLAHFYxTYz7z9VIDRaTGyLkI= github.com/ultraware/funlen v0.2.0/go.mod h1:ZE0q4TsJ8T1SQcjmkhN/w+MceuatI6pBFSxxyteHIJA= github.com/ultraware/whitespace v0.2.0 h1:TYowo2m9Nfj1baEQBjuHzvMRbp19i+RCcRYrSWoFa+g= @@ -947,6 +961,8 @@ github.com/xen0n/gosmopolitan v1.3.0/go.mod h1:rckfr5T6o4lBtM1ga7mLGKZmLxswUoH1z github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yagipy/maintidx v1.0.0 h1:h5NvIsCz+nRDapQ0exNv4aJ0yXSI0420omVANTv3GJM= github.com/yagipy/maintidx v1.0.0/go.mod h1:0qNf/I/CCZXSMhsRsrEPDZ+DkekpKLXAJfsTACwgXLk= github.com/yeya24/promlinter v0.3.0 h1:JVDbMp08lVCP7Y6NP3qHroGAO6z2yGKQtS5JsjqtoFs= diff --git a/main.go b/main.go index 6f01470..271d2e3 100644 --- a/main.go +++ b/main.go @@ -14,6 +14,7 @@ import ( "yellowjacket/backend" "yellowjacket/backend/assets" + "yellowjacket/backend/config" "yellowjacket/backend/logging" "yellowjacket/backend/profiling" "yellowjacket/internal/dev" @@ -86,8 +87,8 @@ func main() { OnBeforeClose: yjApp.OnBeforeClose, OnShutdown: yjApp.OnShutdown, Bind: yjApp.FEBindings, - MinWidth: 512, - MinHeight: 384, + MinWidth: config.MinWidth, + MinHeight: config.MinHeight, MaxWidth: 0, MaxHeight: 0, Linux: &linux.Options{