diff --git a/.gitea/workflows/index-artifact.yml b/.gitea/workflows/index-artifact.yml index 302040b..6b6c97d 100644 --- a/.gitea/workflows/index-artifact.yml +++ b/.gitea/workflows/index-artifact.yml @@ -4,7 +4,7 @@ name: Search index maintenance # trigger below runs the same command: # # no completed import -> build (first run, or resume a partial one) -# import older than 3mo -> rebuild (re-import from the newest dump) +# import older than 6mo -> rebuild (re-import from the newest dump) # otherwise -> refresh (fold in new incremental listens) # # A refresh is cheap and no-ops when nothing new has been published, so @@ -13,7 +13,7 @@ on: push: branches: [main] schedule: - # Weekly update pass. The 3-month rebuild is triggered by the same + # Weekly update pass. The 6-month rebuild is triggered by the same # command when it notices the import has aged out. - cron: '0 4 * * 1' workflow_dispatch: @@ -91,7 +91,12 @@ jobs: - name: Build tools working-directory: /src - run: go build -o /usr/local/bin/ ./cmd/indexbuild ./cmd/indexexport + # The dump importer is behind the `indexbuild` tag so it is not + # linked into the app binary; cmd/indexbuild carries the same tag + # and will not build without it. + run: | + go build -tags indexbuild -o /usr/local/bin/ ./cmd/indexbuild + go build -o /usr/local/bin/ ./cmd/indexexport - name: Maintain index id: maintain @@ -122,12 +127,26 @@ jobs: if: steps.maintain.outputs.complete == 'true' && steps.maintain.outputs.changed == 'true' run: | set -eu - version="$(date -u +%Y%m%d)" - base="${SERVER_URL}/api/packages/${OWNER}/generic/yellowjacket-core-index/${version}" - for f in core-index.db.zst core-index.db.zst.sha256; do - echo "Uploading $f -> $version" - curl --fail-with-body --user "${OWNER}:${PACKAGE_TOKEN}" \ - --upload-file "/tmp/$f" "${base}/${f}" + pkg="${SERVER_URL}/api/packages/${OWNER}/generic/yellowjacket-core-index" + + # Published twice: under a dated version for history, and under + # the fixed "latest" version the client fetches. Clients cannot + # discover the newest dated version on their own — the package + # listing API requires a token, while a plain file GET does not + # — so "latest" is what makes an anonymous first run possible. + # + # A generic package rejects re-uploading a filename that already + # exists, so "latest" is deleted before being rewritten. It is + # absent on the very first publish, hence the tolerated 404. + curl --silent --show-error --user "${OWNER}:${PACKAGE_TOKEN}" \ + --request DELETE "${pkg}/latest" || true + + for version in "$(date -u +%Y%m%d)" latest; do + for f in core-index.db.zst core-index.db.zst.sha256; do + echo "Uploading $f -> $version" + curl --fail-with-body --user "${OWNER}:${PACKAGE_TOKEN}" \ + --upload-file "/tmp/$f" "${pkg}/${version}/${f}" + done done - name: Summary diff --git a/.planning/NOTES.md b/.planning/NOTES.md new file mode 100644 index 0000000..fb37b9b --- /dev/null +++ b/.planning/NOTES.md @@ -0,0 +1,140 @@ +# Notes + +Gotchas, measured facts, and things already considered and rejected. +Measurements carry the date they were taken — several of these are +properties of someone else's server and can change. + +## MetaBrainz caps a client at ~2 MB/s (measured 2026-07-29) + +`data.metabrainz.org` serves a single client at roughly 2.1 MB/s, and +**concurrency does not help**: one Range stream and four concurrent +lanes delivered 32 MB at 2,111,195 B/s and 2,209,000 B/s respectively, +while the same machine pulled 66.9 MB/s from a CDN. One of the four +lanes starved to 0.5 MB/s. The lanes divide a fixed cap; they do not +raise it. + +Consequences: + +- No client-side concurrency change will speed up a dump download. + Pushing harder earns 503s (the reason `dumpLanes` is 4). +- Stage 1 of a full import costs ~11.8 h at best (89 GB after column + projection). Before projection it was 205 GB — about 27 h. + +This is the entire reason the catalog is built centrally and shipped as +an artifact rather than derived per install. + +## Further stage-1 reductions, not yet taken + +Both are CI-side options; neither is safe as a silent client default +because each changes *what gets counted*. + +- **Project `recording_mbid` only** (24.1% of row-group bytes instead of + 43.4%): ~49 GB, ~6.5 h. `canonical_musicbrainz_data.csv` already + carries `recording_mbid`, `release_mbid`, `artist_mbids` and + `release_group_mbid`, so release/RG/artist counts can be rolled up + locally. Cost: listens with no recording MBID are dropped, and artist + totals become "sum of their recordings" rather than direct attribution. +- **Stride-sample members** (1-in-4): ~12 GB, ~1.6 h. The dump is flat + numbered members (`0.parquet`, …). Sampling is viable because the + counts only feed a *ranking* for a top-N cut. Must be a stride, never + a prefix — if members are time-ordered a prefix biases hard toward one + era. + +## Incremental dump retention is 30 days (measured 2026-07-29) + +The incremental directory held 30 dumps (series 2579–2610), and full +dumps land roughly monthly. An artifact older than ~30 days cannot be +topped up: the dailies bridging the gap are gone. That is a permanent +undercount of that window, not corruption — but it pins the artifact +republish cadence at monthly. + +## Anonymous package download is UNVERIFIED + +The client fetches the artifact from a fixed `latest` URL because Gitea's +package *listing* API requires a token while a plain file GET appears not +to — a probe of the not-yet-published artifact returned 404 rather than +401. **That is suggestive, not proof.** No artifact has been published +yet to test against. Confirm before relying on it. + +Also worth deciding deliberately: every install pulling from a personal +Gitea makes its bandwidth and uptime a user-facing dependency. + +## No migration chain + +`applySchema` creates the whole schema from `sql/schemas/*.sql` on every +open; all DDL is `IF NOT EXISTS`. A database written by an older build is +not supported and there is no upgrade path by design. + +Two things this replaced, worth not reintroducing: + +- The 48-step chain was ~3,700 of `database.go`'s 4,061 lines, plus + helpers that existed only to serve it (`backupDatabase`, + `readLibraryDirFromTOML`, `isDuplicateColumnErr`, …). +- `sql/schemas/` had drifted badly from the real schema — it still + described a `genre_recordings` table that migrations had renamed, and + omitted `explore_index`, `http_cache`, `artist_images`, + `similar_artist_map`, `release_to_rg`, `lyrics_index` and + `artist_metadata` entirely. sqlc reads that directory, so it had been + generating against a stale schema and silently missed columns such as + `audio_files.modified_at`. + +**When regenerating schema files from a live database, remember the seed +rows.** `file_types` (the four supported extensions), `player_state` and +`queue` each carry `INSERT OR IGNORE` rows that `sqlite_master` does not +contain. Dropping them breaks every audio-file foreign key. + +## ANALYZE runs after the catalog merge, not at schema creation + +The old migration 45 ran `ANALYZE` once. With the migration chain gone +there is no equivalent moment — an empty database has nothing to measure +— so it runs at the end of the artifact import instead +(`SearchIndex.analyzeIndex`). Without current statistics the planner +mis-estimates the partial expression indexes on `explore_index` +(`idx_explore_title_lower`, `idx_explore_artist_lower`) and scans a +million rows for queries that should seek. + +If another path ever populates the catalog, it needs the same call. + +## Writers, not readers, are responsible for name quality + +`resolveArtistName` falls back to returning the artist MBID when it +cannot find a name. That is fine for a one-off render but must never be +persisted — an MBID stored as a title is unsearchable and shows as a +UUID in the UI. + +This used to be defended at every read (`title != mbid` predicates) and +in the upsert's conflict rules. Those defenses are gone; `AddFromCache` +now refuses to write a name equal to the MBID and lets the upsert's +"non-empty wins" rule fill it in when a real name arrives. +`TestAddFromCacheNeverStoresMBIDAsName` guards this. + +## Attached databases are invisible to the read pool + +`database.DB` holds two handles: a single-writer connection and a +separate query-only pool. `ATTACH` binds to one connection, so anything +touching an attached database must use `ExecContext`/`QueryRowWriter` +(the writer) — `QueryContext` routes to the pool, where the attachment +does not exist and the query fails with "no such table". + +## FTS triggers are defined in Go, not in the schema + +`explore_index`'s three FTS sync triggers live in +`exploreIndexFTSTriggers` in `database.go` rather than in +`sql/schemas/explore_index.sql`, because the bulk-load path drops and +recreates them (`SuspendExploreIndexFTS`). Defining them in both places +would be two copies free to drift. + +Bulk loads must suspend them: measured on a real import, assembly runs at +~31 rows/s with the triggers attached and ~4,700 rows/s without. + +## Explore "library only" toggle was removed (2026-08-06) + +The Explore UI used to have a "library only" mode toggle +(`frontend/src/store/explore-settings.ts`, `explore:libraryOnly` in +localStorage) that filtered the Explore UI to owned content only. It was +removed outright — the app now always shows full (network-enriched) +Explore data. If offline/library-only mode is wanted again, it should be +built from scratch rather than restored; the old implementation gated +several component code paths in ad hoc ways. A separate +`deep_catalog_enabled` backend flag briefly existed for the same idea and +was removed earlier, when the dump importer left the app binary. diff --git a/.planning/plans/pending/001-ship-core-index.md b/.planning/plans/completed/001-ship-core-index.md similarity index 68% rename from .planning/plans/pending/001-ship-core-index.md rename to .planning/plans/completed/001-ship-core-index.md index 35f29c0..4cf8559 100644 --- a/.planning/plans/pending/001-ship-core-index.md +++ b/.planning/plans/completed/001-ship-core-index.md @@ -1,8 +1,27 @@ # 001 — Ship a prebuilt "core" explore index -**Status:** pending -**Branch:** wip +**Status:** complete +**Branch:** cleanup/fresh-start-schema **Created:** 2026-07-25 +**Completed:** 2026-07-30 + +## Outcome + +A fresh install downloads a 70.6 MB artifact and merges 1,076,133 rows +in ~43 s, instead of streaming 205 GB over ~27 h. The dump importer that +produces the artifact left the app binary entirely — it is behind the +`indexbuild` build tag and runs only in CI. + +Phase 5 landed differently than planned: rather than a user-facing +setting gating the deep import, the deep import is simply not in the +app. `deep_catalog_enabled` existed briefly and was removed with it. + +Two things remain unverified or undone, both recorded in +`.planning/NOTES.md`: anonymous package download on git.ljones.me has +not been confirmed against a real published artifact, and installs whose +index was built by older code (no `listens_applied_series`) have no +rescue path — though with no migration chain, those databases are now +unsupported anyway. ## Problem @@ -250,17 +269,96 @@ egress if it is self-hosted on a home connection. columns dropped, metadata stamped, vacuumed. Verified against a synthetic index: no personal columns leak, no orphaned rows, caps respected. -3. ⬜ **One real build.** Run `indexbuild` against a persistent volume - until it converges. This yields the first genuine dump-built index and - with it true row counts, on-disk size, real FTS share, and - `release_to_rg` size. **Every tier number above is still an - extrapolation from synthetic rows until this exists.** -4. ⬜ **Import path.** First-run download + attach + upsert, with checksum - verification, resumability, and clean degradation on failure. Report - it as a job in the Jobs panel — the plumbing for that already exists. -5. ⬜ **Gate the dump build.** Add the setting that makes the full import - opt-in, so a shipped core index isn't immediately followed by the - multi-GB download it was meant to replace. +3. ✅ **One real build.** Superseded by a real dump-built index that + already existed on the dev machine (`dump_import_done` 2026-07-17). + Measured 2026-07-29 — these replace every extrapolation above: + + | | rows | on disk | + |---|---|---| + | `explore_index` | 2,052,168 (227,359 artists / 400,675 RGs / 1,424,134 recordings) | 383 MB | + | its indexes | | 395 MB | + | FTS | | 80 MB | + + 187 B/row for the shippable table, 418 B/row all-in — so the ~900 MB + full-budget estimate was right. Two real exports: + + | tier | rows | artifact | zstd -19 | + |---|---|---|---| + | 50K artists (default) | 1,076,133 | 191.5 MB | **70.6 MB** | + | 25K artists / 10 RG / 20 rec | 620,973 | 110.6 MB | **37.8 MB** | + + `release_to_rg` was empty in that index — it predates the code that + persists it — so its size is still unmeasured. +4. ✅ **Import path.** `backend/explore/artifactfetch.go` (download, + Range-resume, sha256, zstd) and `artifactimport.go` (validate, ATTACH, + batched merge, FTS rebuild, meta stamping). Reported in the Jobs panel + under its own two stages. Measured end to end on the real 50K-artist + artifact against a disk-backed DB: **1,076,133 rows merged in 43.2s** + (24,900 rows/s), yielding a 455 MB `yj.db`, FTS populated and + searchable. In-memory the same merge runs in 28.3s. +5. ✅ **Gate the dump build.** `deep_catalog_enabled` in + `explore_index_meta` (beside `index_build_paused` — it is build state, + read at one decision point). Off by default; exposed as + `DeepCatalogEnabled` / `SetDeepCatalogEnabled` on the explore Service. + An interrupted dump import resumes regardless of the setting, so the + gate never discards a checkpoint that already cost hours. + +## Measured 2026-07-29: why the client cannot fix this itself + +`data.metabrainz.org` caps a client at ~2.1 MB/s. One Range stream and +four concurrent Range lanes both delivered 32 MB at the same aggregate +rate (2,111,195 B/s vs 2,209,000 B/s) while the same machine pulled +66.9 MB/s from a CDN. **Parallelism buys nothing** — the four lanes just +divide the same cap, and one of them starved to 0.5 MB/s. + +So stage 1 costs, unavoidably: + +| | bytes | wall clock | +|---|---|---| +| Whole tar (what shipped before column projection) | 205 GB | ~27 h | +| Column projection, 3 columns (43.4%) | 89 GB | ~11.8 h | +| `recording_mbid` only (24.1%), rolled up via canonical | 49 GB | ~6.5 h | +| + 1-in-4 member stride sample | 12 GB | ~1.6 h | + +The last two are CI-side options, not client defaults: recording-only +drops listens carrying no recording MBID and re-derives artist totals as +a sum over recordings, and sampling trades exact counts for a ranking. +Both are only safe because the selection they feed is a top-N cut. + +## Distribution: the "latest" version trick + +The client cannot enumerate package versions — Gitea's package listing +API requires a token, while an anonymous file GET does not (a probe of a +non-existent artifact returns 404, not 401). So `index-artifact.yml` +publishes each artifact twice: under a dated version for history, and +under a fixed `latest` version that the client fetches from a +predictable URL. Generic packages reject overwriting an existing +filename, so `latest` is DELETEd before each rewrite. + +⚠️ **Unverified:** that anonymous package *download* is actually enabled +on git.ljones.me. The 404-vs-401 probe is suggestive, not proof — no +artifact has been published yet to test against. Confirm before relying +on it, and note that every install pulling from a personal Gitea makes +its bandwidth and uptime a user-facing dependency. + +## Incremental retention bounds artifact staleness + +The incremental dump directory holds 30 dumps (series 2579–2610 as of +2026-07-29) and full dumps land roughly monthly. An artifact older than +~30 days therefore cannot be topped up: the dailies bridging the gap are +gone. That is a permanent undercount of that window's listens, not +corruption — but it pins the republish cadence at monthly. + +## Upgrade path for indexes built by older code + +The dev machine's index has `dump_import_done` set but **no** +`listens_applied_series` and an empty `release_to_rg`, because it was +built before the code that writes them. That combination is a dead end: +`RefreshListenCounts` bails with "no baseline series recorded", and +`runDumpBuild` short-circuits on the done marker, so popularity can +never update again. Current code writes both, so this affects only +pre-existing installs — but the artifact import is the natural place to +rescue them, since merging one stamps a fresh baseline series. 6. ✅ **CI wiring.** `.gitea/workflows/index-artifact.yml` — push + weekly cron + manual, concurrency-guarded, publishes only when `complete && changed` so identical artifacts don't accumulate. diff --git a/.planning/plans/completed/002-data-lifecycle-architecture.md b/.planning/plans/completed/002-data-lifecycle-architecture.md new file mode 100644 index 0000000..60d530e --- /dev/null +++ b/.planning/plans/completed/002-data-lifecycle-architecture.md @@ -0,0 +1,155 @@ +# 002 — Data lifecycle architecture + +**Status:** completed (first tranche); follow-ups tracked below +**Branch:** main +**Created:** 2026-07-26 + +## Problem + +An audit of asset and row cleanup found five leaks, four of which shared +one root cause: **deletion logic was hand-written per call site and lived +far from the thing being deleted.** `RemoveLibrary` knew about ten tables +because someone enumerated them once; migration 32 added an eleventh and +nothing noticed. Files written by `explore` had no cleanup counterpart +anywhere. A function that evicted expired cache rows was written and +never called. + +Findings, in severity order: + +1. **`RemoveLibrary` was broken for any scanned library.** `tagging_items` + holds `FOREIGN KEY(library_id) REFERENCES libraries(id)` with no + `ON DELETE` clause and was never cleared, so `DELETE FROM libraries` + failed with `FOREIGN KEY constraint failed (787)` and rolled back the + whole removal. Every scanned library has `tagging_items` rows (the + scan upserts one per album folder), so this fired on essentially every + real removal. `RemoveLibrary` had zero test coverage. +2. **Artist images were never deleted by anything.** No `os.Remove` in + `explore`, no `DELETE FROM artist_images` in the codebase. Unbounded + in the number of artists ever browsed in Explore, most of whom are not + in the library. +3. **Cover art size variants leaked on removal.** Only the base + `cover_art.file_path` was unlinked; the `_sm/_md/_lg` files beside it + are derived filenames, not rows, so three files per cover survived. +4. **`http_cache` was never pruned.** `Cache.Evict()` existed with no + callers. Reads filter on `expires_at`, so expired rows were inert but + accumulated for the life of the install. +5. **Cover-art proxy cache was never pruned.** No eviction, no size cap. + +## Approach + +Rather than patch five holes, classify the data so the *class* of bug +becomes hard to write. Everything persisted falls on two axes — +regenerability and cost of regeneration — which collapse to four kinds: + +| Kind | Regenerable? | Deletion policy | +|---|---|---| +| **Owned** — projection of the user's files | Yes, by rescan | Follows the files | +| **Authored** — user-created, no other copy | **No** | Explicit user action only | +| **Derived** — computed from owned | Yes, cheaply | Free; must never block owned deletion | +| **Cache** — network or dump sourced | Yes, expensively | TTL/age eviction, never cascade | + +The classification is not just vocabulary — it produces the right fix for +each finding. Finding 1 is derived data acting as a referential parent of +owned data, which the taxonomy makes categorically illegal. Finding 2 is +cache data that never needed owner-linked cleanup at all; it wants age +eviction. Finding 3 is derived data that must be swept against a live set +rather than tracked individually. + +A Go interface was considered and rejected: the only polymorphic consumer +is the janitor, the substrates have nothing in common (SQL rows, an FTS +virtual table, a view, three directories of JPEGs, a 900 MB index), and +provenance is a static fact better enforced by package boundaries than by +methods an implementation may lie about. A declarative catalog gets the +same benefit for a tenth of the cost. + +## What shipped + +**`backend/datamap`** — the catalog. Every table, view, and asset +directory declared with its `Kind`, its `Lifetime` (`cascade`, `set-null`, +`swept`, `retained`), and a note explaining the classification. Plain data +with no service dependencies, so tests can assert it against a live +schema. FTS5 shadow tables resolve to their parent. + +Tests that give it teeth (`backend/datamap/datamap_test.go`): + +- `TestCatalogCoversSchema` — every table in `sqlite_master` is claimed by + exactly one entry. **A new table fails the build until somebody states + what it is and how it dies.** +- `TestCatalogHasNoStaleEntries` — the reverse, catching drift. +- `TestNoActionForeignKeysAreDeclaredSwept` — a `NO ACTION` foreign key + blocks its parent's deletion, so its table must declare `swept`. This is + the exact shape of finding 1, now caught at CI time. +- `TestLifetimesMatchSchema` — declared cascade/set-null must match what + SQLite actually enforces. +- `TestAuthoredCascadesAreDeliberate` — authored data is unrecoverable, so + a cascade onto it needs an explicit exemption. + +**`backend/maintenance`** — the janitor. A registry of named jobs with +per-job minimum intervals, run at startup-idle and on a 6h tick. Policies +follow the taxonomy: derived data sweeps against a live set, cache data +ages out. Registered in one place (`app.go: startJanitor`) so the full set +of janitorial work is a single visible list. + +Jobs: `http-cache-evict` (6h), `covers-sweep` (24h, live set from +`cover_art` expanded via `CoverArtFileSet`), `artist-images-sweep` (24h, +keeps art for library artists indefinitely, evicts browsed-artist art +after 90d), `cover-art-proxy-sweep` (24h, 30d age eviction). + +The covers sweep refuses to act on an empty live set — that means the +query failed to see the table, not that every cover is garbage. + +**Leak tests** (`backend/library/leak_test.go`) — driven by the catalog +rather than a hardcoded list, so new tables are covered the moment they +are catalogued: + +- `TestRemoveLibraryLeavesNoOwnedOrDerivedRows` — removing the only + library leaves no owned or derived rows, except those in + `staleTolerated` with a written reason. +- `TestRemoveLibraryPreservesAuthoredData` — authored data survives. +- `TestSweptTablesAreActuallySwept` — a table declaring `swept` that + nothing sweeps is caught. + +All three were verified to fail when the finding-1 fix is reverted. + +**Fixes** — `tagging_items` cleared inside the removal transaction +(`crud.go` step 17); `CoverArtFileSet` expands originals to variants and +the legacy `_thumb` name; `Cache.Evict` logic moved into a registered job. + +**Incidental:** `Library.emit` — `runtime.EventsEmit` calls `log.Fatalf` +on a context without a Wails runtime, which killed the test binary and +made the whole package untestable. All ten emits in the package now route +through a nil-safe helper. This also removes a real crash risk for +background workers that outlive their context. + +## Follow-ups + +**`audio_files` is a mixed-kind table.** `play_count`, `last_played`, and +`tag_status` are *authored* data living in an *owned* table. Orphan +cleanup treats the whole row as regenerable, which is why renaming a file +destroys its play count — the row is deleted and re-imported fresh. This +is the strongest argument for splitting authored per-track state into its +own table keyed by something more stable than a path. Related: an +audio-stream content hash (excluding tag blocks, so it survives +retagging) would let a rename be recognised as the same file. Deliberately +out of scope here; it is a schema change plus a rename-detection pass, not +a cleanup fix. + +**Cascade adoption.** Fourteen of nineteen foreign keys are `NO ACTION`. +Converting them to `CASCADE` would delete a lot of hand-written orphan +sweeps, but SQLite cannot add `ON DELETE` via `ALTER TABLE` — each needs +the 12-step table rebuild. Note the ordering constraint: cascades delete +rows silently, so any code that collects file paths *before* deleting rows +(as `RemoveLibrary` does for cover art) breaks under cascade. Mark-and- +sweep must land first; the two compose, cascade plus path-collection does +not. + +**Consolidate the ten orphan sweeps.** `DELETE ... WHERE id NOT IN (...)` +appears ten times across `crud.go`, `dbsync.go`, `smartplaylist.go`, and +`database.go`. One shared `sweepOrphans(tx)` would shrink the surface where +a new table can be forgotten. Worth doing opportunistically rather than as +a big-bang refactor. + +**Storage settings pane.** The catalog knows every table and directory and +its kind; the janitor already computes bytes freed. A settings pane showing +per-kind disk usage with "clear cache" and "rebuild derived data" buttons +is now mostly a UI job. diff --git a/.planning/plans/completed/003-download-clients.md b/.planning/plans/completed/003-download-clients.md new file mode 100644 index 0000000..eea824e --- /dev/null +++ b/.planning/plans/completed/003-download-clients.md @@ -0,0 +1,279 @@ +# 003 — Download clients + +**Status:** implemented (v1); follow-ups tracked below +**Branch:** main +**Created:** 2026-07-27 + +## Problem + +YellowJacket can find music (`explore`), identify it (`autotag`), and +manage it (`library`) — but it can't acquire it. The one gap between +"you're missing this album" and "you own this album" is filled today by +the user alt-tabbing to some other tool. + +The naive fix is an HTTP client for Soulseek and a shell-out to yt-dlp. +That produces two bespoke code paths with duplicated queueing, retry, +staging and import logic, and a third service means a third copy. The +services users want to connect are also not the same *kind* of thing — +some search, some transfer bytes, some are entire automation systems we +delegate to — so a single `DownloadClient` interface would be a lie that +every adapter partially implements. + +## The role decomposition + +Every candidate integration fills one or two of three roles: + +| Service | Searches | Transports | Delegates | +|---|---|---|---| +| slskd (Soulseek) | ✅ | ✅ | | +| yt-dlp | ✅ | ✅ | | +| Lidarr | | | ✅ | +| Prowlarr | ✅ | | | +| qBittorrent / Transmission | | ✅ | | +| SABnzbd / NZBGet | | ✅ | | + +So: three small interfaces, not one big one. A provider implements +whichever it supports and declares that in a capability struct, the same +way `jobs.Caps` lets the frontend render controls without switching on +`Kind`. + +```go +// Searcher turns a request into ranked candidates. +type Searcher interface { + Search(ctx context.Context, req Request) ([]Candidate, error) +} + +// Transporter moves a candidate's bytes to a local staging directory. +type Transporter interface { + Grab(ctx context.Context, c Candidate, dst string, p ProgressFunc) (Result, error) + Cancel(ctx context.Context, grabID string) error +} + +// Delegator hands the whole request to an external manager and +// reports back when files land. +type Delegator interface { + Request(ctx context.Context, req Request) (string, error) + Poll(ctx context.Context, externalID string) (DelegateStatus, error) +} +``` + +A `Provider` is the registry entry: identity, config, health check, caps, +plus whichever of the three it satisfies. Search-only providers +(Prowlarr) are paired with a transport at grab time by protocol match +(`torrent` → qBittorrent, `usenet` → SABnzbd); providers that do both +are self-pairing. + +## v1 decisions (settled) + +- **On-demand only.** User-initiated "find this album" from an Explore + artist/album page or a missing-album row. No wanted list, no artist + monitoring, no quality-cutoff upgrades. The queue and pipeline built + here are exactly what monitoring would later sit on top of — see + Deferred. +- **Soulseek via slskd's REST API**, not a native protocol client. Same + adapter shape as everything else, no wire protocol, no credentials in + our process, fully testable against an `httptest` server. A native + provider can slot in behind `Searcher`/`Transporter` later with no + pipeline changes. +- **Stage → autotag → import.** Downloads land in a staging directory, + are matched against the intended release with the existing `autotag` + scorer, tagged, then moved into the library and scanned. Never write + into the library root directly. +- **All four provider families in v1**, sequenced so each phase proves a + different role shape (see Phases). + +## Pipeline + +``` +Request (MBID-anchored where possible) + └─> fan-out Search across enabled providers (per-provider timeout) + └─> merge + rank Candidates + └─> user picks (or auto-pick above confidence threshold) + └─> Grab into staging// + └─> verify (audio decodes, expected track count) + └─> autotag against the intended release + └─> tagwriter writes tags + └─> move into library layout + └─> targeted incremental scan +``` + +The `Request` should carry a release-group or release MBID whenever the +user started from an Explore page, because that anchor is what makes the +autotag step reliable instead of a second guess. Free-text requests are +supported but flagged lower-confidence, and never auto-pick. + +Staging lives under the user data dir, not the library. Partial grabs are +resumable where the provider supports it and swept on startup where it +doesn't. + +## Candidate ranking + +Two independent scores, kept separate: + +1. **Match confidence** — does this candidate contain the release the + user asked for? Reuse `autotag`'s distance/alignment machinery on the + candidate's *filenames* (Soulseek gives paths, not tags), against the + expected tracklist from the explore index. +2. **Source quality** — format (FLAC > V0 > 320 > lower), bitrate, + completeness (file count vs. expected track count), source health + (slskd queue length and upload slots; seeders for torrents), and a + user-set per-provider priority. + +Ranking presents both, because they trade off — a perfectly-matched +128kbps rip should lose to a well-matched FLAC, and the user should be +able to see why. Reusing `autotag.ScoreBreakdown`'s "explain the ranking" +pattern here is deliberate. + +## Persistence + +New tables (migration TBD, next free number): + +- `download_providers` — id, kind, name, enabled, priority, config blob + (JSON), `created_at`. Non-secret config only. +- `download_requests` — id, source (`explore-album`, `explore-artist`, + `manual`), release_mbid / release_group_mbid, free-text query, + requested_at, state, resolved_download_id. +- `download_items` — one row per grab attempt: request_id, provider_id, + candidate JSON, state, bytes/total, staging path, error, timestamps. + +**Secrets** (slskd API key, Lidarr/Prowlarr API keys, qBittorrent +password) do not go in the TOML config or the DB in plaintext. Use the OS +keyring where available with a clearly-labelled encrypted-file fallback, +and never log a config value from a provider's secret field. Open +question below on the exact library. + +## Jobs integration + +Add `jobs.KindDownload`. One job per request (not per file), with +`Stages` for search → grab → import so the existing detail panel renders +the pipeline for free. `Caps{Cancellable: true}`; pausable only for +providers that can resume. Per-provider concurrency caps and a global +cap, both configurable — hammering a Soulseek peer with eight parallel +transfers gets you queued or banned. + +## Frontend + +- New `download-providers` section in `config-page` (HTMX + templ, same + as existing settings) for provider CRUD, test-connection, priority. +- New `download-picker` Lit component: the ranked-candidate dialog, + invoked from Explore album/artist pages and from a missing-album row. +- `download-store.ts` subscribing to the existing `JobsChanged` event — + no new event channel needed for progress. + +## Phases + +Each phase is independently shippable and proves a distinct role shape. + +1. **Core.** Interfaces, registry, `Request`/`Candidate`/`Result` types, + staging dir, ranking, the stage→autotag→import tail, jobs wiring, + schema, secret storage. Ships with a fake provider and full test + coverage of the pipeline. No real network. +2. **yt-dlp.** Subprocess provider: search + transport, no server for the + user to run, so it's the fastest path to an end-to-end working + feature. Proves the local-subprocess shape (binary discovery, + version checks, stdout progress parsing, sandboxing the arg list). +3. **slskd.** Remote search + transport over REST. Proves the remote + HTTP shape and is the highest-value source. This is where filename- + based match confidence earns its keep. +4. **Lidarr.** Delegate. Proves the fire-and-poll shape, where we don't + own the transfer and the "import" step is really "detect what Lidarr + already imported and reconcile". +5. **Prowlarr + qBittorrent/SABnzbd.** Proves split search/transport + pairing — the one case where two providers cooperate on a single + request. + +## Risks and constraints + +- **No bundled credentials, no default-on providers, no preconfigured + indexers.** Every provider is off until the user configures it. The + app ships the ability to connect to services the user already runs. +- **yt-dlp is a moving target.** Pin a minimum version, check it at + provider-enable time, and fail with a clear message rather than + parsing garbage output. +- **Filename-only matching is genuinely hard.** Soulseek results are + `\Music\Album (1997) [FLAC]\01 - Track.flac` at best. Budget real + effort for the path-parsing heuristics; `autotag/normalize.go` is the + starting point. +- **Partial and failed grabs must never reach the library.** The import + step is the only writer into library paths, and it runs after + verification. Staging sweep on startup. +- **Tests must not hit the network.** `httptest` servers for slskd/ + Lidarr/Prowlarr, a stub binary for yt-dlp. + +## Deferred + +- Wanted list with background retry (the natural next plan). +- Artist monitoring + auto-grab of new releases — cheap once the wanted + list exists, because `explore`'s dump index already knows the full + discography and `library` already knows what's owned. +- Quality profiles and upgrade-if-better. +- Native Soulseek protocol client. +- Transmission/Deluge/NZBGet (same shape as their shipped siblings — + add on demand). +- Internet Archive / Bandcamp-collection providers: cheap REST adapters, + worth adding once the core is proven. + +## Resolved questions + +1. **Secret storage.** No keyring dependency was added. Credentials go + in a 0600 JSON file in the user data directory (`download-secrets.json`), + keyed by provider row ID. This is deliberately *not* encryption — a + key stored beside the data it unlocks protects nothing, and claiming + otherwise would be worse than being clear about it. What the file + mode buys is protection from other local users and from the config + file being pasted into a bug report. `SecretStore` is an interface so + an OS keyring backend can be added later without touching any + provider. +2. **Auto-pick.** Implemented behind `Downloads.AutoPick`, default off. + It requires an MBID-anchored request, match ≥ 0.85, quality ≥ 0.5, + and ≥ 0.08 of daylight over second place. Free-text requests can + never auto-pick, because there is no tracklist to be right about. +3. **Library layout.** Configurable path template, default + `{albumartist}/{album}/{track} {title}`. Segments are sanitized for + Windows-reserved characters and trailing dots/spaces so a library + synced between platforms does not produce unopenable files. Existing + files are never overwritten — a collision gets a numbered variant, + because the file already there may be a better copy the user owns. +4. **Entry point.** "Find this album" on the Explore album page, shown + only when a client is connected and the album is not already owned. + The artist-discography right-click is not wired up yet. + +## What shipped + +All five phases, ~4,500 lines with tests, `make lint` clean and the full +backend suite green (including under `-race`). + +**Core** (`backend/download/`): `Searcher`/`Transporter`/`Delegator` +interfaces with capability-driven composition; `Request`/`Candidate`/ +`Result` types; provider registry with self-registering adapters; +two-axis ranking; staging area with escape-guards and startup sweep; +verify → tag → import tail; jobs integration under `KindDownload`; +three tables catalogued in `datamap`. + +**Providers**: yt-dlp (subprocess; assembles albums from per-track +searches, since a "full album" video cannot be imported as tracks), +slskd (remote search + transport, peer-health scoring, collects from the +daemon's own downloads folder), Lidarr (delegate; reconciles in place +rather than moving files out from under a system still managing them), +Prowlarr (search-only) paired at grab time with qBittorrent or SABnzbd. + +**Frontend**: `download-store.ts`, `download-picker` + `candidate-row` +(two meters, not one blended score), `download-clients` settings section +rendering its forms from backend descriptors so a new adapter needs no +frontend change. + +## Follow-ups + +- **Resume across restart.** Live transfers are currently marked failed + on startup and their staging swept, because the transports do not + survive the process. slskd and qBittorrent can both resume in + principle; the item rows already carry what would be needed. +- ~~**Per-provider concurrency caps.**~~ Done in 004: per-kind defaults + (slskd 1, yt-dlp 2, torrent/usenet 4) with a per-provider override, + and the provider's slot is taken before the global one. +- **Prowlarr candidates score blind.** Indexer results carry no file + list, so match scoring has only the release title. Fetching the + torrent metadata before ranking would fix this and is the single + biggest ranking improvement available. +- ~~Wanted list, artist monitoring~~ — done in 004. Quality profiles + and upgrade-if-better remain deferred. diff --git a/.planning/plans/completed/004-wanted-list.md b/.planning/plans/completed/004-wanted-list.md new file mode 100644 index 0000000..b9fa921 --- /dev/null +++ b/.planning/plans/completed/004-wanted-list.md @@ -0,0 +1,163 @@ +# 004 — Wanted list + +**Status:** implemented +**Branch:** main +**Created:** 2026-07-29 +**Follows:** 003-download-clients + +## Problem + +Plan 003 shipped a request as a heavyweight row: library, anchors, +cached tracklist, state machine, error text, cascading items. That is +the right shape for *one attempt to acquire something* and the wrong +shape for *the user wanting something*, and 003 used it for both. + +The consequences showed up immediately. A request that found nothing was +marked `failed`, which is a lie — the album exists, no source had it +today. Retrying meant the user remembering to press a button. Wanting an +artist's future releases was not expressible at all. And a user who +acquired an album by other means kept a failed row about it forever. + +## The model + +A **want** is an MBID, what that MBID names, and retry bookkeeping. +That is all. + +``` +download_wants(mbid, entity, library_id, scope, secondary, state, + parent_id, attempts, last_error, next_try_at, + external_ids) +``` + +`entity` is the only type distinction, and it carries all the policy: + +| entity | meaning | +|---|---| +| `artist` | a subscription. Never satisfied; each pass expands the discography into child wants | +| `release-group` | an album in the abstract — any release satisfies it | +| `release` | one specific edition | +| `recording` | one track | + +`UNIQUE(mbid, library_id)` is load-bearing: it is what makes artist +expansion idempotent, so a subscription can re-run every pass and add +only what is genuinely new. + +Requests did not go away — they became what they always were, the +ephemeral record of one attempt, with a nullable `want_id` back-link. +The lifetimes are now opposite and explicit: **a request is history, a +want is intent.** + +### Nothing here fails + +There is no `failed` want state. An attempt can fail; a want cannot. A +want that found nothing gets `attempts + 1`, a reason the user can read, +and a longer backoff — 6h doubling to a 7-day ceiling, jittered so a +list added in one sitting does not come due in one burst. + +### Satisfaction is ownership, not download + +A want retires when the *library* owns what it names, however it got +there — bought, ripped, copied in. Inferring satisfaction from our own +completed downloads would keep hunting for music already on disk. + +### Artist scope defaults to `future` + +Following an artist takes new releases only, and skips compilations, +live albums and remixes. `all` backfills the discography, and the user +can widen it from the wanted list. Subscribing should not silently queue +forty albums. + +## The reconciler + +A 6-hourly loop (plus on-demand, plus a 3-minute startup delay so the +explore index has loaded). Four steps, in this order: + +1. **Expand** artist subscriptions into album wants — first, so step 2 + sees them this pass rather than next. +2. **Retire** wants the library already owns. +3. **Sync** to clients that keep their own list. +4. **Attempt** a bounded batch (25) of due wants. + +Everything the loop needs about music comes through a four-method +`CatalogPort`, adapted to the explore index in `backend/downloadcatalog.go` +— the composition root, so neither package learns about the other. + +### Unattended grabs, and what stops them + +`Manager.Attempt` is `Start` without the parking: it searches, and grabs +only if `AutoPickable` clears. When it does not, **nothing is +persisted** — no request row. A want retried weekly for a year would +otherwise leave fifty identical failed rows, none of them anything the +user can act on. + +`AutoPickable` gained one condition: an anchored request with an empty +`Expected` is refused. An anchor with no tracklist behind it is an +anchor in name only, and match then rests on album/artist text — exactly +the evidence a wrong-album candidate also has. Nobody is watching a +reconcile pass. + +## Per-provider concurrency + +`Downloads.MaxConcurrent` was the only limit, and was never actually +applied (`SetMaxConcurrent` did not exist). Now: + +- **slskd defaults to 1.** A Soulseek peer serves one file at a time + from one person's upload slot; asking for more gets you queued behind + everyone else at best. One is both the polite number and usually the + fastest. +- yt-dlp 2, torrent/usenet clients 4, overridable per provider via a + `maxConcurrent` field that `Register` appends automatically to any + descriptor declaring `CanTransport`. +- A grab takes its **provider's** slot before the global one, so a queue + on a busy slskd cannot sit on a global slot a usenet transfer could + have used. The transport is resolved before either slot is taken; + delegates take neither, since the transfer is happening inside another + system that is doing its own limiting. + +## The Lister role + +The fourth role, alongside Searcher/Transporter/Delegator. Lidarr +already models a want — a monitored artist or album — and it is always +on, where a desktop player is not. A subscription mirrored there keeps +working while the app is closed. + +- `artist` → Lidarr artist, `monitor: future|missing` per scope +- `release-group`/`release` → monitored album +- `recording` → not pushed. Lidarr cannot say "one track", and + monitoring the album to get it downloads far more than was asked. + +Sync is push-only in the loop; pulling happens only when the user +explicitly imports ("adopt the artists Lidarr already monitors", which +arrive at `future` scope). Removal **unmonitors**, never deletes — the +user's Lidarr may predate this app. + +## Frontend + +- `Wanted` view in the sidebar: Following / Looking for / Paused / + Found, with pause, remove, scope toggle and "Check now". +- "Want this" on the album page, "Follow for new releases" on the artist + page. The want button shows whether or not a client is connected — + wanting is durable and stays queued until one exists. +- `WantedListChanged` event, since a background pass changes the list + without the UI doing anything. + +## Files + +`backend/download/want.go`, `wantstore.go`, `reconcile.go`, +`provider_lidarr_list.go`; `backend/downloadcatalog.go`; +schema `download_wants.sql` + migration 48 for the two new +`download_requests` columns; `frontend/src/components/wanted-view/`. + +## Deferred + +- **Release-group wants are not retired by ownership of a specific + release.** The library indexes release groups and recordings, not + editions, so a `release` want is only satisfied by its own download + completing. +- **No recording lookup on the explore index**, so a track want relies + on the title the UI passed in. A want added as a bare recording MBID + has no tracklist and waits. +- Quality profiles and upgrade-if-better (from 003). +- Resume across restart (from 003) — still the largest gap, and it now + matters more: an unattended grab that dies on restart is retried by + the reconciler, but from zero bytes. diff --git a/CLAUDE.md b/CLAUDE.md index a23a31f..d9ed9aa 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -10,7 +10,6 @@ YellowJacket is a cross-platform desktop music player built with Go (backend) an Active and historical plans live in `.planning/`: -- `.planning/ROADMAP.md` — vision, capability set, milestone sequence. - `.planning/NOTES.md` — gotchas, deferred items, open architecture questions, the "we already considered and rejected" list. - `.planning/plans/active/` — work currently in progress (read first). - `.planning/plans/pending/` — sequenced future work. @@ -18,8 +17,6 @@ Active and historical plans live in `.planning/`: Numbering is sequential and stable across status moves (a plan keeps its `NNN-` prefix as it migrates between `pending → active → completed`). Abandoned plans are deleted; paused work stays in `pending/`. -The legacy `.gsd/` directory is a snapshot of the prior GSD-CLI planning system. It's gitignored and will be removed once nothing relies on it. - ## Commands ```bash @@ -28,8 +25,8 @@ make dev-debug # Same as dev but with YJ_LOG_LEVEL=debug make build-dev # Debug build with symbols make build-prod # Production build (stripped, UPX-compressed) make generate # Run code generators (sqlc + templ via go generate) -make lint # golangci-lint v2 (strict) -make test # All tests with race detector, 2min timeout +make lint # golangci-lint v2 (strict), both build configurations +make test # All tests with race detector, both build configurations make vulncheck # govulncheck for CVEs make setup # Install go tools, frontend deps, git hooks (lefthook) ``` @@ -44,6 +41,14 @@ go test -tags webkit2_41 ./backend/player/ # Single package go test -tags webkit2_41 -run TestName ./backend/player/ # Single test ``` +The central index builder is behind a second tag and is **not** covered +by the command above — `make test` runs both passes, but a manual run +needs it spelled out: + +```bash +go test -tags "webkit2_41 indexbuild" ./backend/explore/... ./cmd/... +``` + Audio playback integration tests require `YELLOWJACKET_INTEGRATION=1`. ## Architecture @@ -55,13 +60,39 @@ Audio playback integration tests require `YELLOWJACKET_INTEGRATION=1`. - `queue` — Track queue with shuffle (Fisher-Yates), repeat modes, auto-advance, and session persistence. - `library` — Concurrent library scanning, metadata extraction, cover art deduplication, incremental rescan. - `database` — SQLite via pure-Go driver. Schema in `database/sql/schemas/`, queries in `database/sql/queries/`. **sqlc** generates Go code into `database/sql/sqlcgen/` — never edit that directory by hand. + There is **no migration chain**: `applySchema` creates everything from + the schema files on every open (all DDL is `IF NOT EXISTS`), and a + database written by an older build is not supported. Changing the + schema means editing the file in `sql/schemas/`, not adding a step. - `metadata` — Tag extraction (ID3v2, Vorbis Comments, FLAC). - `config` — TOML-based settings. Settings page uses HTMX + templ for server-rendered HTML fragments. - `playlist` / `smartplaylist` — Playlist CRUD and rule-based smart playlists. - `mediacontrols` — MPRIS integration on Linux via D-Bus. - `system` — OS-specific paths (XDG on Linux, `%LOCALAPPDATA%` on Windows). +- `explore` — Catalog search and browse over `explore_index`. See below. - `profiling` — pprof server on `:6060`, compiled out in non-dev builds via build tags (`internal/dev/`). +**Explore catalog** (`backend/explore/`): the searchable MusicBrainz/ +ListenBrainz catalog in `explore_index`. Deriving it from the MetaBrainz +dumps means streaming ~89 GB from a server that caps a client near +2 MB/s — half a day, for a catalog identical for every user. So that +work happens **once, centrally**, and users download the result: + +- `cmd/indexbuild` builds the catalog from the dumps; `cmd/indexexport` + cuts it down to a shippable core and stamps its provenance. + `.gitea/workflows/index-artifact.yml` runs both and publishes the + compressed artifact under a fixed `latest` version. +- The app fetches and merges that artifact (`artifactfetch.go`, + `artifactimport.go`) — about a minute, versus a day. +- Everything the app does **not** need is behind the `indexbuild` build + tag (`dumpimport.go`, `dumpcounts.go`, `dumpcatalog.go`, + `dumpproject.go`, `dumpparallel.go`, `indexpatch.go`) so it is not + linked into the binary. `dumpbuild_stub.go` is the app-side entry + point; `dumpshared.go` holds what both sides use. +- The app keeps popularity current with the daily incremental dumps + (`dumpincremental.go`), and resolves artists outside the artifact's + coverage lazily on first view. + **Frontend** (`frontend/`): Lit 3.2 web components + Web Awesome UI library + HTMX. State management via singleton reactive stores in `src/store/`. Wails bindings auto-generated in `frontend/wailsjs/` — don't edit by hand. **Event-driven communication**: Backend emits events via Wails runtime; frontend stores subscribe to them. Event names are constants in `backend/events/`. @@ -82,7 +113,8 @@ Pre-commit hooks verify generated code is fresh — always run `make generate` a ## Testing -Tests use `database.NewTestDB(t)` for in-memory SQLite with full schema. Test audio fixtures live in `test_data/music_library_test/`. Table-driven tests are the norm. +Tests use `database.NewTestDB(t)` for in-memory SQLite, built by the same +`applySchema` production uses so the two cannot diverge. Test audio fixtures live in `test_data/music_library_test/`. Table-driven tests are the norm. ## Git Workflow diff --git a/Makefile b/Makefile index ff7737b..a413eee 100644 --- a/Makefile +++ b/Makefile @@ -36,6 +36,73 @@ fresh-install: setup generate clean esac; \ go tool wails dev -tags webkit2_41 -loglevel Debug -v 2 +# Named, persistent sandboxes: `make sandbox foo` runs dev against +# $(FRESH_HOME_BASE)/yellowjacket-sandbox-foo, creating it on first use +# and reusing it (never deleting) afterward, so you can keep several +# long-lived states around — one with an imported search index, one with +# a small library, etc. `make sandbox-foo` is the same thing. +# +# `make sandboxes` lists the ones that exist. +# +# `make sandbox-rm foo [bar ...]` deletes them again, after confirming. +# +# The bare words after `sandbox` / `sandbox-rm` are extra make goals, so +# they need do-nothing rules to keep make from complaining. Those rules +# exist only when one of those is the first goal, so typos in other +# targets still fail loudly. +SANDBOX_DIR = $(FRESH_HOME_BASE)/yellowjacket-sandbox +ifneq (,$(filter $(firstword $(MAKECMDGOALS)),sandbox sandbox-rm)) +SANDBOX_ARGS := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS)) +SANDBOX_NAME := $(firstword $(SANDBOX_ARGS)) +$(foreach a,$(SANDBOX_ARGS),$(eval $(a):;@:)) +endif + +sandbox: ## Run dev against a named, persistent YJ_HOME: make sandbox + @if [ -z "$(SANDBOX_NAME)" ]; then \ + echo "usage: make sandbox (e.g. make sandbox foo)" >&2; exit 2; \ + fi + @$(MAKE) --no-print-directory sandbox-$(SANDBOX_NAME) + +sandbox-rm: ## Delete named sandboxes: make sandbox-rm [name ...] + @if [ -z "$(SANDBOX_ARGS)" ]; then \ + echo "usage: make sandbox-rm [name ...]" >&2; exit 2; \ + fi + @set -e; \ + targets=""; \ + for n in $(SANDBOX_ARGS); do \ + d="$(SANDBOX_DIR)-$$n"; \ + if [ -d "$$d" ]; then \ + echo " $$(du -sh "$$d" 2>/dev/null | cut -f1) $$d"; \ + targets="$$targets $$d"; \ + else \ + echo " (no such sandbox: $$n)" >&2; \ + fi; \ + done; \ + if [ -z "$$targets" ]; then exit 1; fi; \ + if [ "$(FORCE)" != "1" ]; then \ + printf "delete the above? [y/N] "; read -r ans; \ + case "$$ans" in y|Y|yes|YES) ;; *) echo "aborted"; exit 1 ;; esac; \ + fi; \ + rm -rf $$targets; \ + echo "==> removed" + +sandbox-%: setup generate clean + if [ -f .env ]; then set -a; . ./.env; set +a; fi; \ + export YJ_HOME="$(SANDBOX_DIR)-$*"; \ + mkdir -p "$$YJ_HOME"; \ + echo "==> sandbox '$*' YJ_HOME=$$YJ_HOME"; \ + case "$$(findmnt -no FSTYPE -T "$$YJ_HOME" 2>/dev/null)" in \ + tmpfs|ramfs) echo "==> WARNING: $$YJ_HOME is RAM-backed; the search index import needs ~6GB of real disk. Set FRESH_HOME_BASE to a disk-backed path." ;; \ + esac; \ + go tool wails dev -tags webkit2_41 -loglevel Debug -v 2 + +sandboxes: ## List existing named sandboxes + @ls -d "$(SANDBOX_DIR)"-* 2>/dev/null \ + | sed 's|.*/yellowjacket-sandbox-| |' \ + || echo " (none)" + +.PHONY: sandbox sandbox-rm sandboxes + build-dev: generate go tool wails build -tags webkit2_41 -debug -clean -ldflags "$(LDFLAGS)" @@ -54,9 +121,15 @@ generate: lint: go tool golangci-lint run + go tool golangci-lint run --build-tags indexbuild +# Two passes: the app build, then the `indexbuild` build that adds the +# CI-only dump importer. Without the second pass nothing would compile +# or exercise backend/explore/dump*.go or cmd/indexbuild at all. test: go test -tags webkit2_41 -race -count=1 -timeout 120s ./... + go test -tags "webkit2_41 indexbuild" -race -count=1 -timeout 300s \ + ./backend/explore/... ./cmd/... vulncheck: go tool govulncheck ./... diff --git a/backend/app.go b/backend/app.go index da4e83d..c47a26a 100644 --- a/backend/app.go +++ b/backend/app.go @@ -10,6 +10,7 @@ import ( "log/slog" "net/http" "path/filepath" + "time" wailsruntime "github.com/wailsapp/wails/v2/pkg/runtime" @@ -18,10 +19,13 @@ import ( "yellowjacket/backend/config" "yellowjacket/backend/coverart" "yellowjacket/backend/database" + "yellowjacket/backend/download" + "yellowjacket/backend/events" "yellowjacket/backend/explore" "yellowjacket/backend/frontendutil" "yellowjacket/backend/jobs" "yellowjacket/backend/library" + "yellowjacket/backend/maintenance" "yellowjacket/backend/mediacontrols" "yellowjacket/backend/player" "yellowjacket/backend/playlist" @@ -45,9 +49,13 @@ type YellowJacketApp struct { queue *queue.Queue explore *explore.Service autotag *autotagservice.Service + downloads *download.Manager + downloadSvc *download.Service + wanted *download.Reconciler jobs *jobs.Registry mediaControls mediacontrols.Handler tagWriter *tagwriter.TagWriter + janitor *maintenance.Runner appContext context.Context appConfig *config.Config startupErr error @@ -65,6 +73,7 @@ func NewYellowJacketApp( logger: logger, assetHandler: assetHandler, appContext: context.Background(), + janitor: maintenance.NewRunner(logger), } // create database @@ -168,6 +177,16 @@ func NewYellowJacketApp( yjApp.tagWriter, ) + // Create the download subsystem. Acquiring music is optional: a + // failure here (unwritable data dir, say) must not stop the app + // from playing the library the user already has, so it is logged + // and the feature stays unavailable rather than fatal. + if err := yjApp.initDownloads(); err != nil { + yjApp.logger.Error( + "download clients unavailable", "error", err, + ) + } + yjApp.FEBindings = []any{ yjApp.FrontendUtil, yjApp.appConfig, @@ -181,9 +200,51 @@ func NewYellowJacketApp( jobs.NewService(yjApp.jobs), } + if yjApp.downloadSvc != nil { + yjApp.FEBindings = append(yjApp.FEBindings, yjApp.downloadSvc) + } + return yjApp, nil } +// initDownloads builds the download subsystem: staging area, secret +// store, importer and manager, plus the Wails-bound service. +func (yj *YellowJacketApp) initDownloads() error { + logger := yj.logger.WithGroup("download") + + staging, err := download.NewStaging(logger) + if err != nil { + return fmt.Errorf("could not create download staging: %w", err) + } + + secrets, err := download.NewFileSecretStore() + if err != nil { + return fmt.Errorf("could not create download secret store: %w", err) + } + + store := download.NewStore(yj.database) + + importer := download.NewImporter(logger, staging, yj.tagWriter, yj.library) + + yj.downloads = download.NewManager( + logger, store, secrets, staging, importer, yj.library, + ) + yj.downloads.SetJobRegistry(yj.jobs) + + yj.downloadSvc = download.NewService(logger, yj.downloads, store, secrets) + + // The wanted list needs the explore index to know what an artist + // released and what the library already owns, so it is wired here + // where both exist. The reconcile loop itself is not started until + // the Wails runtime is up. + yj.wanted = download.NewReconciler( + logger, store, yj.downloads, newExploreCatalog(yj.explore), + ) + yj.downloadSvc.SetReconciler(yj.wanted) + + return nil +} + // playerAdapter wraps *player.Player to satisfy the tagwriter.PlayerStopper // interface, breaking the import cycle between tagwriter and player. type playerAdapter struct{ p *player.Player } @@ -194,6 +255,46 @@ func (a *playerAdapter) CurrentFilePath() string { func (a *playerAdapter) StopAndRelease() { a.p.UnloadTrack() } +// initDownloadRuntime brings the download subsystem up once the Wails +// runtime exists: it applies the user's import layout, builds providers +// from stored config, and clears staging left by a previous run. +// +// Provider construction and the sweep both touch the network and the +// filesystem, so they run in the background — a slow or unreachable +// download client must not delay the window appearing. +func (yj *YellowJacketApp) initDownloadRuntime(ctx context.Context) { + cfg := yj.appConfig.Downloads + if cfg == nil { + cfg = &download.UserConfig{} + cfg.ApplyDefaults() + } + + yj.downloads.SetImportOptions(download.ImportOptions{ + LibraryRoot: yj.appConfig.GetLibraryDirectory(), + PathTemplate: cfg.PathTemplate, + }) + yj.downloads.SetMaxConcurrent(cfg.MaxConcurrent) + + go func() { + if err := yj.downloads.Reload(ctx); err != nil { + yj.logger.Warn("could not load download providers", "error", err) + } + + yj.downloads.Sweep(ctx) + }() + + if yj.wanted == nil { + return + } + + yj.wanted.SetInterval(cfg.WantedInterval()) + yj.wanted.SetBatch(cfg.WantedBatch) + yj.wanted.SetOnChange(func() { + wailsruntime.EventsEmit(ctx, events.WantedListChanged) + }) + yj.wanted.Start(ctx) +} + // WindowConfig returns the window configuration for use by the host. func (yj *YellowJacketApp) WindowConfig() *config.WindowConfig { return yj.appConfig.Window @@ -234,6 +335,11 @@ func (yj *YellowJacketApp) OnStartup(ctx context.Context) { yj.autotag.SetContext(ctx) yj.jobs.SetContext(ctx) + if yj.downloadSvc != nil { + yj.downloadSvc.SetContext(ctx) + yj.initDownloadRuntime(ctx) + } + // Bring back jobs the user paused before the last shutdown, still // paused. Must run before the soft scan in OnDomReady, which // checks these records so it does not restart a paused library. @@ -480,5 +586,54 @@ func (yj *YellowJacketApp) OnDomReady(ctx context.Context) { // every app launch is fine; previously-scored items are // skipped (the worker filters score IS NULL). yj.autotag.StartBackgroundPrefetch() + + // Start the janitor last: its sweeps compare against live data, + // so running them after the scan and index work has settled + // avoids deleting something a running import is about to + // reference. Each job enforces its own minimum interval, so the + // daily tick is a cheap no-op most of the time. + yj.startJanitor() }() } + +// janitorTick is how often the maintenance runner wakes up. Individual +// jobs enforce their own minimum intervals, so most ticks do nothing. +const janitorTick = 6 * time.Hour + +// startJanitor registers the maintenance jobs and starts the background +// runner. Every job is registered here rather than at each package's +// init, so the full set of janitorial work is one visible list — a cache +// that forgets to register is missing from this function, which is +// harder to overlook than a function nobody calls. +func (yj *YellowJacketApp) startJanitor() { + coversDir, err := coverart.CoversDir() + if err != nil { + yj.logger.Warn("janitor: could not resolve covers directory", + "err", err) + + return + } + + dataDir, err := system.GetUserDataDirPath() + if err != nil { + yj.logger.Warn("janitor: could not resolve user data directory", + "err", err) + + return + } + + yj.janitor.Register(maintenance.ExpiredHTTPCacheJob(yj.database)) + yj.janitor.Register(maintenance.OrphanedCoverFilesJob( + yj.database, coversDir, library.CoverArtFileSet, + )) + yj.janitor.Register(maintenance.OrphanedArtistImagesJob( + yj.database, filepath.Join(dataDir, explore.ArtistImageDirName), + )) + yj.janitor.Register(maintenance.ExpiredProxyCacheJob( + filepath.Join(dataDir, explore.CoverArtCacheDirName), + )) + + yj.logger.Info("janitor started", "jobs", yj.janitor.JobNames()) + + yj.janitor.Start(yj.appContext, janitorTick) +} diff --git a/backend/autotag/distance.go b/backend/autotag/distance.go index 16adb23..069d2ef 100644 --- a/backend/autotag/distance.go +++ b/backend/autotag/distance.go @@ -192,6 +192,16 @@ func rotateEndWord(s string) string { return s } +// TitleSimilarity exposes titleSimilarity for callers outside the +// package that compare music metadata strings and should get the same +// answer the tagger would. The download pipeline uses it to match +// candidate filenames against an expected tracklist — Soulseek and +// torrent results carry paths, not tags, so filename comparison is the +// only signal available before the bytes arrive. +func TitleSimilarity(a, b string) float64 { + return titleSimilarity(a, b) +} + // titleSimilarity returns a score in [0, 1] from stringDist. 1.0 // means identical after normalization, 0.0 means fully dissimilar. func titleSimilarity(a, b string) float64 { diff --git a/backend/config/config.go b/backend/config/config.go index 267ce0c..01e079e 100644 --- a/backend/config/config.go +++ b/backend/config/config.go @@ -12,6 +12,7 @@ import ( "github.com/BurntSushi/toml" "github.com/wailsapp/wails/v2/pkg/runtime" + "yellowjacket/backend/download" "yellowjacket/backend/events" "yellowjacket/backend/favorites" "yellowjacket/backend/library" @@ -31,14 +32,15 @@ var errSaveBeforeLoad = errors.New("refusing to save: config not loaded from dis type Config struct { ctx context.Context logger *slog.Logger - filePath string // required - loaded bool // true once Load() succeeds - Library *library.Config `toml:"Library"` - Theme *theme.Config `toml:"Theme"` - Window *WindowConfig `toml:"Window"` - TrackList *tracklist.Config `toml:"TrackList"` - Favorites *favorites.Config `toml:"Favorites"` - Shortcuts *shortcuts.Config `toml:"Shortcuts"` + filePath string // required + loaded bool // true once Load() succeeds + Library *library.Config `toml:"Library"` + Theme *theme.Config `toml:"Theme"` + Window *WindowConfig `toml:"Window"` + TrackList *tracklist.Config `toml:"TrackList"` + Favorites *favorites.Config `toml:"Favorites"` + Shortcuts *shortcuts.Config `toml:"Shortcuts"` + Downloads *download.UserConfig `toml:"Downloads"` } // NewConfig creates a new config by loading it from disk. @@ -258,6 +260,12 @@ func (c *Config) applyDefaults() { } c.Shortcuts.ApplyDefaults() + + if c.Downloads == nil { + c.Downloads = &download.UserConfig{} + } + + c.Downloads.ApplyDefaults() } // SetContext sets the Wails runtime context for event emission. diff --git a/backend/database/database.go b/backend/database/database.go index a63becb..854b5e1 100644 --- a/backend/database/database.go +++ b/backend/database/database.go @@ -6,19 +6,13 @@ import ( "database/sql" "embed" "fmt" - "io" "io/fs" "log/slog" - "os" "path" - "path/filepath" "strings" - "time" - "github.com/BurntSushi/toml" _ "modernc.org/sqlite" // Register sqlite driver. - "yellowjacket/backend/autotag" "yellowjacket/backend/database/sql/sqlcgen" "yellowjacket/backend/profiling" "yellowjacket/backend/system" @@ -69,7 +63,7 @@ const ( readPoolConns = 4 ) -// NewDB opens the database and applies schema migrations. +// NewDB opens the database and creates the schema if it is not there. func NewDB(logger *slog.Logger) (*DB, error) { defer profiling.TimeOp(logger, "database.NewDB")() @@ -95,64 +89,8 @@ func NewDB(logger *slog.Logger) (*DB, error) { return nil, fmt.Errorf("could not apply PRAGMAs: %w", err) } - // Execute SQL files from the embedded schemas directory - logger.Debug("reading sql schema files from embedded directory") - - dirEntries, err := schemas.ReadDir("sql/schemas") - if err != nil { - return nil, fmt.Errorf("could not read schemas directory: %w", err) - } - - logger.Debug("executing all sql schema files") - - for _, dirEntry := range dirEntries { - if !dirEntry.IsDir() { - filePath := path.Join("sql/schemas", dirEntry.Name()) - - sqlContent, err := fs.ReadFile(schemas, filePath) - if err != nil { - return nil, fmt.Errorf("could not read file %s: %w", filePath, err) - } - - logger.Debug( - "executing sql schema file", - "filepath", - filePath, - "sql", - string(sqlContent), - ) - - _, err = db.ExecContext(dbCtx, string(sqlContent)) // Execute the SQL - if err != nil { - return nil, fmt.Errorf("error executing sql from file %s: %w", filePath, err) - } - } - } - - // Run versioned schema migrations for columns that cannot be - // added with CREATE TABLE IF NOT EXISTS on existing databases. - if err := runMigrations(dbCtx, db, logger, sqliteDBFilePath); err != nil { - return nil, fmt.Errorf( - "could not run schema migrations: %w", err, - ) - } - - // Remove orphaned playlist_tracks left behind by past deletes - // that ran without foreign key enforcement. - orphanResult, err := db.ExecContext( - dbCtx, - "DELETE FROM playlist_tracks WHERE playlist_id NOT IN (SELECT id FROM playlists)", - ) - if err != nil { - logger.Warn( - "could not clean orphaned playlist tracks", - "err", err, - ) - } else if n, _ := orphanResult.RowsAffected(); n > 0 { - logger.Info( - "Cleaned orphaned playlist tracks", - "deleted", n, - ) + if err := applySchema(dbCtx, db); err != nil { + return nil, err } // Get generated queries @@ -215,12 +153,144 @@ func (d *DB) QueryContextWith(ctx context.Context, query string, args ...any) (* return d.reader().QueryContext(ctx, query, args...) } +// QueryRowWriter runs a single-row query on the writer connection rather +// than the read pool. +// +// Almost every read should use QueryContext instead. This exists for +// the one case that cannot: statements referencing a database ATTACHed +// to the writer. The read pool is a separate sql.DB over the same file, +// so an attachment made on the writer is invisible there and the query +// would fail with "no such table". +func (d *DB) QueryRowWriter(query string, args ...any) *sql.Row { + return d.db.QueryRowContext(d.Ctx, query, args...) +} + // Logger returns the structured logger bound to this DB. Callers can // use it to emit timing or diagnostic logs from query-adjacent code. func (d *DB) Logger() *slog.Logger { return d.logger } +// exploreIndexFTSTriggers is the sole definition of the explore_index → +// FTS5 sync triggers. +// +// They live here rather than in sql/schemas/explore_index.sql because a +// bulk load drops and recreates them (see SuspendExploreIndexFTS), so +// the runtime needs them as statements either way. Defining them in +// both places would be two copies free to drift. +var exploreIndexFTSTriggers = []string{ + `CREATE TRIGGER explore_index_ai AFTER INSERT ON explore_index BEGIN + INSERT INTO explore_index_fts(rowid, title, artist_name, aliases) + VALUES (new.id, new.title, new.artist_name, new.aliases); + END`, + `CREATE TRIGGER explore_index_ad AFTER DELETE ON explore_index BEGIN + INSERT INTO explore_index_fts(explore_index_fts, rowid, title, artist_name, aliases) + VALUES ('delete', old.id, old.title, old.artist_name, old.aliases); + END`, + `CREATE TRIGGER explore_index_au AFTER UPDATE ON explore_index BEGIN + INSERT INTO explore_index_fts(explore_index_fts, rowid, title, artist_name, aliases) + VALUES ('delete', old.id, old.title, old.artist_name, old.aliases); + INSERT INTO explore_index_fts(rowid, title, artist_name, aliases) + VALUES (new.id, new.title, new.artist_name, new.aliases); + END`, +} + +// createExploreIndexFTSTriggers installs the sync triggers. Safe to +// call on a database that already has them. +func createExploreIndexFTSTriggers(ctx context.Context, db *sql.DB) error { + for _, stmt := range exploreIndexFTSTriggers { + if _, err := db.ExecContext(ctx, stmt); err != nil && + !strings.Contains(err.Error(), "already exists") { + return fmt.Errorf("create explore FTS trigger: %w", err) + } + } + + return nil +} + +// SuspendExploreIndexFTS drops the FTS sync triggers so a bulk load can +// write explore_index without paying per-row FTS maintenance. +// +// Row-at-a-time FTS upkeep is what makes a full dump import take +// nearly a day: a bulk DELETE fires the delete trigger once per row, +// which buries the FTS5 index in delete markers and stale segments, and +// every subsequent upsert then works against that debris. Measured on +// a real import, assembly runs at ~31 rows/s with the triggers attached +// and ~4,700 rows/s without. +// +// Callers MUST pair this with ResumeExploreIndexFTS — while suspended, +// explore_index_fts stops tracking the table and search goes stale. +func (d *DB) SuspendExploreIndexFTS() error { + for _, name := range []string{ + "explore_index_ai", "explore_index_ad", "explore_index_au", + } { + if _, err := d.db.ExecContext(d.Ctx, "DROP TRIGGER IF EXISTS "+name); err != nil { + return fmt.Errorf("suspend explore FTS: drop %s: %w", name, err) + } + } + + return nil +} + +// ResumeExploreIndexFTS reinstates the sync triggers and rebuilds the +// FTS index from the content table, discarding whatever accumulated +// while it was suspended. The rebuild is a single linear pass and is +// far cheaper than the per-row maintenance it replaces. +// +// Safe to call when the triggers are already present, so it can run +// from a defer on both the success and failure paths. +func (d *DB) ResumeExploreIndexFTS() error { + if err := createExploreIndexFTSTriggers(d.Ctx, d.db); err != nil { + return fmt.Errorf("resume explore FTS: %w", err) + } + + if _, err := d.db.ExecContext( + d.Ctx, "INSERT INTO explore_index_fts(explore_index_fts) VALUES('rebuild')", + ); err != nil { + return fmt.Errorf("resume explore FTS: rebuild: %w", err) + } + + return nil +} + +// applySchema creates the full schema on a fresh database. +// +// Every statement is CREATE ... IF NOT EXISTS, so this is idempotent and +// runs unconditionally at open. There is no migration chain: the files +// in sql/schemas describe the only schema the app has, and a database +// written by an older build is not supported. +func applySchema(ctx context.Context, db *sql.DB) error { + dirEntries, err := schemas.ReadDir("sql/schemas") + if err != nil { + return fmt.Errorf("could not read schemas directory: %w", err) + } + + for _, dirEntry := range dirEntries { + if dirEntry.IsDir() { + continue + } + + filePath := path.Join("sql/schemas", dirEntry.Name()) + + sqlContent, err := fs.ReadFile(schemas, filePath) + if err != nil { + return fmt.Errorf("could not read file %s: %w", filePath, err) + } + + if _, err := db.ExecContext(ctx, string(sqlContent)); err != nil { + return fmt.Errorf("error executing sql from file %s: %w", filePath, err) + } + } + + // The FTS sync triggers are defined in Go, not in the schema files, + // because the bulk-load path drops and recreates them. + if err := createExploreIndexFTSTriggers(ctx, db); err != nil { + return fmt.Errorf("could not create explore FTS triggers: %w", err) + } + + return nil +} + // applyPRAGMAs configures SQLite connection settings. Called by both // NewDB and NewTestDB to ensure identical behavior. func applyPRAGMAs(ctx context.Context, db *sql.DB) error { @@ -241,3647 +311,3 @@ func applyPRAGMAs(ctx context.Context, db *sql.DB) error { return nil } - -// runMigrations applies incremental schema changes using SQLite's -// PRAGMA user_version as the version tracker. Each migration runs -// once and bumps the version so it is never re-applied. -func runMigrations( - ctx context.Context, - db *sql.DB, - logger *slog.Logger, - dbPath string, -) error { - var version int - - if err := db.QueryRowContext( - ctx, "PRAGMA user_version", - ).Scan(&version); err != nil { - return fmt.Errorf( - "could not read user_version: %w", err, - ) - } - - logger.Debug( - "current schema version", - "user_version", version, - ) - - // Migration 1: add audio-property columns to audio_files. - if version < 1 { - logger.Info("applying migration 1: audio file properties") - - cols := []string{ - "sample_rate int NOT NULL DEFAULT 0", - "bit_depth int NOT NULL DEFAULT 0", - "channels int NOT NULL DEFAULT 0", - "bitrate int NOT NULL DEFAULT 0", - "file_size int NOT NULL DEFAULT 0", - } - - for _, col := range cols { - stmt := "ALTER TABLE audio_files ADD COLUMN " + col - - if _, err := db.ExecContext(ctx, stmt); err != nil { - // Column may already exist on a fresh DB that - // ran the updated CREATE TABLE. SQLite returns - // "duplicate column name" in that case. - if isDuplicateColumnErr(err) { - continue - } - - return fmt.Errorf( - "migration 1 failed (%s): %w", col, err, - ) - } - } - - if _, err := db.ExecContext( - ctx, "PRAGMA user_version = 1", - ); err != nil { - return fmt.Errorf( - "could not set user_version to 1: %w", err, - ) - } - } - - // Migration 2: add basename column and populate search index. - if version < 2 { - if err := migration2BasenameAndFTS( - ctx, db, logger, - ); err != nil { - return err - } - } - - // Migration 3: add UNIQUE constraint to artist_credit_artist. - if version < 3 { - logger.Info( - "applying migration 3: artist_credit_artist unique constraint", - ) - - // Remove duplicates first (keep lowest ID per pair). - if _, err := db.ExecContext(ctx, ` - DELETE FROM artist_credit_artist - WHERE id NOT IN ( - SELECT MIN(id) - FROM artist_credit_artist - GROUP BY artist_id, credit_id - ) - `); err != nil { - return fmt.Errorf( - "migration 3: could not deduplicate: %w", err, - ) - } - - if _, err := db.ExecContext(ctx, ` - CREATE UNIQUE INDEX IF NOT EXISTS - idx_artist_credit_artist_unique - ON artist_credit_artist(artist_id, credit_id) - `); err != nil { - return fmt.Errorf( - "migration 3: could not create unique index: %w", - err, - ) - } - - if _, err := db.ExecContext( - ctx, "PRAGMA user_version = 3", - ); err != nil { - return fmt.Errorf( - "could not set user_version to 3: %w", err, - ) - } - - logger.Info("migration 3 complete") - } - - // Migration 4: create track_metadata VIEW. - if version < 4 { - if err := migration4TrackMetadataView( - ctx, db, logger, - ); err != nil { - return err - } - } - - // Migration 5: rebuild release_groups with composite unique - // constraint on (name, album_artist_credit_id) instead of - // name alone, so albums with the same name by different - // artists are stored as separate rows. - if version < 5 { - if err := migration5ReleaseGroupCompositeUnique( - ctx, db, logger, - ); err != nil { - return err - } - } - - // Migration 6: multi-library support. - if version < 6 { - if err := migration6MultiLibrary( - ctx, db, logger, dbPath, - ); err != nil { - return err - } - } - - // Migration 7: add phantom_file_path to playlist_tracks - // for automatic phantom resolution after library re-scans. - if version < 7 { - if err := migration7PhantomFilePath( - ctx, db, logger, - ); err != nil { - return err - } - } - - // Migration 8: rebuild FTS5 search_index with - // contentless_delete=1 so individual rows can be deleted. - if version < 8 { - if err := migration8ContentlessDelete( - ctx, db, logger, - ); err != nil { - return err - } - } - - // Migration 9: add smart playlist columns to playlists. - if version < 9 { - if err := migration9SmartPlaylists( - ctx, db, logger, - ); err != nil { - return err - } - } - - if version < 10 { - if err := migration10PlayHistory( - ctx, db, logger, - ); err != nil { - return err - } - } - - // Migration 11: explore_cache table for MusicBrainz/ListenBrainz - // API response caching with TTL expiry and MBID lookups. - if version < 11 { - if err := migration11ExploreCache( - ctx, db, logger, - ); err != nil { - return err - } - } - - // Migration 12: explore_index + FTS5 for the popularity search - // index. Stores the top albums and tracks from the most popular - // ListenBrainz artists for instant local search. - if version < 12 { //nolint:mnd - if err := migration12ExploreSearchIndex( - ctx, db, logger, - ); err != nil { - return err - } - } - - // Migration 13: add MusicBrainz ID columns to artists, - // release_groups, and recordings for library↔explore linking. - if version < 13 { //nolint:mnd - if err := migration13MBIDColumns( - ctx, db, logger, - ); err != nil { - return err - } - } - - // Migration 14: add aliases column to explore_index and rebuild - // the FTS5 virtual table with 3 searchable columns. - if version < 14 { //nolint:mnd - if err := migration14ExploreAliases( - ctx, db, logger, - ); err != nil { - return err - } - } - - // Migration 15: add in_library and is_similar columns to - // explore_index for personalized search ranking. - if version < 15 { //nolint:mnd - if err := migration15PersonalizationColumns( - ctx, db, logger, - ); err != nil { - return err - } - } - - // Migration 16: artist_images table for multi-source artist photos. - if version < 16 { //nolint:mnd - if err := migration16ArtistImages( - ctx, db, logger, - ); err != nil { - return err - } - } - - if version < 17 { //nolint:mnd - if err := migration17SimilarArtistMap( - ctx, db, logger, - ); err != nil { - return err - } - } - - if version < 18 { - if err := migration18TrackCoverArt( - ctx, db, logger, - ); err != nil { - return err - } - } - - if version < 19 { - if err := migration19TrackMBIDs( - ctx, db, logger, - ); err != nil { - return err - } - } - - if version < 20 { - if err := migration20TrackRecordingMBID( - ctx, db, logger, - ); err != nil { - return err - } - } - - if version < 21 { //nolint:mnd - logger.Info("applying migration 21: explore_index mbid-only index") - - if _, err := db.ExecContext(ctx, ` - CREATE INDEX IF NOT EXISTS idx_explore_index_mbid_only - ON explore_index(mbid) - `); err != nil { - return fmt.Errorf("migration 21: create mbid-only index: %w", err) - } - - if _, err := db.ExecContext(ctx, - "PRAGMA user_version = 21", - ); err != nil { - return fmt.Errorf("migration 21: set user_version: %w", err) - } - } - - if version < 22 { //nolint:mnd - logger.Info("applying migration 22: replace composite index with UNIQUE(mbid)") - - // Remove any rows with empty MBIDs — they can't be looked up - // and would violate the new UNIQUE(mbid) constraint. - if _, err := db.ExecContext(ctx, ` - DELETE FROM explore_index WHERE mbid = '' - `); err != nil { - return fmt.Errorf("migration 22: delete empty mbids: %w", err) - } - - // Drop the over-engineered composite — MBIDs are globally - // unique, so entity_type in the key adds nothing. - if _, err := db.ExecContext(ctx, ` - DROP INDEX IF EXISTS idx_explore_index_mbid - `); err != nil { - return fmt.Errorf("migration 22: drop composite index: %w", err) - } - - // Drop the plain index from migration 21 and recreate as UNIQUE. - if _, err := db.ExecContext(ctx, ` - DROP INDEX IF EXISTS idx_explore_index_mbid_only - `); err != nil { - return fmt.Errorf("migration 22: drop plain mbid index: %w", err) - } - - if _, err := db.ExecContext(ctx, ` - CREATE UNIQUE INDEX IF NOT EXISTS idx_explore_index_mbid_only - ON explore_index(mbid) - `); err != nil { - return fmt.Errorf("migration 22: create unique mbid index: %w", err) - } - - if _, err := db.ExecContext(ctx, - "PRAGMA user_version = 22", - ); err != nil { - return fmt.Errorf("migration 22: set user_version: %w", err) - } - } - - if version < 23 { //nolint:mnd - logger.Info("applying migration 23: search_clicks table") - - if _, err := db.ExecContext(ctx, ` - CREATE TABLE IF NOT EXISTS search_clicks ( - query TEXT NOT NULL, - entity_mbid TEXT NOT NULL, - entity_type TEXT NOT NULL, - click_count INTEGER NOT NULL DEFAULT 1, - last_clicked DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (query, entity_mbid) - ) - `); err != nil { - return fmt.Errorf("migration 23: create search_clicks: %w", err) - } - - if _, err := db.ExecContext(ctx, ` - CREATE INDEX IF NOT EXISTS idx_search_clicks_query - ON search_clicks(query) - `); err != nil { - return fmt.Errorf("migration 23: create query index: %w", err) - } - - if _, err := db.ExecContext(ctx, - "PRAGMA user_version = 23", - ); err != nil { - return fmt.Errorf("migration 23: set user_version: %w", err) - } - } - - if version < 24 { //nolint:mnd - logger.Info("applying migration 24: explore_index listener_count + duration columns") - - if _, err := db.ExecContext(ctx, ` - ALTER TABLE explore_index ADD COLUMN listener_count INTEGER NOT NULL DEFAULT 0 - `); err != nil { - // Column may already exist from a partial migration. - if !strings.Contains(err.Error(), "duplicate column") { - return fmt.Errorf("migration 24: add listener_count: %w", err) - } - } - - if _, err := db.ExecContext(ctx, ` - ALTER TABLE explore_index ADD COLUMN duration INTEGER NOT NULL DEFAULT 0 - `); err != nil { - if !strings.Contains(err.Error(), "duplicate column") { - return fmt.Errorf("migration 24: add duration: %w", err) - } - } - - if _, err := db.ExecContext(ctx, - "PRAGMA user_version = 24", - ); err != nil { - return fmt.Errorf("migration 24: set user_version: %w", err) - } - } - - if version < 25 { //nolint:mnd - logger.Info("applying migration 25: explore_index duration column") - - if _, err := db.ExecContext(ctx, ` - ALTER TABLE explore_index ADD COLUMN duration INTEGER NOT NULL DEFAULT 0 - `); err != nil { - if !strings.Contains(err.Error(), "duplicate column") { - return fmt.Errorf("migration 25: add duration: %w", err) - } - } - - if _, err := db.ExecContext(ctx, - "PRAGMA user_version = 25", - ); err != nil { - return fmt.Errorf("migration 25: set user_version: %w", err) - } - } - - if version < 26 { //nolint:mnd - logger.Info("applying migration 26: comprehensive explore schema overhaul") - - // Nuke the existing index — we're changing the schema enough - // that a clean rebuild is simpler than trying to migrate in place. - if _, err := db.ExecContext(ctx, `DROP TABLE IF EXISTS explore_index_fts`); err != nil { - return fmt.Errorf("migration 26: drop fts: %w", err) - } - - if _, err := db.ExecContext(ctx, `DROP TABLE IF EXISTS explore_index`); err != nil { - return fmt.Errorf("migration 26: drop explore_index: %w", err) - } - - // Create the new explore_index with all typed columns. - // No more extra_json — every field that matters has its own column. - if _, err := db.ExecContext(ctx, ` - CREATE TABLE explore_index ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - entity_type TEXT NOT NULL, - mbid TEXT NOT NULL, - title TEXT NOT NULL, - artist_name TEXT NOT NULL, - artist_mbid TEXT NOT NULL, - aliases TEXT NOT NULL DEFAULT '', - - -- Popularity signals (from LB popularity API, uncapped). - popularity INTEGER NOT NULL DEFAULT 0, - listener_count INTEGER NOT NULL DEFAULT 0, - - -- Recording-specific fields. - duration INTEGER NOT NULL DEFAULT 0, - caa_release_mbid TEXT NOT NULL DEFAULT '', - release_name TEXT NOT NULL DEFAULT '', - - -- Release-group-specific fields. - primary_type TEXT NOT NULL DEFAULT '', - secondary_types TEXT NOT NULL DEFAULT '', - release_date TEXT NOT NULL DEFAULT '', - - -- Artist-specific fields. - artist_type TEXT NOT NULL DEFAULT '', - country TEXT NOT NULL DEFAULT '', - disambiguation TEXT NOT NULL DEFAULT '', - sort_name TEXT NOT NULL DEFAULT '', - - -- Personalization flags. - in_library INTEGER NOT NULL DEFAULT 0, - is_similar INTEGER NOT NULL DEFAULT 0, - - -- Cross-reference to local library tables. NULL when the - -- entity has no corresponding row in the library. - local_artist_id INTEGER, - local_release_group_id INTEGER, - local_recording_id INTEGER, - - -- Set to 1 by indexOneArtist after fetching the full - -- discography (release groups + recordings). Used by - -- indexedArtistMBIDs() so the AddFromCache organic-growth - -- path doesn't shadow artists from later tier 2/3 runs. - discog_fetched INTEGER NOT NULL DEFAULT 0, - - -- Schema version — lets us mark rows as stale after schema changes. - schema_version INTEGER NOT NULL DEFAULT 1, - - UNIQUE(mbid) - ) - `); err != nil { - return fmt.Errorf("migration 26: create explore_index: %w", err) - } - - if _, err := db.ExecContext(ctx, ` - CREATE INDEX idx_explore_index_artist_mbid - ON explore_index(artist_mbid, entity_type, popularity DESC) - `); err != nil { - return fmt.Errorf("migration 26: create artist_mbid index: %w", err) - } - - if _, err := db.ExecContext(ctx, ` - CREATE INDEX idx_explore_index_entity_pop - ON explore_index(entity_type, popularity DESC) - `); err != nil { - return fmt.Errorf("migration 26: create entity_pop index: %w", err) - } - - // FTS5 virtual table for text search. - if _, err := db.ExecContext(ctx, ` - CREATE VIRTUAL TABLE explore_index_fts USING fts5( - title, artist_name, aliases, - content='explore_index', - content_rowid='id' - ) - `); err != nil { - return fmt.Errorf("migration 26: create fts: %w", err) - } - - // Triggers to keep FTS in sync with the main table. - if _, err := db.ExecContext(ctx, ` - CREATE TRIGGER explore_index_ai AFTER INSERT ON explore_index BEGIN - INSERT INTO explore_index_fts(rowid, title, artist_name, aliases) - VALUES (new.id, new.title, new.artist_name, new.aliases); - END - `); err != nil { - return fmt.Errorf("migration 26: create ai trigger: %w", err) - } - - if _, err := db.ExecContext(ctx, ` - CREATE TRIGGER explore_index_ad AFTER DELETE ON explore_index BEGIN - INSERT INTO explore_index_fts(explore_index_fts, rowid, title, artist_name, aliases) - VALUES ('delete', old.id, old.title, old.artist_name, old.aliases); - END - `); err != nil { - return fmt.Errorf("migration 26: create ad trigger: %w", err) - } - - if _, err := db.ExecContext(ctx, ` - CREATE TRIGGER explore_index_au AFTER UPDATE ON explore_index BEGIN - INSERT INTO explore_index_fts(explore_index_fts, rowid, title, artist_name, aliases) - VALUES ('delete', old.id, old.title, old.artist_name, old.aliases); - INSERT INTO explore_index_fts(rowid, title, artist_name, aliases) - VALUES (new.id, new.title, new.artist_name, new.aliases); - END - `); err != nil { - return fmt.Errorf("migration 26: create au trigger: %w", err) - } - - // Clear the tier metadata so the next build repopulates everything. - if _, err := db.ExecContext(ctx, `DELETE FROM explore_index_meta`); err != nil { - return fmt.Errorf("migration 26: clear meta: %w", err) - } - - if _, err := db.ExecContext(ctx, - "PRAGMA user_version = 26", - ); err != nil { - return fmt.Errorf("migration 26: set user_version: %w", err) - } - } - - if version < 27 { //nolint:mnd - logger.Info( - "applying migration 27: split explore_cache into http_cache and artist_metadata", - ) - - // Create the new tables (no-op if schemas/*.sql already created them). - if _, err := db.ExecContext(ctx, ` - CREATE TABLE IF NOT EXISTS artist_metadata ( - mbid TEXT NOT NULL, - source TEXT NOT NULL, - data BLOB NOT NULL, - fetched_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (mbid, source) - ) - `); err != nil { - return fmt.Errorf("migration 27: create artist_metadata: %w", err) - } - - if _, err := db.ExecContext(ctx, ` - CREATE INDEX IF NOT EXISTS idx_artist_metadata_mbid - ON artist_metadata(mbid) - `); err != nil { - return fmt.Errorf("migration 27: create artist_metadata index: %w", err) - } - - if _, err := db.ExecContext(ctx, ` - CREATE TABLE IF NOT EXISTS http_cache ( - url_key TEXT PRIMARY KEY, - response BLOB NOT NULL, - expires_at DATETIME NOT NULL, - entity_mbid TEXT NOT NULL DEFAULT '', - entity_type TEXT NOT NULL DEFAULT '' - ) - `); err != nil { - return fmt.Errorf("migration 27: create http_cache: %w", err) - } - - if _, err := db.ExecContext(ctx, ` - CREATE INDEX IF NOT EXISTS idx_http_cache_expires - ON http_cache(expires_at) - `); err != nil { - return fmt.Errorf("migration 27: create http_cache index: %w", err) - } - - // Only migrate existing data if explore_cache exists (not a fresh install). - var exploreCacheExists bool - { - row, err := db.QueryContext(ctx, - "SELECT 1 FROM sqlite_master WHERE type='table' AND name='explore_cache'", - ) - if err == nil { - if row.Next() { - exploreCacheExists = true - } - - _ = row.Close() - } - } - - if exploreCacheExists { - // Migrate long-lived sources into artist_metadata. - for _, src := range []string{"audiodb", "fanart", "wikidata-p18", "wikipedia-lead"} { - if _, err := db.ExecContext(ctx, ` - INSERT OR IGNORE INTO artist_metadata (mbid, source, data, fetched_at) - SELECT substr(url_key, ?+1), ?, response, COALESCE(expires_at, CURRENT_TIMESTAMP) - FROM explore_cache - WHERE url_key LIKE ? - `, len(src)+1, src, src+":%"); err != nil { - return fmt.Errorf("migration 27: migrate %s: %w", src, err) - } - } - - // Migrate remaining (short-lived) entries into http_cache. - if _, err := db.ExecContext(ctx, ` - INSERT OR IGNORE INTO http_cache (url_key, response, expires_at, entity_mbid, entity_type) - SELECT url_key, response, expires_at, - COALESCE(mbid, ''), COALESCE(entity_type, '') - FROM explore_cache - `); err != nil { - return fmt.Errorf("migration 27: migrate http_cache: %w", err) - } - - // Drop the old table. - if _, err := db.ExecContext(ctx, `DROP TABLE IF EXISTS explore_cache`); err != nil { - return fmt.Errorf("migration 27: drop explore_cache: %w", err) - } - } - - if _, err := db.ExecContext(ctx, - "PRAGMA user_version = 27", - ); err != nil { - return fmt.Errorf("migration 27: set user_version: %w", err) - } - } - - if version < 28 { //nolint:mnd - logger.Info( - "applying migration 28: repair broken similar_artist_map data from multi-seed labs bug", - ) - - // The multi-seed POST form of the labs similar-artists endpoint - // returns mis-grouped results — each seed ends up with a random - // subset of the shared result pool (1-2 artists for most seeds, - // hundreds for a few). Clear the bad rows and invalidate the - // tier4 timestamp so the next index build refetches per-seed. - if _, err := db.ExecContext(ctx, - "DELETE FROM similar_artist_map", - ); err != nil { - return fmt.Errorf("migration 28: clear similar_artist_map: %w", err) - } - - // Invalidate the tier4 build timestamp so the next startup - // triggers a Tier 4 rebuild. Also clear is_similar markers - // so they get recomputed. - if _, err := db.ExecContext(ctx, - "DELETE FROM explore_index_meta WHERE key = 'tier4_built'", - ); err != nil { - // Not fatal — the meta table might not exist yet. - logger.Warn( - "migration 28: clear tier4_built failed (ok on fresh install)", - "error", - err, - ) - } - - if _, err := db.ExecContext(ctx, - "UPDATE explore_index SET is_similar = 0 WHERE is_similar = 1", - ); err != nil { - // Not fatal — explore_index might not exist yet on a - // fresh install where migration 26 just ran. - logger.Warn("migration 28: clear is_similar failed", "error", err) - } - - if _, err := db.ExecContext(ctx, - "PRAGMA user_version = 28", - ); err != nil { - return fmt.Errorf("migration 28: set user_version: %w", err) - } - } - - if version < 29 { //nolint:mnd - logger.Info( - "applying migration 29: discog_fetched column to track full indexer pipeline coverage", - ) - - // Add a discog_fetched column to explore_index. When set to 1 - // on an artist row, the indexer's fetchTopRecordings/ - // fetchTopReleaseGroups pipeline has run for that artist. - // AddFromCache (the frontend-visit organic-growth path) does - // NOT set this flag — it only writes the artist row plus - // browse-result release groups, so recordings are missing. - // - // indexedArtistMBIDs() filters by discog_fetched=1, so artists - // who only got their row from AddFromCache will still be - // processed by Tier 2/3 and have their full discography fetched - // (including recordings). - if _, err := db.ExecContext(ctx, ` - ALTER TABLE explore_index - ADD COLUMN discog_fetched INTEGER NOT NULL DEFAULT 0 - `); err != nil { - // May fail if migration runs against a fresh schema (column - // will be created by the schema file instead). Don't bail. - logger.Warn( - "migration 29: add discog_fetched column failed (ok if fresh)", - "error", - err, - ) - } - - // Backfill: any artist with at least 5 recordings was almost - // certainly hit by fetchTopRecordings (the floor is 5). Use - // this as a heuristic to mark existing data as "discog fetched" - // so the migration is non-disruptive — only the broken - // AddFromCache-only artists get re-indexed. - if _, err := db.ExecContext(ctx, ` - UPDATE explore_index - SET discog_fetched = 1 - WHERE entity_type = 'artist' - AND mbid IN ( - SELECT artist_mbid - FROM explore_index - WHERE entity_type = 'recording' - GROUP BY artist_mbid - HAVING COUNT(*) >= 5 - ) - `); err != nil { - logger.Warn("migration 29: backfill discog_fetched failed", "error", err) - } - - if _, err := db.ExecContext(ctx, - "PRAGMA user_version = 29", - ); err != nil { - return fmt.Errorf("migration 29: set user_version: %w", err) - } - } - - if version < 30 { //nolint:mnd - logger.Info( - "applying migration 30: invalidate MB browse-releases cache for recording MBID fix", - ) - - // Earlier versions of convertRelease used the MusicBrainz - // track MBID instead of the recording MBID for MBTrack.MBID. - // Tracks and recordings have distinct MBIDs in MB, and the - // local library tags files with the recording MBID, so the - // library-status indicator on album detail pages was always - // showing "not in library" for cached results. Clear the - // http_cache entries for MB browse-releases so the next - // visit refetches with the fixed converter. - if _, err := db.ExecContext(ctx, - "DELETE FROM http_cache WHERE url_key LIKE 'mb:browse:releases:%'", - ); err != nil { - // Not fatal — cache might not exist on fresh installs. - logger.Warn("migration 30: clear browse-releases cache failed", "error", err) - } - - if _, err := db.ExecContext(ctx, - "PRAGMA user_version = 30", - ); err != nil { - return fmt.Errorf("migration 30: set user_version: %w", err) - } - } - - if version < 31 { //nolint:mnd - if err := migration31TagStatus(ctx, db, logger); err != nil { - return err - } - } - - if version < 32 { //nolint:mnd - if err := migration32TaggingItems(ctx, db, logger); err != nil { - return err - } - } - - if version < 33 { //nolint:mnd - if err := migration33AutotagWarning(ctx, db, logger); err != nil { - return err - } - } - - if version < 34 { //nolint:mnd - if err := migration34FolderBasedGroupKey(ctx, db, logger); err != nil { - return err - } - } - - if version < 35 { //nolint:mnd - if err := migration35OriginalYear(ctx, db, logger); err != nil { - return err - } - } - - if version < 36 { //nolint:mnd - if err := migration36ClearedAt(ctx, db, logger); err != nil { - return err - } - } - - if version < 37 { //nolint:mnd - if err := migration37ExploreFTSDiacritics(ctx, db, logger); err != nil { - return err - } - } - - if version < 38 { //nolint:mnd - if err := migration38TaggingCandidates(ctx, db, logger); err != nil { - return err - } - } - - if version < 39 { //nolint:mnd - if err := migration39LyricsIndex(ctx, db, logger); err != nil { - return err - } - } - - if version < 40 { //nolint:mnd - if err := migration40ExploreExactMatchIndexes(ctx, db, logger); err != nil { - return err - } - } - - if version < 41 { //nolint:mnd - if err := migration41ExploreChampionFTS(ctx, db, logger); err != nil { - return err - } - } - - if version < 42 { //nolint:mnd - if err := migration42ReleaseToRG(ctx, db, logger); err != nil { - return err - } - } - - if version < 43 { //nolint:mnd - if err := migration43MergeArtistCredits(ctx, db, logger); err != nil { - return err - } - } - - if version < 44 { //nolint:mnd - if err := migration44ExploreCAAReleaseIndex(ctx, db, logger); err != nil { - return err - } - } - - if version < 45 { //nolint:mnd - if err := migration45Analyze(ctx, db, logger); err != nil { - return err - } - } - - if version < 46 { //nolint:mnd - if err := migration46SmartSnapshot(ctx, db, logger); err != nil { - return err - } - } - - return nil -} - -// migration46SmartSnapshot adds the smart_snapshot_at column to the -// playlists table. Smart playlists now materialize their evaluated -// membership into playlist_tracks and only re-evaluate on demand; the -// timestamp records when that snapshot was last taken (NULL means the -// playlist has never been materialized, so it is backfilled on first -// open). -func migration46SmartSnapshot( - ctx context.Context, - db *sql.DB, - logger *slog.Logger, -) error { - logger.Info("applying migration 46: smart playlist snapshot column") - - if _, err := db.ExecContext(ctx, - `ALTER TABLE playlists - ADD COLUMN smart_snapshot_at DATETIME`, - ); err != nil { - if !isDuplicateColumnErr(err) { - return fmt.Errorf( - "migration 46: could not add smart_snapshot_at column: %w", - err, - ) - } - } - - if _, err := db.ExecContext( - ctx, "PRAGMA user_version = 46", - ); err != nil { - return fmt.Errorf( - "migration 46: set user_version: %w", err, - ) - } - - logger.Info("migration 46 complete") - - return nil -} - -// migration45Analyze runs ANALYZE so SQLite's query planner has real -// table/index statistics. Without stats the planner guesses from row -// counts alone and mis-chose indexes on the ~2M-row explore_index — e.g. -// the top-result parent-release lookup scanned all 400k release_group -// rows via idx_explore_index_entity_pop instead of seeking the new -// idx_explore_caa_release, costing seconds per search. ANALYZE populates -// sqlite_stat1 (a one-time ~1.5s scan) and the planner then picks the -// right index for that query and every other query on these large tables. -func migration45Analyze( - ctx context.Context, - db *sql.DB, - logger *slog.Logger, -) error { - logger.Info("applying migration 45: ANALYZE for query planner statistics") - - if _, err := db.ExecContext(ctx, "ANALYZE"); err != nil { - return fmt.Errorf("migration 45: analyze: %w", err) - } - - if _, err := db.ExecContext(ctx, "PRAGMA user_version = 45"); err != nil { - return fmt.Errorf("migration 45: set user_version: %w", err) - } - - logger.Info("migration 45 complete") - - return nil -} - -// migration44ExploreCAAReleaseIndex adds a partial index on -// caa_release_mbid so the top-result resolver's parent-release-group -// lookup (SearchIndex.ReleaseGroupMBIDsForCAAReleaseMBIDs) seeks the -// index instead of scanning every release_group row in explore_index -// (~150k) on the hot search path. The index is partial, mirroring the -// query's own filter (entity_type = 'release_group' AND -// caa_release_mbid is non-empty), so it stays small and covers exactly the -// rows that lookup can match. Without it, a generic query whose top -// results include recordings with cover art (e.g. "big") spends -// seconds in this scan. -func migration44ExploreCAAReleaseIndex( - ctx context.Context, - db *sql.DB, - logger *slog.Logger, -) error { - logger.Info("applying migration 44: explore caa_release_mbid index") - - if _, err := db.ExecContext(ctx, ` - CREATE INDEX IF NOT EXISTS idx_explore_caa_release - ON explore_index(caa_release_mbid) - WHERE entity_type = 'release_group' AND caa_release_mbid != '' - `); err != nil { - return fmt.Errorf("migration 44: create caa_release index: %w", err) - } - - if _, err := db.ExecContext(ctx, "PRAGMA user_version = 44"); err != nil { - return fmt.Errorf("migration 44: set user_version: %w", err) - } - - logger.Info("migration 44 complete") - - return nil -} - -// migration43MergeArtistCredits repairs artist rows that were created -// from full credit strings. Before the scanner resolved a track's -// primary artist, a credit like "Lana Del Rey ft. Sean Lennon" was -// stored as its own artists row and stamped with the primary artist's -// single MBID — so one MusicBrainz artist fanned out into many rows that -// shared an MBID, and the explore index (last-write-wins per MBID) then -// displayed a featured-credit string as the artist's name. -// -// This collapses every set of artists rows that share an MBID into the -// one "clean" member (a name with no featuring clause), repoints the -// artist_credit_artist links, deletes the redundant rows, and refreshes -// the explore index's artist titles from the survivors. Clusters with -// no clean member (all names carry a marker) are left untouched. -func migration43MergeArtistCredits( - ctx context.Context, - db *sql.DB, - logger *slog.Logger, -) error { - logger.Info("applying migration 43: merge credit-string artists") - - // Map each redundant artist row to the clean canonical row for its - // MBID. "Clean" = a name carrying no featuring marker; the lowest - // id among those is the canonical survivor. - if _, err := db.ExecContext(ctx, ` - CREATE TEMP TABLE artist_merge_map AS - SELECT a.id AS dirty_id, canon.canon_id AS canon_id - FROM artists a - JOIN ( - SELECT mbid, MIN(id) AS canon_id - FROM artists - WHERE mbid IS NOT NULL AND mbid != '' - AND lower(name) NOT LIKE '% feat %' - AND lower(name) NOT LIKE '% feat. %' - AND lower(name) NOT LIKE '% featuring %' - AND lower(name) NOT LIKE '% ft %' - AND lower(name) NOT LIKE '% ft. %' - GROUP BY mbid - ) canon ON canon.mbid = a.mbid - WHERE a.id != canon.canon_id - `); err != nil { - return fmt.Errorf("migration 43: build merge map: %w", err) - } - - // Drop links that would collide with an existing (canonical, credit) - // link after repointing — the unique index would otherwise reject - // the UPDATE. - if _, err := db.ExecContext(ctx, ` - DELETE FROM artist_credit_artist - WHERE id IN ( - SELECT aca.id - FROM artist_credit_artist aca - JOIN artist_merge_map m ON m.dirty_id = aca.artist_id - WHERE EXISTS ( - SELECT 1 FROM artist_credit_artist keep - WHERE keep.artist_id = m.canon_id - AND keep.credit_id = aca.credit_id - ) - ) - `); err != nil { - return fmt.Errorf("migration 43: prune colliding links: %w", err) - } - - // Repoint surviving links to the canonical artist. - if _, err := db.ExecContext(ctx, ` - UPDATE artist_credit_artist - SET artist_id = ( - SELECT canon_id FROM artist_merge_map - WHERE dirty_id = artist_credit_artist.artist_id - ) - WHERE artist_id IN (SELECT dirty_id FROM artist_merge_map) - `); err != nil { - return fmt.Errorf("migration 43: repoint links: %w", err) - } - - // Remove the now-orphaned credit-string artist rows. - if _, err := db.ExecContext(ctx, ` - DELETE FROM artists WHERE id IN (SELECT dirty_id FROM artist_merge_map) - `); err != nil { - return fmt.Errorf("migration 43: delete merged artists: %w", err) - } - - // Refresh explore-index artist rows from the surviving library - // artists so their (previously clobbered) titles show the clean - // name. The AFTER UPDATE trigger keeps explore_index_fts in sync. - // Only rows backed by a library artist are touched; dump-only rows - // are left alone. - if _, err := db.ExecContext(ctx, ` - UPDATE explore_index - SET title = ( - SELECT name FROM artists - WHERE artists.mbid = explore_index.mbid ORDER BY id LIMIT 1), - artist_name = ( - SELECT name FROM artists - WHERE artists.mbid = explore_index.mbid ORDER BY id LIMIT 1), - local_artist_id = ( - SELECT id FROM artists - WHERE artists.mbid = explore_index.mbid ORDER BY id LIMIT 1) - WHERE entity_type = 'artist' - AND EXISTS (SELECT 1 FROM artists WHERE artists.mbid = explore_index.mbid) - `); err != nil { - return fmt.Errorf("migration 43: refresh explore titles: %w", err) - } - - if _, err := db.ExecContext(ctx, "DROP TABLE IF EXISTS artist_merge_map"); err != nil { - return fmt.Errorf("migration 43: drop temp table: %w", err) - } - - if _, err := db.ExecContext(ctx, "PRAGMA user_version = 43"); err != nil { - return fmt.Errorf("migration 43: set user_version: %w", err) - } - - logger.Info("migration 43 complete") - - return nil -} - -// migration42ReleaseToRG creates the release_to_rg mapping table: for -// every release under an indexed release group, which release-group it -// belongs to. It is populated from the canonical dump during a full -// import (the mapping is otherwise in-memory only and discarded). The -// incremental-dump popularity refresh uses it to roll per-release listen -// deltas up to their release group, so album popularity stays fresh -// without any API call. -func migration42ReleaseToRG( - ctx context.Context, - db *sql.DB, - logger *slog.Logger, -) error { - logger.Info("applying migration 42: release_to_rg table") - - if _, err := db.ExecContext(ctx, ` - CREATE TABLE IF NOT EXISTS release_to_rg ( - release_mbid TEXT PRIMARY KEY, - rg_mbid TEXT NOT NULL - ) WITHOUT ROWID - `); err != nil { - return fmt.Errorf("migration 42: create release_to_rg: %w", err) - } - - if _, err := db.ExecContext(ctx, - "PRAGMA user_version = 42", - ); err != nil { - return fmt.Errorf("migration 42: set user_version: %w", err) - } - - logger.Info("migration 42 complete") - - return nil -} - -// migration41ExploreChampionFTS creates the "champion" full-text index: -// a second external-content FTS5 over explore_index that holds only the -// high-popularity / owned rows. Short generic prefixes ("the", "a") -// match hundreds of thousands of rows in the full index, and the -// popularity-blended ORDER BY must score every one of them — seconds of -// work. Routing those queries at the champion index instead scores only -// the ~90k rows that could plausibly win, cutting the query from seconds -// to tens of milliseconds. The table is created empty here; the search -// index populates it at runtime (see SearchIndex.RebuildChampionIndex) -// because the row set derives from popularity, which changes as the -// index is (re)built. -func migration41ExploreChampionFTS( - ctx context.Context, - db *sql.DB, - logger *slog.Logger, -) error { - logger.Info("applying migration 41: explore champion FTS") - - if _, err := db.ExecContext(ctx, ` - CREATE VIRTUAL TABLE IF NOT EXISTS explore_champion_fts USING fts5( - title, artist_name, aliases, - content='explore_index', - content_rowid='id', - tokenize='unicode61 remove_diacritics 2' - ) - `); err != nil { - return fmt.Errorf("migration 41: create champion fts: %w", err) - } - - if _, err := db.ExecContext(ctx, "PRAGMA user_version = 41"); err != nil { - return fmt.Errorf("migration 41: set user_version: %w", err) - } - - logger.Info("migration 41 complete") - - return nil -} - -// migration39LyricsIndex creates the contentless FTS5 lyrics_index -// (see lyrics_index.sql) and back-populates it from any recordings -// that already have embedded lyrics, so lyric search works on -// existing libraries without waiting for a rescan. -func migration39LyricsIndex( - ctx context.Context, - db *sql.DB, - logger *slog.Logger, -) error { - logger.Info("applying migration 39: lyrics_index FTS") - - if _, err := db.ExecContext(ctx, ` - CREATE VIRTUAL TABLE IF NOT EXISTS lyrics_index USING fts5( - lyrics, - content='', - contentless_delete=1, - tokenize='unicode61 remove_diacritics 2' - ) - `); err != nil { - return fmt.Errorf("migration 39: create lyrics_index: %w", err) - } - - // Back-populate from recordings that already carry lyrics. The - // rowid is the recording id so it stays stable across rebuilds. - if _, err := db.ExecContext(ctx, ` - INSERT INTO lyrics_index(rowid, lyrics) - SELECT id, lyrics - FROM recordings - WHERE lyrics IS NOT NULL AND lyrics != '' - `); err != nil { - return fmt.Errorf("migration 39: populate lyrics_index: %w", err) - } - - if _, err := db.ExecContext(ctx, "PRAGMA user_version = 39"); err != nil { - return fmt.Errorf("migration 39: set user_version: %w", err) - } - - logger.Info("migration 39 complete") - - return nil -} - -// migration40ExploreExactMatchIndexes adds partial expression indexes -// on LOWER(title) and LOWER(artist_name) so the interactive top-result -// resolver's exact-match lookup (SearchIndex.ExactMatches) seeks the -// index instead of scanning all ~240k explore_index rows on every -// keystroke. The indexes are partial (WHERE popularity > 0) because -// that lookup always filters on popularity, keeping them small; the -// UNION-of-equalities query shape in ExactMatches is what lets SQLite -// use them (an OR across the two columns forces a scan instead). -func migration40ExploreExactMatchIndexes( - ctx context.Context, - db *sql.DB, - logger *slog.Logger, -) error { - logger.Info("applying migration 40: explore exact-match indexes") - - if _, err := db.ExecContext(ctx, ` - CREATE INDEX IF NOT EXISTS idx_explore_title_lower - ON explore_index(LOWER(title)) - WHERE popularity > 0 - `); err != nil { - return fmt.Errorf("migration 40: create title index: %w", err) - } - - if _, err := db.ExecContext(ctx, ` - CREATE INDEX IF NOT EXISTS idx_explore_artist_lower - ON explore_index(LOWER(artist_name)) - WHERE popularity > 0 - `); err != nil { - return fmt.Errorf("migration 40: create artist index: %w", err) - } - - if _, err := db.ExecContext(ctx, "PRAGMA user_version = 40"); err != nil { - return fmt.Errorf("migration 40: set user_version: %w", err) - } - - logger.Info("migration 40 complete") - - return nil -} - -// migration38TaggingCandidates creates the tagging_candidates table — -// a durable per-group store for the scored candidate list so it is -// computed once and reused across restarts instead of re-hitting -// MusicBrainz every session (see tagging_candidates.sql). A plain -// CREATE TABLE IF NOT EXISTS is safe on both fresh and existing DBs. -func migration38TaggingCandidates( - ctx context.Context, - db *sql.DB, - logger *slog.Logger, -) error { - logger.Info("applying migration 38: tagging_candidates") - - if _, err := db.ExecContext(ctx, ` - CREATE TABLE IF NOT EXISTS tagging_candidates ( - group_key TEXT PRIMARY KEY, - candidates TEXT NOT NULL, - computed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - FOREIGN KEY(group_key) REFERENCES tagging_items(group_key) ON DELETE CASCADE - ) - `); err != nil { - return fmt.Errorf("migration 38: create tagging_candidates: %w", err) - } - - if _, err := db.ExecContext(ctx, "PRAGMA user_version = 38"); err != nil { - return fmt.Errorf("migration 38: set user_version: %w", err) - } - - logger.Info("migration 38 complete") - - return nil -} - -// migration36ClearedAt adds tagging_items.cleared_at — a nullable -// timestamp set when the user invokes "clear completed entries". -// Cleared rows stay in the table (so a re-scan doesn't resurrect -// them as pending) but get filtered from the review queue. -func migration36ClearedAt( - ctx context.Context, - db *sql.DB, - logger *slog.Logger, -) error { - logger.Info("applying migration 36: tagging_items.cleared_at") - - if _, err := db.ExecContext( - ctx, - `ALTER TABLE tagging_items ADD COLUMN cleared_at DATETIME`, - ); err != nil { - if !strings.Contains(err.Error(), "duplicate column name") { - return fmt.Errorf("migration 36: add cleared_at: %w", err) - } - - logger.Warn("migration 36: cleared_at already present (ok if fresh)", "err", err) - } - - if _, err := db.ExecContext(ctx, "PRAGMA user_version = 36"); err != nil { - return fmt.Errorf("migration 36: set user_version: %w", err) - } - - logger.Info("migration 36 complete") - - return nil -} - -// migration37ExploreFTSDiacritics rebuilds explore_index_fts with the -// "unicode61 remove_diacritics 2" tokeniser so accented queries match -// their unaccented forms (e.g. "beyonce" finds "Beyoncé"), matching the -// library search_index tokeniser. The original table (migration 26) -// was created with the default tokeniser, which does not fold -// diacritics. -// -// Because explore_index_fts is an external-content table over -// explore_index, the rebuild repopulates from the existing content -// rows — no data loss and no need to re-run the expensive tiered index -// build. -func migration37ExploreFTSDiacritics( - ctx context.Context, - db *sql.DB, - logger *slog.Logger, -) error { - logger.Info("applying migration 37: explore_index_fts diacritic folding") - - // Drop the sync triggers and the FTS table, then recreate both. - // The triggers must go first — they reference the FTS table. - stmts := []string{ - `DROP TRIGGER IF EXISTS explore_index_ai`, - `DROP TRIGGER IF EXISTS explore_index_ad`, - `DROP TRIGGER IF EXISTS explore_index_au`, - `DROP TABLE IF EXISTS explore_index_fts`, - `CREATE VIRTUAL TABLE explore_index_fts USING fts5( - title, artist_name, aliases, - content='explore_index', - content_rowid='id', - tokenize='unicode61 remove_diacritics 2' - )`, - `CREATE TRIGGER explore_index_ai AFTER INSERT ON explore_index BEGIN - INSERT INTO explore_index_fts(rowid, title, artist_name, aliases) - VALUES (new.id, new.title, new.artist_name, new.aliases); - END`, - `CREATE TRIGGER explore_index_ad AFTER DELETE ON explore_index BEGIN - INSERT INTO explore_index_fts(explore_index_fts, rowid, title, artist_name, aliases) - VALUES ('delete', old.id, old.title, old.artist_name, old.aliases); - END`, - `CREATE TRIGGER explore_index_au AFTER UPDATE ON explore_index BEGIN - INSERT INTO explore_index_fts(explore_index_fts, rowid, title, artist_name, aliases) - VALUES ('delete', old.id, old.title, old.artist_name, old.aliases); - INSERT INTO explore_index_fts(rowid, title, artist_name, aliases) - VALUES (new.id, new.title, new.artist_name, new.aliases); - END`, - // Repopulate the FTS index from the content table. - `INSERT INTO explore_index_fts(explore_index_fts) VALUES('rebuild')`, - } - - for _, stmt := range stmts { - if _, err := db.ExecContext(ctx, stmt); err != nil { - return fmt.Errorf("migration 37: %w", err) - } - } - - if _, err := db.ExecContext(ctx, "PRAGMA user_version = 37"); err != nil { - return fmt.Errorf("migration 37: set user_version: %w", err) - } - - logger.Info("migration 37 complete") - - return nil -} - -// backfills it from file_path, creates the basename index, and -// populates the FTS5 search_index table. -func migration2BasenameAndFTS( - ctx context.Context, - db *sql.DB, - logger *slog.Logger, -) error { - logger.Info( - "applying migration 2: basename column + FTS5 search index", - ) - - // Add basename column (may already exist on fresh DBs). - if _, err := db.ExecContext( - ctx, - "ALTER TABLE audio_files ADD COLUMN basename text NOT NULL DEFAULT ''", - ); err != nil && !isDuplicateColumnErr(err) { - return fmt.Errorf( - "migration 2: could not add basename column: %w", - err, - ) - } - - // Backfill basename from file_path for existing rows. - // SQLite doesn't have a basename function, so we use - // REPLACE to strip directories by finding everything - // after the last '/'. - if _, err := db.ExecContext(ctx, ` - UPDATE audio_files - SET basename = CASE - WHEN INSTR(file_path, '/') > 0 - THEN SUBSTR( - file_path, - LENGTH(file_path) - - LENGTH( - REPLACE(file_path, '/', '') - ) - + 1 - ) - ELSE file_path - END - WHERE basename = '' - `); err != nil { - return fmt.Errorf( - "migration 2: could not backfill basename: %w", - err, - ) - } - - // Create index (IF NOT EXISTS handles fresh DBs). - if _, err := db.ExecContext(ctx, ` - CREATE INDEX IF NOT EXISTS idx_audio_files_basename - ON audio_files(basename) - `); err != nil { - return fmt.Errorf( - "migration 2: could not create basename index: %w", - err, - ) - } - - // Populate FTS5 search index from existing data. - if _, err := db.ExecContext(ctx, ` - INSERT INTO search_index(rowid, file_path, title, artist, album) - SELECT - af.id, - af.file_path, - COALESCE(r.name, ''), - COALESCE(ac.text, ''), - COALESCE(rg.name, '') - FROM audio_files af - LEFT JOIN recordings r ON af.recording_id = r.id - LEFT JOIN artist_credit ac ON r.artist_credit_id = ac.id - LEFT JOIN ( - SELECT recording_id, - MIN(release_group_id) AS release_group_id - FROM release_group_recordings - GROUP BY recording_id - ) rgr ON r.id = rgr.recording_id - LEFT JOIN release_groups rg - ON rgr.release_group_id = rg.id - `); err != nil { - return fmt.Errorf( - "migration 2: could not populate search index: %w", - err, - ) - } - - if _, err := db.ExecContext( - ctx, "PRAGMA user_version = 2", - ); err != nil { - return fmt.Errorf( - "could not set user_version to 2: %w", err, - ) - } - - logger.Info("migration 2 complete") - - return nil -} - -// migration4TrackMetadataView creates the track_metadata VIEW that -// consolidates the 5-table JOIN used by FTS5 search queries. -// Fresh databases get the VIEW from the embedded schema file; -// this migration covers databases created before the VIEW existed. -func migration4TrackMetadataView( - ctx context.Context, - db *sql.DB, - logger *slog.Logger, -) error { - logger.Info( - "applying migration 4: track_metadata VIEW", - ) - - if _, err := db.ExecContext(ctx, ` - CREATE VIEW IF NOT EXISTS track_metadata AS - SELECT - af.id, - af.file_path, - af.length_milliseconds, - COALESCE(r.name, '') AS title, - COALESCE(ac.text, '') AS artist_name, - r.track_number, - r.disc_number, - COALESCE(rg.name, '') AS album, - CAST(COALESCE( - (SELECT GROUP_CONCAT(g.name, '||') - FROM recording_genres rg_sub - JOIN genres g ON rg_sub.genre_id = g.id - WHERE rg_sub.recording_id = r.id), - '' - ) AS TEXT) AS genre, - COALESCE(r.year, 0) AS year, - COALESCE(r.composer, '') AS composer, - COALESCE(ft.extension, '') AS file_type, - af.sample_rate, - af.bit_depth, - af.channels, - af.bitrate, - af.file_size - FROM audio_files af - LEFT JOIN recordings r ON af.recording_id = r.id - LEFT JOIN artist_credit ac ON r.artist_credit_id = ac.id - LEFT JOIN ( - SELECT recording_id, - MIN(release_group_id) AS release_group_id - FROM release_group_recordings - GROUP BY recording_id - ) rgr ON r.id = rgr.recording_id - LEFT JOIN release_groups rg ON rgr.release_group_id = rg.id - LEFT JOIN file_types ft ON af.file_type_id = ft.id - `); err != nil { - return fmt.Errorf( - "migration 4: could not create track_metadata VIEW: %w", - err, - ) - } - - if _, err := db.ExecContext( - ctx, "PRAGMA user_version = 4", - ); err != nil { - return fmt.Errorf( - "could not set user_version to 4: %w", err, - ) - } - - logger.Info("migration 4 complete") - - return nil -} - -// migration5ReleaseGroupCompositeUnique rebuilds the release_groups -// table with UNIQUE(name, album_artist_credit_id) instead of -// UNIQUE(name). SQLite cannot ALTER a UNIQUE constraint, so we -// must rebuild the table. -// -// SAFETY: Hand-crafted SQL for schema migration. -func migration5ReleaseGroupCompositeUnique( - ctx context.Context, - db *sql.DB, - logger *slog.Logger, -) error { - logger.Info( - "applying migration 5: release_groups composite unique constraint", - ) - - // Temporarily disable FK checks for table rebuild. - if _, err := db.ExecContext( - ctx, "PRAGMA foreign_keys = OFF", - ); err != nil { - return fmt.Errorf( - "migration 5: could not disable foreign keys: %w", - err, - ) - } - - // Drop the track_metadata VIEW that references release_groups - // so the table rebuild can proceed without SQLite complaining - // about a dangling VIEW reference. - if _, err := db.ExecContext( - ctx, "DROP VIEW IF EXISTS track_metadata", - ); err != nil { - return fmt.Errorf( - "migration 5: could not drop track_metadata VIEW: %w", - err, - ) - } - - // Create new table with composite unique constraint. - if _, err := db.ExecContext(ctx, ` - CREATE TABLE release_groups_new ( - id INTEGER PRIMARY KEY, - name TEXT NOT NULL, - cover_art_id INTEGER, - album_artist_credit_id INTEGER, - year INTEGER, - total_tracks INTEGER, - total_discs INTEGER, - FOREIGN KEY(cover_art_id) REFERENCES cover_art(id), - FOREIGN KEY(album_artist_credit_id) REFERENCES artist_credit(id), - UNIQUE(name, album_artist_credit_id) - ) - `); err != nil { - return fmt.Errorf( - "migration 5: could not create release_groups_new: %w", - err, - ) - } - - // Copy all data. Columns are listed explicitly so later schema - // additions (e.g. migration 13's mbid column) don't break this - // migration when it runs on a fresh DB where CREATE TABLE IF NOT - // EXISTS has already materialized the current schema. - if _, err := db.ExecContext(ctx, ` - INSERT INTO release_groups_new - (id, name, cover_art_id, album_artist_credit_id, - year, total_tracks, total_discs) - SELECT id, name, cover_art_id, album_artist_credit_id, - year, total_tracks, total_discs - FROM release_groups - `); err != nil { - return fmt.Errorf( - "migration 5: could not copy data: %w", err, - ) - } - - // Drop old table. - if _, err := db.ExecContext( - ctx, "DROP TABLE release_groups", - ); err != nil { - return fmt.Errorf( - "migration 5: could not drop old table: %w", err, - ) - } - - // Rename new table. - if _, err := db.ExecContext(ctx, ` - ALTER TABLE release_groups_new - RENAME TO release_groups - `); err != nil { - return fmt.Errorf( - "migration 5: could not rename table: %w", err, - ) - } - - // Recreate indexes. - if _, err := db.ExecContext(ctx, ` - CREATE INDEX IF NOT EXISTS idx_release_groups_cover_art_id - ON release_groups(cover_art_id) - `); err != nil { - return fmt.Errorf( - "migration 5: could not create cover_art_id index: %w", - err, - ) - } - - if _, err := db.ExecContext(ctx, ` - CREATE INDEX IF NOT EXISTS idx_release_groups_album_artist_credit_id - ON release_groups(album_artist_credit_id) - `); err != nil { - return fmt.Errorf( - "migration 5: could not create album_artist_credit_id index: %w", - err, - ) - } - - // Recreate the track_metadata VIEW that was dropped above. - // The definition must match the embedded schema file - // (sql/schemas/track_metadata_view.sql) exactly. - if _, err := db.ExecContext(ctx, ` - CREATE VIEW IF NOT EXISTS track_metadata AS - SELECT - af.id, - af.file_path, - af.length_milliseconds, - COALESCE(r.name, '') AS title, - COALESCE(ac.text, '') AS artist_name, - r.track_number, - r.disc_number, - COALESCE(rg.name, '') AS album, - CAST(COALESCE( - (SELECT GROUP_CONCAT(g.name, '||') - FROM recording_genres rg_sub - JOIN genres g ON rg_sub.genre_id = g.id - WHERE rg_sub.recording_id = r.id), - '' - ) AS TEXT) AS genre, - COALESCE(r.year, 0) AS year, - COALESCE(r.composer, '') AS composer, - COALESCE(ft.extension, '') AS file_type, - af.sample_rate, - af.bit_depth, - af.channels, - af.bitrate, - af.file_size - FROM audio_files af - LEFT JOIN recordings r ON af.recording_id = r.id - LEFT JOIN artist_credit ac ON r.artist_credit_id = ac.id - LEFT JOIN ( - SELECT recording_id, - MIN(release_group_id) AS release_group_id - FROM release_group_recordings - GROUP BY recording_id - ) rgr ON r.id = rgr.recording_id - LEFT JOIN release_groups rg ON rgr.release_group_id = rg.id - LEFT JOIN file_types ft ON af.file_type_id = ft.id - `); err != nil { - return fmt.Errorf( - "migration 5: could not recreate track_metadata VIEW: %w", - err, - ) - } - - // Re-enable FK checks. - if _, err := db.ExecContext( - ctx, "PRAGMA foreign_keys = ON", - ); err != nil { - return fmt.Errorf( - "migration 5: could not re-enable foreign keys: %w", - err, - ) - } - - if _, err := db.ExecContext( - ctx, "PRAGMA user_version = 5", - ); err != nil { - return fmt.Errorf( - "could not set user_version to 5: %w", err, - ) - } - - logger.Info("migration 5 complete") - - return nil -} - -// isDuplicateColumnErr returns true when the error is SQLite's -// "duplicate column name" error from an ALTER TABLE ADD COLUMN -// on a column that already exists. -func isDuplicateColumnErr(err error) bool { - return err != nil && - strings.Contains( - err.Error(), "duplicate column name", - ) -} - -// backupDatabase copies the database file to a timestamped backup -// before running a destructive migration. Returns the backup path. -func backupDatabase( - dbPath string, logger *slog.Logger, -) (string, error) { - backupPath := dbPath + ".bak." + time.Now().Format("20060102") - - src, err := os.Open(dbPath) - if err != nil { - return "", fmt.Errorf( - "could not open database for backup: %w", err, - ) - } - - defer func() { _ = src.Close() }() - - dst, err := os.Create(backupPath) - if err != nil { - return "", fmt.Errorf( - "could not create backup file: %w", err, - ) - } - - defer func() { _ = dst.Close() }() - - if _, err := io.Copy(dst, src); err != nil { - return "", fmt.Errorf( - "could not copy database to backup: %w", err, - ) - } - - logger.Info("database backup created", "path", backupPath) - - return backupPath, nil -} - -// migration6MultiLibrary adds multi-library support: creates the -// libraries table, adds library_id FK to audio_files, rebuilds -// playlist_tracks with SET NULL FK and phantom metadata columns, -// and recreates the track_metadata VIEW with library_id. -// -// SAFETY: Hand-crafted SQL for schema migration. -func migration6MultiLibrary( - ctx context.Context, - db *sql.DB, - logger *slog.Logger, - dbPath string, -) error { - logger.Info("applying migration 6: multi-library support") - - // 1. Backup database BEFORE any changes (skip for in-memory DBs). - if dbPath != "" && dbPath != ":memory:" { - if _, err := backupDatabase(dbPath, logger); err != nil { - return fmt.Errorf( - "migration 6: backup failed: %w", err, - ) - } - } - - // 2. Read TOML config to get existing library directory. - existingDir := readLibraryDirFromTOML(logger) - - // 3. Disable FK checks for table rebuild. - // SAFETY: PRAGMA foreign_keys cannot run inside a transaction. - if _, err := db.ExecContext( - ctx, "PRAGMA foreign_keys = OFF", - ); err != nil { - return fmt.Errorf( - "migration 6: could not disable foreign keys: %w", - err, - ) - } - - // 4. Create libraries table. - // SAFETY: Hand-crafted DDL for new table. - if _, err := db.ExecContext(ctx, ` - CREATE TABLE IF NOT EXISTS libraries ( - id INTEGER PRIMARY KEY, - name TEXT NOT NULL, - path TEXT NOT NULL UNIQUE, - created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP - ) - `); err != nil { - return fmt.Errorf( - "migration 6: could not create libraries table: %w", - err, - ) - } - - // 5. Insert default library from TOML (if existingDir is not empty). - var defaultLibID int64 - - if existingDir != "" { - libName := filepath.Base(existingDir) - - // SAFETY: Hand-crafted INSERT for migrated default library. - result, err := db.ExecContext(ctx, - "INSERT INTO libraries (name, path) VALUES (?, ?)", - libName, existingDir, - ) - if err != nil { - return fmt.Errorf( - "migration 6: could not insert default library: %w", - err, - ) - } - - defaultLibID, _ = result.LastInsertId() - - logger.Info("migrated existing library", - "name", libName, - "path", existingDir, - "id", defaultLibID, - ) - } - - // 6. Add library_id column to audio_files. - // SAFETY: ALTER TABLE ADD COLUMN with dynamic DEFAULT for backfill. - stmt := fmt.Sprintf( - "ALTER TABLE audio_files ADD COLUMN library_id INTEGER NOT NULL DEFAULT %d", - defaultLibID, - ) - - if _, err := db.ExecContext(ctx, stmt); err != nil { - if !isDuplicateColumnErr(err) { - return fmt.Errorf( - "migration 6: could not add library_id column: %w", - err, - ) - } - } - - // 7. Create index on library_id. - // SAFETY: Hand-crafted index for FK performance. - if _, err := db.ExecContext(ctx, ` - CREATE INDEX IF NOT EXISTS idx_audio_files_library_id - ON audio_files(library_id) - `); err != nil { - return fmt.Errorf( - "migration 6: could not create library_id index: %w", - err, - ) - } - - // 8. Drop track_metadata VIEW before table rebuild. - if _, err := db.ExecContext( - ctx, "DROP VIEW IF EXISTS track_metadata", - ); err != nil { - return fmt.Errorf( - "migration 6: could not drop track_metadata VIEW: %w", - err, - ) - } - - // 9. Rebuild playlist_tracks for SET NULL FK + phantom columns. - // SAFETY: Table rebuild — playlist_tracks changes to SET NULL, - // queue_tracks keeps CASCADE (ephemeral, not rebuilt). - - // SAFETY: Hand-crafted DDL for rebuilt playlist_tracks. - if _, err := db.ExecContext(ctx, ` - CREATE TABLE playlist_tracks_new ( - id INTEGER PRIMARY KEY, - playlist_id INTEGER NOT NULL, - audio_file_id INTEGER, - position INTEGER NOT NULL, - phantom_title TEXT, - phantom_artist TEXT, - phantom_album TEXT, - phantom_duration_ms INTEGER, - phantom_genre TEXT, - phantom_cover_art_path TEXT, - FOREIGN KEY(playlist_id) REFERENCES playlists(id) ON DELETE CASCADE, - FOREIGN KEY(audio_file_id) REFERENCES audio_files(id) ON DELETE SET NULL - ) - `); err != nil { - return fmt.Errorf( - "migration 6: could not create playlist_tracks_new: %w", - err, - ) - } - - // Copy existing data (phantom columns get NULL). - // SAFETY: Hand-crafted INSERT-SELECT for data migration. - if _, err := db.ExecContext(ctx, ` - INSERT INTO playlist_tracks_new (id, playlist_id, audio_file_id, position) - SELECT id, playlist_id, audio_file_id, position FROM playlist_tracks - `); err != nil { - return fmt.Errorf( - "migration 6: could not copy playlist_tracks data: %w", - err, - ) - } - - // Drop old table. - if _, err := db.ExecContext( - ctx, "DROP TABLE playlist_tracks", - ); err != nil { - return fmt.Errorf( - "migration 6: could not drop old playlist_tracks: %w", - err, - ) - } - - // Rename. - if _, err := db.ExecContext(ctx, - "ALTER TABLE playlist_tracks_new RENAME TO playlist_tracks", - ); err != nil { - return fmt.Errorf( - "migration 6: could not rename playlist_tracks_new: %w", - err, - ) - } - - // Recreate indexes. - if _, err := db.ExecContext(ctx, ` - CREATE INDEX IF NOT EXISTS idx_playlist_tracks_playlist_id - ON playlist_tracks(playlist_id) - `); err != nil { - return fmt.Errorf( - "migration 6: could not create playlist_id index: %w", - err, - ) - } - - if _, err := db.ExecContext(ctx, ` - CREATE INDEX IF NOT EXISTS idx_playlist_tracks_audio_file_id - ON playlist_tracks(audio_file_id) - `); err != nil { - return fmt.Errorf( - "migration 6: could not create audio_file_id index: %w", - err, - ) - } - - // 10. Backfill phantom metadata on existing playlist_tracks - // from audio_files JOINs. Eager population per user decision. - // SAFETY: Hand-crafted UPDATE-FROM-SELECT for phantom backfill. - if _, err := db.ExecContext(ctx, ` - UPDATE playlist_tracks SET - phantom_title = sub.title, - phantom_artist = sub.artist, - phantom_album = sub.album, - phantom_duration_ms = sub.duration, - phantom_genre = sub.genre, - phantom_cover_art_path = sub.cover_art_path - FROM ( - SELECT - pt.id AS pt_id, - COALESCE(r.name, '') AS title, - COALESCE(ac.text, '') AS artist, - COALESCE(rg.name, '') AS album, - af.length_milliseconds AS duration, - CAST(COALESCE( - (SELECT GROUP_CONCAT(g.name, '||') - FROM recording_genres rg_sub - JOIN genres g ON rg_sub.genre_id = g.id - WHERE rg_sub.recording_id = r.id), - '' - ) AS TEXT) AS genre, - COALESCE(ca.file_path, '') AS cover_art_path - FROM playlist_tracks pt - JOIN audio_files af ON pt.audio_file_id = af.id - LEFT JOIN recordings r ON af.recording_id = r.id - LEFT JOIN artist_credit ac ON r.artist_credit_id = ac.id - LEFT JOIN ( - SELECT recording_id, MIN(release_group_id) AS release_group_id - FROM release_group_recordings - GROUP BY recording_id - ) rgr ON r.id = rgr.recording_id - LEFT JOIN release_groups rg ON rgr.release_group_id = rg.id - LEFT JOIN cover_art ca ON rg.cover_art_id = ca.id - ) sub - WHERE playlist_tracks.id = sub.pt_id - `); err != nil { - return fmt.Errorf( - "migration 6: could not backfill phantom metadata: %w", - err, - ) - } - - // 11. Recreate track_metadata VIEW with library_id. - // SAFETY: Hand-crafted VIEW recreation matching schema file. - if _, err := db.ExecContext(ctx, ` - CREATE VIEW IF NOT EXISTS track_metadata AS - SELECT - af.id, - af.file_path, - af.length_milliseconds, - COALESCE(r.name, '') AS title, - COALESCE(ac.text, '') AS artist_name, - r.track_number, - r.disc_number, - COALESCE(rg.name, '') AS album, - CAST(COALESCE( - (SELECT GROUP_CONCAT(g.name, '||') - FROM recording_genres rg_sub - JOIN genres g ON rg_sub.genre_id = g.id - WHERE rg_sub.recording_id = r.id), - '' - ) AS TEXT) AS genre, - COALESCE(r.year, 0) AS year, - COALESCE(r.composer, '') AS composer, - COALESCE(ft.extension, '') AS file_type, - af.sample_rate, - af.bit_depth, - af.channels, - af.bitrate, - af.file_size, - af.library_id - FROM audio_files af - LEFT JOIN recordings r ON af.recording_id = r.id - LEFT JOIN artist_credit ac ON r.artist_credit_id = ac.id - LEFT JOIN ( - SELECT recording_id, - MIN(release_group_id) AS release_group_id - FROM release_group_recordings - GROUP BY recording_id - ) rgr ON r.id = rgr.recording_id - LEFT JOIN release_groups rg ON rgr.release_group_id = rg.id - LEFT JOIN file_types ft ON af.file_type_id = ft.id - `); err != nil { - return fmt.Errorf( - "migration 6: could not recreate track_metadata VIEW: %w", - err, - ) - } - - // 12. Re-enable FK checks. - if _, err := db.ExecContext( - ctx, "PRAGMA foreign_keys = ON", - ); err != nil { - return fmt.Errorf( - "migration 6: could not re-enable foreign keys: %w", - err, - ) - } - - // 13. Remove DirectoryPath from TOML config (libraries table - // is now the source of truth). - if existingDir != "" { - removeLibraryDirFromTOML(logger) - } - - // 14. Set version. - if _, err := db.ExecContext( - ctx, "PRAGMA user_version = 6", - ); err != nil { - return fmt.Errorf( - "could not set user_version to 6: %w", err, - ) - } - - logger.Info("migration 6 complete") - - return nil -} - -// migration7PhantomFilePath adds the phantom_file_path column to -// playlist_tracks so that phantom entries can be automatically -// re-linked to audio_files after a library scan. -func migration7PhantomFilePath( - ctx context.Context, - db *sql.DB, - logger *slog.Logger, -) error { - logger.Info( - "applying migration 7: phantom_file_path column", - ) - - // SAFETY: ALTER TABLE ADD COLUMN for new nullable column. - if _, err := db.ExecContext(ctx, - `ALTER TABLE playlist_tracks - ADD COLUMN phantom_file_path TEXT`, - ); err != nil { - if !isDuplicateColumnErr(err) { - return fmt.Errorf( - "migration 7: could not add column: %w", - err, - ) - } - } - - if _, err := db.ExecContext( - ctx, "PRAGMA user_version = 7", - ); err != nil { - return fmt.Errorf( - "could not set user_version to 7: %w", err, - ) - } - - logger.Info("migration 7 complete") - - return nil -} - -// migration8ContentlessDelete rebuilds the FTS5 search_index with -// contentless_delete=1 so that individual rows can be deleted. -// This is a prerequisite for inline tag edit → DB sync in Phase 16. -// -// SAFETY: Hand-crafted SQL for FTS5 schema migration. -func migration8ContentlessDelete( - ctx context.Context, - db *sql.DB, - logger *slog.Logger, -) error { - logger.Info( - "applying migration 8: rebuilding FTS5 search_index with contentless_delete=1", - ) - - // Drop the old contentless FTS5 table (content='' only). - if _, err := db.ExecContext( - ctx, `DROP TABLE IF EXISTS search_index`, - ); err != nil { - return fmt.Errorf( - "migration 8: could not drop search_index: %w", err, - ) - } - - // Recreate with contentless_delete=1 added. - // SAFETY: Must match backend/database/sql/schemas/search_index.sql exactly. - if _, err := db.ExecContext(ctx, ` - CREATE VIRTUAL TABLE IF NOT EXISTS search_index USING fts5( - file_path, - title, - artist, - album, - content='', - contentless_delete=1, - tokenize='unicode61 remove_diacritics 2' - ) - `); err != nil { - return fmt.Errorf( - "migration 8: could not recreate search_index: %w", err, - ) - } - - // Repopulate from track_metadata VIEW. - // SAFETY: FTS5 INSERT from VIEW; no user input. - if _, err := db.ExecContext(ctx, ` - INSERT INTO search_index(rowid, file_path, title, artist, album) - SELECT id, file_path, title, artist_name, album - FROM track_metadata - `); err != nil { - return fmt.Errorf( - "migration 8: could not repopulate search_index: %w", err, - ) - } - - if _, err := db.ExecContext( - ctx, "PRAGMA user_version = 8", - ); err != nil { - return fmt.Errorf( - "migration 8: could not set user_version: %w", err, - ) - } - - logger.Info("migration 8 complete") - - return nil -} - -// migration9SmartPlaylists adds the is_smart and smart_rules -// columns to the playlists table for smart playlist support. -func migration9SmartPlaylists( - ctx context.Context, - db *sql.DB, - logger *slog.Logger, -) error { - logger.Info( - "applying migration 9: smart playlist columns", - ) - - if _, err := db.ExecContext(ctx, - `ALTER TABLE playlists - ADD COLUMN is_smart INTEGER NOT NULL DEFAULT 0`, - ); err != nil { - if !isDuplicateColumnErr(err) { - return fmt.Errorf( - "migration 9: could not add is_smart column: %w", - err, - ) - } - } - - if _, err := db.ExecContext(ctx, - `ALTER TABLE playlists - ADD COLUMN smart_rules TEXT`, - ); err != nil { - if !isDuplicateColumnErr(err) { - return fmt.Errorf( - "migration 9: could not add smart_rules column: %w", - err, - ) - } - } - - if _, err := db.ExecContext( - ctx, "PRAGMA user_version = 9", - ); err != nil { - return fmt.Errorf( - "could not set user_version to 9: %w", err, - ) - } - - logger.Info("migration 9 complete") - - return nil -} - -// migration10PlayHistory adds play history tracking: -// - play_history table for timestamped play log -// - play_count and last_played columns on audio_files -// - Recreates track_metadata VIEW to expose the new columns. -func migration10PlayHistory( - ctx context.Context, - db *sql.DB, - logger *slog.Logger, -) error { - logger.Info( - "applying migration 10: play history tracking", - ) - - // 1. Create play_history table. - if _, err := db.ExecContext(ctx, ` - CREATE TABLE IF NOT EXISTS play_history ( - id INTEGER PRIMARY KEY, - audio_file_id INTEGER NOT NULL, - played_at DATETIME NOT NULL DEFAULT (datetime('now')), - FOREIGN KEY(audio_file_id) REFERENCES audio_files(id) ON DELETE CASCADE - )`, - ); err != nil { - return fmt.Errorf( - "migration 10: could not create play_history table: %w", - err, - ) - } - - if _, err := db.ExecContext(ctx, ` - CREATE INDEX IF NOT EXISTS idx_play_history_audio_file_id - ON play_history(audio_file_id)`, - ); err != nil { - return fmt.Errorf( - "migration 10: could not create play_history index: %w", - err, - ) - } - - // 2. Add play_count and last_played columns to audio_files. - if _, err := db.ExecContext(ctx, - `ALTER TABLE audio_files - ADD COLUMN play_count INTEGER NOT NULL DEFAULT 0`, - ); err != nil { - if !isDuplicateColumnErr(err) { - return fmt.Errorf( - "migration 10: could not add play_count column: %w", - err, - ) - } - } - - if _, err := db.ExecContext(ctx, - `ALTER TABLE audio_files - ADD COLUMN last_played DATETIME`, - ); err != nil { - if !isDuplicateColumnErr(err) { - return fmt.Errorf( - "migration 10: could not add last_played column: %w", - err, - ) - } - } - - // 3. Recreate track_metadata VIEW to include play_count and last_played. - if _, err := db.ExecContext( - ctx, "DROP VIEW IF EXISTS track_metadata", - ); err != nil { - return fmt.Errorf( - "migration 10: could not drop track_metadata VIEW: %w", - err, - ) - } - - if _, err := db.ExecContext(ctx, ` - CREATE VIEW IF NOT EXISTS track_metadata AS - SELECT - af.id, - af.file_path, - af.length_milliseconds, - COALESCE(r.name, '') AS title, - COALESCE(ac.text, '') AS artist_name, - r.track_number, - r.disc_number, - COALESCE(rg.name, '') AS album, - CAST(COALESCE( - (SELECT GROUP_CONCAT(g.name, '||') - FROM recording_genres rg_sub - JOIN genres g ON rg_sub.genre_id = g.id - WHERE rg_sub.recording_id = r.id), - '' - ) AS TEXT) AS genre, - COALESCE(r.year, 0) AS year, - COALESCE(r.composer, '') AS composer, - COALESCE(ft.extension, '') AS file_type, - af.sample_rate, - af.bit_depth, - af.channels, - af.bitrate, - af.file_size, - af.library_id, - af.play_count, - af.last_played - FROM audio_files af - LEFT JOIN recordings r ON af.recording_id = r.id - LEFT JOIN artist_credit ac ON r.artist_credit_id = ac.id - LEFT JOIN ( - SELECT recording_id, - MIN(release_group_id) AS release_group_id - FROM release_group_recordings - GROUP BY recording_id - ) rgr ON r.id = rgr.recording_id - LEFT JOIN release_groups rg ON rgr.release_group_id = rg.id - LEFT JOIN file_types ft ON af.file_type_id = ft.id`, - ); err != nil { - return fmt.Errorf( - "migration 10: could not create track_metadata VIEW: %w", - err, - ) - } - - if _, err := db.ExecContext( - ctx, "PRAGMA user_version = 10", - ); err != nil { - return fmt.Errorf( - "could not set user_version to 10: %w", err, - ) - } - - logger.Info("migration 10 complete") - - return nil -} - -// migration11ExploreCache creates the explore_cache table for -// MusicBrainz and ListenBrainz API response caching. The table -// stores raw JSON keyed by URL with TTL-based expiry and optional -// MBID columns for future autotagging lookups. -func migration11ExploreCache( - ctx context.Context, - db *sql.DB, - logger *slog.Logger, -) error { - logger.Info( - "applying migration 11: explore_cache table", - ) - - if _, err := db.ExecContext(ctx, ` - CREATE TABLE IF NOT EXISTS explore_cache ( - url_key TEXT PRIMARY KEY, - response TEXT NOT NULL, - mbid TEXT, - entity_type TEXT, - expires_at DATETIME NOT NULL, - created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP - ) - `); err != nil { - return fmt.Errorf( - "migration 11: could not create explore_cache table: %w", - err, - ) - } - - if _, err := db.ExecContext(ctx, ` - CREATE INDEX IF NOT EXISTS idx_explore_cache_expires - ON explore_cache(expires_at) - `); err != nil { - return fmt.Errorf( - "migration 11: could not create expires index: %w", - err, - ) - } - - if _, err := db.ExecContext(ctx, ` - CREATE INDEX IF NOT EXISTS idx_explore_cache_mbid - ON explore_cache(mbid) - `); err != nil { - return fmt.Errorf( - "migration 11: could not create mbid index: %w", - err, - ) - } - - if _, err := db.ExecContext( - ctx, "PRAGMA user_version = 11", - ); err != nil { - return fmt.Errorf( - "could not set user_version to 11: %w", err, - ) - } - - logger.Info("migration 11 complete") - - return nil -} - -// migration12ExploreSearchIndex creates the explore_index table, -// the FTS5 virtual table for full-text search, sync triggers, and -// the explore_index_meta table for build tracking. -func migration12ExploreSearchIndex( - ctx context.Context, - db *sql.DB, - logger *slog.Logger, -) error { - logger.Info("applying migration 12: explore search index") - - // Content table — slim denormalized rows for search. - if _, err := db.ExecContext(ctx, ` - CREATE TABLE IF NOT EXISTS explore_index ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - entity_type TEXT NOT NULL, - mbid TEXT NOT NULL, - title TEXT NOT NULL, - artist_name TEXT NOT NULL, - artist_mbid TEXT NOT NULL, - popularity INTEGER NOT NULL DEFAULT 0, - extra_json TEXT - ) - `); err != nil { - return fmt.Errorf("migration 12: create explore_index: %w", err) - } - - if _, err := db.ExecContext(ctx, ` - CREATE UNIQUE INDEX IF NOT EXISTS idx_explore_index_mbid - ON explore_index(entity_type, mbid) - `); err != nil { - return fmt.Errorf("migration 12: create mbid index: %w", err) - } - - // FTS5 virtual table backed by the content table. - if _, err := db.ExecContext(ctx, ` - CREATE VIRTUAL TABLE IF NOT EXISTS explore_index_fts USING fts5( - title, artist_name, - content='explore_index', - content_rowid='id' - ) - `); err != nil { - return fmt.Errorf("migration 12: create FTS5 table: %w", err) - } - - // Triggers to keep FTS in sync. - if _, err := db.ExecContext(ctx, ` - CREATE TRIGGER IF NOT EXISTS explore_index_ai AFTER INSERT ON explore_index BEGIN - INSERT INTO explore_index_fts(rowid, title, artist_name) - VALUES (new.id, new.title, new.artist_name); - END - `); err != nil { - return fmt.Errorf("migration 12: create insert trigger: %w", err) - } - - if _, err := db.ExecContext(ctx, ` - CREATE TRIGGER IF NOT EXISTS explore_index_ad AFTER DELETE ON explore_index BEGIN - INSERT INTO explore_index_fts(explore_index_fts, rowid, title, artist_name) - VALUES ('delete', old.id, old.title, old.artist_name); - END - `); err != nil { - return fmt.Errorf("migration 12: create delete trigger: %w", err) - } - - if _, err := db.ExecContext(ctx, ` - CREATE TRIGGER IF NOT EXISTS explore_index_au AFTER UPDATE ON explore_index BEGIN - INSERT INTO explore_index_fts(explore_index_fts, rowid, title, artist_name) - VALUES ('delete', old.id, old.title, old.artist_name); - INSERT INTO explore_index_fts(rowid, title, artist_name) - VALUES (new.id, new.title, new.artist_name); - END - `); err != nil { - return fmt.Errorf("migration 12: create update trigger: %w", err) - } - - // Metadata table for build tracking. - if _, err := db.ExecContext(ctx, ` - CREATE TABLE IF NOT EXISTS explore_index_meta ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL - ) - `); err != nil { - return fmt.Errorf("migration 12: create meta table: %w", err) - } - - if _, err := db.ExecContext( - ctx, "PRAGMA user_version = 12", - ); err != nil { - return fmt.Errorf("could not set user_version to 12: %w", err) - } - - logger.Info("migration 12 complete") - - return nil -} - -// migration13MBIDColumns adds MusicBrainz ID columns to artists, -// release_groups, and recordings for linking local library entities -// to MusicBrainz/ListenBrainz explore data. -func migration13MBIDColumns( - ctx context.Context, - db *sql.DB, - logger *slog.Logger, -) error { - logger.Info("applying migration 13: MusicBrainz ID columns") - - alterStmts := []struct { - table string - column string - }{ - {"artists", "mbid"}, - {"release_groups", "mbid"}, - {"recordings", "mbid"}, - } - - for _, s := range alterStmts { - stmt := fmt.Sprintf( - "ALTER TABLE %s ADD COLUMN %s TEXT", s.table, s.column, - ) - - if _, err := db.ExecContext(ctx, stmt); err != nil { - // Column may already exist from a partial migration. - if !strings.Contains(err.Error(), "duplicate column") { - return fmt.Errorf("migration 13: alter %s: %w", s.table, err) - } - } - } - - // Partial indexes for MBID lookups (only index non-NULL rows). - indexes := []string{ - "CREATE INDEX IF NOT EXISTS idx_artists_mbid ON artists(mbid) WHERE mbid IS NOT NULL", - "CREATE INDEX IF NOT EXISTS idx_release_groups_mbid ON release_groups(mbid) WHERE mbid IS NOT NULL", - "CREATE INDEX IF NOT EXISTS idx_recordings_mbid ON recordings(mbid) WHERE mbid IS NOT NULL", - } - - for _, stmt := range indexes { - if _, err := db.ExecContext(ctx, stmt); err != nil { - return fmt.Errorf("migration 13: create index: %w", err) - } - } - - if _, err := db.ExecContext( - ctx, "PRAGMA user_version = 13", - ); err != nil { - return fmt.Errorf("could not set user_version to 13: %w", err) - } - - logger.Info("migration 13 complete") - - return nil -} - -// migration14ExploreAliases adds an aliases column to explore_index -// and rebuilds the FTS5 virtual table with three searchable columns -// (title, artist_name, aliases) for alias-aware search. -func migration14ExploreAliases( - ctx context.Context, - db *sql.DB, - logger *slog.Logger, -) error { - logger.Info("applying migration 14: explore index aliases + FTS5 rebuild") - - // Add aliases column to content table. - if _, err := db.ExecContext(ctx, - "ALTER TABLE explore_index ADD COLUMN aliases TEXT DEFAULT ''", - ); err != nil { - if !strings.Contains(err.Error(), "duplicate column") { - return fmt.Errorf("migration 14: alter explore_index: %w", err) - } - } - - // Drop old triggers. - for _, name := range []string{ - "explore_index_ai", "explore_index_ad", "explore_index_au", - } { - if _, err := db.ExecContext(ctx, - "DROP TRIGGER IF EXISTS "+name, - ); err != nil { - return fmt.Errorf("migration 14: drop trigger %s: %w", name, err) - } - } - - // Drop and recreate FTS5 with 3 columns. - if _, err := db.ExecContext(ctx, - "DROP TABLE IF EXISTS explore_index_fts", - ); err != nil { - return fmt.Errorf("migration 14: drop FTS5: %w", err) - } - - if _, err := db.ExecContext(ctx, ` - CREATE VIRTUAL TABLE explore_index_fts USING fts5( - title, artist_name, aliases, - content='explore_index', - content_rowid='id' - ) - `); err != nil { - return fmt.Errorf("migration 14: create FTS5: %w", err) - } - - // Recreate triggers with 3 columns. - if _, err := db.ExecContext(ctx, ` - CREATE TRIGGER explore_index_ai AFTER INSERT ON explore_index BEGIN - INSERT INTO explore_index_fts(rowid, title, artist_name, aliases) - VALUES (new.id, new.title, new.artist_name, new.aliases); - END - `); err != nil { - return fmt.Errorf("migration 14: create insert trigger: %w", err) - } - - if _, err := db.ExecContext(ctx, ` - CREATE TRIGGER explore_index_ad AFTER DELETE ON explore_index BEGIN - INSERT INTO explore_index_fts(explore_index_fts, rowid, title, artist_name, aliases) - VALUES ('delete', old.id, old.title, old.artist_name, old.aliases); - END - `); err != nil { - return fmt.Errorf("migration 14: create delete trigger: %w", err) - } - - if _, err := db.ExecContext(ctx, ` - CREATE TRIGGER explore_index_au AFTER UPDATE ON explore_index BEGIN - INSERT INTO explore_index_fts(explore_index_fts, rowid, title, artist_name, aliases) - VALUES ('delete', old.id, old.title, old.artist_name, old.aliases); - INSERT INTO explore_index_fts(rowid, title, artist_name, aliases) - VALUES (new.id, new.title, new.artist_name, new.aliases); - END - `); err != nil { - return fmt.Errorf("migration 14: create update trigger: %w", err) - } - - // Rebuild FTS5 index from existing content table rows. - if _, err := db.ExecContext(ctx, - "INSERT INTO explore_index_fts(explore_index_fts) VALUES ('rebuild')", - ); err != nil { - return fmt.Errorf("migration 14: rebuild FTS5: %w", err) - } - - if _, err := db.ExecContext( - ctx, "PRAGMA user_version = 14", - ); err != nil { - return fmt.Errorf("could not set user_version to 14: %w", err) - } - - // Clear the index build timestamp so the next build populates aliases. - _, _ = db.ExecContext(ctx, - "DELETE FROM explore_index_meta WHERE key IN ('tier1_built', 'discog_built')", - ) - - logger.Info("migration 14 complete") - - return nil -} - -// migration15PersonalizationColumns adds in_library and is_similar -// columns to explore_index for personalized search ranking. -func migration15PersonalizationColumns( - ctx context.Context, - db *sql.DB, - logger *slog.Logger, -) error { - logger.Info("applying migration 15: personalization columns") - - for _, col := range []string{"in_library", "is_similar"} { - stmt := fmt.Sprintf( - "ALTER TABLE explore_index ADD COLUMN %s INTEGER NOT NULL DEFAULT 0", col, - ) - - if _, err := db.ExecContext(ctx, stmt); err != nil { - if !strings.Contains(err.Error(), "duplicate column") { - return fmt.Errorf("migration 15: alter explore_index: %w", err) - } - } - } - - // Backfill in_library for artists already in the library. - if _, err := db.ExecContext(ctx, ` - UPDATE explore_index SET in_library = 1 - WHERE entity_type = 'artist' - AND mbid IN (SELECT mbid FROM artists WHERE mbid IS NOT NULL AND mbid != '') - `); err != nil { - logger.Warn("migration 15: backfill in_library artists", "error", err) - } - - // Backfill in_library for release groups already in the library. - if _, err := db.ExecContext(ctx, ` - UPDATE explore_index SET in_library = 1 - WHERE entity_type = 'release_group' - AND mbid IN (SELECT mbid FROM release_groups WHERE mbid IS NOT NULL AND mbid != '') - `); err != nil { - logger.Warn("migration 15: backfill in_library release_groups", "error", err) - } - - // Clear discog_built so the next index build populates these flags. - _, _ = db.ExecContext(ctx, - "DELETE FROM explore_index_meta WHERE key = 'discog_built'", - ) - - if _, err := db.ExecContext( - ctx, "PRAGMA user_version = 15", - ); err != nil { - return fmt.Errorf("could not set user_version to 15: %w", err) - } - - logger.Info("migration 15 complete") - - return nil -} - -// migration16ArtistImages creates the artist_images table for -// storing multiple artist photos from multiple sources, with -// thumbnail generation for the primary image. -func migration16ArtistImages( - ctx context.Context, - db *sql.DB, - logger *slog.Logger, -) error { - logger.Info("applying migration 16: artist_images table") - - if _, err := db.ExecContext(ctx, ` - CREATE TABLE IF NOT EXISTS artist_images ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - artist_mbid TEXT NOT NULL, - source TEXT NOT NULL, - source_url TEXT NOT NULL, - file_path TEXT NOT NULL, - is_primary INTEGER NOT NULL DEFAULT 0, - sort_order INTEGER NOT NULL DEFAULT 0, - width INTEGER, - height INTEGER, - file_size INTEGER, - created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP - ) - `); err != nil { - return fmt.Errorf("migration 16: create artist_images: %w", err) - } - - if _, err := db.ExecContext(ctx, ` - CREATE INDEX IF NOT EXISTS idx_artist_images_mbid - ON artist_images(artist_mbid) - `); err != nil { - return fmt.Errorf("migration 16: create mbid index: %w", err) - } - - if _, err := db.ExecContext(ctx, ` - CREATE UNIQUE INDEX IF NOT EXISTS idx_artist_images_source - ON artist_images(artist_mbid, source, source_url) - `); err != nil { - return fmt.Errorf("migration 16: create source index: %w", err) - } - - if _, err := db.ExecContext( - ctx, "PRAGMA user_version = 16", - ); err != nil { - return fmt.Errorf("could not set user_version to 16: %w", err) - } - - logger.Info("migration 16 complete") - - return nil -} - -func migration17SimilarArtistMap( - ctx context.Context, - db *sql.DB, - logger *slog.Logger, -) error { - logger.Info("applying migration 17: similar_artist_map table") - - if _, err := db.ExecContext(ctx, ` - CREATE TABLE IF NOT EXISTS similar_artist_map ( - source_artist_mbid TEXT NOT NULL, - similar_artist_mbid TEXT NOT NULL, - similar_artist_name TEXT NOT NULL, - score INTEGER NOT NULL DEFAULT 0, - PRIMARY KEY (source_artist_mbid, similar_artist_mbid) - ) - `); err != nil { - return fmt.Errorf("migration 17: create similar_artist_map: %w", err) - } - - if _, err := db.ExecContext(ctx, ` - CREATE INDEX IF NOT EXISTS idx_similar_artist_map_source - ON similar_artist_map(source_artist_mbid) - `); err != nil { - return fmt.Errorf("migration 17: create source index: %w", err) - } - - if _, err := db.ExecContext(ctx, - "PRAGMA user_version = 17", - ); err != nil { - return fmt.Errorf("could not set user_version to 17: %w", err) - } - - logger.Info("migration 17 complete") - - return nil -} - -// migration18TrackCoverArt recreates the track_metadata VIEW to -// include cover_art_path via a JOIN to the cover_art table. -func migration18TrackCoverArt( - ctx context.Context, - db *sql.DB, - logger *slog.Logger, -) error { - logger.Info("applying migration 18: track_metadata cover_art_path") - - if _, err := db.ExecContext( - ctx, "DROP VIEW IF EXISTS track_metadata", - ); err != nil { - return fmt.Errorf("migration 18: drop view: %w", err) - } - - if _, err := db.ExecContext(ctx, ` - CREATE VIEW IF NOT EXISTS track_metadata AS - SELECT - af.id, - af.file_path, - af.length_milliseconds, - COALESCE(r.name, '') AS title, - COALESCE(ac.text, '') AS artist_name, - r.track_number, - r.disc_number, - COALESCE(rg.name, '') AS album, - CAST(COALESCE( - (SELECT GROUP_CONCAT(g.name, '||') - FROM recording_genres rg_sub - JOIN genres g ON rg_sub.genre_id = g.id - WHERE rg_sub.recording_id = r.id), - '' - ) AS TEXT) AS genre, - COALESCE(r.year, 0) AS year, - COALESCE(r.composer, '') AS composer, - COALESCE(ft.extension, '') AS file_type, - af.sample_rate, - af.bit_depth, - af.channels, - af.bitrate, - af.file_size, - af.library_id, - af.play_count, - af.last_played, - COALESCE(ca.file_path, '') AS cover_art_path - FROM audio_files af - LEFT JOIN recordings r ON af.recording_id = r.id - LEFT JOIN artist_credit ac ON r.artist_credit_id = ac.id - LEFT JOIN ( - SELECT recording_id, - MIN(release_group_id) AS release_group_id - FROM release_group_recordings - GROUP BY recording_id - ) rgr ON r.id = rgr.recording_id - LEFT JOIN release_groups rg ON rgr.release_group_id = rg.id - LEFT JOIN cover_art ca ON rg.cover_art_id = ca.id - LEFT JOIN file_types ft ON af.file_type_id = ft.id - `); err != nil { - return fmt.Errorf("migration 18: create view: %w", err) - } - - if _, err := db.ExecContext(ctx, - "PRAGMA user_version = 18", - ); err != nil { - return fmt.Errorf("could not set user_version to 18: %w", err) - } - - logger.Info("migration 18 complete") - - return nil -} - -// readLibraryDirFromTOML reads the TOML config file and returns -// the Library.DirectoryPath value, or "" if not configured. -func readLibraryDirFromTOML(logger *slog.Logger) string { - configDir, err := system.GetUserConfigDirPath() - if err != nil { - logger.Debug( - "could not get config dir for TOML read", - "err", err, - ) - - return "" - } - - configPath := path.Join(configDir, "config.toml") - - data, err := os.ReadFile(configPath) - if err != nil { - logger.Debug( - "could not read config.toml", - "path", configPath, - "err", err, - ) - - return "" - } - - // Minimal struct to extract only the Library.DirectoryPath field. - var cfg struct { - Library struct { - DirectoryPath string `toml:"DirectoryPath"` - } `toml:"Library"` - } - - if _, err := toml.Decode(string(data), &cfg); err != nil { - logger.Debug( - "could not parse config.toml", - "path", configPath, - "err", err, - ) - - return "" - } - - return cfg.Library.DirectoryPath -} - -// removeLibraryDirFromTOML reads the TOML config, removes the -// Library.DirectoryPath field, and writes the config back. This -// ensures the libraries table is the sole source of truth after -// migration. -func removeLibraryDirFromTOML(logger *slog.Logger) { - configDir, err := system.GetUserConfigDirPath() - if err != nil { - logger.Warn( - "could not get config dir for TOML cleanup", - "err", err, - ) - - return - } - - configPath := path.Join(configDir, "config.toml") - - data, err := os.ReadFile(configPath) - if err != nil { - logger.Warn( - "could not read config.toml for cleanup", - "path", configPath, - "err", err, - ) - - return - } - - // Parse the full config as a generic map to preserve all fields. - var cfg map[string]any - - if _, err := toml.Decode(string(data), &cfg); err != nil { - logger.Warn( - "could not parse config.toml for cleanup", - "err", err, - ) - - return - } - - // Remove DirectoryPath from [Library] section. - if lib, ok := cfg["Library"].(map[string]any); ok { - delete(lib, "DirectoryPath") - - // If Library section is now empty, remove it entirely. - if len(lib) == 0 { - delete(cfg, "Library") - } - } - - // Write updated config back. - out, err := toml.Marshal(cfg) - if err != nil { - logger.Warn( - "could not marshal updated config.toml", - "err", err, - ) - - return - } - - if err := os.WriteFile(configPath, out, 0o644); err != nil { - logger.Warn( - "could not write updated config.toml", - "path", configPath, - "err", err, - ) - - return - } - - logger.Info( - "removed Library.DirectoryPath from config.toml", - "path", configPath, - ) -} - -// migration19TrackMBIDs recreates the track_metadata VIEW to include -// artist_mbid and release_group_mbid columns via the relational -// chain: recording → artist_credit → artist_credit_artist → artist. -func migration19TrackMBIDs( - ctx context.Context, - db *sql.DB, - logger *slog.Logger, -) error { - logger.Info("applying migration 19: track_metadata MBID columns") - - if _, err := db.ExecContext( - ctx, "DROP VIEW IF EXISTS track_metadata", - ); err != nil { - return fmt.Errorf("migration 19: drop view: %w", err) - } - - if _, err := db.ExecContext(ctx, ` - CREATE VIEW IF NOT EXISTS track_metadata AS - SELECT - af.id, - af.file_path, - af.length_milliseconds, - COALESCE(r.name, '') AS title, - COALESCE(ac.text, '') AS artist_name, - r.track_number, - r.disc_number, - COALESCE(rg.name, '') AS album, - CAST(COALESCE( - (SELECT GROUP_CONCAT(g.name, '||') - FROM recording_genres rg_sub - JOIN genres g ON rg_sub.genre_id = g.id - WHERE rg_sub.recording_id = r.id), - '' - ) AS TEXT) AS genre, - COALESCE(r.year, 0) AS year, - COALESCE(r.composer, '') AS composer, - COALESCE(ft.extension, '') AS file_type, - af.sample_rate, - af.bit_depth, - af.channels, - af.bitrate, - af.file_size, - af.library_id, - af.play_count, - af.last_played, - COALESCE(ca.file_path, '') AS cover_art_path, - COALESCE(a.mbid, '') AS artist_mbid, - COALESCE(rg.mbid, '') AS release_group_mbid, - COALESCE(r.mbid, '') AS recording_mbid - FROM audio_files af - LEFT JOIN recordings r ON af.recording_id = r.id - LEFT JOIN artist_credit ac ON r.artist_credit_id = ac.id - LEFT JOIN artist_credit_artist aca ON aca.credit_id = ac.id - LEFT JOIN artists a ON a.id = aca.artist_id - LEFT JOIN ( - SELECT recording_id, - MIN(release_group_id) AS release_group_id - FROM release_group_recordings - GROUP BY recording_id - ) rgr ON r.id = rgr.recording_id - LEFT JOIN release_groups rg ON rgr.release_group_id = rg.id - LEFT JOIN cover_art ca ON rg.cover_art_id = ca.id - LEFT JOIN file_types ft ON af.file_type_id = ft.id - `); err != nil { - return fmt.Errorf("migration 19: create view: %w", err) - } - - if _, err := db.ExecContext(ctx, - "PRAGMA user_version = 19", - ); err != nil { - return fmt.Errorf("could not set user_version to 19: %w", err) - } - - logger.Info("migration 19 complete") - - return nil -} - -// migration20TrackRecordingMBID recreates the track_metadata VIEW to -// add the recording_mbid column (missed in migration 19). -func migration20TrackRecordingMBID( - ctx context.Context, - db *sql.DB, - logger *slog.Logger, -) error { - logger.Info("applying migration 20: track_metadata recording_mbid") - - if _, err := db.ExecContext( - ctx, "DROP VIEW IF EXISTS track_metadata", - ); err != nil { - return fmt.Errorf("migration 20: drop view: %w", err) - } - - if _, err := db.ExecContext(ctx, ` - CREATE VIEW IF NOT EXISTS track_metadata AS - SELECT - af.id, - af.file_path, - af.length_milliseconds, - COALESCE(r.name, '') AS title, - COALESCE(ac.text, '') AS artist_name, - r.track_number, - r.disc_number, - COALESCE(rg.name, '') AS album, - CAST(COALESCE( - (SELECT GROUP_CONCAT(g.name, '||') - FROM recording_genres rg_sub - JOIN genres g ON rg_sub.genre_id = g.id - WHERE rg_sub.recording_id = r.id), - '' - ) AS TEXT) AS genre, - COALESCE(r.year, 0) AS year, - COALESCE(r.composer, '') AS composer, - COALESCE(ft.extension, '') AS file_type, - af.sample_rate, - af.bit_depth, - af.channels, - af.bitrate, - af.file_size, - af.library_id, - af.play_count, - af.last_played, - COALESCE(ca.file_path, '') AS cover_art_path, - COALESCE(a.mbid, '') AS artist_mbid, - COALESCE(rg.mbid, '') AS release_group_mbid, - COALESCE(r.mbid, '') AS recording_mbid - FROM audio_files af - LEFT JOIN recordings r ON af.recording_id = r.id - LEFT JOIN artist_credit ac ON r.artist_credit_id = ac.id - LEFT JOIN artist_credit_artist aca ON aca.credit_id = ac.id - LEFT JOIN artists a ON a.id = aca.artist_id - LEFT JOIN ( - SELECT recording_id, - MIN(release_group_id) AS release_group_id - FROM release_group_recordings - GROUP BY recording_id - ) rgr ON r.id = rgr.recording_id - LEFT JOIN release_groups rg ON rgr.release_group_id = rg.id - LEFT JOIN cover_art ca ON rg.cover_art_id = ca.id - LEFT JOIN file_types ft ON af.file_type_id = ft.id - `); err != nil { - return fmt.Errorf("migration 20: create view: %w", err) - } - - // Purge stale ListenBrainz top-recordings cache entries that - // were written before the caaReleaseMbid field was added to - // the LBTopRecording struct. Without this, cached entries - // render without cover art thumbnails in the top tracks section. - if _, err := db.ExecContext(ctx, - "DELETE FROM explore_cache WHERE url_key LIKE 'lb:top-recordings:%'", - ); err != nil { - logger.Warn("migration 20: could not purge stale top-recordings cache", "err", err) - // Non-fatal — entries will expire naturally via TTL. - } - - if _, err := db.ExecContext(ctx, - "PRAGMA user_version = 20", - ); err != nil { - return fmt.Errorf("could not set user_version to 20: %w", err) - } - - logger.Info("migration 20 complete") - - return nil -} - -// migration31TagStatus adds the tag_status column to audio_files, -// indexes the "untagged" slice for the pending-count badge, and -// backfills rows whose recording already carries an MBID as -// `user_confirmed`. Everything else stays at the `untagged` -// default. The column-level CHECK constraint is added inline with -// the ALTER TABLE — SQLite supports column constraints in ADD -// COLUMN, so existing DBs pick it up too. -func migration31TagStatus( - ctx context.Context, - db *sql.DB, - logger *slog.Logger, -) error { - logger.Info("applying migration 31: tag_status column") - - if _, err := db.ExecContext(ctx, ` - ALTER TABLE audio_files - ADD COLUMN tag_status TEXT NOT NULL DEFAULT 'untagged' - CHECK(tag_status IN ( - 'untagged', 'auto_matched', 'user_confirmed', 'user_skipped_permanent' - )) - `); err != nil && !isDuplicateColumnErr(err) { - return fmt.Errorf("migration 31: add tag_status: %w", err) - } - - if _, err := db.ExecContext(ctx, ` - CREATE INDEX IF NOT EXISTS idx_audio_files_tag_status_untagged - ON audio_files(library_id) WHERE tag_status = 'untagged' - `); err != nil { - return fmt.Errorf("migration 31: create index: %w", err) - } - - if _, err := db.ExecContext(ctx, ` - UPDATE audio_files - SET tag_status = 'user_confirmed' - WHERE tag_status = 'untagged' - AND recording_id IN ( - SELECT id FROM recordings - WHERE mbid IS NOT NULL AND mbid != '' - ) - `); err != nil { - return fmt.Errorf("migration 31: backfill tag_status: %w", err) - } - - if _, err := db.ExecContext(ctx, - "PRAGMA user_version = 31", - ); err != nil { - return fmt.Errorf("migration 31: set user_version: %w", err) - } - - logger.Info("migration 31 complete") - - return nil -} - -// migration32TaggingItems creates the tagging_items table and adds -// the group_key column to audio_files, then backfills both from the -// current `audio_files` / `recordings` / `release_groups` state. -// The Go-side autotag.GroupKey helper is the single source of truth -// for the key format (keeps the hash algorithm decoupled from SQL). -// -// SAFETY: Hand-crafted ALTER TABLE + CREATE TABLE + streaming -// backfill inside a single transaction. -func migration32TaggingItems( - ctx context.Context, - db *sql.DB, - logger *slog.Logger, -) error { - logger.Info("applying migration 32: tagging_items + group_key") - - if _, err := db.ExecContext(ctx, ` - CREATE TABLE IF NOT EXISTS tagging_items ( - group_key TEXT PRIMARY KEY, - library_id INTEGER NOT NULL, - track_count INTEGER NOT NULL DEFAULT 0, - album_name TEXT NOT NULL DEFAULT '', - album_artist TEXT NOT NULL DEFAULT '', - disc_number INTEGER NOT NULL DEFAULT 0, - best_match_release_mbid TEXT, - score REAL, - last_checked_at DATETIME, - status TEXT NOT NULL DEFAULT 'pending' - CHECK(status IN ('pending', 'matched', 'confirmed', 'skipped')), - created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - FOREIGN KEY(library_id) REFERENCES libraries(id) - ) - `); err != nil { - return fmt.Errorf("migration 32: create tagging_items: %w", err) - } - - if _, err := db.ExecContext(ctx, ` - CREATE INDEX IF NOT EXISTS idx_tagging_items_library_status - ON tagging_items(library_id, status) - `); err != nil { - return fmt.Errorf("migration 32: create library_status index: %w", err) - } - - if _, err := db.ExecContext(ctx, ` - CREATE INDEX IF NOT EXISTS idx_tagging_items_status_pending - ON tagging_items(library_id) WHERE status = 'pending' - `); err != nil { - return fmt.Errorf("migration 32: create pending index: %w", err) - } - - if _, err := db.ExecContext(ctx, ` - ALTER TABLE audio_files - ADD COLUMN group_key TEXT NOT NULL DEFAULT '' - `); err != nil && !isDuplicateColumnErr(err) { - return fmt.Errorf("migration 32: add group_key: %w", err) - } - - if _, err := db.ExecContext(ctx, ` - CREATE INDEX IF NOT EXISTS idx_audio_files_group_key - ON audio_files(group_key) WHERE group_key != '' - `); err != nil { - return fmt.Errorf("migration 32: create group_key index: %w", err) - } - - if err := backfillGroupKeys(ctx, db, logger); err != nil { - return fmt.Errorf("migration 32: backfill group_key: %w", err) - } - - if err := aggregateTaggingItems(ctx, db, logger); err != nil { - return fmt.Errorf("migration 32: aggregate tagging_items: %w", err) - } - - if _, err := db.ExecContext(ctx, - "PRAGMA user_version = 32", - ); err != nil { - return fmt.Errorf("migration 32: set user_version: %w", err) - } - - logger.Info("migration 32 complete") - - return nil -} - -// backfillGroupKeys streams existing audio_files rows in batches of -// ~500 and writes the computed group_key back via a single UPDATE -// per row inside one transaction. It joins to release_groups for -// the album name and recordings for the disc number; both fall back -// to the zero value when absent. -func backfillGroupKeys( - ctx context.Context, - db *sql.DB, - logger *slog.Logger, -) error { - const batchSize = 500 - - type row struct { - id int64 - libraryID int64 - filePath string - discNumber int64 - } - - for { - // Each pass reads the next N rows with group_key still empty; - // once updated, they drop out of the filter, so no OFFSET - // bookkeeping is needed. - rows, err := db.QueryContext(ctx, ` - SELECT af.id, af.library_id, af.file_path, - COALESCE(r.disc_number, 0) - FROM audio_files af - LEFT JOIN recordings r ON af.recording_id = r.id - WHERE af.group_key = '' - ORDER BY af.id - LIMIT ? - `, batchSize) - if err != nil { - return fmt.Errorf("select batch: %w", err) - } - - batch := make([]row, 0, batchSize) - - for rows.Next() { - var r row - if scanErr := rows.Scan( - &r.id, &r.libraryID, &r.filePath, &r.discNumber, - ); scanErr != nil { - _ = rows.Close() - - return fmt.Errorf("scan row: %w", scanErr) - } - - batch = append(batch, r) - } - - if closeErr := rows.Close(); closeErr != nil { - return fmt.Errorf("close rows: %w", closeErr) - } - - if len(batch) == 0 { - break - } - - tx, err := db.BeginTx(ctx, nil) - if err != nil { - return fmt.Errorf("begin tx: %w", err) - } - - for _, r := range batch { - key := autotag.GroupKey( - r.libraryID, r.filePath, int(r.discNumber), - ) - if _, err := tx.ExecContext(ctx, - `UPDATE audio_files SET group_key = ? WHERE id = ?`, - key, r.id, - ); err != nil { - _ = tx.Rollback() - - return fmt.Errorf("update row %d: %w", r.id, err) - } - } - - if err := tx.Commit(); err != nil { - return fmt.Errorf("commit tx: %w", err) - } - - logger.Debug( - "migration 32: backfilled group_key batch", "count", len(batch), - ) - - if len(batch) < batchSize { - break - } - } - - return nil -} - -// aggregateTaggingItems populates tagging_items from the now- -// populated audio_files.group_key, one row per (group_key, -// library_id) pair. Status defaults to `confirmed` when every -// track in the group already has tag_status `user_confirmed`, -// otherwise `pending`. -func aggregateTaggingItems( - ctx context.Context, - db *sql.DB, - logger *slog.Logger, -) error { - result, err := db.ExecContext(ctx, ` - INSERT INTO tagging_items ( - group_key, library_id, track_count, - album_name, album_artist, disc_number, status - ) - SELECT - af.group_key, - af.library_id, - COUNT(*) AS track_count, - COALESCE(MAX(rg.name), '') AS album_name, - COALESCE(MAX(ac.text), '') AS album_artist, - COALESCE(MAX(r.disc_number), 0) AS disc_number, - CASE WHEN SUM(CASE WHEN af.tag_status = 'user_confirmed' THEN 0 ELSE 1 END) = 0 - THEN 'confirmed' ELSE 'pending' END AS status - FROM audio_files af - LEFT JOIN recordings r ON af.recording_id = r.id - LEFT JOIN release_group_recordings rgr ON rgr.recording_id = r.id - LEFT JOIN release_groups rg ON rgr.release_group_id = rg.id - LEFT JOIN artist_credit ac ON rg.album_artist_credit_id = ac.id - WHERE af.group_key != '' - GROUP BY af.group_key, af.library_id - ON CONFLICT(group_key) DO NOTHING - `) - if err != nil { - return fmt.Errorf("insert aggregates: %w", err) - } - - if n, rowsErr := result.RowsAffected(); rowsErr == nil { - logger.Debug( - "migration 32: aggregated tagging_items rows", - "count", n, - ) - } - - return nil -} - -// migration33AutotagWarning adds the per-library flag that records -// whether the user has seen (and dismissed) the first-time autotag -// apply warning. Zero means "still warn"; one means acknowledged. -func migration33AutotagWarning( - ctx context.Context, - db *sql.DB, - logger *slog.Logger, -) error { - logger.Info("applying migration 33: libraries.autotag_warning_acked") - - if _, err := db.ExecContext(ctx, ` - ALTER TABLE libraries - ADD COLUMN autotag_warning_acked INTEGER NOT NULL DEFAULT 0 - `); err != nil && !isDuplicateColumnErr(err) { - return fmt.Errorf("migration 33: add column: %w", err) - } - - if _, err := db.ExecContext(ctx, - "PRAGMA user_version = 33", - ); err != nil { - return fmt.Errorf("migration 33: set user_version: %w", err) - } - - logger.Info("migration 33 complete") - - return nil -} - -// migration35OriginalYear adds release_groups.original_year (the -// release-group's MusicBrainz first-release-date year) and rebuilds -// the track_metadata view so its "year" column prefers the original -// release year over the file-tag year. This makes a 1973 album -// show as 1973 in the tracklist and smart-playlist year rules even -// when the user owns the 2010 remaster. release_year is added as -// a separate view column for callers that need the file-tag year. -func migration35OriginalYear( - ctx context.Context, - db *sql.DB, - logger *slog.Logger, -) error { - logger.Info("applying migration 35: release_groups.original_year + view rebuild") - - if _, err := db.ExecContext( - ctx, - `ALTER TABLE release_groups ADD COLUMN original_year INTEGER`, - ); err != nil { - // Tolerate duplicate-column on re-run / fresh-DB schema race. - if !strings.Contains(err.Error(), "duplicate column name") { - return fmt.Errorf("migration 35: add original_year: %w", err) - } - - logger.Warn("migration 35: original_year already present (ok if fresh)", "err", err) - } - - // Drop and recreate the track_metadata view so its year column - // picks up the new fallback chain. CREATE VIEW IF NOT EXISTS - // in the schema file is a no-op once the view exists, so we have - // to do this explicitly here for existing DBs. - // - // The body must match sql/schemas/track_metadata_view.sql. - if _, err := db.ExecContext(ctx, `DROP VIEW IF EXISTS track_metadata`); err != nil { - return fmt.Errorf("migration 35: drop view: %w", err) - } - - if _, err := db.ExecContext(ctx, ` - CREATE VIEW track_metadata AS - SELECT - af.id, - af.file_path, - af.length_milliseconds, - COALESCE(r.name, '') AS title, - COALESCE(ac.text, '') AS artist_name, - r.track_number, - r.disc_number, - COALESCE(rg.name, '') AS album, - CAST(COALESCE( - (SELECT GROUP_CONCAT(g.name, '||') - FROM recording_genres rg_sub - JOIN genres g ON rg_sub.genre_id = g.id - WHERE rg_sub.recording_id = r.id), - '' - ) AS TEXT) AS genre, - COALESCE(rg.original_year, rg.year, r.year, 0) AS year, - COALESCE(rg.year, r.year, 0) AS release_year, - COALESCE(r.composer, '') AS composer, - COALESCE(ft.extension, '') AS file_type, - af.sample_rate, - af.bit_depth, - af.channels, - af.bitrate, - af.file_size, - af.library_id, - af.play_count, - af.last_played, - COALESCE(ca.file_path, '') AS cover_art_path, - COALESCE(a.mbid, '') AS artist_mbid, - COALESCE(rg.mbid, '') AS release_group_mbid, - COALESCE(r.mbid, '') AS recording_mbid - FROM audio_files af - LEFT JOIN recordings r ON af.recording_id = r.id - LEFT JOIN artist_credit ac ON r.artist_credit_id = ac.id - LEFT JOIN artist_credit_artist aca ON aca.credit_id = ac.id - LEFT JOIN artists a ON a.id = aca.artist_id - LEFT JOIN ( - SELECT recording_id, - MIN(release_group_id) AS release_group_id - FROM release_group_recordings - GROUP BY recording_id - ) rgr ON r.id = rgr.recording_id - LEFT JOIN release_groups rg ON rgr.release_group_id = rg.id - LEFT JOIN cover_art ca ON rg.cover_art_id = ca.id - LEFT JOIN file_types ft ON af.file_type_id = ft.id - `); err != nil { - return fmt.Errorf("migration 35: recreate view: %w", err) - } - - if _, err := db.ExecContext(ctx, "PRAGMA user_version = 35"); err != nil { - return fmt.Errorf("migration 35: set user_version: %w", err) - } - - logger.Info("migration 35 complete") - - return nil -} - -// migration34FolderBasedGroupKey recomputes every audio_files -// row's group_key with the new folder-based algorithm (album tag -// dropped from the hash inputs). Tracks in the same parent -// directory + same disc number now share a key regardless of any -// per-track variation in their album tag — fixes the fragmenting -// behaviour where one album would produce N one-track tagging -// groups when its tracks carried slightly different album strings. -// -// After the recompute, tagging_items is wiped and re-aggregated -// from the new keys. The user's review state is reset; this is a -// blunt instrument but the right one — a partial migration would -// leave fragments of the old shape stranded in pending status. -// -// SAFETY: clears tagging_items unconditionally on existing DBs. -// On fresh DBs (test_data) the tagging_items aggregate at the end -// of the migration just no-ops since audio_files is empty. -func migration34FolderBasedGroupKey( - ctx context.Context, - db *sql.DB, - logger *slog.Logger, -) error { - logger.Info("applying migration 34: folder-based group_key") - - // Wipe stale state first so the recompute can stream into a - // clean tagging_items table. - if _, err := db.ExecContext(ctx, `DELETE FROM tagging_items`); err != nil { - return fmt.Errorf("migration 34: clear tagging_items: %w", err) - } - - // Force every audio_files.group_key back to '' so the existing - // backfill logic (which filters WHERE group_key = '') can - // recompute every row. - if _, err := db.ExecContext(ctx, - `UPDATE audio_files SET group_key = ''`, - ); err != nil { - return fmt.Errorf("migration 34: clear group_keys: %w", err) - } - - if err := backfillGroupKeys(ctx, db, logger); err != nil { - return fmt.Errorf("migration 34: backfill: %w", err) - } - - if err := aggregateTaggingItems(ctx, db, logger); err != nil { - return fmt.Errorf("migration 34: aggregate: %w", err) - } - - if _, err := db.ExecContext(ctx, - "PRAGMA user_version = 34", - ); err != nil { - return fmt.Errorf("migration 34: set user_version: %w", err) - } - - logger.Info("migration 34 complete") - - return nil -} diff --git a/backend/database/database_test.go b/backend/database/database_test.go index 9ee7fff..960238b 100644 --- a/backend/database/database_test.go +++ b/backend/database/database_test.go @@ -12,7 +12,7 @@ import ( // Migration 6 integration tests // --------------------------------------------------------------------------- -func TestMigration6FreshDB(t *testing.T) { +func TestSchemaCreatesLibrariesTable(t *testing.T) { t.Parallel() db := NewTestDB(t) @@ -172,32 +172,6 @@ func TestMigration6FreshDB(t *testing.T) { t.Error("track_metadata VIEW does not contain library_id") } - // Verify user_version >= 7. - var version int - - verRows, err := db.QueryContext("PRAGMA user_version") - if err != nil { - t.Fatalf("PRAGMA user_version: %v", err) - } - - if !verRows.Next() { - _ = verRows.Close() - - t.Fatal("PRAGMA user_version: no row returned") - } - - if err := verRows.Scan(&version); err != nil { - _ = verRows.Close() - - t.Fatalf("scan user_version: %v", err) - } - - _ = verRows.Close() - - if version < 7 { - t.Errorf("user_version = %d, want >= 7", version) - } - // Verify libraries table has only the sentinel row on fresh DB. count, err := db.Queries.CountLibraries(db.Ctx) if err != nil { @@ -213,7 +187,7 @@ func TestMigration6FreshDB(t *testing.T) { } } -func TestMigration6LibraryQueries(t *testing.T) { +func TestLibraryQueries(t *testing.T) { t.Parallel() db := NewTestDB(t) @@ -331,7 +305,7 @@ func TestMigration6LibraryQueries(t *testing.T) { } } -func TestMigration6PhantomPlaylistTracks(t *testing.T) { +func TestPhantomPlaylistTracksAreCleaned(t *testing.T) { t.Parallel() db, libID := NewTestDBWithLibrary(t, "Test", "/test/music") @@ -462,7 +436,7 @@ func TestMigration6PhantomPlaylistTracks(t *testing.T) { } } -func TestMigration6AudioFilesLibraryFK(t *testing.T) { +func TestAudioFilesLibraryForeignKey(t *testing.T) { t.Parallel() db, libID := NewTestDBWithLibrary(t, "Test", "/test/fk-lib") @@ -523,7 +497,7 @@ func TestMigration6AudioFilesLibraryFK(t *testing.T) { } } -func TestMigration6TrackMetadataViewHasLibraryID(t *testing.T) { +func TestTrackMetadataViewHasLibraryID(t *testing.T) { t.Parallel() db, libID := NewTestDBWithLibrary(t, "Test", "/test/view-lib") @@ -592,37 +566,11 @@ func TestMigration6TrackMetadataViewHasLibraryID(t *testing.T) { // Migration 9 integration tests // --------------------------------------------------------------------------- -func TestMigration9SmartPlaylistColumns(t *testing.T) { +func TestSmartPlaylistColumns(t *testing.T) { t.Parallel() db := NewTestDB(t) - // Verify user_version >= 9. - var version int - - verRows, err := db.QueryContext("PRAGMA user_version") - if err != nil { - t.Fatalf("PRAGMA user_version: %v", err) - } - - if !verRows.Next() { - _ = verRows.Close() - - t.Fatal("PRAGMA user_version: no row returned") - } - - if err := verRows.Scan(&version); err != nil { - _ = verRows.Close() - - t.Fatalf("scan user_version: %v", err) - } - - _ = verRows.Close() - - if version < 9 { - t.Errorf("user_version = %d, want >= 9", version) - } - // Verify playlists table has is_smart and smart_rules columns. hasSmart := false hasRules := false @@ -774,37 +722,11 @@ func TestMigration9SmartPlaylistColumns(t *testing.T) { // Migration 10 — play history tracking // --------------------------------------------------------------------------- -func TestMigration10PlayHistory(t *testing.T) { +func TestPlayHistoryTable(t *testing.T) { t.Parallel() db := NewTestDB(t) - // Verify user_version >= 10. - var version int - - verRows, err := db.QueryContext("PRAGMA user_version") - if err != nil { - t.Fatalf("PRAGMA user_version: %v", err) - } - - if !verRows.Next() { - _ = verRows.Close() - - t.Fatal("PRAGMA user_version: no row returned") - } - - if err := verRows.Scan(&version); err != nil { - _ = verRows.Close() - - t.Fatalf("scan user_version: %v", err) - } - - _ = verRows.Close() - - if version < 10 { - t.Errorf("user_version = %d, want >= 10", version) - } - // Verify play_history table exists. var tableCount int64 @@ -1058,7 +980,7 @@ func TestMigration10PlayHistory(t *testing.T) { // Migration 11 — explore_cache table // --------------------------------------------------------------------------- -func TestMigration11ExploreCache(t *testing.T) { +func TestHTTPCacheTable(t *testing.T) { t.Parallel() // explore_cache was split into http_cache + artist_metadata by @@ -1069,32 +991,6 @@ func TestMigration11ExploreCache(t *testing.T) { db := NewTestDB(t) - // Verify user_version >= 11. - var version int - - verRows, err := db.QueryContext("PRAGMA user_version") - if err != nil { - t.Fatalf("PRAGMA user_version: %v", err) - } - - if !verRows.Next() { - _ = verRows.Close() - - t.Fatal("PRAGMA user_version: no row returned") - } - - if err := verRows.Scan(&version); err != nil { - _ = verRows.Close() - - t.Fatalf("scan user_version: %v", err) - } - - _ = verRows.Close() - - if version < 11 { - t.Errorf("user_version = %d, want >= 11", version) - } - // Verify explore_cache table exists. var tableCount int64 diff --git a/backend/database/explorefts_test.go b/backend/database/explorefts_test.go new file mode 100644 index 0000000..1b4e069 --- /dev/null +++ b/backend/database/explorefts_test.go @@ -0,0 +1,159 @@ +package database + +import ( + "testing" +) + +// seedExploreRow inserts one explore_index row. +func seedExploreRow(t *testing.T, db *DB, mbid, title, artist string) { + t.Helper() + + if _, err := db.ExecContext(` + INSERT INTO explore_index (entity_type, mbid, title, artist_name, artist_mbid) + VALUES ('recording', ?, ?, ?, '') + `, mbid, title, artist); err != nil { + t.Fatalf("seed %s: %v", mbid, err) + } +} + +// ftsMatches returns how many FTS rows match a query. +func ftsMatches(t *testing.T, db *DB, query string) int { + t.Helper() + + rows, err := db.QueryContext( + "SELECT COUNT(*) FROM explore_index_fts WHERE explore_index_fts MATCH ?", query, + ) + if err != nil { + t.Fatalf("fts query %q: %v", query, err) + } + + defer func() { _ = rows.Close() }() + + n := 0 + + if rows.Next() { + if err := rows.Scan(&n); err != nil { + t.Fatalf("scan fts count: %v", err) + } + } + + if err := rows.Err(); err != nil { + t.Fatalf("fts rows: %v", err) + } + + return n +} + +// Rows written while FTS sync is suspended are invisible to search +// until the window closes — and fully searchable afterwards. This is +// the contract the dump import's bulk-load path depends on. +func TestExploreFTSSuspendResumeIndexesBulkRows(t *testing.T) { + db := NewTestDB(t) + + seedExploreRow(t, db, "mbid-before", "Before Suspend", "Artist One") + + if got := ftsMatches(t, db, "Before"); got != 1 { + t.Fatalf("matches for pre-suspend row = %d, want 1", got) + } + + if err := db.SuspendExploreIndexFTS(); err != nil { + t.Fatalf("suspend: %v", err) + } + + seedExploreRow(t, db, "mbid-during", "During Suspend", "Artist Two") + + if got := ftsMatches(t, db, "During"); got != 0 { + t.Errorf("matches while suspended = %d, want 0 (triggers should be off)", got) + } + + if err := db.ResumeExploreIndexFTS(); err != nil { + t.Fatalf("resume: %v", err) + } + + if got := ftsMatches(t, db, "During"); got != 1 { + t.Errorf("matches for bulk-loaded row after resume = %d, want 1", got) + } + + if got := ftsMatches(t, db, "Before"); got != 1 { + t.Errorf("matches for pre-suspend row after resume = %d, want 1", got) + } +} + +// The import wipes explore_index before reassembling it. With the +// triggers suspended that DELETE writes no FTS delete-markers, so the +// rebuild must be what clears the old rows out of search. +func TestExploreFTSResumeDropsDeletedRows(t *testing.T) { + db := NewTestDB(t) + + seedExploreRow(t, db, "mbid-stale", "Stale Recording", "Old Artist") + + if err := db.SuspendExploreIndexFTS(); err != nil { + t.Fatalf("suspend: %v", err) + } + + if _, err := db.ExecContext("DELETE FROM explore_index"); err != nil { + t.Fatalf("wipe: %v", err) + } + + seedExploreRow(t, db, "mbid-fresh", "Fresh Recording", "New Artist") + + if err := db.ResumeExploreIndexFTS(); err != nil { + t.Fatalf("resume: %v", err) + } + + if got := ftsMatches(t, db, "Stale"); got != 0 { + t.Errorf("matches for wiped row = %d, want 0", got) + } + + if got := ftsMatches(t, db, "Fresh"); got != 1 { + t.Errorf("matches for reassembled row = %d, want 1", got) + } +} + +// resumeFTS runs from a defer as well as at its natural point in the +// pipeline, so a second call must be harmless. +func TestExploreFTSResumeIsIdempotent(t *testing.T) { + db := NewTestDB(t) + + if err := db.SuspendExploreIndexFTS(); err != nil { + t.Fatalf("suspend: %v", err) + } + + seedExploreRow(t, db, "mbid-a", "Repeatable Resume", "Artist") + + if err := db.ResumeExploreIndexFTS(); err != nil { + t.Fatalf("first resume: %v", err) + } + + if err := db.ResumeExploreIndexFTS(); err != nil { + t.Fatalf("second resume: %v", err) + } + + if got := ftsMatches(t, db, "Repeatable"); got != 1 { + t.Errorf("matches after repeated resume = %d, want 1", got) + } + + // Triggers must still be live for ordinary writes after the window. + seedExploreRow(t, db, "mbid-b", "Postwindow Row", "Artist") + + if got := ftsMatches(t, db, "Postwindow"); got != 1 { + t.Errorf("matches for row written after resume = %d, want 1", got) + } +} + +// Suspending twice must not fail — the triggers are simply already gone. +func TestExploreFTSSuspendIsIdempotent(t *testing.T) { + db := NewTestDB(t) + + if err := db.SuspendExploreIndexFTS(); err != nil { + t.Fatalf("first suspend: %v", err) + } + + if err := db.SuspendExploreIndexFTS(); err != nil { + t.Fatalf("second suspend: %v", err) + } + + if err := db.ResumeExploreIndexFTS(); err != nil { + t.Fatalf("resume: %v", err) + } +} diff --git a/backend/database/search_test.go b/backend/database/search_test.go index 07c46ab..26018a5 100644 --- a/backend/database/search_test.go +++ b/backend/database/search_test.go @@ -1062,45 +1062,17 @@ func TestClearSearchIndexPreservesSchema(t *testing.T) { } // --------------------------------------------------------------------------- -// Migration test +// Schema constraint tests // --------------------------------------------------------------------------- -func TestMigrationsApplied(t *testing.T) { +func TestSearchIndexSchema(t *testing.T) { t.Parallel() db := NewTestDB(t) - // Verify user_version >= 3 (all 3 migrations applied). - // Use QueryContext + immediate Scan + Close to release the - // single connection before subsequent ExecContext calls. - var version int - - rows, err := db.QueryContext("PRAGMA user_version") - if err != nil { - t.Fatalf("PRAGMA user_version: %v", err) - } - - if !rows.Next() { - _ = rows.Close() - - t.Fatal("PRAGMA user_version: no row returned") - } - - if err := rows.Scan(&version); err != nil { - _ = rows.Close() - - t.Fatalf("scan user_version: %v", err) - } - - _ = rows.Close() - - if version < 3 { - t.Errorf("user_version = %d, want >= 3", version) - } - - // Verify the UNIQUE index from migration 3 exists by attempting - // a duplicate insert. First, create the prerequisite rows. - _, err = db.ExecContext( + // Verify the UNIQUE index on artist_credit_artist exists by + // attempting a duplicate insert. First, create the prerequisites. + _, err := db.ExecContext( "INSERT INTO artists (id, name) VALUES (1, 'Test')", ) if err != nil { diff --git a/backend/database/sql/queries/audio_files.sql b/backend/database/sql/queries/audio_files.sql index 8c7a98d..e2c0cd2 100644 --- a/backend/database/sql/queries/audio_files.sql +++ b/backend/database/sql/queries/audio_files.sql @@ -6,8 +6,8 @@ RETURNING *; INSERT INTO audio_files ( file_path, length_milliseconds, file_type_id, recording_id, sample_rate, bit_depth, channels, bitrate, file_size, basename, - library_id, group_key, tag_status -) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + library_id, group_key, tag_status, modified_at +) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING *; -- name: GetAudioFileGroupKey :one @@ -32,9 +32,23 @@ WHERE id = ?; -- name: UpdateAudioFileRecording :exec UPDATE audio_files -SET recording_id = ?, sample_rate = ?, bit_depth = ?, channels = ?, bitrate = ?, file_size = ? +SET recording_id = ?, sample_rate = ?, bit_depth = ?, channels = ?, bitrate = ?, file_size = ?, length_milliseconds = ?, modified_at = ? WHERE id = ?; +-- name: UpdateAudioFileStat :exec +-- Records the on-disk mtime/size without re-reading tags. Used to +-- backfill the staleness baseline for files the scan skipped, and to +-- re-baseline after YellowJacket's own tag writer rewrites a file. +UPDATE audio_files +SET modified_at = ?, file_size = ? +WHERE id = ?; + +-- name: GetLibraryMaxModifiedAt :one +-- Newest recorded mtime in a library, for the startup soft scan. 0 when +-- the library is empty or no row has a baseline yet. +SELECT CAST(COALESCE(MAX(modified_at), 0) AS INTEGER) FROM audio_files +WHERE library_id = ?; + -- name: DeleteAudioFile :exec DELETE FROM audio_files WHERE id = ?; diff --git a/backend/database/sql/queries/download.sql b/backend/database/sql/queries/download.sql new file mode 100644 index 0000000..c55dbb1 --- /dev/null +++ b/backend/database/sql/queries/download.sql @@ -0,0 +1,203 @@ +-- name: ListDownloadProviders :many +SELECT id, kind, name, enabled, priority, settings, created_at +FROM download_providers +ORDER BY priority DESC, name; + +-- name: GetDownloadProvider :one +SELECT id, kind, name, enabled, priority, settings, created_at +FROM download_providers +WHERE id = ?; + +-- name: CreateDownloadProvider :one +INSERT INTO download_providers (kind, name, enabled, priority, settings) +VALUES (?, ?, ?, ?, ?) +RETURNING id; + +-- name: UpdateDownloadProvider :exec +UPDATE download_providers +SET name = ?, enabled = ?, priority = ?, settings = ? +WHERE id = ?; + +-- name: DeleteDownloadProvider :exec +DELETE FROM download_providers +WHERE id = ?; + +-- name: CreateDownloadRequest :exec +INSERT INTO download_requests ( + id, library_id, source, want_id, release_mbid, release_group_mbid, + recording_mbid, artist, album, query, expected, state +) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + +-- name: GetDownloadRequest :one +SELECT id, library_id, source, want_id, release_mbid, release_group_mbid, + recording_mbid, artist, album, query, expected, state, error, + created_at, updated_at +FROM download_requests +WHERE id = ?; + +-- name: ListDownloadRequests :many +SELECT id, library_id, source, want_id, release_mbid, release_group_mbid, + recording_mbid, artist, album, query, expected, state, error, + created_at, updated_at +FROM download_requests +ORDER BY created_at DESC +LIMIT ?; + +-- name: ListLiveDownloadRequests :many +SELECT id, library_id, source, want_id, release_mbid, release_group_mbid, + recording_mbid, artist, album, query, expected, state, error, + created_at, updated_at +FROM download_requests +WHERE state NOT IN ('complete', 'cancelled', 'failed') +ORDER BY created_at; + +-- name: SetDownloadRequestState :exec +UPDATE download_requests +SET state = ?, error = ?, updated_at = CURRENT_TIMESTAMP +WHERE id = ?; + +-- name: DeleteDownloadRequest :exec +DELETE FROM download_requests +WHERE id = ?; + +-- name: CreateDownloadItem :exec +INSERT INTO download_items ( + id, request_id, provider_id, transport_id, external_id, + candidate, state, staging_dir, bytes_total +) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?); + +-- name: GetDownloadItem :one +SELECT id, request_id, provider_id, transport_id, external_id, candidate, + state, staging_dir, bytes_done, bytes_total, imported_paths, + error, created_at, updated_at +FROM download_items +WHERE id = ?; + +-- name: ListDownloadItemsForRequest :many +SELECT id, request_id, provider_id, transport_id, external_id, candidate, + state, staging_dir, bytes_done, bytes_total, imported_paths, + error, created_at, updated_at +FROM download_items +WHERE request_id = ? +ORDER BY created_at; + +-- name: ListLiveDownloadItems :many +SELECT id, request_id, provider_id, transport_id, external_id, candidate, + state, staging_dir, bytes_done, bytes_total, imported_paths, + error, created_at, updated_at +FROM download_items +WHERE state NOT IN ('complete', 'cancelled', 'failed') +ORDER BY created_at; + +-- name: SetDownloadItemState :exec +UPDATE download_items +SET state = ?, error = ?, updated_at = CURRENT_TIMESTAMP +WHERE id = ?; + +-- name: SetDownloadItemProgress :exec +UPDATE download_items +SET bytes_done = ?, bytes_total = ?, updated_at = CURRENT_TIMESTAMP +WHERE id = ?; + +-- name: SetDownloadItemExternalID :exec +UPDATE download_items +SET external_id = ?, updated_at = CURRENT_TIMESTAMP +WHERE id = ?; + +-- name: SetDownloadItemImported :exec +UPDATE download_items +SET imported_paths = ?, state = 'complete', error = '', + updated_at = CURRENT_TIMESTAMP +WHERE id = ?; + +-- name: DeleteFinishedDownloadRequests :exec +DELETE FROM download_requests +WHERE state IN ('complete', 'cancelled', 'failed'); + +-- --------------------------------------------------------------------- +-- Wants +-- --------------------------------------------------------------------- + +-- name: UpsertDownloadWant :one +-- Adding something already wanted is not an error and must not reset +-- the retry clock, so the conflict path only refreshes display text and +-- un-pauses nothing. scope and secondary are updated because asking +-- again with a wider scope is a real change of intent. +INSERT INTO download_wants ( + mbid, entity, library_id, artist, title, scope, secondary, + parent_id, next_try_at +) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) +ON CONFLICT(mbid, library_id) DO UPDATE SET + artist = CASE WHEN excluded.artist <> '' THEN excluded.artist + ELSE download_wants.artist END, + title = CASE WHEN excluded.title <> '' THEN excluded.title + ELSE download_wants.title END, + scope = excluded.scope, + secondary = excluded.secondary, + updated_at = CURRENT_TIMESTAMP +RETURNING id; + +-- name: GetDownloadWant :one +SELECT * FROM download_wants WHERE id = ?; + +-- name: GetDownloadWantByMBID :one +SELECT * FROM download_wants WHERE mbid = ? AND library_id = ?; + +-- name: ListDownloadWants :many +SELECT * FROM download_wants +ORDER BY + CASE state WHEN 'wanted' THEN 0 WHEN 'paused' THEN 1 ELSE 2 END, + artist, title; + +-- name: ListDownloadWantsByEntity :many +SELECT * FROM download_wants +WHERE entity = ? AND state = ? +ORDER BY id; + +-- name: ListDueDownloadWants :many +-- Everything the reconciler should act on this pass: wanted, not an +-- artist subscription (those expand rather than download), and either +-- never tried or past its backoff. +SELECT * FROM download_wants +WHERE state = 'wanted' + AND entity <> 'artist' + AND (next_try_at IS NULL OR next_try_at <= CURRENT_TIMESTAMP) +ORDER BY attempts, created_at +LIMIT ?; + +-- name: ListChildDownloadWants :many +SELECT * FROM download_wants WHERE parent_id = ? ORDER BY id; + +-- name: SetDownloadWantState :exec +UPDATE download_wants +SET state = ?, last_error = ?, updated_at = CURRENT_TIMESTAMP +WHERE id = ?; + +-- name: RecordDownloadWantAttempt :exec +UPDATE download_wants +SET attempts = attempts + 1, + last_error = ?, + last_tried_at = CURRENT_TIMESTAMP, + next_try_at = ?, + updated_at = CURRENT_TIMESTAMP +WHERE id = ?; + +-- name: SatisfyDownloadWant :exec +UPDATE download_wants +SET state = 'satisfied', last_error = '', next_try_at = NULL, + updated_at = CURRENT_TIMESTAMP +WHERE id = ?; + +-- name: SetDownloadWantExternalIDs :exec +UPDATE download_wants +SET external_ids = ?, updated_at = CURRENT_TIMESTAMP +WHERE id = ?; + +-- name: DeleteDownloadWant :exec +DELETE FROM download_wants WHERE id = ?; + +-- name: DeleteSatisfiedDownloadWants :exec +DELETE FROM download_wants WHERE state = 'satisfied'; diff --git a/backend/database/sql/schemas/_libraries.sql b/backend/database/sql/schemas/_libraries.sql deleted file mode 100644 index 4507c1f..0000000 --- a/backend/database/sql/schemas/_libraries.sql +++ /dev/null @@ -1,7 +0,0 @@ -CREATE TABLE IF NOT EXISTS libraries ( - id INTEGER PRIMARY KEY, - name TEXT NOT NULL, - path TEXT NOT NULL UNIQUE, - created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - autotag_warning_acked INTEGER NOT NULL DEFAULT 0 -); diff --git a/backend/database/sql/schemas/artist_credit_artist.sql b/backend/database/sql/schemas/artist_credit_artist.sql index 730ad0c..200a044 100644 --- a/backend/database/sql/schemas/artist_credit_artist.sql +++ b/backend/database/sql/schemas/artist_credit_artist.sql @@ -11,3 +11,6 @@ CREATE INDEX IF NOT EXISTS idx_artist_credit_artist_artist_id CREATE INDEX IF NOT EXISTS idx_artist_credit_artist_credit_id ON artist_credit_artist(credit_id); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_artist_credit_artist_unique + ON artist_credit_artist(artist_id, credit_id); diff --git a/backend/database/sql/schemas/artist_images.sql b/backend/database/sql/schemas/artist_images.sql new file mode 100644 index 0000000..3b69083 --- /dev/null +++ b/backend/database/sql/schemas/artist_images.sql @@ -0,0 +1,19 @@ +CREATE TABLE IF NOT EXISTS artist_images ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + artist_mbid TEXT NOT NULL, + source TEXT NOT NULL, + source_url TEXT NOT NULL, + file_path TEXT NOT NULL, + is_primary INTEGER NOT NULL DEFAULT 0, + sort_order INTEGER NOT NULL DEFAULT 0, + width INTEGER, + height INTEGER, + file_size INTEGER, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP + ); + +CREATE INDEX IF NOT EXISTS idx_artist_images_mbid + ON artist_images(artist_mbid); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_artist_images_source + ON artist_images(artist_mbid, source, source_url); diff --git a/backend/database/sql/schemas/artist_metadata.sql b/backend/database/sql/schemas/artist_metadata.sql index 1941db7..9b86c69 100644 --- a/backend/database/sql/schemas/artist_metadata.sql +++ b/backend/database/sql/schemas/artist_metadata.sql @@ -2,6 +2,8 @@ -- Sources: audiodb, fanart, wikidata-p18, wikipedia-lead, mb:artist-rels. -- No TTL — this data changes very rarely and is the backing store for -- the artist detail page. + + CREATE TABLE IF NOT EXISTS artist_metadata ( mbid TEXT NOT NULL, source TEXT NOT NULL, @@ -9,4 +11,5 @@ CREATE TABLE IF NOT EXISTS artist_metadata ( fetched_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (mbid, source) ); + CREATE INDEX IF NOT EXISTS idx_artist_metadata_mbid ON artist_metadata(mbid); diff --git a/backend/database/sql/schemas/artists.sql b/backend/database/sql/schemas/artists.sql index 09d2cf7..6441400 100644 --- a/backend/database/sql/schemas/artists.sql +++ b/backend/database/sql/schemas/artists.sql @@ -3,3 +3,5 @@ CREATE TABLE IF NOT EXISTS artists ( name TEXT NOT NULL UNIQUE, mbid TEXT ); + +CREATE INDEX IF NOT EXISTS idx_artists_mbid ON artists(mbid) WHERE mbid IS NOT NULL; diff --git a/backend/database/sql/schemas/audio_files.sql b/backend/database/sql/schemas/audio_files.sql index f5a68be..b422c06 100644 --- a/backend/database/sql/schemas/audio_files.sql +++ b/backend/database/sql/schemas/audio_files.sql @@ -18,22 +18,28 @@ CREATE TABLE IF NOT EXISTS audio_files ( 'untagged', 'auto_matched', 'user_confirmed', 'user_skipped_permanent' )), group_key TEXT NOT NULL DEFAULT '', + -- File mtime as a Unix timestamp in seconds, captured at import. + -- Compared against the on-disk mtime during a scan to detect files + -- another application retagged in place. 0 means "never recorded" + -- (rows predating migration 47) and is treated as not-stale so an + -- upgrade does not re-import the whole library. + modified_at int NOT NULL DEFAULT 0, FOREIGN KEY(file_type_id) REFERENCES file_types(id), FOREIGN KEY(recording_id) REFERENCES recordings(id), FOREIGN KEY(library_id) REFERENCES libraries(id) ); +CREATE INDEX IF NOT EXISTS idx_audio_files_basename + ON audio_files(basename); + +CREATE INDEX IF NOT EXISTS idx_audio_files_group_key + ON audio_files(group_key) WHERE group_key != ''; + +CREATE INDEX IF NOT EXISTS idx_audio_files_library_id + ON audio_files(library_id); + CREATE INDEX IF NOT EXISTS idx_audio_files_recording_id ON audio_files(recording_id); --- idx_audio_files_library_id is created by migration 6 (not here) because --- on existing databases this schema file is a no-op (CREATE TABLE IF NOT EXISTS) --- and the library_id column doesn't exist until the migration adds it. --- --- idx_audio_files_tag_status_untagged + idx_audio_files_group_key are --- created by migrations 31 and 32 for the same reason — on a pre-31 --- database the partial index predicates (`WHERE tag_status = '...'` --- and `WHERE group_key != ''`) would reference columns that don't --- yet exist, since CREATE TABLE IF NOT EXISTS does not add columns --- to existing tables. sqlc still sees the columns above, and fresh --- DBs pick up the indexes inside the migrations. +CREATE INDEX IF NOT EXISTS idx_audio_files_tag_status_untagged + ON audio_files(library_id) WHERE tag_status = 'untagged'; diff --git a/backend/database/sql/schemas/download_items.sql b/backend/database/sql/schemas/download_items.sql new file mode 100644 index 0000000..eb974df --- /dev/null +++ b/backend/database/sql/schemas/download_items.sql @@ -0,0 +1,49 @@ +-- One row per grab attempt against one candidate. A request can have +-- several: the first pick stalls, the user picks another, or a +-- search-only provider's candidate is fetched by a separate transport +-- (in which case provider_id is the searcher and transport_id is the +-- fetcher). +-- +-- `candidate` is the full ranked Candidate as JSON. It is stored +-- rather than re-derived because the provider's result set is +-- ephemeral — a Soulseek peer that had the files an hour ago may be +-- offline now, and the item still has to render in the UI and explain +-- why it was chosen. +-- +-- external_id holds a delegating manager's own identifier (a Lidarr +-- queue id), which is how polling finds the record again after a +-- restart. + + +CREATE TABLE IF NOT EXISTS download_items ( + id TEXT PRIMARY KEY, + request_id TEXT NOT NULL, + provider_id INTEGER NOT NULL, + transport_id INTEGER, + external_id TEXT NOT NULL DEFAULT '', + candidate TEXT NOT NULL DEFAULT '{}', + state TEXT NOT NULL DEFAULT 'queued' + CHECK(state IN ('searching', 'found', 'queued', 'grabbing', + 'verifying', 'tagging', 'importing', + 'complete', 'cancelled', 'failed')), + staging_dir TEXT NOT NULL DEFAULT '', + bytes_done INTEGER NOT NULL DEFAULT 0, + bytes_total INTEGER NOT NULL DEFAULT 0, + -- imported_paths is a JSON array of the library paths the files + -- ended up at, so an import can be undone without guessing. + imported_paths TEXT NOT NULL DEFAULT '[]', + error TEXT NOT NULL DEFAULT '', + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY(request_id) REFERENCES download_requests(id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_download_items_live + ON download_items(state) + WHERE state NOT IN ('complete', 'cancelled', 'failed'); + +CREATE INDEX IF NOT EXISTS idx_download_items_request + ON download_items(request_id); + +CREATE INDEX IF NOT EXISTS idx_download_items_state + ON download_items(state); diff --git a/backend/database/sql/schemas/download_providers.sql b/backend/database/sql/schemas/download_providers.sql new file mode 100644 index 0000000..ba38eda --- /dev/null +++ b/backend/database/sql/schemas/download_providers.sql @@ -0,0 +1,28 @@ +-- Download clients the user has connected: an slskd daemon, a Lidarr +-- instance, yt-dlp on PATH. One row per configured instance, so two +-- Prowlarr servers or two Soulseek accounts coexist. +-- +-- Secrets (API keys, passwords) are NOT stored here. They live in a +-- 0600 file keyed by this row's id, so this table can be dumped into a +-- bug report without redaction. `settings` holds only non-sensitive +-- values (host, port, category, output format) as a JSON object. +-- +-- `kind` names the adapter implementation and is looked up in the +-- provider registry at startup; a row whose kind no longer exists is +-- reported to the user rather than silently dropped. + + +CREATE TABLE IF NOT EXISTS download_providers ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + kind TEXT NOT NULL, + name TEXT NOT NULL, + enabled INTEGER NOT NULL DEFAULT 1, + -- priority breaks ties between providers that found equally good + -- candidates. Higher wins; 50 is the neutral default. + priority INTEGER NOT NULL DEFAULT 50, + settings TEXT NOT NULL DEFAULT '{}', + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_download_providers_enabled + ON download_providers(enabled); diff --git a/backend/database/sql/schemas/download_requests.sql b/backend/database/sql/schemas/download_requests.sql new file mode 100644 index 0000000..21981d9 --- /dev/null +++ b/backend/database/sql/schemas/download_requests.sql @@ -0,0 +1,49 @@ +-- One row per "go find me this", from the moment the user asks until +-- the files are in the library or the attempt is abandoned. +-- +-- release_mbid / release_group_mbid are the anchor: a request that +-- carries one can be matched against a known tracklist at import time, +-- which is what makes unattended completion safe. Free-text requests +-- (both NULL) are always presented to the user for confirmation. +-- +-- `expected` caches the anchor's tracklist as JSON so ranking and +-- import do not have to re-resolve it, and so a request survives the +-- explore index being rebuilt underneath it. + + +CREATE TABLE IF NOT EXISTS download_requests ( + id TEXT PRIMARY KEY, + library_id INTEGER NOT NULL, + -- source records where the request came from: 'explore-album', + -- 'explore-artist', 'missing-album', 'wanted', 'manual'. + source TEXT NOT NULL DEFAULT 'manual', + -- want_id is set when the reconciler raised this request from the + -- wanted list, so the outcome can be written back to the want. + -- NULL for one-off requests the user started by hand. Requests are + -- disposable and wants are not, so the delete is a SET NULL rather + -- than a cascade in either direction. + want_id INTEGER REFERENCES download_wants(id) ON DELETE SET NULL, + release_mbid TEXT, + release_group_mbid TEXT, + -- recording_mbid anchors a single-track request raised from a + -- track-level want. + recording_mbid TEXT, + artist TEXT NOT NULL DEFAULT '', + album TEXT NOT NULL DEFAULT '', + query TEXT NOT NULL DEFAULT '', + expected TEXT NOT NULL DEFAULT '[]', + state TEXT NOT NULL DEFAULT 'searching' + CHECK(state IN ('searching', 'found', 'queued', 'grabbing', + 'verifying', 'tagging', 'importing', + 'complete', 'cancelled', 'failed')), + error TEXT NOT NULL DEFAULT '', + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY(library_id) REFERENCES libraries(id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_download_requests_created + ON download_requests(created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_download_requests_state + ON download_requests(state); diff --git a/backend/database/sql/schemas/download_wants.sql b/backend/database/sql/schemas/download_wants.sql new file mode 100644 index 0000000..9d59e56 --- /dev/null +++ b/backend/database/sql/schemas/download_wants.sql @@ -0,0 +1,76 @@ +-- One row per "go find me this", from the moment the user asks until +-- the files are in the library or the attempt is abandoned. +-- +-- release_mbid / release_group_mbid are the anchor: a request that +-- carries one can be matched against a known tracklist at import time, +-- which is what makes unattended completion safe. Free-text requests +-- (both NULL) are always presented to the user for confirmation. +-- +-- `expected` caches the anchor's tracklist as JSON so ranking and +-- import do not have to re-resolve it, and so a request survives the +-- explore index being rebuilt underneath it. + + +CREATE TABLE IF NOT EXISTS download_wants ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + mbid TEXT NOT NULL, + entity TEXT NOT NULL + CHECK(entity IN ('artist', 'release-group', 'release', 'recording')), + library_id INTEGER NOT NULL, + + -- Display text, cached so the wanted list renders without touching + -- the explore index. Neither is authoritative; the MBID is. + artist TEXT NOT NULL DEFAULT '', + title TEXT NOT NULL DEFAULT '', + + scope TEXT NOT NULL DEFAULT 'future' + CHECK(scope IN ('future', 'all')), + + -- secondary controls whether an artist want's expansion includes + -- compilations, live albums and remixes. Off by default: someone + -- subscribing to an artist wants the albums, not six versions of + -- the same greatest-hits package. + secondary INTEGER NOT NULL DEFAULT 0, + + state TEXT NOT NULL DEFAULT 'wanted' + CHECK(state IN ('wanted', 'satisfied', 'paused')), + + -- parent_id links a want the reconciler derived from an artist + -- want. Deleting the artist takes its derived children with it, + -- but children the user pinned themselves have no parent and stay. + parent_id INTEGER, + + -- Retry bookkeeping. attempts drives the backoff; last_error is + -- the most recent reason it did not work out, which for a wanted + -- item is information rather than a failure. + attempts INTEGER NOT NULL DEFAULT 0, + last_error TEXT NOT NULL DEFAULT '', + last_tried_at DATETIME, + next_try_at DATETIME, + + -- external_ids maps provider row ID to that provider's own + -- identifier for this want, for clients that keep their own + -- persistent list (a Lidarr artist ID). JSON object. + external_ids TEXT NOT NULL DEFAULT '{}', + + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + + -- One want per thing per library. Asking twice is not two wants, + -- and this is what lets an artist expansion re-run every reconcile + -- without accumulating duplicates. + UNIQUE(mbid, library_id), + + FOREIGN KEY(library_id) REFERENCES libraries(id) ON DELETE CASCADE, + FOREIGN KEY(parent_id) REFERENCES download_wants(id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_download_wants_due + ON download_wants(next_try_at) + WHERE state = 'wanted'; + +CREATE INDEX IF NOT EXISTS idx_download_wants_entity + ON download_wants(entity, state); + +CREATE INDEX IF NOT EXISTS idx_download_wants_parent + ON download_wants(parent_id); diff --git a/backend/database/sql/schemas/explore_champion_fts.sql b/backend/database/sql/schemas/explore_champion_fts.sql new file mode 100644 index 0000000..020fe12 --- /dev/null +++ b/backend/database/sql/schemas/explore_champion_fts.sql @@ -0,0 +1,6 @@ +CREATE VIRTUAL TABLE IF NOT EXISTS explore_champion_fts USING fts5( + title, artist_name, aliases, + content='explore_index', + content_rowid='id', + tokenize='unicode61 remove_diacritics 2' + ); diff --git a/backend/database/sql/schemas/explore_index.sql b/backend/database/sql/schemas/explore_index.sql new file mode 100644 index 0000000..cdb61c5 --- /dev/null +++ b/backend/database/sql/schemas/explore_index.sql @@ -0,0 +1,65 @@ +CREATE TABLE IF NOT EXISTS explore_index ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + entity_type TEXT NOT NULL, + mbid TEXT NOT NULL, + title TEXT NOT NULL, + artist_name TEXT NOT NULL, + artist_mbid TEXT NOT NULL, + aliases TEXT NOT NULL DEFAULT '', + + -- Popularity signals, derived from the ListenBrainz listens dump. + popularity INTEGER NOT NULL DEFAULT 0, + listener_count INTEGER NOT NULL DEFAULT 0, + + -- Recording-specific fields. + duration INTEGER NOT NULL DEFAULT 0, + caa_release_mbid TEXT NOT NULL DEFAULT '', + release_name TEXT NOT NULL DEFAULT '', + + -- Release-group-specific fields. + primary_type TEXT NOT NULL DEFAULT '', + secondary_types TEXT NOT NULL DEFAULT '', + release_date TEXT NOT NULL DEFAULT '', + + -- Artist-specific fields. + artist_type TEXT NOT NULL DEFAULT '', + country TEXT NOT NULL DEFAULT '', + disambiguation TEXT NOT NULL DEFAULT '', + sort_name TEXT NOT NULL DEFAULT '', + + -- Personalization flags. + in_library INTEGER NOT NULL DEFAULT 0, + is_similar INTEGER NOT NULL DEFAULT 0, + + -- Cross-reference to local library tables. NULL when the + -- entity has no corresponding row in the library. + local_artist_id INTEGER, + local_release_group_id INTEGER, + local_recording_id INTEGER, + + -- Set once an artist's full discography (release groups + + -- recordings) has been fetched, so EnsureArtistDiscography can + -- skip artists the catalog already covers. + discog_fetched INTEGER NOT NULL DEFAULT 0, + + + UNIQUE(mbid) + ); + +CREATE INDEX IF NOT EXISTS idx_explore_artist_lower + ON explore_index(LOWER(artist_name)) + WHERE popularity > 0; + +CREATE INDEX IF NOT EXISTS idx_explore_caa_release + ON explore_index(caa_release_mbid) + WHERE entity_type = 'release_group' AND caa_release_mbid != ''; + +CREATE INDEX IF NOT EXISTS idx_explore_index_artist_mbid + ON explore_index(artist_mbid, entity_type, popularity DESC); + +CREATE INDEX IF NOT EXISTS idx_explore_index_entity_pop + ON explore_index(entity_type, popularity DESC); + +CREATE INDEX IF NOT EXISTS idx_explore_title_lower + ON explore_index(LOWER(title)) + WHERE popularity > 0; diff --git a/backend/database/sql/schemas/explore_index_fts.sql b/backend/database/sql/schemas/explore_index_fts.sql new file mode 100644 index 0000000..ea82d84 --- /dev/null +++ b/backend/database/sql/schemas/explore_index_fts.sql @@ -0,0 +1,6 @@ +CREATE VIRTUAL TABLE IF NOT EXISTS explore_index_fts USING fts5( + title, artist_name, aliases, + content='explore_index', + content_rowid='id', + tokenize='unicode61 remove_diacritics 2' + ); diff --git a/backend/database/sql/schemas/explore_index_meta.sql b/backend/database/sql/schemas/explore_index_meta.sql new file mode 100644 index 0000000..e056b8c --- /dev/null +++ b/backend/database/sql/schemas/explore_index_meta.sql @@ -0,0 +1,4 @@ +CREATE TABLE IF NOT EXISTS explore_index_meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); diff --git a/backend/database/sql/schemas/file_types.sql b/backend/database/sql/schemas/file_types.sql index d71b7c5..6749311 100644 --- a/backend/database/sql/schemas/file_types.sql +++ b/backend/database/sql/schemas/file_types.sql @@ -3,6 +3,8 @@ CREATE TABLE IF NOT EXISTS file_types ( extension text NOT NULL UNIQUE ); +-- Seed rows: the supported audio formats, referenced by +-- audio_files.file_type_id. INSERT OR IGNORE INTO file_types (id, extension) VALUES (0, '.mp3'); INSERT OR IGNORE INTO file_types (id, extension) VALUES (1, '.flac'); INSERT OR IGNORE INTO file_types (id, extension) VALUES (2, '.ogg'); diff --git a/backend/database/sql/schemas/http_cache.sql b/backend/database/sql/schemas/http_cache.sql index 4e42333..be232bc 100644 --- a/backend/database/sql/schemas/http_cache.sql +++ b/backend/database/sql/schemas/http_cache.sql @@ -1,5 +1,7 @@ -- Short-lived HTTP response cache (search results, MB/LB lookups, etc). -- For long-lived enrichment data keyed by MBID, see artist_metadata.sql. + + CREATE TABLE IF NOT EXISTS http_cache ( url_key TEXT PRIMARY KEY, response BLOB NOT NULL, @@ -7,5 +9,7 @@ CREATE TABLE IF NOT EXISTS http_cache ( entity_mbid TEXT NOT NULL DEFAULT '', entity_type TEXT NOT NULL DEFAULT '' ); + CREATE INDEX IF NOT EXISTS idx_http_cache_expires ON http_cache(expires_at); + CREATE INDEX IF NOT EXISTS idx_http_cache_mbid ON http_cache(entity_mbid); diff --git a/backend/database/sql/schemas/job_state.sql b/backend/database/sql/schemas/job_state.sql index 6a5caee..a86a44d 100644 --- a/backend/database/sql/schemas/job_state.sql +++ b/backend/database/sql/schemas/job_state.sql @@ -6,6 +6,8 @@ -- Rows are written when a durable job enters the paused state and -- deleted on resume, cancel, or completion — this is not a job history -- table, and it stays at zero rows in the common case. + + CREATE TABLE IF NOT EXISTS job_state ( id TEXT PRIMARY KEY, kind TEXT NOT NULL, diff --git a/backend/database/sql/schemas/libraries.sql b/backend/database/sql/schemas/libraries.sql new file mode 100644 index 0000000..6c065dd --- /dev/null +++ b/backend/database/sql/schemas/libraries.sql @@ -0,0 +1,20 @@ +-- One row per "go find me this", from the moment the user asks until +-- the files are in the library or the attempt is abandoned. +-- +-- release_mbid / release_group_mbid are the anchor: a request that +-- carries one can be matched against a known tracklist at import time, +-- which is what makes unattended completion safe. Free-text requests +-- (both NULL) are always presented to the user for confirmation. +-- +-- `expected` caches the anchor's tracklist as JSON so ranking and +-- import do not have to re-resolve it, and so a request survives the +-- explore index being rebuilt underneath it. + + +CREATE TABLE IF NOT EXISTS libraries ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + path TEXT NOT NULL UNIQUE, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + autotag_warning_acked INTEGER NOT NULL DEFAULT 0 +); diff --git a/backend/database/sql/schemas/lyrics_index.sql b/backend/database/sql/schemas/lyrics_index.sql index 04a8816..a2e00b5 100644 --- a/backend/database/sql/schemas/lyrics_index.sql +++ b/backend/database/sql/schemas/lyrics_index.sql @@ -7,6 +7,8 @@ -- tokenised inverted index, so it stays compact even for large -- libraries. contentless_delete=1 lets us delete/reinsert a single -- row when a track's lyrics change (scan update or LRCLIB backfill). + + CREATE VIRTUAL TABLE IF NOT EXISTS lyrics_index USING fts5( lyrics, content='', diff --git a/backend/database/sql/schemas/player_state.sql b/backend/database/sql/schemas/player_state.sql index 5cfe9ed..757d519 100644 --- a/backend/database/sql/schemas/player_state.sql +++ b/backend/database/sql/schemas/player_state.sql @@ -6,4 +6,5 @@ CREATE TABLE IF NOT EXISTS player_state ( last_position_seconds INTEGER NOT NULL DEFAULT 0 ); +-- Singleton row: player state is a single mutable record. INSERT OR IGNORE INTO player_state (id) VALUES (1); diff --git a/backend/database/sql/schemas/playlist_tracks.sql b/backend/database/sql/schemas/playlist_tracks.sql index 938ff55..700af25 100644 --- a/backend/database/sql/schemas/playlist_tracks.sql +++ b/backend/database/sql/schemas/playlist_tracks.sql @@ -1,21 +1,20 @@ -CREATE TABLE IF NOT EXISTS playlist_tracks ( - id INTEGER PRIMARY KEY, - playlist_id INTEGER NOT NULL, - audio_file_id INTEGER, - position INTEGER NOT NULL, - phantom_title TEXT, - phantom_artist TEXT, - phantom_album TEXT, - phantom_duration_ms INTEGER, - phantom_genre TEXT, - phantom_cover_art_path TEXT, - phantom_file_path TEXT, - FOREIGN KEY(playlist_id) REFERENCES playlists(id) ON DELETE CASCADE, - FOREIGN KEY(audio_file_id) REFERENCES audio_files(id) ON DELETE SET NULL -); - -CREATE INDEX IF NOT EXISTS idx_playlist_tracks_playlist_id - ON playlist_tracks(playlist_id); +CREATE TABLE IF NOT EXISTS "playlist_tracks" ( + id INTEGER PRIMARY KEY, + playlist_id INTEGER NOT NULL, + audio_file_id INTEGER, + position INTEGER NOT NULL, + phantom_title TEXT, + phantom_artist TEXT, + phantom_album TEXT, + phantom_duration_ms INTEGER, + phantom_genre TEXT, + phantom_cover_art_path TEXT, phantom_file_path TEXT, + FOREIGN KEY(playlist_id) REFERENCES playlists(id) ON DELETE CASCADE, + FOREIGN KEY(audio_file_id) REFERENCES audio_files(id) ON DELETE SET NULL + ); CREATE INDEX IF NOT EXISTS idx_playlist_tracks_audio_file_id - ON playlist_tracks(audio_file_id); + ON playlist_tracks(audio_file_id); + +CREATE INDEX IF NOT EXISTS idx_playlist_tracks_playlist_id + ON playlist_tracks(playlist_id); diff --git a/backend/database/sql/schemas/queue.sql b/backend/database/sql/schemas/queue.sql index 08c5658..19c783a 100644 --- a/backend/database/sql/schemas/queue.sql +++ b/backend/database/sql/schemas/queue.sql @@ -8,4 +8,5 @@ CREATE TABLE IF NOT EXISTS queue ( FOREIGN KEY(source_playlist_id) REFERENCES playlists(id) ON DELETE SET NULL ); +-- Singleton row: there is exactly one playback queue. INSERT OR IGNORE INTO queue (id) VALUES (1); diff --git a/backend/database/sql/schemas/genre_recordings.sql b/backend/database/sql/schemas/recording_genres.sql similarity index 100% rename from backend/database/sql/schemas/genre_recordings.sql rename to backend/database/sql/schemas/recording_genres.sql index 64fc0fe..f658af0 100644 --- a/backend/database/sql/schemas/genre_recordings.sql +++ b/backend/database/sql/schemas/recording_genres.sql @@ -7,8 +7,8 @@ CREATE TABLE IF NOT EXISTS recording_genres ( UNIQUE(recording_id, genre_id) ); -CREATE INDEX IF NOT EXISTS idx_recording_genres_recording_id - ON recording_genres(recording_id); - CREATE INDEX IF NOT EXISTS idx_recording_genres_genre_id ON recording_genres(genre_id); + +CREATE INDEX IF NOT EXISTS idx_recording_genres_recording_id + ON recording_genres(recording_id); diff --git a/backend/database/sql/schemas/recordings.sql b/backend/database/sql/schemas/recordings.sql index 58c66cd..c372209 100644 --- a/backend/database/sql/schemas/recordings.sql +++ b/backend/database/sql/schemas/recordings.sql @@ -15,3 +15,5 @@ CREATE TABLE IF NOT EXISTS recordings ( CREATE INDEX IF NOT EXISTS idx_recordings_artist_credit_id ON recordings(artist_credit_id); + +CREATE INDEX IF NOT EXISTS idx_recordings_mbid ON recordings(mbid) WHERE mbid IS NOT NULL; diff --git a/backend/database/sql/schemas/release_groups.sql b/backend/database/sql/schemas/release_groups.sql index 66d3fdc..3859593 100644 --- a/backend/database/sql/schemas/release_groups.sql +++ b/backend/database/sql/schemas/release_groups.sql @@ -1,30 +1,20 @@ -CREATE TABLE IF NOT EXISTS release_groups ( +CREATE TABLE IF NOT EXISTS "release_groups" ( id INTEGER PRIMARY KEY, name TEXT NOT NULL, cover_art_id INTEGER, album_artist_credit_id INTEGER, - -- year is the *technical release year* of the album as it lives - -- in the user's library — typically the file's ID3 year tag, - -- which for remasters/reissues is the reissue year. year INTEGER, - -- original_year is the album's *first-release-date* year sourced - -- from MusicBrainz' release-group.first-release-date. For a 2010 - -- remaster of a 1973 album, year=2010 and original_year=1973. - -- Populated by autotag apply; NULL until the user accepts a - -- candidate (or for libraries that have never been autotagged). - -- Reads should COALESCE(original_year, year) to get the - -- preferred user-facing year. - original_year INTEGER, total_tracks INTEGER, - total_discs INTEGER, - mbid TEXT, + total_discs INTEGER, mbid TEXT, original_year INTEGER, FOREIGN KEY(cover_art_id) REFERENCES cover_art(id), FOREIGN KEY(album_artist_credit_id) REFERENCES artist_credit(id), UNIQUE(name, album_artist_credit_id) -); - -CREATE INDEX IF NOT EXISTS idx_release_groups_cover_art_id - ON release_groups(cover_art_id); + ); CREATE INDEX IF NOT EXISTS idx_release_groups_album_artist_credit_id - ON release_groups(album_artist_credit_id); + ON release_groups(album_artist_credit_id); + +CREATE INDEX IF NOT EXISTS idx_release_groups_cover_art_id + ON release_groups(cover_art_id); + +CREATE INDEX IF NOT EXISTS idx_release_groups_mbid ON release_groups(mbid) WHERE mbid IS NOT NULL; diff --git a/backend/database/sql/schemas/release_to_rg.sql b/backend/database/sql/schemas/release_to_rg.sql new file mode 100644 index 0000000..471d9e7 --- /dev/null +++ b/backend/database/sql/schemas/release_to_rg.sql @@ -0,0 +1,4 @@ +CREATE TABLE IF NOT EXISTS release_to_rg ( + release_mbid TEXT PRIMARY KEY, + rg_mbid TEXT NOT NULL + ) WITHOUT ROWID; diff --git a/backend/database/sql/schemas/search_clicks.sql b/backend/database/sql/schemas/search_clicks.sql new file mode 100644 index 0000000..95821b7 --- /dev/null +++ b/backend/database/sql/schemas/search_clicks.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS search_clicks ( + query TEXT NOT NULL, + entity_mbid TEXT NOT NULL, + entity_type TEXT NOT NULL, + click_count INTEGER NOT NULL DEFAULT 1, + last_clicked DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (query, entity_mbid) + ); + +CREATE INDEX IF NOT EXISTS idx_search_clicks_query + ON search_clicks(query); diff --git a/backend/database/sql/schemas/search_index.sql b/backend/database/sql/schemas/search_index.sql index efbaf35..2adc734 100644 --- a/backend/database/sql/schemas/search_index.sql +++ b/backend/database/sql/schemas/search_index.sql @@ -1,9 +1,9 @@ CREATE VIRTUAL TABLE IF NOT EXISTS search_index USING fts5( - file_path, - title, - artist, - album, - content='', - contentless_delete=1, - tokenize='unicode61 remove_diacritics 2' -); + file_path, + title, + artist, + album, + content='', + contentless_delete=1, + tokenize='unicode61 remove_diacritics 2' + ); diff --git a/backend/database/sql/schemas/similar_artist_map.sql b/backend/database/sql/schemas/similar_artist_map.sql new file mode 100644 index 0000000..66b94b8 --- /dev/null +++ b/backend/database/sql/schemas/similar_artist_map.sql @@ -0,0 +1,10 @@ +CREATE TABLE IF NOT EXISTS similar_artist_map ( + source_artist_mbid TEXT NOT NULL, + similar_artist_mbid TEXT NOT NULL, + similar_artist_name TEXT NOT NULL, + score INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (source_artist_mbid, similar_artist_mbid) + ); + +CREATE INDEX IF NOT EXISTS idx_similar_artist_map_source + ON similar_artist_map(source_artist_mbid); diff --git a/backend/database/sql/schemas/tagging_candidates.sql b/backend/database/sql/schemas/tagging_candidates.sql index afc6b5e..27739fb 100644 --- a/backend/database/sql/schemas/tagging_candidates.sql +++ b/backend/database/sql/schemas/tagging_candidates.sql @@ -10,6 +10,8 @@ -- CASCADE ties the blob's lifetime to its tagging_items row: when a -- group's tracks change, the scan path deletes the old group_key row -- (and SQLite, with foreign_keys = ON, drops the stale blob with it). + + CREATE TABLE IF NOT EXISTS tagging_candidates ( group_key TEXT PRIMARY KEY, candidates TEXT NOT NULL, diff --git a/backend/database/sql/schemas/track_metadata.sql b/backend/database/sql/schemas/track_metadata.sql new file mode 100644 index 0000000..a94d709 --- /dev/null +++ b/backend/database/sql/schemas/track_metadata.sql @@ -0,0 +1,47 @@ +CREATE VIEW IF NOT EXISTS track_metadata AS + SELECT + af.id, + af.file_path, + af.length_milliseconds, + COALESCE(r.name, '') AS title, + COALESCE(ac.text, '') AS artist_name, + r.track_number, + r.disc_number, + COALESCE(rg.name, '') AS album, + CAST(COALESCE( + (SELECT GROUP_CONCAT(g.name, '||') + FROM recording_genres rg_sub + JOIN genres g ON rg_sub.genre_id = g.id + WHERE rg_sub.recording_id = r.id), + '' + ) AS TEXT) AS genre, + COALESCE(rg.original_year, rg.year, r.year, 0) AS year, + COALESCE(rg.year, r.year, 0) AS release_year, + COALESCE(r.composer, '') AS composer, + COALESCE(ft.extension, '') AS file_type, + af.sample_rate, + af.bit_depth, + af.channels, + af.bitrate, + af.file_size, + af.library_id, + af.play_count, + af.last_played, + COALESCE(ca.file_path, '') AS cover_art_path, + COALESCE(a.mbid, '') AS artist_mbid, + COALESCE(rg.mbid, '') AS release_group_mbid, + COALESCE(r.mbid, '') AS recording_mbid + FROM audio_files af + LEFT JOIN recordings r ON af.recording_id = r.id + LEFT JOIN artist_credit ac ON r.artist_credit_id = ac.id + LEFT JOIN artist_credit_artist aca ON aca.credit_id = ac.id + LEFT JOIN artists a ON a.id = aca.artist_id + LEFT JOIN ( + SELECT recording_id, + MIN(release_group_id) AS release_group_id + FROM release_group_recordings + GROUP BY recording_id + ) rgr ON r.id = rgr.recording_id + LEFT JOIN release_groups rg ON rgr.release_group_id = rg.id + LEFT JOIN cover_art ca ON rg.cover_art_id = ca.id + LEFT JOIN file_types ft ON af.file_type_id = ft.id; diff --git a/backend/database/sql/schemas/track_metadata_view.sql b/backend/database/sql/schemas/track_metadata_view.sql deleted file mode 100644 index 04d6fd9..0000000 --- a/backend/database/sql/schemas/track_metadata_view.sql +++ /dev/null @@ -1,52 +0,0 @@ -CREATE VIEW IF NOT EXISTS track_metadata AS -SELECT - af.id, - af.file_path, - af.length_milliseconds, - COALESCE(r.name, '') AS title, - COALESCE(ac.text, '') AS artist_name, - r.track_number, - r.disc_number, - COALESCE(rg.name, '') AS album, - CAST(COALESCE( - (SELECT GROUP_CONCAT(g.name, '||') - FROM recording_genres rg_sub - JOIN genres g ON rg_sub.genre_id = g.id - WHERE rg_sub.recording_id = r.id), - '' - ) AS TEXT) AS genre, - -- Year defaults to the release group's original release year - -- (MusicBrainz first-release-date) so a 1973 album shows as - -- 1973 even if the user owns the 2010 remaster. Falls back - -- to release-group year (file tag), then to recording year. - -- See release_groups.original_year for full semantics. - COALESCE(rg.original_year, rg.year, r.year, 0) AS year, - COALESCE(rg.year, r.year, 0) AS release_year, - COALESCE(r.composer, '') AS composer, - COALESCE(ft.extension, '') AS file_type, - af.sample_rate, - af.bit_depth, - af.channels, - af.bitrate, - af.file_size, - af.library_id, - af.play_count, - af.last_played, - COALESCE(ca.file_path, '') AS cover_art_path, - COALESCE(a.mbid, '') AS artist_mbid, - COALESCE(rg.mbid, '') AS release_group_mbid, - COALESCE(r.mbid, '') AS recording_mbid -FROM audio_files af -LEFT JOIN recordings r ON af.recording_id = r.id -LEFT JOIN artist_credit ac ON r.artist_credit_id = ac.id -LEFT JOIN artist_credit_artist aca ON aca.credit_id = ac.id -LEFT JOIN artists a ON a.id = aca.artist_id -LEFT JOIN ( - SELECT recording_id, - MIN(release_group_id) AS release_group_id - FROM release_group_recordings - GROUP BY recording_id -) rgr ON r.id = rgr.recording_id -LEFT JOIN release_groups rg ON rgr.release_group_id = rg.id -LEFT JOIN cover_art ca ON rg.cover_art_id = ca.id -LEFT JOIN file_types ft ON af.file_type_id = ft.id; diff --git a/backend/database/sql/sqlcgen/audio_files.sql.go b/backend/database/sql/sqlcgen/audio_files.sql.go index f2c3332..63305e3 100644 --- a/backend/database/sql/sqlcgen/audio_files.sql.go +++ b/backend/database/sql/sqlcgen/audio_files.sql.go @@ -35,7 +35,7 @@ func (q *Queries) CountAudioFilesByLibrary(ctx context.Context, libraryID int64) const createAudioFile = `-- name: CreateAudioFile :one INSERT INTO audio_files (file_path, length_milliseconds, file_type_id, recording_id, sample_rate, bit_depth, channels, bitrate, file_size, basename, library_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) -RETURNING id, file_path, length_milliseconds, file_type_id, recording_id, sample_rate, bit_depth, channels, bitrate, file_size, basename, library_id, play_count, last_played, tag_status, group_key +RETURNING id, file_path, length_milliseconds, file_type_id, recording_id, sample_rate, bit_depth, channels, bitrate, file_size, basename, library_id, play_count, last_played, tag_status, group_key, modified_at ` type CreateAudioFileParams struct { @@ -84,6 +84,7 @@ func (q *Queries) CreateAudioFile(ctx context.Context, arg CreateAudioFileParams &i.LastPlayed, &i.TagStatus, &i.GroupKey, + &i.ModifiedAt, ) return i, err } @@ -92,9 +93,9 @@ const createAudioFileWithGroupKey = `-- name: CreateAudioFileWithGroupKey :one INSERT INTO audio_files ( file_path, length_milliseconds, file_type_id, recording_id, sample_rate, bit_depth, channels, bitrate, file_size, basename, - library_id, group_key, tag_status -) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) -RETURNING id, file_path, length_milliseconds, file_type_id, recording_id, sample_rate, bit_depth, channels, bitrate, file_size, basename, library_id, play_count, last_played, tag_status, group_key + library_id, group_key, tag_status, modified_at +) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +RETURNING id, file_path, length_milliseconds, file_type_id, recording_id, sample_rate, bit_depth, channels, bitrate, file_size, basename, library_id, play_count, last_played, tag_status, group_key, modified_at ` type CreateAudioFileWithGroupKeyParams struct { @@ -111,6 +112,7 @@ type CreateAudioFileWithGroupKeyParams struct { LibraryID int64 GroupKey string TagStatus string + ModifiedAt int64 } func (q *Queries) CreateAudioFileWithGroupKey(ctx context.Context, arg CreateAudioFileWithGroupKeyParams) (AudioFile, error) { @@ -128,6 +130,7 @@ func (q *Queries) CreateAudioFileWithGroupKey(ctx context.Context, arg CreateAud arg.LibraryID, arg.GroupKey, arg.TagStatus, + arg.ModifiedAt, ) var i AudioFile err := row.Scan( @@ -147,6 +150,7 @@ func (q *Queries) CreateAudioFileWithGroupKey(ctx context.Context, arg CreateAud &i.LastPlayed, &i.TagStatus, &i.GroupKey, + &i.ModifiedAt, ) return i, err } @@ -203,7 +207,7 @@ func (q *Queries) GetAllAudioFilePaths(ctx context.Context) ([]GetAllAudioFilePa } const getAllAudioFiles = `-- name: GetAllAudioFiles :many -SELECT id, file_path, length_milliseconds, file_type_id, recording_id, sample_rate, bit_depth, channels, bitrate, file_size, basename, library_id, play_count, last_played, tag_status, group_key FROM audio_files +SELECT id, file_path, length_milliseconds, file_type_id, recording_id, sample_rate, bit_depth, channels, bitrate, file_size, basename, library_id, play_count, last_played, tag_status, group_key, modified_at FROM audio_files ` func (q *Queries) GetAllAudioFiles(ctx context.Context) ([]AudioFile, error) { @@ -232,6 +236,7 @@ func (q *Queries) GetAllAudioFiles(ctx context.Context) ([]AudioFile, error) { &i.LastPlayed, &i.TagStatus, &i.GroupKey, + &i.ModifiedAt, ); err != nil { return nil, err } @@ -527,7 +532,7 @@ func (q *Queries) GetAllTracksWithFullMetadataByLibrary(ctx context.Context, lib } const getAudioFile = `-- name: GetAudioFile :one -SELECT id, file_path, length_milliseconds, file_type_id, recording_id, sample_rate, bit_depth, channels, bitrate, file_size, basename, library_id, play_count, last_played, tag_status, group_key FROM audio_files +SELECT id, file_path, length_milliseconds, file_type_id, recording_id, sample_rate, bit_depth, channels, bitrate, file_size, basename, library_id, play_count, last_played, tag_status, group_key, modified_at FROM audio_files WHERE id = ? LIMIT 1 ` @@ -551,12 +556,13 @@ func (q *Queries) GetAudioFile(ctx context.Context, id int64) (AudioFile, error) &i.LastPlayed, &i.TagStatus, &i.GroupKey, + &i.ModifiedAt, ) return i, err } const getAudioFileByPath = `-- name: GetAudioFileByPath :one -SELECT id, file_path, length_milliseconds, file_type_id, recording_id, sample_rate, bit_depth, channels, bitrate, file_size, basename, library_id, play_count, last_played, tag_status, group_key FROM audio_files +SELECT id, file_path, length_milliseconds, file_type_id, recording_id, sample_rate, bit_depth, channels, bitrate, file_size, basename, library_id, play_count, last_played, tag_status, group_key, modified_at FROM audio_files WHERE file_path = ? LIMIT 1 ` @@ -580,6 +586,7 @@ func (q *Queries) GetAudioFileByPath(ctx context.Context, filePath string) (Audi &i.LastPlayed, &i.TagStatus, &i.GroupKey, + &i.ModifiedAt, ) return i, err } @@ -597,7 +604,7 @@ func (q *Queries) GetAudioFileGroupKey(ctx context.Context, id int64) (string, e } const getAudioFilesByLibrary = `-- name: GetAudioFilesByLibrary :many -SELECT id, file_path, length_milliseconds, file_type_id, recording_id, sample_rate, bit_depth, channels, bitrate, file_size, basename, library_id, play_count, last_played, tag_status, group_key FROM audio_files WHERE library_id = ? +SELECT id, file_path, length_milliseconds, file_type_id, recording_id, sample_rate, bit_depth, channels, bitrate, file_size, basename, library_id, play_count, last_played, tag_status, group_key, modified_at FROM audio_files WHERE library_id = ? ` func (q *Queries) GetAudioFilesByLibrary(ctx context.Context, libraryID int64) ([]AudioFile, error) { @@ -626,6 +633,7 @@ func (q *Queries) GetAudioFilesByLibrary(ctx context.Context, libraryID int64) ( &i.LastPlayed, &i.TagStatus, &i.GroupKey, + &i.ModifiedAt, ); err != nil { return nil, err } @@ -854,7 +862,7 @@ func (q *Queries) GetAudioFilesByReleaseGroupByLibrary(ctx context.Context, arg } const getAudioFilesNeedingMetadata = `-- name: GetAudioFilesNeedingMetadata :many -SELECT id, file_path, length_milliseconds, file_type_id, recording_id, sample_rate, bit_depth, channels, bitrate, file_size, basename, library_id, play_count, last_played, tag_status, group_key FROM audio_files +SELECT id, file_path, length_milliseconds, file_type_id, recording_id, sample_rate, bit_depth, channels, bitrate, file_size, basename, library_id, play_count, last_played, tag_status, group_key, modified_at FROM audio_files WHERE recording_id = 0 ` @@ -884,6 +892,7 @@ func (q *Queries) GetAudioFilesNeedingMetadata(ctx context.Context) ([]AudioFile &i.LastPlayed, &i.TagStatus, &i.GroupKey, + &i.ModifiedAt, ); err != nil { return nil, err } @@ -898,6 +907,20 @@ func (q *Queries) GetAudioFilesNeedingMetadata(ctx context.Context) ([]AudioFile return items, nil } +const getLibraryMaxModifiedAt = `-- name: GetLibraryMaxModifiedAt :one +SELECT CAST(COALESCE(MAX(modified_at), 0) AS INTEGER) FROM audio_files +WHERE library_id = ? +` + +// Newest recorded mtime in a library, for the startup soft scan. 0 when +// the library is empty or no row has a baseline yet. +func (q *Queries) GetLibraryMaxModifiedAt(ctx context.Context, libraryID int64) (int64, error) { + row := q.db.QueryRowContext(ctx, getLibraryMaxModifiedAt, libraryID) + var column_1 int64 + err := row.Scan(&column_1) + return column_1, err +} + const getRandomAudioFilePath = `-- name: GetRandomAudioFilePath :one SELECT file_path FROM audio_files ORDER BY RANDOM() @@ -1139,18 +1162,20 @@ func (q *Queries) UpdateAudioFile(ctx context.Context, arg UpdateAudioFileParams const updateAudioFileRecording = `-- name: UpdateAudioFileRecording :exec UPDATE audio_files -SET recording_id = ?, sample_rate = ?, bit_depth = ?, channels = ?, bitrate = ?, file_size = ? +SET recording_id = ?, sample_rate = ?, bit_depth = ?, channels = ?, bitrate = ?, file_size = ?, length_milliseconds = ?, modified_at = ? WHERE id = ? ` type UpdateAudioFileRecordingParams struct { - RecordingID int64 - SampleRate int64 - BitDepth int64 - Channels int64 - Bitrate int64 - FileSize int64 - ID int64 + RecordingID int64 + SampleRate int64 + BitDepth int64 + Channels int64 + Bitrate int64 + FileSize int64 + LengthMilliseconds int64 + ModifiedAt int64 + ID int64 } func (q *Queries) UpdateAudioFileRecording(ctx context.Context, arg UpdateAudioFileRecordingParams) error { @@ -1161,7 +1186,29 @@ func (q *Queries) UpdateAudioFileRecording(ctx context.Context, arg UpdateAudioF arg.Channels, arg.Bitrate, arg.FileSize, + arg.LengthMilliseconds, + arg.ModifiedAt, arg.ID, ) return err } + +const updateAudioFileStat = `-- name: UpdateAudioFileStat :exec +UPDATE audio_files +SET modified_at = ?, file_size = ? +WHERE id = ? +` + +type UpdateAudioFileStatParams struct { + ModifiedAt int64 + FileSize int64 + ID int64 +} + +// Records the on-disk mtime/size without re-reading tags. Used to +// backfill the staleness baseline for files the scan skipped, and to +// re-baseline after YellowJacket's own tag writer rewrites a file. +func (q *Queries) UpdateAudioFileStat(ctx context.Context, arg UpdateAudioFileStatParams) error { + _, err := q.db.ExecContext(ctx, updateAudioFileStat, arg.ModifiedAt, arg.FileSize, arg.ID) + return err +} diff --git a/backend/database/sql/sqlcgen/download.sql.go b/backend/database/sql/sqlcgen/download.sql.go new file mode 100644 index 0000000..1760217 --- /dev/null +++ b/backend/database/sql/sqlcgen/download.sql.go @@ -0,0 +1,959 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 +// source: download.sql + +package sqlcgen + +import ( + "context" + "database/sql" +) + +const createDownloadItem = `-- name: CreateDownloadItem :exec +INSERT INTO download_items ( + id, request_id, provider_id, transport_id, external_id, + candidate, state, staging_dir, bytes_total +) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) +` + +type CreateDownloadItemParams struct { + ID string + RequestID string + ProviderID int64 + TransportID sql.NullInt64 + ExternalID string + Candidate string + State string + StagingDir string + BytesTotal int64 +} + +func (q *Queries) CreateDownloadItem(ctx context.Context, arg CreateDownloadItemParams) error { + _, err := q.db.ExecContext(ctx, createDownloadItem, + arg.ID, + arg.RequestID, + arg.ProviderID, + arg.TransportID, + arg.ExternalID, + arg.Candidate, + arg.State, + arg.StagingDir, + arg.BytesTotal, + ) + return err +} + +const createDownloadProvider = `-- name: CreateDownloadProvider :one +INSERT INTO download_providers (kind, name, enabled, priority, settings) +VALUES (?, ?, ?, ?, ?) +RETURNING id +` + +type CreateDownloadProviderParams struct { + Kind string + Name string + Enabled int64 + Priority int64 + Settings string +} + +func (q *Queries) CreateDownloadProvider(ctx context.Context, arg CreateDownloadProviderParams) (int64, error) { + row := q.db.QueryRowContext(ctx, createDownloadProvider, + arg.Kind, + arg.Name, + arg.Enabled, + arg.Priority, + arg.Settings, + ) + var id int64 + err := row.Scan(&id) + return id, err +} + +const createDownloadRequest = `-- name: CreateDownloadRequest :exec +INSERT INTO download_requests ( + id, library_id, source, want_id, release_mbid, release_group_mbid, + recording_mbid, artist, album, query, expected, state +) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +` + +type CreateDownloadRequestParams struct { + ID string + LibraryID int64 + Source string + WantID sql.NullInt64 + ReleaseMbid sql.NullString + ReleaseGroupMbid sql.NullString + RecordingMbid sql.NullString + Artist string + Album string + Query string + Expected string + State string +} + +func (q *Queries) CreateDownloadRequest(ctx context.Context, arg CreateDownloadRequestParams) error { + _, err := q.db.ExecContext(ctx, createDownloadRequest, + arg.ID, + arg.LibraryID, + arg.Source, + arg.WantID, + arg.ReleaseMbid, + arg.ReleaseGroupMbid, + arg.RecordingMbid, + arg.Artist, + arg.Album, + arg.Query, + arg.Expected, + arg.State, + ) + return err +} + +const deleteDownloadProvider = `-- name: DeleteDownloadProvider :exec +DELETE FROM download_providers +WHERE id = ? +` + +func (q *Queries) DeleteDownloadProvider(ctx context.Context, id int64) error { + _, err := q.db.ExecContext(ctx, deleteDownloadProvider, id) + return err +} + +const deleteDownloadRequest = `-- name: DeleteDownloadRequest :exec +DELETE FROM download_requests +WHERE id = ? +` + +func (q *Queries) DeleteDownloadRequest(ctx context.Context, id string) error { + _, err := q.db.ExecContext(ctx, deleteDownloadRequest, id) + return err +} + +const deleteDownloadWant = `-- name: DeleteDownloadWant :exec +DELETE FROM download_wants WHERE id = ? +` + +func (q *Queries) DeleteDownloadWant(ctx context.Context, id int64) error { + _, err := q.db.ExecContext(ctx, deleteDownloadWant, id) + return err +} + +const deleteFinishedDownloadRequests = `-- name: DeleteFinishedDownloadRequests :exec +DELETE FROM download_requests +WHERE state IN ('complete', 'cancelled', 'failed') +` + +func (q *Queries) DeleteFinishedDownloadRequests(ctx context.Context) error { + _, err := q.db.ExecContext(ctx, deleteFinishedDownloadRequests) + return err +} + +const deleteSatisfiedDownloadWants = `-- name: DeleteSatisfiedDownloadWants :exec +DELETE FROM download_wants WHERE state = 'satisfied' +` + +func (q *Queries) DeleteSatisfiedDownloadWants(ctx context.Context) error { + _, err := q.db.ExecContext(ctx, deleteSatisfiedDownloadWants) + return err +} + +const getDownloadItem = `-- name: GetDownloadItem :one +SELECT id, request_id, provider_id, transport_id, external_id, candidate, + state, staging_dir, bytes_done, bytes_total, imported_paths, + error, created_at, updated_at +FROM download_items +WHERE id = ? +` + +func (q *Queries) GetDownloadItem(ctx context.Context, id string) (DownloadItem, error) { + row := q.db.QueryRowContext(ctx, getDownloadItem, id) + var i DownloadItem + err := row.Scan( + &i.ID, + &i.RequestID, + &i.ProviderID, + &i.TransportID, + &i.ExternalID, + &i.Candidate, + &i.State, + &i.StagingDir, + &i.BytesDone, + &i.BytesTotal, + &i.ImportedPaths, + &i.Error, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} + +const getDownloadProvider = `-- name: GetDownloadProvider :one +SELECT id, kind, name, enabled, priority, settings, created_at +FROM download_providers +WHERE id = ? +` + +func (q *Queries) GetDownloadProvider(ctx context.Context, id int64) (DownloadProvider, error) { + row := q.db.QueryRowContext(ctx, getDownloadProvider, id) + var i DownloadProvider + err := row.Scan( + &i.ID, + &i.Kind, + &i.Name, + &i.Enabled, + &i.Priority, + &i.Settings, + &i.CreatedAt, + ) + return i, err +} + +const getDownloadRequest = `-- name: GetDownloadRequest :one +SELECT id, library_id, source, want_id, release_mbid, release_group_mbid, + recording_mbid, artist, album, query, expected, state, error, + created_at, updated_at +FROM download_requests +WHERE id = ? +` + +func (q *Queries) GetDownloadRequest(ctx context.Context, id string) (DownloadRequest, error) { + row := q.db.QueryRowContext(ctx, getDownloadRequest, id) + var i DownloadRequest + err := row.Scan( + &i.ID, + &i.LibraryID, + &i.Source, + &i.WantID, + &i.ReleaseMbid, + &i.ReleaseGroupMbid, + &i.RecordingMbid, + &i.Artist, + &i.Album, + &i.Query, + &i.Expected, + &i.State, + &i.Error, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} + +const getDownloadWant = `-- name: GetDownloadWant :one +SELECT id, mbid, entity, library_id, artist, title, scope, secondary, state, parent_id, attempts, last_error, last_tried_at, next_try_at, external_ids, created_at, updated_at FROM download_wants WHERE id = ? +` + +func (q *Queries) GetDownloadWant(ctx context.Context, id int64) (DownloadWant, error) { + row := q.db.QueryRowContext(ctx, getDownloadWant, id) + var i DownloadWant + err := row.Scan( + &i.ID, + &i.Mbid, + &i.Entity, + &i.LibraryID, + &i.Artist, + &i.Title, + &i.Scope, + &i.Secondary, + &i.State, + &i.ParentID, + &i.Attempts, + &i.LastError, + &i.LastTriedAt, + &i.NextTryAt, + &i.ExternalIds, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} + +const getDownloadWantByMBID = `-- name: GetDownloadWantByMBID :one +SELECT id, mbid, entity, library_id, artist, title, scope, secondary, state, parent_id, attempts, last_error, last_tried_at, next_try_at, external_ids, created_at, updated_at FROM download_wants WHERE mbid = ? AND library_id = ? +` + +type GetDownloadWantByMBIDParams struct { + Mbid string + LibraryID int64 +} + +func (q *Queries) GetDownloadWantByMBID(ctx context.Context, arg GetDownloadWantByMBIDParams) (DownloadWant, error) { + row := q.db.QueryRowContext(ctx, getDownloadWantByMBID, arg.Mbid, arg.LibraryID) + var i DownloadWant + err := row.Scan( + &i.ID, + &i.Mbid, + &i.Entity, + &i.LibraryID, + &i.Artist, + &i.Title, + &i.Scope, + &i.Secondary, + &i.State, + &i.ParentID, + &i.Attempts, + &i.LastError, + &i.LastTriedAt, + &i.NextTryAt, + &i.ExternalIds, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} + +const listChildDownloadWants = `-- name: ListChildDownloadWants :many +SELECT id, mbid, entity, library_id, artist, title, scope, secondary, state, parent_id, attempts, last_error, last_tried_at, next_try_at, external_ids, created_at, updated_at FROM download_wants WHERE parent_id = ? ORDER BY id +` + +func (q *Queries) ListChildDownloadWants(ctx context.Context, parentID sql.NullInt64) ([]DownloadWant, error) { + rows, err := q.db.QueryContext(ctx, listChildDownloadWants, parentID) + if err != nil { + return nil, err + } + defer rows.Close() + var items []DownloadWant + for rows.Next() { + var i DownloadWant + if err := rows.Scan( + &i.ID, + &i.Mbid, + &i.Entity, + &i.LibraryID, + &i.Artist, + &i.Title, + &i.Scope, + &i.Secondary, + &i.State, + &i.ParentID, + &i.Attempts, + &i.LastError, + &i.LastTriedAt, + &i.NextTryAt, + &i.ExternalIds, + &i.CreatedAt, + &i.UpdatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listDownloadItemsForRequest = `-- name: ListDownloadItemsForRequest :many +SELECT id, request_id, provider_id, transport_id, external_id, candidate, + state, staging_dir, bytes_done, bytes_total, imported_paths, + error, created_at, updated_at +FROM download_items +WHERE request_id = ? +ORDER BY created_at +` + +func (q *Queries) ListDownloadItemsForRequest(ctx context.Context, requestID string) ([]DownloadItem, error) { + rows, err := q.db.QueryContext(ctx, listDownloadItemsForRequest, requestID) + if err != nil { + return nil, err + } + defer rows.Close() + var items []DownloadItem + for rows.Next() { + var i DownloadItem + if err := rows.Scan( + &i.ID, + &i.RequestID, + &i.ProviderID, + &i.TransportID, + &i.ExternalID, + &i.Candidate, + &i.State, + &i.StagingDir, + &i.BytesDone, + &i.BytesTotal, + &i.ImportedPaths, + &i.Error, + &i.CreatedAt, + &i.UpdatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listDownloadProviders = `-- name: ListDownloadProviders :many +SELECT id, kind, name, enabled, priority, settings, created_at +FROM download_providers +ORDER BY priority DESC, name +` + +func (q *Queries) ListDownloadProviders(ctx context.Context) ([]DownloadProvider, error) { + rows, err := q.db.QueryContext(ctx, listDownloadProviders) + if err != nil { + return nil, err + } + defer rows.Close() + var items []DownloadProvider + for rows.Next() { + var i DownloadProvider + if err := rows.Scan( + &i.ID, + &i.Kind, + &i.Name, + &i.Enabled, + &i.Priority, + &i.Settings, + &i.CreatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listDownloadRequests = `-- name: ListDownloadRequests :many +SELECT id, library_id, source, want_id, release_mbid, release_group_mbid, + recording_mbid, artist, album, query, expected, state, error, + created_at, updated_at +FROM download_requests +ORDER BY created_at DESC +LIMIT ? +` + +func (q *Queries) ListDownloadRequests(ctx context.Context, limit int64) ([]DownloadRequest, error) { + rows, err := q.db.QueryContext(ctx, listDownloadRequests, limit) + if err != nil { + return nil, err + } + defer rows.Close() + var items []DownloadRequest + for rows.Next() { + var i DownloadRequest + if err := rows.Scan( + &i.ID, + &i.LibraryID, + &i.Source, + &i.WantID, + &i.ReleaseMbid, + &i.ReleaseGroupMbid, + &i.RecordingMbid, + &i.Artist, + &i.Album, + &i.Query, + &i.Expected, + &i.State, + &i.Error, + &i.CreatedAt, + &i.UpdatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listDownloadWants = `-- name: ListDownloadWants :many +SELECT id, mbid, entity, library_id, artist, title, scope, secondary, state, parent_id, attempts, last_error, last_tried_at, next_try_at, external_ids, created_at, updated_at FROM download_wants +ORDER BY + CASE state WHEN 'wanted' THEN 0 WHEN 'paused' THEN 1 ELSE 2 END, + artist, title +` + +func (q *Queries) ListDownloadWants(ctx context.Context) ([]DownloadWant, error) { + rows, err := q.db.QueryContext(ctx, listDownloadWants) + if err != nil { + return nil, err + } + defer rows.Close() + var items []DownloadWant + for rows.Next() { + var i DownloadWant + if err := rows.Scan( + &i.ID, + &i.Mbid, + &i.Entity, + &i.LibraryID, + &i.Artist, + &i.Title, + &i.Scope, + &i.Secondary, + &i.State, + &i.ParentID, + &i.Attempts, + &i.LastError, + &i.LastTriedAt, + &i.NextTryAt, + &i.ExternalIds, + &i.CreatedAt, + &i.UpdatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listDownloadWantsByEntity = `-- name: ListDownloadWantsByEntity :many +SELECT id, mbid, entity, library_id, artist, title, scope, secondary, state, parent_id, attempts, last_error, last_tried_at, next_try_at, external_ids, created_at, updated_at FROM download_wants +WHERE entity = ? AND state = ? +ORDER BY id +` + +type ListDownloadWantsByEntityParams struct { + Entity string + State string +} + +func (q *Queries) ListDownloadWantsByEntity(ctx context.Context, arg ListDownloadWantsByEntityParams) ([]DownloadWant, error) { + rows, err := q.db.QueryContext(ctx, listDownloadWantsByEntity, arg.Entity, arg.State) + if err != nil { + return nil, err + } + defer rows.Close() + var items []DownloadWant + for rows.Next() { + var i DownloadWant + if err := rows.Scan( + &i.ID, + &i.Mbid, + &i.Entity, + &i.LibraryID, + &i.Artist, + &i.Title, + &i.Scope, + &i.Secondary, + &i.State, + &i.ParentID, + &i.Attempts, + &i.LastError, + &i.LastTriedAt, + &i.NextTryAt, + &i.ExternalIds, + &i.CreatedAt, + &i.UpdatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listDueDownloadWants = `-- name: ListDueDownloadWants :many +SELECT id, mbid, entity, library_id, artist, title, scope, secondary, state, parent_id, attempts, last_error, last_tried_at, next_try_at, external_ids, created_at, updated_at FROM download_wants +WHERE state = 'wanted' + AND entity <> 'artist' + AND (next_try_at IS NULL OR next_try_at <= CURRENT_TIMESTAMP) +ORDER BY attempts, created_at +LIMIT ? +` + +// Everything the reconciler should act on this pass: wanted, not an +// artist subscription (those expand rather than download), and either +// never tried or past its backoff. +func (q *Queries) ListDueDownloadWants(ctx context.Context, limit int64) ([]DownloadWant, error) { + rows, err := q.db.QueryContext(ctx, listDueDownloadWants, limit) + if err != nil { + return nil, err + } + defer rows.Close() + var items []DownloadWant + for rows.Next() { + var i DownloadWant + if err := rows.Scan( + &i.ID, + &i.Mbid, + &i.Entity, + &i.LibraryID, + &i.Artist, + &i.Title, + &i.Scope, + &i.Secondary, + &i.State, + &i.ParentID, + &i.Attempts, + &i.LastError, + &i.LastTriedAt, + &i.NextTryAt, + &i.ExternalIds, + &i.CreatedAt, + &i.UpdatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listLiveDownloadItems = `-- name: ListLiveDownloadItems :many +SELECT id, request_id, provider_id, transport_id, external_id, candidate, + state, staging_dir, bytes_done, bytes_total, imported_paths, + error, created_at, updated_at +FROM download_items +WHERE state NOT IN ('complete', 'cancelled', 'failed') +ORDER BY created_at +` + +func (q *Queries) ListLiveDownloadItems(ctx context.Context) ([]DownloadItem, error) { + rows, err := q.db.QueryContext(ctx, listLiveDownloadItems) + if err != nil { + return nil, err + } + defer rows.Close() + var items []DownloadItem + for rows.Next() { + var i DownloadItem + if err := rows.Scan( + &i.ID, + &i.RequestID, + &i.ProviderID, + &i.TransportID, + &i.ExternalID, + &i.Candidate, + &i.State, + &i.StagingDir, + &i.BytesDone, + &i.BytesTotal, + &i.ImportedPaths, + &i.Error, + &i.CreatedAt, + &i.UpdatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listLiveDownloadRequests = `-- name: ListLiveDownloadRequests :many +SELECT id, library_id, source, want_id, release_mbid, release_group_mbid, + recording_mbid, artist, album, query, expected, state, error, + created_at, updated_at +FROM download_requests +WHERE state NOT IN ('complete', 'cancelled', 'failed') +ORDER BY created_at +` + +func (q *Queries) ListLiveDownloadRequests(ctx context.Context) ([]DownloadRequest, error) { + rows, err := q.db.QueryContext(ctx, listLiveDownloadRequests) + if err != nil { + return nil, err + } + defer rows.Close() + var items []DownloadRequest + for rows.Next() { + var i DownloadRequest + if err := rows.Scan( + &i.ID, + &i.LibraryID, + &i.Source, + &i.WantID, + &i.ReleaseMbid, + &i.ReleaseGroupMbid, + &i.RecordingMbid, + &i.Artist, + &i.Album, + &i.Query, + &i.Expected, + &i.State, + &i.Error, + &i.CreatedAt, + &i.UpdatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const recordDownloadWantAttempt = `-- name: RecordDownloadWantAttempt :exec +UPDATE download_wants +SET attempts = attempts + 1, + last_error = ?, + last_tried_at = CURRENT_TIMESTAMP, + next_try_at = ?, + updated_at = CURRENT_TIMESTAMP +WHERE id = ? +` + +type RecordDownloadWantAttemptParams struct { + LastError string + NextTryAt sql.NullTime + ID int64 +} + +func (q *Queries) RecordDownloadWantAttempt(ctx context.Context, arg RecordDownloadWantAttemptParams) error { + _, err := q.db.ExecContext(ctx, recordDownloadWantAttempt, arg.LastError, arg.NextTryAt, arg.ID) + return err +} + +const satisfyDownloadWant = `-- name: SatisfyDownloadWant :exec +UPDATE download_wants +SET state = 'satisfied', last_error = '', next_try_at = NULL, + updated_at = CURRENT_TIMESTAMP +WHERE id = ? +` + +func (q *Queries) SatisfyDownloadWant(ctx context.Context, id int64) error { + _, err := q.db.ExecContext(ctx, satisfyDownloadWant, id) + return err +} + +const setDownloadItemExternalID = `-- name: SetDownloadItemExternalID :exec +UPDATE download_items +SET external_id = ?, updated_at = CURRENT_TIMESTAMP +WHERE id = ? +` + +type SetDownloadItemExternalIDParams struct { + ExternalID string + ID string +} + +func (q *Queries) SetDownloadItemExternalID(ctx context.Context, arg SetDownloadItemExternalIDParams) error { + _, err := q.db.ExecContext(ctx, setDownloadItemExternalID, arg.ExternalID, arg.ID) + return err +} + +const setDownloadItemImported = `-- name: SetDownloadItemImported :exec +UPDATE download_items +SET imported_paths = ?, state = 'complete', error = '', + updated_at = CURRENT_TIMESTAMP +WHERE id = ? +` + +type SetDownloadItemImportedParams struct { + ImportedPaths string + ID string +} + +func (q *Queries) SetDownloadItemImported(ctx context.Context, arg SetDownloadItemImportedParams) error { + _, err := q.db.ExecContext(ctx, setDownloadItemImported, arg.ImportedPaths, arg.ID) + return err +} + +const setDownloadItemProgress = `-- name: SetDownloadItemProgress :exec +UPDATE download_items +SET bytes_done = ?, bytes_total = ?, updated_at = CURRENT_TIMESTAMP +WHERE id = ? +` + +type SetDownloadItemProgressParams struct { + BytesDone int64 + BytesTotal int64 + ID string +} + +func (q *Queries) SetDownloadItemProgress(ctx context.Context, arg SetDownloadItemProgressParams) error { + _, err := q.db.ExecContext(ctx, setDownloadItemProgress, arg.BytesDone, arg.BytesTotal, arg.ID) + return err +} + +const setDownloadItemState = `-- name: SetDownloadItemState :exec +UPDATE download_items +SET state = ?, error = ?, updated_at = CURRENT_TIMESTAMP +WHERE id = ? +` + +type SetDownloadItemStateParams struct { + State string + Error string + ID string +} + +func (q *Queries) SetDownloadItemState(ctx context.Context, arg SetDownloadItemStateParams) error { + _, err := q.db.ExecContext(ctx, setDownloadItemState, arg.State, arg.Error, arg.ID) + return err +} + +const setDownloadRequestState = `-- name: SetDownloadRequestState :exec +UPDATE download_requests +SET state = ?, error = ?, updated_at = CURRENT_TIMESTAMP +WHERE id = ? +` + +type SetDownloadRequestStateParams struct { + State string + Error string + ID string +} + +func (q *Queries) SetDownloadRequestState(ctx context.Context, arg SetDownloadRequestStateParams) error { + _, err := q.db.ExecContext(ctx, setDownloadRequestState, arg.State, arg.Error, arg.ID) + return err +} + +const setDownloadWantExternalIDs = `-- name: SetDownloadWantExternalIDs :exec +UPDATE download_wants +SET external_ids = ?, updated_at = CURRENT_TIMESTAMP +WHERE id = ? +` + +type SetDownloadWantExternalIDsParams struct { + ExternalIds string + ID int64 +} + +func (q *Queries) SetDownloadWantExternalIDs(ctx context.Context, arg SetDownloadWantExternalIDsParams) error { + _, err := q.db.ExecContext(ctx, setDownloadWantExternalIDs, arg.ExternalIds, arg.ID) + return err +} + +const setDownloadWantState = `-- name: SetDownloadWantState :exec +UPDATE download_wants +SET state = ?, last_error = ?, updated_at = CURRENT_TIMESTAMP +WHERE id = ? +` + +type SetDownloadWantStateParams struct { + State string + LastError string + ID int64 +} + +func (q *Queries) SetDownloadWantState(ctx context.Context, arg SetDownloadWantStateParams) error { + _, err := q.db.ExecContext(ctx, setDownloadWantState, arg.State, arg.LastError, arg.ID) + return err +} + +const updateDownloadProvider = `-- name: UpdateDownloadProvider :exec +UPDATE download_providers +SET name = ?, enabled = ?, priority = ?, settings = ? +WHERE id = ? +` + +type UpdateDownloadProviderParams struct { + Name string + Enabled int64 + Priority int64 + Settings string + ID int64 +} + +func (q *Queries) UpdateDownloadProvider(ctx context.Context, arg UpdateDownloadProviderParams) error { + _, err := q.db.ExecContext(ctx, updateDownloadProvider, + arg.Name, + arg.Enabled, + arg.Priority, + arg.Settings, + arg.ID, + ) + return err +} + +const upsertDownloadWant = `-- name: UpsertDownloadWant :one + +INSERT INTO download_wants ( + mbid, entity, library_id, artist, title, scope, secondary, + parent_id, next_try_at +) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) +ON CONFLICT(mbid, library_id) DO UPDATE SET + artist = CASE WHEN excluded.artist <> '' THEN excluded.artist + ELSE download_wants.artist END, + title = CASE WHEN excluded.title <> '' THEN excluded.title + ELSE download_wants.title END, + scope = excluded.scope, + secondary = excluded.secondary, + updated_at = CURRENT_TIMESTAMP +RETURNING id +` + +type UpsertDownloadWantParams struct { + Mbid string + Entity string + LibraryID int64 + Artist string + Title string + Scope string + Secondary int64 + ParentID sql.NullInt64 +} + +// --------------------------------------------------------------------- +// Wants +// --------------------------------------------------------------------- +// Adding something already wanted is not an error and must not reset +// the retry clock, so the conflict path only refreshes display text and +// un-pauses nothing. scope and secondary are updated because asking +// again with a wider scope is a real change of intent. +func (q *Queries) UpsertDownloadWant(ctx context.Context, arg UpsertDownloadWantParams) (int64, error) { + row := q.db.QueryRowContext(ctx, upsertDownloadWant, + arg.Mbid, + arg.Entity, + arg.LibraryID, + arg.Artist, + arg.Title, + arg.Scope, + arg.Secondary, + arg.ParentID, + ) + var id int64 + err := row.Scan(&id) + return id, err +} diff --git a/backend/database/sql/sqlcgen/models.go b/backend/database/sql/sqlcgen/models.go index ff7ab29..6af50e8 100644 --- a/backend/database/sql/sqlcgen/models.go +++ b/backend/database/sql/sqlcgen/models.go @@ -26,6 +26,20 @@ type ArtistCreditArtist struct { CreditID int64 } +type ArtistImage struct { + ID int64 + ArtistMbid string + Source string + SourceUrl string + FilePath string + IsPrimary int64 + SortOrder int64 + Width sql.NullInt64 + Height sql.NullInt64 + FileSize sql.NullInt64 + CreatedAt time.Time +} + type ArtistMetadatum struct { Mbid string Source string @@ -50,6 +64,7 @@ type AudioFile struct { LastPlayed sql.NullTime TagStatus string GroupKey string + ModifiedAt int64 } type CoverArt struct { @@ -59,6 +74,116 @@ type CoverArt struct { MimeType string } +type DownloadItem struct { + ID string + RequestID string + ProviderID int64 + TransportID sql.NullInt64 + ExternalID string + Candidate string + State string + StagingDir string + BytesDone int64 + BytesTotal int64 + ImportedPaths string + Error string + CreatedAt time.Time + UpdatedAt time.Time +} + +type DownloadProvider struct { + ID int64 + Kind string + Name string + Enabled int64 + Priority int64 + Settings string + CreatedAt time.Time +} + +type DownloadRequest struct { + ID string + LibraryID int64 + Source string + WantID sql.NullInt64 + ReleaseMbid sql.NullString + ReleaseGroupMbid sql.NullString + RecordingMbid sql.NullString + Artist string + Album string + Query string + Expected string + State string + Error string + CreatedAt time.Time + UpdatedAt time.Time +} + +type DownloadWant struct { + ID int64 + Mbid string + Entity string + LibraryID int64 + Artist string + Title string + Scope string + Secondary int64 + State string + ParentID sql.NullInt64 + Attempts int64 + LastError string + LastTriedAt sql.NullTime + NextTryAt sql.NullTime + ExternalIds string + CreatedAt time.Time + UpdatedAt time.Time +} + +type ExploreChampionFt struct { + Title string + ArtistName string + Aliases string +} + +type ExploreIndex struct { + ID int64 + EntityType string + Mbid string + Title string + ArtistName string + ArtistMbid string + Aliases string + Popularity int64 + ListenerCount int64 + Duration int64 + CaaReleaseMbid string + ReleaseName string + PrimaryType string + SecondaryTypes string + ReleaseDate string + ArtistType string + Country string + Disambiguation string + SortName string + InLibrary int64 + IsSimilar int64 + LocalArtistID sql.NullInt64 + LocalReleaseGroupID sql.NullInt64 + LocalRecordingID sql.NullInt64 + DiscogFetched int64 +} + +type ExploreIndexFt struct { + Title string + ArtistName string + Aliases string +} + +type ExploreIndexMetum struct { + Key string + Value string +} + type FileType struct { ID int64 Extension string @@ -176,10 +301,10 @@ type ReleaseGroup struct { CoverArtID sql.NullInt64 AlbumArtistCreditID sql.NullInt64 Year sql.NullInt64 - OriginalYear sql.NullInt64 TotalTracks sql.NullInt64 TotalDiscs sql.NullInt64 Mbid sql.NullString + OriginalYear sql.NullInt64 } type ReleaseGroupRecording struct { @@ -190,6 +315,19 @@ type ReleaseGroupRecording struct { DiscNumber sql.NullInt64 } +type ReleaseToRg struct { + ReleaseMbid string + RgMbid string +} + +type SearchClick struct { + Query string + EntityMbid string + EntityType string + ClickCount int64 + LastClicked time.Time +} + type SearchIndex struct { FilePath string Title string @@ -197,6 +335,13 @@ type SearchIndex struct { Album string } +type SimilarArtistMap struct { + SourceArtistMbid string + SimilarArtistMbid string + SimilarArtistName string + Score int64 +} + type TaggingCandidate struct { GroupKey string Candidates string diff --git a/backend/database/sql/sqlcgen/release_groups.sql.go b/backend/database/sql/sqlcgen/release_groups.sql.go index d191b73..5e12cb2 100644 --- a/backend/database/sql/sqlcgen/release_groups.sql.go +++ b/backend/database/sql/sqlcgen/release_groups.sql.go @@ -23,7 +23,7 @@ func (q *Queries) CountReleaseGroupRecordings(ctx context.Context, releaseGroupI const createReleaseGroup = `-- name: CreateReleaseGroup :one INSERT INTO release_groups (name) VALUES (?) -RETURNING id, name, cover_art_id, album_artist_credit_id, year, original_year, total_tracks, total_discs, mbid +RETURNING id, name, cover_art_id, album_artist_credit_id, year, total_tracks, total_discs, mbid, original_year ` func (q *Queries) CreateReleaseGroup(ctx context.Context, name string) (ReleaseGroup, error) { @@ -35,10 +35,10 @@ func (q *Queries) CreateReleaseGroup(ctx context.Context, name string) (ReleaseG &i.CoverArtID, &i.AlbumArtistCreditID, &i.Year, - &i.OriginalYear, &i.TotalTracks, &i.TotalDiscs, &i.Mbid, + &i.OriginalYear, ) return i, err } @@ -47,7 +47,7 @@ const createReleaseGroupFull = `-- name: CreateReleaseGroupFull :one INSERT INTO release_groups ( name, cover_art_id, album_artist_credit_id, year, total_tracks, total_discs ) VALUES (?, ?, ?, ?, ?, ?) -RETURNING id, name, cover_art_id, album_artist_credit_id, year, original_year, total_tracks, total_discs, mbid +RETURNING id, name, cover_art_id, album_artist_credit_id, year, total_tracks, total_discs, mbid, original_year ` type CreateReleaseGroupFullParams struct { @@ -75,10 +75,10 @@ func (q *Queries) CreateReleaseGroupFull(ctx context.Context, arg CreateReleaseG &i.CoverArtID, &i.AlbumArtistCreditID, &i.Year, - &i.OriginalYear, &i.TotalTracks, &i.TotalDiscs, &i.Mbid, + &i.OriginalYear, ) return i, err } @@ -432,7 +432,7 @@ func (q *Queries) GetAllAlbumsWithDetailsByLibrary(ctx context.Context, libraryI } const getAllReleaseGroups = `-- name: GetAllReleaseGroups :many -SELECT id, name, cover_art_id, album_artist_credit_id, year, original_year, total_tracks, total_discs, mbid FROM release_groups +SELECT id, name, cover_art_id, album_artist_credit_id, year, total_tracks, total_discs, mbid, original_year FROM release_groups ORDER BY name ` @@ -451,10 +451,10 @@ func (q *Queries) GetAllReleaseGroups(ctx context.Context) ([]ReleaseGroup, erro &i.CoverArtID, &i.AlbumArtistCreditID, &i.Year, - &i.OriginalYear, &i.TotalTracks, &i.TotalDiscs, &i.Mbid, + &i.OriginalYear, ); err != nil { return nil, err } @@ -470,7 +470,7 @@ func (q *Queries) GetAllReleaseGroups(ctx context.Context) ([]ReleaseGroup, erro } const getReleaseGroup = `-- name: GetReleaseGroup :one -SELECT id, name, cover_art_id, album_artist_credit_id, year, original_year, total_tracks, total_discs, mbid FROM release_groups +SELECT id, name, cover_art_id, album_artist_credit_id, year, total_tracks, total_discs, mbid, original_year FROM release_groups WHERE id = ? LIMIT 1 ` @@ -483,16 +483,16 @@ func (q *Queries) GetReleaseGroup(ctx context.Context, id int64) (ReleaseGroup, &i.CoverArtID, &i.AlbumArtistCreditID, &i.Year, - &i.OriginalYear, &i.TotalTracks, &i.TotalDiscs, &i.Mbid, + &i.OriginalYear, ) return i, err } const getReleaseGroupByNameAndArtist = `-- name: GetReleaseGroupByNameAndArtist :one -SELECT id, name, cover_art_id, album_artist_credit_id, year, original_year, total_tracks, total_discs, mbid FROM release_groups +SELECT id, name, cover_art_id, album_artist_credit_id, year, total_tracks, total_discs, mbid, original_year FROM release_groups WHERE name = ? AND album_artist_credit_id = ? LIMIT 1 ` @@ -510,10 +510,10 @@ func (q *Queries) GetReleaseGroupByNameAndArtist(ctx context.Context, arg GetRel &i.CoverArtID, &i.AlbumArtistCreditID, &i.Year, - &i.OriginalYear, &i.TotalTracks, &i.TotalDiscs, &i.Mbid, + &i.OriginalYear, ) return i, err } @@ -574,7 +574,7 @@ VALUES (?, ?, ?) ON CONFLICT(name, album_artist_credit_id) DO UPDATE SET album_artist_credit_id = COALESCE(excluded.album_artist_credit_id, release_groups.album_artist_credit_id), year = COALESCE(excluded.year, release_groups.year) -RETURNING id, name, cover_art_id, album_artist_credit_id, year, original_year, total_tracks, total_discs, mbid +RETURNING id, name, cover_art_id, album_artist_credit_id, year, total_tracks, total_discs, mbid, original_year ` type UpsertReleaseGroupParams struct { @@ -592,10 +592,10 @@ func (q *Queries) UpsertReleaseGroup(ctx context.Context, arg UpsertReleaseGroup &i.CoverArtID, &i.AlbumArtistCreditID, &i.Year, - &i.OriginalYear, &i.TotalTracks, &i.TotalDiscs, &i.Mbid, + &i.OriginalYear, ) return i, err } diff --git a/backend/database/tagging_items_test.go b/backend/database/tagging_items_test.go index 139d1dd..93b388e 100644 --- a/backend/database/tagging_items_test.go +++ b/backend/database/tagging_items_test.go @@ -13,7 +13,7 @@ import ( // Migration 31: tag_status column // --------------------------------------------------------------------------- -func TestMigration31_TagStatusDefaultAndCheck(t *testing.T) { +func TestTagStatusDefaultAndCheck(t *testing.T) { t.Parallel() db := database.NewTestDB(t) @@ -42,7 +42,7 @@ func TestMigration31_TagStatusDefaultAndCheck(t *testing.T) { } } -func TestMigration31_BackfillFromRecordingMBID(t *testing.T) { +func TestTagStatusBackfillFromRecordingMBID(t *testing.T) { t.Parallel() // Migration 31 runs against a DB that already exists — NewTestDB @@ -97,7 +97,7 @@ func TestMigration31_BackfillFromRecordingMBID(t *testing.T) { // Migration 32: tagging_items table + group_key column // --------------------------------------------------------------------------- -func TestMigration32_TaggingItemsAndGroupKey(t *testing.T) { +func TestTaggingItemsAndGroupKey(t *testing.T) { t.Parallel() db := database.NewTestDB(t) diff --git a/backend/database/testhelper.go b/backend/database/testhelper.go index 0c083b3..e8aef3c 100644 --- a/backend/database/testhelper.go +++ b/backend/database/testhelper.go @@ -2,9 +2,7 @@ package database import ( "database/sql" - "io/fs" "log/slog" - "path" "testing" _ "modernc.org/sqlite" // Register sqlite driver. @@ -34,31 +32,10 @@ func NewTestDB(t *testing.T) *DB { t.Fatalf("could not apply PRAGMAs: %v", err) } - dirEntries, err := schemas.ReadDir("sql/schemas") - if err != nil { - t.Fatalf("could not read schemas directory: %v", err) - } - - for _, dirEntry := range dirEntries { - if !dirEntry.IsDir() { - filePath := path.Join("sql/schemas", dirEntry.Name()) - - sqlContent, err := fs.ReadFile(schemas, filePath) - if err != nil { - t.Fatalf("could not read file %s: %v", filePath, err) - } - - if _, err = db.ExecContext(ctx, string(sqlContent)); err != nil { - t.Fatalf( - "error executing sql from file %s: %v", - filePath, err, - ) - } - } - } - - if err := runMigrations(ctx, db, slog.Default(), ":memory:"); err != nil { - t.Fatalf("could not run migrations: %v", err) + // The same call production uses, so a test database and a real one + // cannot diverge. + if err := applySchema(ctx, db); err != nil { + t.Fatalf("could not apply schema: %v", err) } // Insert a sentinel library row at id=0 so audio_files inserts diff --git a/backend/datamap/datamap.go b/backend/datamap/datamap.go new file mode 100644 index 0000000..fbe5ed1 --- /dev/null +++ b/backend/datamap/datamap.go @@ -0,0 +1,410 @@ +// Package datamap is the catalog of everything YellowJacket persists. +// +// It exists because deletion logic used to be written per call site and +// lived far from the thing being deleted. Nobody adding a table could +// know the full set of places that needed updating, so tables leaked +// (rows nothing ever removed), files leaked (thumbnails whose database +// row was gone), and in one case a new table's foreign key silently made +// libraries unremovable. +// +// Every table, view, and on-disk asset directory is classified along two +// axes that between them determine every policy worth having: +// +// - Kind — what it costs to lose the data. +// - Lifetime — how rows or files are removed. +// +// The catalog is plain data with no dependency on any service, so tests +// can assert it against a live schema. The rule that gives it teeth: +// every table in the database must be claimed by exactly one entry here, +// enforced by TestCatalogCoversSchema. A new table fails the build until +// somebody states what it is and how it dies. +package datamap + +import "strings" + +// Kind classifies persisted data by what losing it costs. +type Kind string + +const ( + // Owned data is a projection of the user's audio files. The files + // on disk are the source of truth; a rescan rebuilds all of it. + Owned Kind = "owned" + + // Authored data was created by the user and exists nowhere else. + // Losing it is unrecoverable data loss — it must never be deleted + // as a side effect of anything. + Authored Kind = "authored" + + // Derived data is computed from owned data and is cheap to rebuild. + // It may be deleted freely and must never block deletion of the + // data it was derived from. + Derived Kind = "derived" + + // Cache data came from the network or a MusicBrainz dump. It is + // rebuildable but expensive — rate limits, or a multi-hour index + // build — so it is evicted on its own schedule rather than being + // tied to the lifetime of anything else. + Cache Kind = "cache" +) + +// Lifetime says how rows leave a table. It is declared here and checked +// against the live schema by TestLifetimesMatchSchema, so a foreign key +// that disagrees with the stated policy fails the tests. +type Lifetime string + +const ( + // Cascade means a foreign key with ON DELETE CASCADE removes rows + // when the parent goes. No application code required. + Cascade Lifetime = "cascade" + + // SetNull means a foreign key with ON DELETE SET NULL orphans the + // row deliberately, keeping it alive without its parent. + SetNull Lifetime = "set-null" + + // Swept means application code must delete these rows explicitly — + // either in a removal path or in the maintenance janitor. Any table + // with a NO ACTION foreign key must declare this, because such a key + // blocks its parent's deletion until someone clears the child rows. + Swept Lifetime = "swept" + + // Retained means rows are never removed automatically. Only an + // explicit user action deletes them. + Retained Lifetime = "retained" +) + +// Table is one catalog entry. +type Table struct { + // Name is the SQL table or view name. + Name string + // Kind is what the data costs to lose. + Kind Kind + // Lifetime is how rows are removed. + Lifetime Lifetime + // FTS marks an FTS5 virtual table, which SQLite backs with four + // shadow tables (_config, _data, _docsize, _idx). Those are + // implementation detail and are resolved to this entry. + FTS bool + // Note records why the classification is what it is, particularly + // where a table is not purely one kind. + Note string +} + +// ftsShadowSuffixes are the tables SQLite creates behind an FTS5 virtual +// table. They belong to their parent and are never catalogued directly. +var ftsShadowSuffixes = []string{ + "_config", "_data", "_docsize", "_idx", +} + +// internalTables are SQLite's own bookkeeping, outside our control. +var internalTables = map[string]bool{ + "sqlite_sequence": true, + "sqlite_stat1": true, + "sqlite_stat4": true, +} + +// tables is the catalog. Keep it alphabetical. +var tables = []Table{ + { + Name: "artist_credit", Kind: Owned, Lifetime: Swept, + Note: "Credit strings parsed from file tags. Orphan-swept when " + + "no recording references them.", + }, + { + Name: "artist_credit_artist", Kind: Owned, Lifetime: Swept, + Note: "Join table between credits and artists.", + }, + { + Name: "artist_images", Kind: Cache, Lifetime: Swept, + Note: "Artist photos from fanart.tv/MusicBrainz. Rows point at " + + "files under the artist-images directory; the janitor sweeps " + + "both together.", + }, + { + Name: "artist_metadata", Kind: Cache, Lifetime: Swept, + Note: "Fetched artist bios and metadata. No TTL — swept when the " + + "artist is no longer referenced.", + }, + { + Name: "artists", Kind: Owned, Lifetime: Swept, + Note: "Artists parsed from file tags. Orphan-swept.", + }, + { + Name: "audio_files", Kind: Owned, Lifetime: Swept, + Note: "MIXED KIND. Mostly an owned projection of files on disk, " + + "but play_count, last_played and tag_status are authored and " + + "exist nowhere else. Deleting a row to rebuild it destroys " + + "that authored state — which is why a file rename currently " + + "loses play counts. See the data architecture plan.", + }, + { + Name: "cover_art", Kind: Owned, Lifetime: Swept, + Note: "Extracted embedded artwork. file_path names the original " + + "only; the sized variants beside it are derived filenames and " + + "must be expanded when deleting (see library.coverArtFileSet).", + }, + { + Name: "download_items", Kind: Authored, Lifetime: Cascade, + Note: "One grab attempt per row, with the ranked candidate stored " + + "as JSON. Cascades from download_requests. The candidate blob " + + "is kept rather than re-derived because a provider's result " + + "set is ephemeral — the peer that had the files may be gone, " + + "and the row still has to explain why it was chosen.", + }, + { + Name: "download_providers", Kind: Authored, Lifetime: Retained, + Note: "Download clients the user connected. Removed only by the " + + "user. Holds no secrets: API keys live in a 0600 file keyed " + + "by this row's id, so the table can be dumped into a bug " + + "report without redaction.", + }, + { + Name: "download_requests", Kind: Authored, Lifetime: Cascade, + Note: "One row per 'go find me this'. Cascades from libraries, " + + "and cascades onward to download_items. Terminal rows are " + + "history the user clears explicitly.", + }, + { + Name: "download_wants", Kind: Authored, Lifetime: Cascade, + Note: "The wanted list: one MBID per row, plus retry bookkeeping. " + + "Cascades from libraries, and from a parent artist want to " + + "the album wants it derived. Unlike download_requests these " + + "are not history — a want outlives every attempt made on it " + + "and is only removed by the user or by the library coming " + + "to own what it names.", + }, + { + Name: "explore_champion_fts", Kind: Cache, Lifetime: Retained, FTS: true, + Note: "Full-text index over the champion entities of the " + + "MusicBrainz dump. Rebuilt only by a full index build.", + }, + { + Name: "explore_index", Kind: Cache, Lifetime: Retained, + Note: "The offline MusicBrainz search index. Rebuilding costs a " + + "~205GB dump stream, so it is never swept automatically.", + }, + { + Name: "explore_index_fts", Kind: Cache, Lifetime: Retained, FTS: true, + Note: "Full-text index over explore_index.", + }, + { + Name: "explore_index_meta", Kind: Cache, Lifetime: Retained, + Note: "Build metadata for explore_index: dump version, coverage " + + "tiers, last refresh.", + }, + { + Name: "file_types", Kind: Derived, Lifetime: Retained, + Note: "Static lookup rows seeded from code, not user data.", + }, + { + Name: "genres", Kind: Owned, Lifetime: Swept, + Note: "Genres parsed from file tags. Orphan-swept.", + }, + { + Name: "http_cache", Kind: Cache, Lifetime: Swept, + Note: "Cached HTTP responses with a TTL. Reads filter on " + + "expires_at; the janitor deletes expired rows.", + }, + { + Name: "job_state", Kind: Authored, Lifetime: Retained, + Note: "Persisted background-job state, including scans the user " + + "paused. Represents user intent, so it survives restarts.", + }, + { + Name: "libraries", Kind: Authored, Lifetime: Retained, + Note: "The directories the user chose. Removed only by explicit " + + "user action via RemoveLibrary.", + }, + { + Name: "lyrics_index", Kind: Derived, Lifetime: Retained, FTS: true, + Note: "Full-text index over embedded and fetched lyrics. Rebuilt " + + "from owned files plus the LRCLIB backfill.", + }, + { + Name: "play_history", Kind: Authored, Lifetime: Cascade, + Note: "Listening history. Authored, but intentionally cascades " + + "with its track — history for a file no longer in the library " + + "has nothing to point at.", + }, + { + Name: "player_state", Kind: Authored, Lifetime: Retained, + Note: "Volume, repeat and shuffle modes, last position.", + }, + { + Name: "playlist_tracks", Kind: Authored, Lifetime: SetNull, + Note: "Playlist membership. Deliberately survives its track: " + + "audio_file_id is nulled and phantom_* columns preserve the " + + "entry so a rescan can re-link it.", + }, + { + Name: "playlists", Kind: Authored, Lifetime: Retained, + Note: "User-created playlists, including smart playlist rules.", + }, + { + Name: "queue", Kind: Authored, Lifetime: Retained, + Note: "The play queue's own state (current position, source).", + }, + { + Name: "queue_tracks", Kind: Authored, Lifetime: Cascade, + Note: "Queue entries. Cascade with their track; the queue is " + + "compacted afterwards.", + }, + { + Name: "recording_genres", Kind: Owned, Lifetime: Swept, + Note: "Join table between recordings and genres.", + }, + { + Name: "recordings", Kind: Owned, Lifetime: Swept, + Note: "Tracks as parsed from file tags. Orphan-swept.", + }, + { + Name: "release_group_recordings", Kind: Owned, Lifetime: Swept, + Note: "Join table between release groups and recordings.", + }, + { + Name: "release_groups", Kind: Owned, Lifetime: Swept, + Note: "Albums as parsed from file tags. Orphan-swept.", + }, + { + Name: "release_to_rg", Kind: Cache, Lifetime: Retained, + Note: "Release to release-group mapping from the dump.", + }, + { + Name: "search_clicks", Kind: Authored, Lifetime: Retained, + Note: "Which results the user picked, used to rank future " + + "searches. Behavioural but unrecoverable if dropped.", + }, + { + Name: "search_index", Kind: Derived, Lifetime: Retained, FTS: true, + Note: "Contentless FTS5 over the library. Individual rows cannot " + + "be deleted, so stale entries are tolerated and filtered by " + + "joining track_metadata; a full rescan rebuilds it.", + }, + { + Name: "similar_artist_map", Kind: Cache, Lifetime: Retained, + Note: "Artist similarity edges derived from the dump.", + }, + { + Name: "tagging_candidates", Kind: Derived, Lifetime: Cascade, + Note: "Scored MusicBrainz candidates for a tagging group. " + + "Cascades with its tagging_items row.", + }, + { + Name: "tagging_items", Kind: Derived, Lifetime: Swept, + Note: "MIXED KIND. The grouping is derived from folder layout, " + + "but status and cleared_at record the user's review " + + "decisions. Its library_id foreign key is NO ACTION, so " + + "RemoveLibrary must delete these rows explicitly or the " + + "library cannot be removed at all.", + }, + { + Name: "track_metadata", Kind: Derived, Lifetime: Retained, + Note: "A view joining audio_files to its entity chain. Holds no " + + "rows of its own.", + }, +} + +// directories are the on-disk asset trees under the user data directory. +// Files there have no foreign keys, so nothing removes them implicitly — +// each needs a sweeper in the janitor. +var directories = []Directory{ + { + Name: "covers", Kind: Derived, + Note: "Extracted cover art plus generated _sm/_md/_lg variants. " + + "Live set is cover_art.file_path expanded to its variants.", + }, + { + Name: "artist-images", Kind: Cache, + Note: "Per-artist-MBID directories of fetched photos, a " + + "primary.jpg with thumbnails, and a .miss marker for artists " + + "known to have no art. Live set is artist_images.file_path.", + }, + { + Name: "cover-art-cache", Kind: Cache, + Note: "Cover Art Archive images fetched for Explore browsing, " + + "keyed by release-group MBID. Not tied to owned data at all.", + }, +} + +// Directory is an on-disk asset tree owned by the application. +type Directory struct { + // Name is the directory name under the user data directory. + Name string + // Kind is what the files cost to lose. + Kind Kind + // Note records what lives there and how the live set is determined. + Note string +} + +// Tables returns the full catalog. +func Tables() []Table { + out := make([]Table, len(tables)) + copy(out, tables) + + return out +} + +// Directories returns the catalogued asset directories. +func Directories() []Directory { + out := make([]Directory, len(directories)) + copy(out, directories) + + return out +} + +// Lookup returns the catalog entry for a table name, resolving FTS +// shadow tables to their parent. Reports false for SQLite's internal +// tables and for anything not catalogued. +func Lookup(name string) (Table, bool) { + if internalTables[name] { + return Table{}, false + } + + for _, t := range tables { + if t.Name == name { + return t, true + } + } + + if parent, ok := ftsParent(name); ok { + return Lookup(parent) + } + + return Table{}, false +} + +// IsInternal reports whether a table is SQLite's own bookkeeping. +func IsInternal(name string) bool { + return internalTables[name] || strings.HasPrefix(name, "sqlite_") +} + +// ftsParent maps an FTS5 shadow table to the virtual table that owns it. +func ftsParent(name string) (string, bool) { + for _, suffix := range ftsShadowSuffixes { + if !strings.HasSuffix(name, suffix) { + continue + } + + parent := strings.TrimSuffix(name, suffix) + + for _, t := range tables { + if t.Name == parent && t.FTS { + return parent, true + } + } + } + + return "", false +} + +// ByKind returns every catalogued table of a given kind. +func ByKind(k Kind) []Table { + var out []Table + + for _, t := range tables { + if t.Kind == k { + out = append(out, t) + } + } + + return out +} diff --git a/backend/datamap/datamap_test.go b/backend/datamap/datamap_test.go new file mode 100644 index 0000000..6f48357 --- /dev/null +++ b/backend/datamap/datamap_test.go @@ -0,0 +1,287 @@ +package datamap_test + +import ( + "fmt" + "testing" + + "yellowjacket/backend/database" + "yellowjacket/backend/datamap" +) + +// liveTables returns every table and view in a freshly migrated schema. +func liveTables(t *testing.T, db *database.DB) []string { + t.Helper() + + rows, err := db.QueryContext( + `SELECT name FROM sqlite_master + WHERE type IN ('table', 'view') + ORDER BY name`, + ) + if err != nil { + t.Fatalf("read sqlite_master: %v", err) + } + + defer func() { _ = rows.Close() }() + + var names []string + + for rows.Next() { + var name string + + if err := rows.Scan(&name); err != nil { + t.Fatalf("scan table name: %v", err) + } + + names = append(names, name) + } + + return names +} + +// Every table in the schema must be claimed by exactly one catalog +// entry. This is the mechanism that stops a new table from silently +// having no deletion policy — the failure mode that made libraries +// unremovable when tagging_items was added. +func TestCatalogCoversSchema(t *testing.T) { + t.Parallel() + + db := database.NewTestDB(t) + + for _, name := range liveTables(t, db) { + if datamap.IsInternal(name) { + continue + } + + if _, ok := datamap.Lookup(name); !ok { + t.Errorf( + "table %q exists in the schema but is not in the datamap "+ + "catalog — add an entry stating its Kind and Lifetime", + name, + ) + } + } +} + +// The reverse direction: a catalog entry naming a table that no longer +// exists means the catalog has drifted. +func TestCatalogHasNoStaleEntries(t *testing.T) { + t.Parallel() + + db := database.NewTestDB(t) + + live := make(map[string]bool) + for _, name := range liveTables(t, db) { + live[name] = true + } + + for _, entry := range datamap.Tables() { + if !live[entry.Name] { + t.Errorf( + "catalog lists %q but it is not in the schema", + entry.Name, + ) + } + } +} + +type foreignKey struct { + child string + from string + parent string + onDelete string +} + +// liveForeignKeys reads every foreign key in the schema. +func liveForeignKeys(t *testing.T, db *database.DB) []foreignKey { + t.Helper() + + var out []foreignKey + + for _, table := range liveTables(t, db) { + if datamap.IsInternal(table) { + continue + } + + rows, err := db.QueryContext( + fmt.Sprintf("PRAGMA foreign_key_list(%q)", table), + ) + if err != nil { + continue // views have none + } + + for rows.Next() { + var ( + id, seq int + parent, from, to, onUpd, onDel, matchOn string + ) + + if err := rows.Scan( + &id, &seq, &parent, &from, &to, &onUpd, &onDel, &matchOn, + ); err != nil { + continue + } + + out = append(out, foreignKey{ + child: table, + from: from, + parent: parent, + onDelete: onDel, + }) + } + + _ = rows.Close() + } + + return out +} + +// A foreign key with NO ACTION blocks its parent's deletion until +// application code clears the child rows. Any table with such a key +// must therefore declare Lifetime "swept" — an assertion that some +// removal path or janitor actually deletes them. Declaring "retained" +// or "cascade" while holding a NO ACTION key is the exact shape of the +// tagging_items bug. +func TestNoActionForeignKeysAreDeclaredSwept(t *testing.T) { + t.Parallel() + + db := database.NewTestDB(t) + + for _, fk := range liveForeignKeys(t, db) { + if fk.onDelete != "NO ACTION" { + continue + } + + entry, ok := datamap.Lookup(fk.child) + if !ok { + continue // TestCatalogCoversSchema reports this + } + + if entry.Lifetime != datamap.Swept { + t.Errorf( + "%s.%s references %s with ON DELETE NO ACTION, so it "+ + "blocks deletion of %s — but the catalog declares "+ + "Lifetime %q. Either declare it %q and delete the rows "+ + "explicitly, or give the key an ON DELETE action.", + fk.child, fk.from, fk.parent, fk.parent, + entry.Lifetime, datamap.Swept, + ) + } + } +} + +// Declared cascade/set-null lifetimes must match the actual schema, so +// the catalog cannot quietly drift from what SQLite enforces. +func TestLifetimesMatchSchema(t *testing.T) { + t.Parallel() + + db := database.NewTestDB(t) + + actual := make(map[string]map[string]bool) + + for _, fk := range liveForeignKeys(t, db) { + if actual[fk.child] == nil { + actual[fk.child] = make(map[string]bool) + } + + actual[fk.child][fk.onDelete] = true + } + + for _, entry := range datamap.Tables() { + switch entry.Lifetime { + case datamap.Cascade: + if !actual[entry.Name]["CASCADE"] { + t.Errorf( + "%s declares Lifetime cascade but has no "+ + "ON DELETE CASCADE foreign key", + entry.Name, + ) + } + case datamap.SetNull: + if !actual[entry.Name]["SET NULL"] { + t.Errorf( + "%s declares Lifetime set-null but has no "+ + "ON DELETE SET NULL foreign key", + entry.Name, + ) + } + case datamap.Swept, datamap.Retained: + // No schema-level obligation. + } + } +} + +// Authored data is unrecoverable, so it must never be removed as a side +// effect of deleting owned data. Cascade is allowed only where the +// catalog explains why (play_history, queue_tracks); this test pins the +// set so a new cascade onto authored data is a deliberate decision. +func TestAuthoredCascadesAreDeliberate(t *testing.T) { + t.Parallel() + + allowed := map[string]bool{ + "play_history": true, + "queue_tracks": true, + + // Download history is scoped to the library it imported into. + // When that library is removed the files it acquired go with + // it, so a request describing "fetch this into library 3" has + // nothing left to mean. Keeping the rows would leave history + // pointing at a library the user deleted. + "download_requests": true, + + // Items belong to their request and have no independent + // meaning; they cascade with it. + "download_items": true, + + // A want says "put this in library 3". Delete that library and + // there is no longer anywhere for it to go, so the want has + // nothing left to mean — the same reasoning as its requests. + // The second cascade, artist want to derived album wants, is + // the point of the subscription: unsubscribing from an artist + // must stop the albums it queued on the user's behalf. + "download_wants": true, + } + + for _, entry := range datamap.ByKind(datamap.Authored) { + if entry.Lifetime == datamap.Cascade && !allowed[entry.Name] { + t.Errorf( + "authored table %q cascades on delete — authored data is "+ + "unrecoverable, so this needs an explicit exemption "+ + "and a note explaining it", + entry.Name, + ) + } + } +} + +// Every catalogued table needs a note; the classification is only useful +// if the reasoning is written down. +func TestEveryEntryHasANote(t *testing.T) { + t.Parallel() + + for _, entry := range datamap.Tables() { + if entry.Note == "" { + t.Errorf("catalog entry %q has no Note", entry.Name) + } + } + + for _, dir := range datamap.Directories() { + if dir.Note == "" { + t.Errorf("catalog directory %q has no Note", dir.Name) + } + } +} + +// FTS shadow tables must resolve to their parent entry rather than +// needing catalogue entries of their own. +func TestFTSShadowResolution(t *testing.T) { + t.Parallel() + + entry, ok := datamap.Lookup("search_index_data") + if !ok { + t.Fatal("search_index_data did not resolve to a catalog entry") + } + + if entry.Name != "search_index" { + t.Errorf("resolved to %q, want search_index", entry.Name) + } +} diff --git a/backend/download/apiclient.go b/backend/download/apiclient.go new file mode 100644 index 0000000..cb86e9b --- /dev/null +++ b/backend/download/apiclient.go @@ -0,0 +1,167 @@ +package download + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +// Every remote provider here talks to a self-hosted service over the +// same shape of API: a base URL, one header carrying an API key, JSON +// in and out, and a handful of status codes that mean the same thing +// everywhere. This is that, once. +// +// Each adapter supplies its own sentinel errors so callers can still +// distinguish "slskd is down" from "Prowlarr is down" with errors.Is. + +// apiClient is a small JSON-over-HTTP client for a self-hosted service. +type apiClient struct { + http *http.Client + + baseURL string + authKey string + authValue string + + // errUnreachable and errAuth are the adapter's sentinels, returned + // for transport failures and rejected credentials respectively. + errUnreachable error + errAuth error +} + +// newAPIClient builds a client for one service. +func newAPIClient( + baseURL, authHeader, authValue string, + timeout time.Duration, + errUnreachable, errAuth error, +) *apiClient { + return &apiClient{ + http: &http.Client{Timeout: timeout}, + baseURL: strings.TrimRight(baseURL, "/"), + authKey: authHeader, + authValue: authValue, + errUnreachable: errUnreachable, + errAuth: errAuth, + } +} + +// get performs a GET, decoding the response into out when non-nil. +func (c *apiClient) get(ctx context.Context, endpoint string, out any) error { + return c.do(ctx, http.MethodGet, endpoint, nil, out) +} + +// post performs a POST with a JSON body. +func (c *apiClient) post( + ctx context.Context, + endpoint string, + body, out any, +) error { + return c.do(ctx, http.MethodPost, endpoint, body, out) +} + +// put performs a PUT with a JSON body. +func (c *apiClient) put( + ctx context.Context, + endpoint string, + body, out any, +) error { + return c.do(ctx, http.MethodPut, endpoint, body, out) +} + +// delete performs a DELETE. +func (c *apiClient) delete(ctx context.Context, endpoint string) error { + return c.do(ctx, http.MethodDelete, endpoint, nil, nil) +} + +// do performs one request. +func (c *apiClient) do( + ctx context.Context, + method, endpoint string, + body, out any, +) error { + var reader io.Reader + + if body != nil { + encoded, err := json.Marshal(body) + if err != nil { + return fmt.Errorf("encode request body: %w", err) + } + + reader = bytes.NewReader(encoded) + } + + req, err := http.NewRequestWithContext( + ctx, method, c.baseURL+endpoint, reader, + ) + if err != nil { + return fmt.Errorf("build request: %w", err) + } + + if c.authKey != "" { + req.Header.Set(c.authKey, c.authValue) + } + + req.Header.Set("Accept", "application/json") + + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + + resp, err := c.http.Do(req) + if err != nil { + return fmt.Errorf("%w: %w", c.errUnreachable, err) + } + + defer func() { _ = resp.Body.Close() }() + + if err := c.checkStatus(resp); err != nil { + return err + } + + if out == nil { + return nil + } + + if err := json.NewDecoder(resp.Body).Decode(out); err != nil { + return fmt.Errorf("decode response: %w", err) + } + + return nil +} + +// checkStatus maps HTTP status onto the adapter's sentinel errors, +// including a snippet of the body — self-hosted services put the useful +// part of a failure there, not in the status line. +func (c *apiClient) checkStatus(resp *http.Response) error { + switch { + case resp.StatusCode == http.StatusUnauthorized, + resp.StatusCode == http.StatusForbidden: + return c.errAuth + case resp.StatusCode >= 400: + snippet, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + + return fmt.Errorf( + "%w: HTTP %d: %s", + c.errUnreachable, + resp.StatusCode, + strings.TrimSpace(string(snippet)), + ) + default: + return nil + } +} + +// decodeJSON decodes a JSON string into out. Providers whose auth or +// response handling does not fit apiClient still parse bodies the same +// way, so the helper lives here rather than being repeated. +func decodeJSON(body string, out any) error { + if err := json.Unmarshal([]byte(body), out); err != nil { + return fmt.Errorf("decode json: %w", err) + } + + return nil +} diff --git a/backend/download/concurrency_test.go b/backend/download/concurrency_test.go new file mode 100644 index 0000000..50780ef --- /dev/null +++ b/backend/download/concurrency_test.go @@ -0,0 +1,215 @@ +package download + +import ( + "context" + "testing" + "time" +) + +func TestConcurrencyForPrefersOverrideThenKind(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + cfg Config + want int + }{ + { + name: "slskd defaults to one", + cfg: Config{Kind: KindSlskd}, + want: 1, + }, + { + name: "usenet defaults higher", + cfg: Config{Kind: KindSABnzbd}, + want: 4, + }, + { + name: "explicit override wins", + cfg: Config{ + Kind: KindSlskd, + Settings: map[string]string{concurrencyKey: "3"}, + }, + want: 3, + }, + { + name: "nonsense override falls back", + cfg: Config{ + Kind: KindSlskd, + Settings: map[string]string{concurrencyKey: "not a number"}, + }, + want: 1, + }, + { + name: "zero override falls back", + cfg: Config{ + Kind: KindSlskd, + Settings: map[string]string{concurrencyKey: "0"}, + }, + want: 1, + }, + { + name: "unknown kind falls back to the global default", + cfg: Config{Kind: Kind("something-new")}, + want: defaultConcurrency, + }, + } + + for _, tt := range tests { + if got := concurrencyFor(tt.cfg); got != tt.want { + t.Errorf("%s: got %d, want %d", tt.name, got, tt.want) + } + } +} + +// The reason the per-provider cap exists: a Soulseek daemon capped at +// one transfer must serialize, even when the global cap would allow +// more and the user has queued several albums at once. +func TestPerProviderCapSerializesTransfers(t *testing.T) { + t.Parallel() + + f := newManagerFixture(t) + f.manager.SetMaxConcurrent(4) + + slow := fakeWithAlbum(1, "slskd-like", ".flac") + slow.GrabGate = make(chan struct{}) + + f.manager.installProvider(Config{ + ID: 1, + Kind: KindSlskd, + Priority: 50, + }, slow) + + ctx := context.Background() + + // Three requests against the same one-at-a-time provider. + for i := range 3 { + req := fourTrackRequest() + req.ID = "req-" + string(rune('a'+i)) + + if err := f.store.CreateRequest(ctx, req); err != nil { + t.Fatalf("CreateRequest: %v", err) + } + + candidate := slow.Candidates[0] + candidate.ProviderID = 1 + + go f.manager.grab(ctx, req, candidate, nil) + } + + // Give all three a chance to reach the transport, then check how + // many actually got through the gate. + waitFor(t, func() bool { return slow.GrabCallCount() >= 1 }, "no grab started") + time.Sleep(150 * time.Millisecond) + + if got := slow.MaxParallelGrabs(); got != 1 { + t.Errorf("%d simultaneous transfers, want 1", got) + } + + close(slow.GrabGate) + + waitFor( + t, + func() bool { return slow.GrabCallCount() == 3 }, + "not every queued transfer ran once the first finished", + ) + + if got := slow.MaxParallelGrabs(); got != 1 { + t.Errorf("%d simultaneous transfers overall, want 1", got) + } +} + +// A provider that tolerates parallelism is not held to Soulseek's +// limit, and the global cap is what bounds it. +func TestPerProviderCapAllowsParallelWhereSafe(t *testing.T) { + t.Parallel() + + f := newManagerFixture(t) + f.manager.SetMaxConcurrent(4) + + fast := fakeWithAlbum(1, "usenet-like", ".flac") + fast.GrabGate = make(chan struct{}) + + f.manager.installProvider(Config{ + ID: 1, + Kind: KindSABnzbd, + Priority: 50, + }, fast) + + ctx := context.Background() + + for i := range 3 { + req := fourTrackRequest() + req.ID = "req-" + string(rune('a'+i)) + + if err := f.store.CreateRequest(ctx, req); err != nil { + t.Fatalf("CreateRequest: %v", err) + } + + candidate := fast.Candidates[0] + candidate.ProviderID = 1 + + go f.manager.grab(ctx, req, candidate, nil) + } + + waitFor( + t, + func() bool { return fast.MaxParallelGrabs() >= 3 }, + "transfers were serialized against a provider that allows parallelism", + ) + + close(fast.GrabGate) +} + +// Reload must not strand a running transfer's slot when a provider's +// limit changes underneath it. +func TestSyncSemaphoresReplacesChangedLimits(t *testing.T) { + t.Parallel() + + f := newManagerFixture(t) + + f.manager.installProvider(Config{ID: 1, Kind: KindSlskd}, nil) + + first := f.manager.semaphoreFor(1) + if cap(first) != 1 { + t.Fatalf("slskd semaphore cap = %d, want 1", cap(first)) + } + + // Same limit: the semaphore is kept, so in-flight accounting is not + // reset by an unrelated settings save. + f.manager.syncSemaphores(map[int64]Config{1: {ID: 1, Kind: KindSlskd}}) + + if again := f.manager.semaphoreFor(1); again != first { + t.Error("semaphore was replaced despite an unchanged limit") + } + + // Changed limit: a new semaphore, with the new capacity. + f.manager.syncSemaphores(map[int64]Config{1: { + ID: 1, + Kind: KindSlskd, + Settings: map[string]string{concurrencyKey: "5"}, + }}) + + f.manager.provMu.Lock() + f.manager.configs[1] = Config{ + ID: 1, + Kind: KindSlskd, + Settings: map[string]string{concurrencyKey: "5"}, + } + f.manager.provMu.Unlock() + + if changed := f.manager.semaphoreFor(1); cap(changed) != 5 { + t.Errorf("semaphore cap = %d after raising the limit, want 5", cap(changed)) + } + + // A provider that is gone leaves no semaphore behind. + f.manager.syncSemaphores(map[int64]Config{}) + + f.manager.semMu.Lock() + _, still := f.manager.provSem[1] + f.manager.semMu.Unlock() + + if still { + t.Error("semaphore survived the provider being removed") + } +} diff --git a/backend/download/config.go b/backend/download/config.go new file mode 100644 index 0000000..4c57435 --- /dev/null +++ b/backend/download/config.go @@ -0,0 +1,60 @@ +package download + +import "time" + +// UserConfig is the download subsystem's slice of the TOML config file. +// Provider connections are not here — they live in the database, keyed +// by row, because there can be many of them and they change through the +// settings UI rather than by hand-editing. +type UserConfig struct { + // PathTemplate lays out imported files under the library root. + // Tokens: {albumartist} {artist} {album} {year} {track} {disc} + // {title}. Empty falls back to DefaultPathTemplate. + PathTemplate string `toml:"PathTemplate"` + + // AutoPick lets a single high-confidence, high-quality candidate + // download without asking. Off by default: an unattended download + // that picks wrong puts the wrong files in the library, and the + // ranking has to earn that trust on a given user's sources first. + AutoPick bool `toml:"AutoPick"` + + // MaxConcurrent bounds simultaneous transfers across all providers. + // Per-provider limits sit underneath it and are set on the provider + // itself, since the right number depends on what is on the other + // end: one Soulseek peer, or a usenet server built for parallelism. + MaxConcurrent int `toml:"MaxConcurrent"` + + // WantedIntervalMinutes is how often the wanted list is reconciled: + // artist subscriptions expanded, owned items retired, due wants + // searched for. Zero uses the default. + WantedIntervalMinutes int `toml:"WantedIntervalMinutes"` + + // WantedBatch bounds how many wants one reconcile pass searches + // for. A large list should be worked through steadily rather than + // in one burst that every provider sees as a flood. + WantedBatch int `toml:"WantedBatch"` +} + +// ApplyDefaults fills unset fields. +func (c *UserConfig) ApplyDefaults() { + if c.PathTemplate == "" { + c.PathTemplate = DefaultPathTemplate + } + + if c.MaxConcurrent <= 0 { + c.MaxConcurrent = defaultConcurrency + } + + if c.WantedIntervalMinutes <= 0 { + c.WantedIntervalMinutes = int(defaultReconcileInterval / time.Minute) + } + + if c.WantedBatch <= 0 { + c.WantedBatch = defaultDueBatch + } +} + +// WantedInterval is the reconcile interval as a duration. +func (c *UserConfig) WantedInterval() time.Duration { + return time.Duration(c.WantedIntervalMinutes) * time.Minute +} diff --git a/backend/download/fake.go b/backend/download/fake.go new file mode 100644 index 0000000..b483df5 --- /dev/null +++ b/backend/download/fake.go @@ -0,0 +1,283 @@ +package download + +import ( + "context" + "errors" + "log/slog" + "os" + "path/filepath" + "sync" +) + +// The fake provider exists so the pipeline can be tested end to end +// without a network, a daemon, or a binary on PATH. It is registered +// like any real adapter and excluded from Descriptors(), so it can +// never be offered in the UI. + +// FakeProvider is an in-memory Provider used by tests. It fills all +// three roles; which ones are active is controlled by its Caps. +type FakeProvider struct { + info ProviderInfo + + mu sync.Mutex + + // Candidates is what Search returns. + Candidates []Candidate + + // SearchErr, GrabErr and CheckErr are returned when set. + SearchErr error + GrabErr error + CheckErr error + + // Written maps a relative file name to the bytes Grab creates in + // the staging directory, simulating a completed transfer. + Written map[string][]byte + + // DelegateStatuses is returned by Poll in order, the last repeating. + DelegateStatuses []DelegateStatus + + // GrabGate, when set, blocks Grab until it is closed or the + // context ends. It lets a test hold transfers open long enough to + // observe how many run at once. + GrabGate chan struct{} + + // concurrent tracks how many Grabs are in flight, and maxParallel + // the high-water mark, which is what a concurrency cap is asserted + // against. + concurrent int + maxParallel int + + // Calls records what happened, for assertions. + SearchCalls int + GrabCalls int + DelegateCalls int + pollIndex int +} + +// NewFakeProvider returns a fake with the given capabilities. +func NewFakeProvider(id int64, name string, caps Caps) *FakeProvider { + return &FakeProvider{ + info: ProviderInfo{ + ID: id, + Kind: KindFake, + Name: name, + Enabled: true, + Priority: 50, + Caps: caps, + }, + Written: map[string][]byte{}, + } +} + +// Info returns the fake's identity. +func (f *FakeProvider) Info() ProviderInfo { + return f.info +} + +// SetPriority adjusts the fake's priority. +func (f *FakeProvider) SetPriority(p int) { + f.info.Priority = p +} + +// Check reports configured health. +func (f *FakeProvider) Check(_ context.Context) error { + return f.CheckErr +} + +// Close is a no-op. +func (f *FakeProvider) Close() error { + return nil +} + +// Search returns the configured candidates. +func (f *FakeProvider) Search( + ctx context.Context, + _ Request, +) ([]Candidate, error) { + f.mu.Lock() + f.SearchCalls++ + f.mu.Unlock() + + if err := ctx.Err(); err != nil { + return nil, err //nolint:wrapcheck // context error, already meaningful + } + + if f.SearchErr != nil { + return nil, f.SearchErr + } + + out := make([]Candidate, len(f.Candidates)) + copy(out, f.Candidates) + + for i := range out { + out[i].ProviderID = f.info.ID + } + + return out, nil +} + +// Grab writes the configured files into dst. +func (f *FakeProvider) Grab( + ctx context.Context, + _ Candidate, + dst string, + onProgress ProgressFunc, +) (Result, error) { + f.mu.Lock() + f.GrabCalls++ + f.concurrent++ + + if f.concurrent > f.maxParallel { + f.maxParallel = f.concurrent + } + + gate := f.GrabGate + f.mu.Unlock() + + defer func() { + f.mu.Lock() + f.concurrent-- + f.mu.Unlock() + }() + + if gate != nil { + select { + case <-gate: + case <-ctx.Done(): + return Result{}, ctx.Err() //nolint:wrapcheck // context error + } + } + + if err := ctx.Err(); err != nil { + return Result{}, err //nolint:wrapcheck // context error + } + + if f.GrabErr != nil { + return Result{}, f.GrabErr + } + + files := make([]string, 0, len(f.Written)) + + var total int64 + + for name, data := range f.Written { + path := filepath.Join(dst, name) + + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + return Result{}, err //nolint:wrapcheck // test helper + } + + if err := os.WriteFile(path, data, 0o640); err != nil { + return Result{}, err //nolint:wrapcheck // test helper + } + + files = append(files, path) + total += int64(len(data)) + + if onProgress != nil { + onProgress(Progress{Current: total, Total: total}) + } + } + + return Result{Dir: dst, Files: files, BytesTransferred: total}, nil +} + +// GrabCallCount reports how many transfers have been started, safe to +// read while transfers are in flight. +func (f *FakeProvider) GrabCallCount() int { + f.mu.Lock() + defer f.mu.Unlock() + + return f.GrabCalls +} + +// MaxParallelGrabs reports the most simultaneous transfers this +// provider ever saw, which is what a per-provider cap is asserted +// against. +func (f *FakeProvider) MaxParallelGrabs() int { + f.mu.Lock() + defer f.mu.Unlock() + + return f.maxParallel +} + +// errNoDelegateStatus is returned when a fake delegator runs out of +// scripted statuses. +var errNoDelegateStatus = errors.New("fake: no delegate status configured") + +// Delegate records the call and returns a fixed external ID. +func (f *FakeProvider) Delegate( + _ context.Context, + _ Request, +) (string, error) { + f.mu.Lock() + defer f.mu.Unlock() + + f.DelegateCalls++ + + return "fake-external-1", nil +} + +// Poll returns the next scripted status, repeating the last. +func (f *FakeProvider) Poll( + _ context.Context, + _ string, +) (DelegateStatus, error) { + f.mu.Lock() + defer f.mu.Unlock() + + if len(f.DelegateStatuses) == 0 { + return DelegateStatus{}, errNoDelegateStatus + } + + i := f.pollIndex + if i >= len(f.DelegateStatuses) { + i = len(f.DelegateStatuses) - 1 + } else { + f.pollIndex++ + } + + return f.DelegateStatuses[i], nil +} + +// Withdraw is a no-op. +func (f *FakeProvider) Withdraw(_ context.Context, _ string) error { + return nil +} + +// fakeRegistry lets tests install providers directly, bypassing the +// database and the constructor registry. +func (m *Manager) installProvider(cfg Config, p Provider) { + m.provMu.Lock() + defer m.provMu.Unlock() + + m.providers[cfg.ID] = p + m.configs[cfg.ID] = cfg +} + +func init() { + Register( + Descriptor{ + Kind: KindFake, + Name: "Fake (testing)", + Summary: "In-memory provider used by the test suite.", + Caps: Caps{ + CanSearch: true, + CanTransport: true, + }, + }, + func(cfg Config, _ SecretLookup, _ *slog.Logger) (Provider, error) { + return NewFakeProvider(cfg.ID, cfg.Name, Caps{ + CanSearch: true, + CanTransport: true, + }), nil + }, + ) +} + +// slogDiscard returns a logger that writes nowhere, for tests. +func slogDiscard() *slog.Logger { + return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{ + Level: slog.LevelError + 1, + })) +} diff --git a/backend/download/importer.go b/backend/download/importer.go new file mode 100644 index 0000000..00e7c71 --- /dev/null +++ b/backend/download/importer.go @@ -0,0 +1,536 @@ +package download + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + + "yellowjacket/backend/tagwriter" +) + +// The import step is the only writer into library paths. Everything +// before it happens in staging, where a bad download is a directory to +// delete rather than a row to un-ingest. +// +// Order matters: tags are written while the files are still staged, so +// the scanner's first sight of a file is already correct. Tagging +// after the move would mean a window where the library holds a track +// titled "01 - Track01.flac", and the user would watch it fix itself. + +// Import errors. +var ( + // ErrNoAudio means the grab produced no playable audio files. + ErrNoAudio = errors.New("download contained no audio files") + + // ErrTooIncomplete means too few of the expected tracks arrived to + // call the download successful. + ErrTooIncomplete = errors.New("download is missing too many tracks") + + // ErrDestinationExists means the computed library path is already + // occupied by a different file. + ErrDestinationExists = errors.New("destination file already exists") +) + +// minCompleteness is the fraction of the expected tracklist that must +// arrive for an anchored import to proceed. Below this the download is +// a different thing than what was asked for — a single, a sampler, a +// partial transfer — and quietly importing it would corrupt the +// library's idea of the album. +const minCompleteness = 0.8 + +// TagWriterPort is the tag-writing capability the importer needs. +// Narrow interface rather than *tagwriter.TagWriter so importer tests +// do not need a database. +type TagWriterPort interface { + WriteUntrackedFileTags(filePath string, changes tagwriter.TagChanges) error +} + +// LibraryPort is the library-side capability the importer needs. +type LibraryPort interface { + // ScanLibrary triggers a rescan so imported files are ingested. + ScanLibrary(id int64) error +} + +// ImportOptions configures how imported files are laid out. +type ImportOptions struct { + // LibraryRoot is the directory imported files are placed under. + LibraryRoot string + + // PathTemplate lays out the destination path. Supported tokens: + // {albumartist} {artist} {album} {year} {track} {disc} {title}. + // Empty means flat: everything into LibraryRoot/{albumartist}/{album}. + PathTemplate string + + // WriteTags controls whether the importer tags files before moving + // them. Off for delegate providers, which have already imported + // and tagged the files themselves. + WriteTags bool +} + +// DefaultPathTemplate is the layout used when none is configured. +const DefaultPathTemplate = "{albumartist}/{album}/{track} {title}" + +// Importer moves verified downloads into the library. +type Importer struct { + logger *slog.Logger + staging *Staging + tags TagWriterPort + library LibraryPort +} + +// NewImporter builds an importer. +func NewImporter( + logger *slog.Logger, + staging *Staging, + tags TagWriterPort, + library LibraryPort, +) *Importer { + return &Importer{ + logger: logger, + staging: staging, + tags: tags, + library: library, + } +} + +// ImportResult reports what an import placed where. +type ImportResult struct { + // Paths are the library paths files ended up at. + Paths []string + + // Tagged counts files whose tags were rewritten. + Tagged int + + // Skipped counts non-audio files left in staging (logs, cue sheets, + // scene .nfo files) — deliberately not imported. + Skipped int +} + +// Import verifies, tags and moves a completed grab into the library. +// +// On any failure the staging directory is left intact so the user can +// retry or inspect it; only a fully successful import releases staging. +func (i *Importer) Import( + ctx context.Context, + req Request, + result Result, + opts ImportOptions, +) (ImportResult, error) { + files, err := i.staging.Verify(result.Dir, result.Files) + if err != nil { + return ImportResult{}, err + } + + audio, skipped := splitAudio(files) + if len(audio) == 0 { + return ImportResult{}, ErrNoAudio + } + + if err := checkCompleteness(len(audio), req); err != nil { + return ImportResult{}, err + } + + // Align staged files to the expected tracklist so tags and + // filenames reflect the release, not the uploader's naming. + plan := i.planFiles(audio, req) + + out := ImportResult{ + Paths: make([]string, 0, len(plan)), + Skipped: skipped, + } + + for _, p := range plan { + if err := ctx.Err(); err != nil { + return out, fmt.Errorf("import cancelled: %w", err) + } + + if opts.WriteTags { + if err := i.tagFile(p, req); err != nil { + // A file that cannot be tagged is still worth importing + // — the scanner will read whatever tags it has, and the + // autotag queue can pick it up later. Losing the whole + // album over one unwritable file would be worse. + i.logger.Warn( + "could not tag downloaded file before import", + "path", p.Source, + "error", err, + ) + } else { + out.Tagged++ + } + } + + dest, err := i.destinationFor(p, req, opts) + if err != nil { + return out, err + } + + if err := movePath(p.Source, dest); err != nil { + return out, err + } + + out.Paths = append(out.Paths, dest) + } + + return out, nil +} + +// plannedFile pairs a staged file with the expected track it matched. +type plannedFile struct { + Source string + + // Track is the matched expected track, or the zero value when the + // file could not be aligned (free-text requests, bonus tracks). + Track ExpectedTrack + Matched bool +} + +// planFiles aligns staged files to the expected tracklist. +func (i *Importer) planFiles(audio []string, req Request) []plannedFile { + files := make([]CandidateFile, 0, len(audio)) + + for _, a := range audio { + format, isAudio := FormatForPath(a) + files = append(files, CandidateFile{ + Path: a, + Format: format, + IsAudio: isAudio, + }) + } + + matched, _ := matchFiles(files, req.Expected) + + byPosition := make(map[int]ExpectedTrack, len(req.Expected)) + for _, e := range req.Expected { + byPosition[e.Position] = e + } + + out := make([]plannedFile, 0, len(matched)) + + for _, m := range matched { + p := plannedFile{Source: m.Path} + + if t, ok := byPosition[m.MatchedTo]; ok && m.MatchedTo != 0 { + p.Track = t + p.Matched = true + } + + out = append(out, p) + } + + // Stable order: matched tracks by position, then unmatched by path, + // so a partial import is reproducible. + sort.SliceStable(out, func(a, b int) bool { + if out[a].Matched != out[b].Matched { + return out[a].Matched + } + + if out[a].Matched { + if out[a].Track.DiscNumber != out[b].Track.DiscNumber { + return out[a].Track.DiscNumber < out[b].Track.DiscNumber + } + + return out[a].Track.Position < out[b].Track.Position + } + + return out[a].Source < out[b].Source + }) + + return out +} + +// tagFile writes the release's metadata onto a staged file. +func (i *Importer) tagFile(p plannedFile, req Request) error { + if i.tags == nil || !p.Matched { + return nil + } + + changes := tagwriter.TagChanges{ + tagwriter.FieldAlbum: req.Album, + tagwriter.FieldAlbumArtist: req.Artist, + tagwriter.FieldTitle: p.Track.Title, + tagwriter.FieldTrackNumber: p.Track.Position, + } + + if p.Track.Artist != "" { + changes[tagwriter.FieldArtist] = p.Track.Artist + } else { + changes[tagwriter.FieldArtist] = req.Artist + } + + if p.Track.DiscNumber > 0 { + changes[tagwriter.FieldDiscNumber] = p.Track.DiscNumber + } + + if err := i.tags.WriteUntrackedFileTags(p.Source, changes); err != nil { + return fmt.Errorf("write tags: %w", err) + } + + return nil +} + +// destinationFor computes a file's library path from the template. +func (i *Importer) destinationFor( + p plannedFile, + req Request, + opts ImportOptions, +) (string, error) { + if opts.LibraryRoot == "" { + return "", fmt.Errorf( + "%w: no library root configured", ErrNotConfigured, + ) + } + + tmpl := opts.PathTemplate + if tmpl == "" { + tmpl = DefaultPathTemplate + } + + ext := filepath.Ext(p.Source) + + title := p.Track.Title + if title == "" { + // Unmatched file: keep the uploader's name rather than + // inventing one, so nothing is silently renamed to a track it + // may not be. + title = strings.TrimSuffix(filepath.Base(p.Source), ext) + } + + artist := p.Track.Artist + if artist == "" { + artist = req.Artist + } + + repl := strings.NewReplacer( + "{albumartist}", sanitizePathPart(fallback(req.Artist, "Unknown Artist")), + "{artist}", sanitizePathPart(fallback(artist, "Unknown Artist")), + "{album}", sanitizePathPart(fallback(req.Album, "Unknown Album")), + "{title}", sanitizePathPart(title), + "{track}", trackToken(p.Track.Position), + "{disc}", strconv.Itoa(p.Track.DiscNumber), + "{year}", "", + ) + + rel := repl.Replace(tmpl) + + // Clean up any empty segments left by unset tokens. + parts := make([]string, 0, 4) + + for _, seg := range strings.Split(rel, "/") { + seg = strings.TrimSpace(seg) + if seg != "" { + parts = append(parts, seg) + } + } + + if len(parts) == 0 { + return "", fmt.Errorf( + "%w: path template produced an empty path", ErrNotConfigured, + ) + } + + dest := filepath.Join(opts.LibraryRoot, filepath.Join(parts...)) + ext + + return uniqueDestination(dest) +} + +// uniqueDestination returns dest, or a numbered variant when dest is +// taken. Overwriting is never right here: the existing file may be a +// better copy the user already owns, and the download is not +// authoritative just because it arrived later. +func uniqueDestination(dest string) (string, error) { + const maxAttempts = 50 + + ext := filepath.Ext(dest) + base := strings.TrimSuffix(dest, ext) + + for n := range maxAttempts { + candidate := dest + if n > 0 { + candidate = base + " (" + strconv.Itoa(n+1) + ")" + ext + } + + _, err := os.Stat(candidate) + if os.IsNotExist(err) { + return candidate, nil + } + + if err != nil { + return "", fmt.Errorf("stat destination: %w", err) + } + } + + return "", fmt.Errorf("%w: %s", ErrDestinationExists, dest) +} + +// movePath moves a file, falling back to copy+remove when the staging +// area and the library are on different filesystems — which is the +// normal case, since staging lives in the user data directory. +func movePath(src, dest string) error { + if err := os.MkdirAll(filepath.Dir(dest), 0o750); err != nil { + return fmt.Errorf("create destination directory: %w", err) + } + + if err := os.Rename(src, dest); err == nil { + return nil + } + + if err := copyFile(src, dest); err != nil { + return err + } + + if err := os.Remove(src); err != nil { + // The copy succeeded, so the import is good; a leftover staged + // file is swept later. + return nil //nolint:nilerr // staging sweep handles the leftover + } + + return nil +} + +// copyFile copies src to dest, writing to a temporary file first so an +// interrupted copy never leaves a partial file at a library path where +// the scanner would find it. +func copyFile(src, dest string) error { + in, err := os.Open(src) + if err != nil { + return fmt.Errorf("open downloaded file: %w", err) + } + + defer func() { _ = in.Close() }() + + tmp := dest + ".part" + + out, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o640) + if err != nil { + return fmt.Errorf("create library file: %w", err) + } + + if _, err := io.Copy(out, in); err != nil { + _ = out.Close() + _ = os.Remove(tmp) + + return fmt.Errorf("copy into library: %w", err) + } + + if err := out.Close(); err != nil { + _ = os.Remove(tmp) + + return fmt.Errorf("close library file: %w", err) + } + + if err := os.Rename(tmp, dest); err != nil { + _ = os.Remove(tmp) + + return fmt.Errorf("finalize library file: %w", err) + } + + return nil +} + +// checkCompleteness rejects an anchored download that is missing too +// much of its tracklist. +func checkCompleteness(got int, req Request) error { + if len(req.Expected) == 0 { + return nil + } + + ratio := float64(got) / float64(len(req.Expected)) + if ratio < minCompleteness { + return fmt.Errorf( + "%w: got %d of %d tracks", + ErrTooIncomplete, got, len(req.Expected), + ) + } + + return nil +} + +// splitAudio partitions verified files into audio and a count of the +// rest. +func splitAudio(files []string) (audio []string, skipped int) { + audio = make([]string, 0, len(files)) + + for _, f := range files { + if _, ok := FormatForPath(f); ok { + audio = append(audio, f) + + continue + } + + skipped++ + } + + return audio, skipped +} + +// trackToken formats a track number as a zero-padded two-digit string, +// or empty when unknown. +func trackToken(n int) string { + if n <= 0 { + return "" + } + + if n < 10 { + return "0" + strconv.Itoa(n) + } + + return strconv.Itoa(n) +} + +// fallback returns s, or alt when s is blank. +func fallback(s, alt string) string { + if strings.TrimSpace(s) == "" { + return alt + } + + return s +} + +// sanitizePathPart makes a string safe as a single path segment on +// every supported platform: Windows reserves characters that are legal +// on Linux, and a library synced between the two must not produce +// unopenable files. +func sanitizePathPart(s string) string { + const maxSegment = 120 + + var b strings.Builder + + b.Grow(len(s)) + + for _, r := range s { + switch r { + case '/', '\\', ':', '*', '?', '"', '<', '>', '|': + b.WriteByte('_') + default: + if r < 0x20 { + continue + } + + b.WriteRune(r) + } + } + + out := strings.TrimSpace(b.String()) + + // Trailing dots and spaces are silently stripped by Windows, which + // turns "Vol. 2 " into a name that no longer round-trips. + out = strings.TrimRight(out, ". ") + + if len(out) > maxSegment { + out = strings.TrimSpace(out[:maxSegment]) + } + + if out == "" { + return "Unknown" + } + + return out +} diff --git a/backend/download/importer_test.go b/backend/download/importer_test.go new file mode 100644 index 0000000..de2a394 --- /dev/null +++ b/backend/download/importer_test.go @@ -0,0 +1,440 @@ +package download + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "yellowjacket/backend/tagwriter" +) + +// recordingTagWriter captures tag writes instead of touching files, so +// importer tests do not need real audio. +type recordingTagWriter struct { + mu sync.Mutex + writes map[string]tagwriter.TagChanges + failFor string +} + +func newRecordingTagWriter() *recordingTagWriter { + return &recordingTagWriter{writes: map[string]tagwriter.TagChanges{}} +} + +var errTagWriteFailed = errors.New("tag write failed") + +func (r *recordingTagWriter) WriteUntrackedFileTags( + path string, + changes tagwriter.TagChanges, +) error { + r.mu.Lock() + defer r.mu.Unlock() + + if r.failFor != "" && strings.Contains(path, r.failFor) { + return errTagWriteFailed + } + + r.writes[filepath.Base(path)] = changes + + return nil +} + +// stubLibrary records scan requests. +type stubLibrary struct { + mu sync.Mutex + scanned []int64 +} + +func (s *stubLibrary) ScanLibrary(id int64) error { + s.mu.Lock() + defer s.mu.Unlock() + + s.scanned = append(s.scanned, id) + + return nil +} + +// importFixture stages a set of files and returns the pieces an import +// needs. +type importFixture struct { + staging *Staging + importer *Importer + tags *recordingTagWriter + lib *stubLibrary + dir string + root string + files []string +} + +func newImportFixture(t *testing.T, names ...string) importFixture { + t.Helper() + + staging := newTestStaging(t) + tags := newRecordingTagWriter() + lib := &stubLibrary{} + imp := NewImporter(slogDiscard(), staging, tags, lib) + + dir, err := staging.Reserve("item-1") + if err != nil { + t.Fatalf("Reserve: %v", err) + } + + files := make([]string, 0, len(names)) + + for _, n := range names { + p := filepath.Join(dir, n) + + if err := os.MkdirAll(filepath.Dir(p), 0o750); err != nil { + t.Fatalf("mkdir: %v", err) + } + + if err := os.WriteFile(p, []byte("audio-data"), 0o600); err != nil { + t.Fatalf("write: %v", err) + } + + files = append(files, p) + } + + return importFixture{ + staging: staging, + importer: imp, + tags: tags, + lib: lib, + dir: dir, + root: t.TempDir(), + files: files, + } +} + +func fourTrackRequest() Request { + return Request{ + ID: "req-1", + LibraryID: 1, + ReleaseMBID: "mbid-1", + Artist: "Radiohead", + Album: "OK Computer", + Expected: []ExpectedTrack{ + {Position: 1, Title: "Airbag"}, + {Position: 2, Title: "Paranoid Android"}, + {Position: 3, Title: "Subterranean Homesick Alien"}, + {Position: 4, Title: "Exit Music (For a Film)"}, + }, + } +} + +func TestImportPlacesAndTagsFiles(t *testing.T) { + t.Parallel() + + f := newImportFixture(t, + "01 - Airbag.flac", + "02 - Paranoid Android.flac", + "03 - Subterranean Homesick Alien.flac", + "04 - Exit Music (For a Film).flac", + ) + + got, err := f.importer.Import( + context.Background(), + fourTrackRequest(), + Result{Dir: f.dir, Files: f.files}, + ImportOptions{LibraryRoot: f.root, WriteTags: true}, + ) + if err != nil { + t.Fatalf("Import: %v", err) + } + + if len(got.Paths) != 4 { + t.Fatalf("imported %d files, want 4", len(got.Paths)) + } + + if got.Tagged != 4 { + t.Errorf("tagged %d files, want 4", got.Tagged) + } + + want := filepath.Join( + f.root, "Radiohead", "OK Computer", "01 Airbag.flac", + ) + + if got.Paths[0] != want { + t.Errorf("first path = %s, want %s", got.Paths[0], want) + } + + for _, p := range got.Paths { + if _, err := os.Stat(p); err != nil { + t.Errorf("imported file missing: %v", err) + } + } +} + +// Tags must be written while the file is still staged: if the library +// ever sees an untagged file, the scanner ingests it and the user +// watches it correct itself. +func TestImportTagsBeforeMoving(t *testing.T) { + t.Parallel() + + f := newImportFixture(t, "01 - Airbag.flac") + + req := fourTrackRequest() + req.Expected = req.Expected[:1] + + if _, err := f.importer.Import( + context.Background(), + req, + Result{Dir: f.dir, Files: f.files}, + ImportOptions{LibraryRoot: f.root, WriteTags: true}, + ); err != nil { + t.Fatalf("Import: %v", err) + } + + changes, ok := f.tags.writes["01 - Airbag.flac"] + if !ok { + t.Fatalf( + "tags were not written to the staged filename; got writes for %v", + keysOf(f.tags.writes), + ) + } + + if changes[tagwriter.FieldTitle] != "Airbag" { + t.Errorf("title = %v, want Airbag", changes[tagwriter.FieldTitle]) + } + + if changes[tagwriter.FieldAlbum] != "OK Computer" { + t.Errorf("album = %v, want OK Computer", changes[tagwriter.FieldAlbum]) + } +} + +// One unwritable file should not cost the whole album. +func TestImportContinuesWhenTaggingFails(t *testing.T) { + t.Parallel() + + f := newImportFixture(t, + "01 - Airbag.flac", + "02 - Paranoid Android.flac", + "03 - Subterranean Homesick Alien.flac", + "04 - Exit Music (For a Film).flac", + ) + f.tags.failFor = "Paranoid" + + got, err := f.importer.Import( + context.Background(), + fourTrackRequest(), + Result{Dir: f.dir, Files: f.files}, + ImportOptions{LibraryRoot: f.root, WriteTags: true}, + ) + if err != nil { + t.Fatalf("Import: %v", err) + } + + if len(got.Paths) != 4 { + t.Errorf("imported %d files, want all 4", len(got.Paths)) + } + + if got.Tagged != 3 { + t.Errorf("tagged %d, want 3 (one failure)", got.Tagged) + } +} + +func TestImportRejectsTooIncomplete(t *testing.T) { + t.Parallel() + + f := newImportFixture(t, "01 - Airbag.flac") + + _, err := f.importer.Import( + context.Background(), + fourTrackRequest(), + Result{Dir: f.dir, Files: f.files}, + ImportOptions{LibraryRoot: f.root, WriteTags: true}, + ) + + if !errors.Is(err, ErrTooIncomplete) { + t.Fatalf("error = %v, want ErrTooIncomplete", err) + } + + entries, _ := os.ReadDir(f.root) + if len(entries) != 0 { + t.Error("failed import wrote into the library root") + } +} + +func TestImportRejectsNoAudio(t *testing.T) { + t.Parallel() + + f := newImportFixture(t, "rip.log", "cover.jpg") + + _, err := f.importer.Import( + context.Background(), + fourTrackRequest(), + Result{Dir: f.dir, Files: f.files}, + ImportOptions{LibraryRoot: f.root, WriteTags: true}, + ) + + if !errors.Is(err, ErrNoAudio) { + t.Fatalf("error = %v, want ErrNoAudio", err) + } +} + +func TestImportSkipsNonAudioFiles(t *testing.T) { + t.Parallel() + + f := newImportFixture(t, + "01 - Airbag.flac", + "02 - Paranoid Android.flac", + "03 - Subterranean Homesick Alien.flac", + "04 - Exit Music (For a Film).flac", + "rip.log", + "cover.jpg", + ) + + got, err := f.importer.Import( + context.Background(), + fourTrackRequest(), + Result{Dir: f.dir, Files: f.files}, + ImportOptions{LibraryRoot: f.root, WriteTags: true}, + ) + if err != nil { + t.Fatalf("Import: %v", err) + } + + if got.Skipped != 2 { + t.Errorf("skipped = %d, want 2", got.Skipped) + } + + if len(got.Paths) != 4 { + t.Errorf("imported %d, want 4 audio files only", len(got.Paths)) + } +} + +// An existing file is never overwritten: it may be a better copy the +// user already owns, and arriving later does not make a download +// authoritative. +func TestImportNeverOverwrites(t *testing.T) { + t.Parallel() + + f := newImportFixture(t, "01 - Airbag.flac") + + req := fourTrackRequest() + req.Expected = req.Expected[:1] + + existing := filepath.Join( + f.root, "Radiohead", "OK Computer", "01 Airbag.flac", + ) + + if err := os.MkdirAll(filepath.Dir(existing), 0o750); err != nil { + t.Fatalf("mkdir: %v", err) + } + + if err := os.WriteFile(existing, []byte("original"), 0o600); err != nil { + t.Fatalf("write: %v", err) + } + + got, err := f.importer.Import( + context.Background(), + req, + Result{Dir: f.dir, Files: f.files}, + ImportOptions{LibraryRoot: f.root, WriteTags: true}, + ) + if err != nil { + t.Fatalf("Import: %v", err) + } + + data, err := os.ReadFile(existing) + if err != nil { + t.Fatalf("read: %v", err) + } + + if string(data) != "original" { + t.Error("import overwrote an existing library file") + } + + if got.Paths[0] == existing { + t.Errorf("imported to the occupied path %s", existing) + } +} + +func TestImportCustomPathTemplate(t *testing.T) { + t.Parallel() + + f := newImportFixture(t, "01 - Airbag.flac") + + req := fourTrackRequest() + req.Expected = req.Expected[:1] + + got, err := f.importer.Import( + context.Background(), + req, + Result{Dir: f.dir, Files: f.files}, + ImportOptions{ + LibraryRoot: f.root, + PathTemplate: "{albumartist} - {album}/{track}. {title}", + WriteTags: true, + }, + ) + if err != nil { + t.Fatalf("Import: %v", err) + } + + want := filepath.Join( + f.root, "Radiohead - OK Computer", "01. Airbag.flac", + ) + + if got.Paths[0] != want { + t.Errorf("path = %s, want %s", got.Paths[0], want) + } +} + +func TestSanitizePathPart(t *testing.T) { + t.Parallel() + + tests := []struct { + in string + want string + }{ + {"AC/DC", "AC_DC"}, + {"Where Are We Now?", "Where Are We Now_"}, + {`Bad: Title*`, "Bad_ Title_"}, + {"Vol. 2 ", "Vol. 2"}, + {"trailing dots...", "trailing dots"}, + {"", "Unknown"}, + } + + for _, tt := range tests { + t.Run(tt.in, func(t *testing.T) { + t.Parallel() + + if got := sanitizePathPart(tt.in); got != tt.want { + t.Errorf("sanitizePathPart(%q) = %q, want %q", tt.in, got, tt.want) + } + }) + } +} + +func TestImportRequiresLibraryRoot(t *testing.T) { + t.Parallel() + + f := newImportFixture(t, "01 - Airbag.flac") + + req := fourTrackRequest() + req.Expected = req.Expected[:1] + + _, err := f.importer.Import( + context.Background(), + req, + Result{Dir: f.dir, Files: f.files}, + ImportOptions{WriteTags: true}, + ) + + if !errors.Is(err, ErrNotConfigured) { + t.Fatalf("error = %v, want ErrNotConfigured", err) + } +} + +func keysOf(m map[string]tagwriter.TagChanges) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + + return out +} diff --git a/backend/download/manager.go b/backend/download/manager.go new file mode 100644 index 0000000..67b2f6e --- /dev/null +++ b/backend/download/manager.go @@ -0,0 +1,1227 @@ +package download + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "log/slog" + "strconv" + "sync" + "time" + + "yellowjacket/backend/jobs" +) + +// Manager owns the download pipeline: it builds providers from stored +// config, fans a request out across them, ranks what comes back, drives +// the chosen candidate through grab → verify → tag → import, and +// reports the whole thing as one job. +// +// One job per request, not per file. The user asked for an album; the +// fact that it arrives as twelve transfers is an implementation detail +// they should not have to read a job list to understand. + +// Timeouts and limits. +const ( + // searchTimeout bounds one provider's search. The fan-out takes + // whatever returned in time rather than blocking on the slowest — + // a wedged Prowlarr indexer must not stall a Soulseek result that + // arrived in 200ms. + searchTimeout = 25 * time.Second + + // grabTimeout bounds one transfer. Soulseek queues are measured in + // hours when a peer is busy, so this is generous by design. + grabTimeout = 6 * time.Hour + + // pollInterval is how often delegating managers are asked for + // status. + pollInterval = 15 * time.Second + + // delegateTimeout bounds how long we wait for a delegate to finish + // before giving up and telling the user to check that system. + delegateTimeout = 12 * time.Hour + + // defaultConcurrency bounds simultaneous grabs across all + // providers. Soulseek peers queue or ban on parallel requests, so + // the default is deliberately low. + defaultConcurrency = 2 +) + +// concurrencyKey is the per-provider setting that overrides its kind's +// default transfer limit. +const concurrencyKey = "maxConcurrent" + +// kindConcurrency is the default number of simultaneous transfers each +// provider kind will tolerate. +// +// A single global cap is the wrong shape here: usenet and torrent +// clients are built to run many transfers at once and are throttled by +// bandwidth, while Soulseek transfers come from one person's home +// upload slot. Hitting the same peer with parallel requests gets you +// queued behind everyone else at best and banned at worst, so slskd is +// capped at one — the polite number, and the one that actually +// completes fastest, because a Soulseek peer serves one file at a time +// regardless of how many you ask for. +var kindConcurrency = map[Kind]int{ + KindSlskd: 1, + KindYtDlp: 2, + KindQBittorrent: 4, + KindSABnzbd: 4, + KindProwlarr: 4, + KindLidarr: 4, + KindFake: 4, +} + +// concurrencyFor returns a provider's transfer limit: its configured +// override, else its kind's default, else the global default. +func concurrencyFor(cfg Config) int { + if raw, ok := cfg.Settings[concurrencyKey]; ok && raw != "" { + if n, err := strconv.Atoi(raw); err == nil && n > 0 { + return n + } + } + + if n, ok := kindConcurrency[cfg.Kind]; ok { + return n + } + + return defaultConcurrency +} + +// Manager errors. +var ( + // ErrNoProviders means nothing is configured and enabled. + ErrNoProviders = errors.New("no download providers are enabled") + + // ErrNoCandidates means every provider searched and found nothing. + ErrNoCandidates = errors.New("no candidates found") + + // ErrCandidateGone means the chosen candidate is no longer in the + // request's result set — usually a stale UI. + ErrCandidateGone = errors.New("candidate is no longer available") + + // ErrDelegateFailed means an external manager reported that it + // could not fulfil the request. + ErrDelegateFailed = errors.New("delegate reported failure") +) + +// Manager coordinates the download subsystem. +type Manager struct { + logger *slog.Logger + store *Store + secrets SecretStore + staging *Staging + importer *Importer + library LibraryPort + + // jobsReg is optional; without it downloads still work but do not + // appear in the background jobs panel. + jobsReg *jobs.Registry + + // opts describes the library layout imports follow. + optsMu sync.RWMutex + opts ImportOptions + + // providers caches built provider instances by config ID. Rebuilt + // whenever config changes, so a settings edit takes effect without + // a restart. + provMu sync.RWMutex + providers map[int64]Provider + configs map[int64]Config + + // results holds the ranked candidates of live requests, so the + // picker can be reopened without re-searching. + resMu sync.RWMutex + results map[string][]Candidate + + // active tracks cancel functions for in-flight requests. + actMu sync.Mutex + active map[string]context.CancelFunc + + // sem bounds concurrent grabs across every provider. + sem chan struct{} + + // provSem bounds concurrent grabs per transporting provider, + // rebuilt on Reload alongside the providers themselves. A grab + // takes its provider's slot before the global one, so a queue on a + // busy Soulseek daemon cannot sit on a global slot that a usenet + // transfer could have used. + semMu sync.Mutex + provSem map[int64]chan struct{} + + // delegatePoll is how often delegating managers are asked for + // status. A field rather than the constant so tests can drive the + // full delegate flow without sleeping through it. + delegatePoll time.Duration +} + +// NewManager builds a download manager. Providers are not constructed +// until Reload is called, so a manager can be created before the Wails +// runtime exists. +func NewManager( + logger *slog.Logger, + store *Store, + secrets SecretStore, + staging *Staging, + importer *Importer, + library LibraryPort, +) *Manager { + return &Manager{ + logger: logger, + store: store, + secrets: secrets, + staging: staging, + importer: importer, + library: library, + providers: map[int64]Provider{}, + configs: map[int64]Config{}, + results: map[string][]Candidate{}, + active: map[string]context.CancelFunc{}, + sem: make(chan struct{}, defaultConcurrency), + provSem: map[int64]chan struct{}{}, + delegatePoll: pollInterval, + } +} + +// SetJobRegistry wires the background jobs panel. +func (m *Manager) SetJobRegistry(reg *jobs.Registry) { + m.jobsReg = reg +} + +// SetImportOptions configures the library layout imports follow. +func (m *Manager) SetImportOptions(opts ImportOptions) { + m.optsMu.Lock() + defer m.optsMu.Unlock() + + m.opts = opts +} + +// importOptions returns the current layout options. +func (m *Manager) importOptions() ImportOptions { + m.optsMu.RLock() + defer m.optsMu.RUnlock() + + return m.opts +} + +// Reload rebuilds every provider from stored config. Called at startup +// and after any provider settings change. +// +// A provider that fails to build is logged and skipped rather than +// failing the reload: one misconfigured client must not disable the +// others. +func (m *Manager) Reload(ctx context.Context) error { + configs, err := m.store.ListProviders(ctx) + if err != nil { + return err + } + + built := make(map[int64]Provider, len(configs)) + kept := make(map[int64]Config, len(configs)) + + for _, cfg := range configs { + kept[cfg.ID] = cfg + + if !cfg.Enabled { + continue + } + + p, err := New(cfg, lookupFor(m.secrets, cfg.ID), m.logger) + if err != nil { + m.logger.Warn( + "could not build download provider", + "provider", cfg.Name, + "kind", cfg.Kind, + "error", err, + ) + + continue + } + + built[cfg.ID] = p + } + + m.provMu.Lock() + old := m.providers + m.providers = built + m.configs = kept + m.provMu.Unlock() + + m.syncSemaphores(kept) + + for id, p := range old { + if _, reused := built[id]; reused { + continue + } + + if err := p.Close(); err != nil { + m.logger.Debug( + "error closing replaced provider", "id", id, "error", err, + ) + } + } + + return nil +} + +// Sweep cleans staging directories left by a previous run. Called at +// startup after the store is available. +func (m *Manager) Sweep(ctx context.Context) { + live, err := m.store.ListLiveItems(ctx) + if err != nil { + m.logger.Warn("could not list live download items", "error", err) + + return + } + + // Anything the database still thinks is live cannot be resumed: the + // transports do not survive a restart. Mark them failed so the UI + // does not show a phantom transfer, then let staging be swept. + liveIDs := make(map[string]bool, len(live)) + + for _, item := range live { + liveIDs[item.ID] = true + + if err := m.store.SetItemState( + ctx, item.ID, StateFailed, "interrupted by restart", + ); err != nil { + m.logger.Warn( + "could not fail interrupted download item", + "item", item.ID, "error", err, + ) + } + + if err := m.store.SetRequestState( + ctx, item.RequestID, StateFailed, "interrupted by restart", + ); err != nil { + m.logger.Warn( + "could not fail interrupted download request", + "request", item.RequestID, "error", err, + ) + } + } + + if _, err := m.staging.Sweep(); err != nil { + m.logger.Warn("could not sweep staging directory", "error", err) + } + + if _, err := m.staging.SweepOrphans(map[string]bool{}); err != nil { + m.logger.Warn("could not sweep orphaned staging dirs", "error", err) + } +} + +// enabledProviders returns a snapshot of built providers with their +// configs. +func (m *Manager) enabledProviders() map[int64]Provider { + m.provMu.RLock() + defer m.provMu.RUnlock() + + out := make(map[int64]Provider, len(m.providers)) + for id, p := range m.providers { + out[id] = p + } + + return out +} + +// SetMaxConcurrent sets the global transfer limit. Called once at +// startup from the user's config; a change takes effect for transfers +// that start afterwards, since a transfer already running holds a slot +// in the semaphore it acquired. +func (m *Manager) SetMaxConcurrent(n int) { + if n <= 0 { + n = defaultConcurrency + } + + m.semMu.Lock() + defer m.semMu.Unlock() + + m.sem = make(chan struct{}, n) +} + +// globalSem returns the current global semaphore. Callers must hold on +// to what they get: releasing into a semaphore that was replaced in the +// meantime would return a slot to the wrong pool. +func (m *Manager) globalSem() chan struct{} { + m.semMu.Lock() + defer m.semMu.Unlock() + + return m.sem +} + +// semaphoreFor returns a provider's own transfer semaphore, creating it +// on first use from that provider's configured or default limit. +func (m *Manager) semaphoreFor(id int64) chan struct{} { + m.provMu.RLock() + cfg, known := m.configs[id] + m.provMu.RUnlock() + + m.semMu.Lock() + defer m.semMu.Unlock() + + if sem, ok := m.provSem[id]; ok { + return sem + } + + limit := defaultConcurrency + if known { + limit = concurrencyFor(cfg) + } + + sem := make(chan struct{}, limit) + m.provSem[id] = sem + + return sem +} + +// syncSemaphores drops semaphores for providers that no longer exist +// and for providers whose limit changed. Transfers already holding a +// slot keep their own reference to the old channel, so replacing the +// map entry cannot strand them; it only means the new limit applies +// from the next transfer on. +func (m *Manager) syncSemaphores(configs map[int64]Config) { + m.semMu.Lock() + defer m.semMu.Unlock() + + for id, sem := range m.provSem { + cfg, ok := configs[id] + if !ok { + delete(m.provSem, id) + + continue + } + + if cap(sem) != concurrencyFor(cfg) { + delete(m.provSem, id) + } + } +} + +// listers returns every enabled provider that keeps a persistent wanted +// list of its own, keyed by provider ID. +func (m *Manager) listers() map[int64]Lister { + m.provMu.RLock() + defer m.provMu.RUnlock() + + out := map[int64]Lister{} + + for id, p := range m.providers { + if l, ok := asLister(p); ok { + out[id] = l + } + } + + return out +} + +// priorityFor returns a provider's configured priority. +func (m *Manager) priorityFor(id int64) int { + m.provMu.RLock() + defer m.provMu.RUnlock() + + if cfg, ok := m.configs[id]; ok { + return cfg.Priority + } + + return 50 +} + +// Search fans a request out across every enabled searching provider and +// returns ranked candidates. Providers are searched concurrently with +// a per-provider timeout; a provider that errors or times out is logged +// and skipped, because partial results beat no results. +func (m *Manager) Search( + ctx context.Context, + req Request, +) ([]Candidate, error) { + providers := m.enabledProviders() + if len(providers) == 0 { + return nil, ErrNoProviders + } + + type found struct { + candidates []Candidate + err error + id int64 + } + + results := make(chan found) + searched := 0 + + for id, p := range providers { + s, ok := asSearcher(p) + if !ok { + continue + } + + searched++ + + go func(id int64, s Searcher) { + sctx, cancel := context.WithTimeout(ctx, searchTimeout) + defer cancel() + + c, err := s.Search(sctx, req) + results <- found{candidates: c, err: err, id: id} + }(id, s) + } + + if searched == 0 { + return nil, fmt.Errorf("%w: none can search", ErrNoProviders) + } + + all := make([]Candidate, 0, searched*8) + + for range searched { + r := <-results + + if r.err != nil { + m.logger.Warn( + "download provider search failed", + "provider", r.id, + "error", r.err, + ) + + continue + } + + for i := range r.candidates { + r.candidates[i].ProviderID = r.id + } + + all = append(all, r.candidates...) + } + + if len(all) == 0 { + return nil, ErrNoCandidates + } + + return Rank(req, all, m.priorityFor), nil +} + +// Start creates a request, searches for it, and either grabs the clear +// winner automatically or parks the ranked list for the user to pick +// from. It returns as soon as the search completes; the transfer runs +// in the background under a job. +func (m *Manager) Start( + ctx context.Context, + req Request, +) ([]Candidate, error) { + if req.ID == "" { + req.ID = newID() + } + + if err := m.store.CreateRequest(ctx, req); err != nil { + return nil, err + } + + job := m.startJob(req) + + ranked, err := m.Search(ctx, req) + if err != nil { + m.failRequest(ctx, job, req.ID, err) + + return nil, err + } + + m.resMu.Lock() + m.results[req.ID] = ranked + m.resMu.Unlock() + + if err := m.store.SetRequestState( + ctx, req.ID, StateFound, "", + ); err != nil { + m.logger.Warn("could not record found state", "error", err) + } + + if job != nil { + job.Logf(jobs.LevelInfo, fmt.Sprintf( + "Found %d candidates across enabled providers", len(ranked), + )) + } + + if AutoPickable(req, ranked) { + if job != nil { + job.Logf(jobs.LevelInfo, "Auto-selected best candidate") + } + + go m.grab(context.WithoutCancel(ctx), req, ranked[0], job) + + return ranked, nil + } + + if job != nil { + job.SetPhase("Waiting for you to pick") + job.SetState(jobs.StatePaused) + } + + return ranked, nil +} + +// Attempt searches on behalf of the wanted list and starts a download +// only if there is a clear winner. It returns whether it started and, +// when it did not, a sentence the wanted list can show the user. +// +// Unlike Start it persists nothing when it does not act. A want that +// is retried weekly for a year would otherwise leave fifty failed +// request rows behind it, all saying the same thing the want itself +// already says — and none of them anything the user can do something +// about. Nobody is watching a reconcile pass, so the only two honest +// outcomes are "downloading it now" and "still looking". +func (m *Manager) Attempt( + ctx context.Context, + req Request, +) (bool, string, error) { + if req.ID == "" { + req.ID = newID() + } + + ranked, err := m.Search(ctx, req) + if err != nil { + return false, "", err + } + + if !AutoPickable(req, ranked) { + best := ranked[0] + + return false, fmt.Sprintf( + "best of %d found is not a confident enough match "+ + "(match %.0f%%, quality %.0f%%)", + len(ranked), + best.Match.Overall*100, //nolint:mnd // percent + best.Quality.Overall*100, + ), nil + } + + if err := m.store.CreateRequest(ctx, req); err != nil { + return false, "", err + } + + m.resMu.Lock() + m.results[req.ID] = ranked + m.resMu.Unlock() + + if err := m.store.SetRequestState(ctx, req.ID, StateFound, ""); err != nil { + m.logger.Warn("could not record found state", "error", err) + } + + job := m.startJob(req) + + if job != nil { + job.Logf(jobs.LevelInfo, fmt.Sprintf( + "Wanted list: auto-selected the best of %d candidates", + len(ranked), + )) + } + + go m.grab(context.WithoutCancel(ctx), req, ranked[0], job) + + return true, "", nil +} + +// Pick starts the transfer for a candidate the user chose. +func (m *Manager) Pick( + ctx context.Context, + requestID, candidateID string, +) error { + req, err := m.store.GetRequest(ctx, requestID) + if err != nil { + return err + } + + m.resMu.RLock() + ranked := m.results[requestID] + m.resMu.RUnlock() + + var chosen *Candidate + + for i := range ranked { + if ranked[i].ID == candidateID { + chosen = &ranked[i] + + break + } + } + + if chosen == nil { + return fmt.Errorf("%w: %s", ErrCandidateGone, candidateID) + } + + job := m.startJob(req) + + go m.grab(context.WithoutCancel(ctx), req, *chosen, job) + + return nil +} + +// Cancel aborts a live request. +func (m *Manager) Cancel(ctx context.Context, requestID string) error { + m.actMu.Lock() + cancel, ok := m.active[requestID] + m.actMu.Unlock() + + if ok { + cancel() + } + + if err := m.store.SetRequestState( + ctx, requestID, StateCancelled, "", + ); err != nil { + return err + } + + return nil +} + +// grab drives one candidate all the way to the library. It runs on its +// own goroutine and owns the job from here on. +func (m *Manager) grab( + ctx context.Context, + req Request, + c Candidate, + job *jobs.Handle, +) { + ctx, cancel := context.WithTimeout(ctx, grabTimeout) + defer cancel() + + m.actMu.Lock() + m.active[req.ID] = cancel + m.actMu.Unlock() + + defer func() { + m.actMu.Lock() + delete(m.active, req.ID) + m.actMu.Unlock() + }() + + // Who will move the bytes is decided before any slot is taken, so + // the transfer waits in its own provider's queue rather than in a + // global one. A delegate takes no slot at all: the transfer is + // happening inside another system, which is doing its own limiting, + // and blocking a local slot on it would be counting someone else's + // work against our budget. + plan, err := m.planTransfer(req, c) + if err != nil { + m.failRequest(ctx, job, req.ID, err) + + return + } + + if !plan.delegated() { + provSem := m.semaphoreFor(plan.transportID) + + select { + case provSem <- struct{}{}: + defer func() { <-provSem }() + case <-ctx.Done(): + m.failRequest(ctx, job, req.ID, ctx.Err()) + + return + } + + globalSem := m.globalSem() + + select { + case globalSem <- struct{}{}: + defer func() { <-globalSem }() + case <-ctx.Done(): + m.failRequest(ctx, job, req.ID, ctx.Err()) + + return + } + } + + item := Item{ + ID: newID(), + RequestID: req.ID, + ProviderID: c.ProviderID, + Candidate: c, + State: StateQueued, + BytesTotal: c.TotalSize, + } + + dir, err := m.staging.Reserve(item.ID) + if err != nil { + m.failRequest(ctx, job, req.ID, err) + + return + } + + item.StagingDir = dir + + if err := m.store.CreateItem(ctx, item); err != nil { + m.failRequest(ctx, job, req.ID, err) + + return + } + + result, err := m.transfer(ctx, req, item, plan, job) + if err != nil { + m.failItem(ctx, job, item, req.ID, err) + + return + } + + m.setStates(ctx, req.ID, item.ID, StateImporting) + + if job != nil { + job.SetPhase("Importing") + job.SetStages(importStages(2)) + } + + var imported ImportResult + + if result.Delegated { + // The external manager already placed and tagged these files in + // its own library. Moving them out from under a system that is + // still managing them would be worse than useless, so the files + // are recorded where they are and the library scan picks them + // up in place. + imported = ImportResult{Paths: result.Files} + + if job != nil { + job.Logf(jobs.LevelInfo, fmt.Sprintf( + "External manager imported %d files; recording them in place", + len(result.Files), + )) + } + } else { + opts := m.importOptions() + opts.WriteTags = true + + imported, err = m.importer.Import(ctx, req, result, opts) + if err != nil { + m.failItem(ctx, job, item, req.ID, err) + + return + } + } + + if err := m.store.SetItemImported( + ctx, item.ID, imported.Paths, + ); err != nil { + m.logger.Warn("could not record imported paths", "error", err) + } + + if err := m.store.SetRequestState( + ctx, req.ID, StateComplete, "", + ); err != nil { + m.logger.Warn("could not record complete state", "error", err) + } + + // A request raised from the wanted list retires its want here + // rather than waiting for the next reconcile pass to notice the + // files, so the wanted list is right the moment the download + // finishes. The pass would reach the same conclusion by asking the + // library; this is the same answer, sooner. + if req.WantID != 0 { + if err := m.store.SatisfyWant(ctx, req.WantID); err != nil { + m.logger.Warn( + "could not satisfy want", "want", req.WantID, "error", err, + ) + } + } + + // The ranked list only existed so the picker could be reopened + // mid-flight. Holding it after the download completes would leak a + // few hundred candidates per request for the life of the process. + m.resMu.Lock() + delete(m.results, req.ID) + m.resMu.Unlock() + + // Staging is only released on a fully successful import; a failure + // leaves the files for retry or inspection. + if err := m.staging.Release(item.StagingDir); err != nil { + m.logger.Warn("could not release staging dir", "error", err) + } + + if m.library != nil { + if err := m.library.ScanLibrary(req.LibraryID); err != nil { + m.logger.Warn( + "could not trigger scan after import", + "library", req.LibraryID, + "error", err, + ) + } + } + + if job != nil { + job.SetStats([]jobs.Stat{ + {Label: "Imported", Value: itoa(len(imported.Paths))}, + {Label: "Tagged", Value: itoa(imported.Tagged)}, + }) + job.Logf(jobs.LevelInfo, fmt.Sprintf( + "Imported %d files into the library", len(imported.Paths), + )) + job.Complete() + } +} + +// transfer moves the bytes, dispatching on whether the candidate's +// provider fetches its own results, needs a separate transport, or +// delegates the whole thing. +func (m *Manager) transfer( + ctx context.Context, + req Request, + item Item, + plan transferPlan, + job *jobs.Handle, +) (Result, error) { + if plan.delegated() { + return m.delegate(ctx, req, item, plan.delegate, job) + } + + m.setStates(ctx, req.ID, item.ID, StateGrabbing) + + if job != nil { + job.SetPhase("Downloading") + job.SetProgress(0, item.Candidate.TotalSize) + } + + onProgress := m.progressReporter(ctx, item.ID, job) + + result, err := plan.transport.Grab( + ctx, item.Candidate, item.StagingDir, onProgress, + ) + if err != nil { + return Result{}, fmt.Errorf("grab failed: %w", err) + } + + m.setStates(ctx, req.ID, item.ID, StateVerifying) + + if job != nil { + job.SetPhase("Verifying") + } + + return result, nil +} + +// transportFor picks the transport that will fetch a candidate: the +// finding provider itself when it can, otherwise the highest-priority +// enabled provider that handles the candidate's protocol. +func (m *Manager) transportFor( + providers map[int64]Provider, + sourceID int64, + source Provider, + c Candidate, +) (Transporter, int64, error) { + if c.Protocol == ProtocolDirect { + t, ok := asTransporter(source) + if !ok { + return nil, 0, fmt.Errorf( + "%w: %s cannot fetch its own results", + ErrUnsupported, source.Info().Kind, + ) + } + + return t, sourceID, nil + } + + var ( + best Transporter + bestID int64 + bestPrio = -1 + ) + + for id, p := range providers { + t, ok := asTransporter(p) + if !ok || !p.Info().Caps.Handles(c.Protocol) { + continue + } + + if prio := m.priorityFor(id); prio > bestPrio { + best, bestID, bestPrio = t, id, prio + } + } + + if best == nil { + return nil, 0, fmt.Errorf("%w: %s", ErrNoTransport, c.Protocol) + } + + return best, bestID, nil +} + +// transferPlan is who will move a candidate's bytes, resolved before +// any concurrency slot is taken so a transfer queues against the +// provider that will actually do the work. +type transferPlan struct { + // delegate is set when an external manager owns the whole transfer. + delegate Delegator + + // transport and transportID are set otherwise. + transport Transporter + transportID int64 +} + +// delegated reports whether this plan hands the work to another system. +func (p transferPlan) delegated() bool { return p.delegate != nil } + +// planTransfer decides how a candidate will be fetched. +func (m *Manager) planTransfer(_ Request, c Candidate) (transferPlan, error) { + providers := m.enabledProviders() + + source, ok := providers[c.ProviderID] + if !ok { + return transferPlan{}, fmt.Errorf( + "%w: provider %d", ErrNotConfigured, c.ProviderID, + ) + } + + if d, ok := asDelegator(source); ok { + return transferPlan{delegate: d}, nil + } + + transport, id, err := m.transportFor(providers, c.ProviderID, source, c) + if err != nil { + return transferPlan{}, err + } + + return transferPlan{transport: transport, transportID: id}, nil +} + +// delegate hands the request to an external manager and polls until it +// reports terminal state. +func (m *Manager) delegate( + ctx context.Context, + req Request, + item Item, + d Delegator, + job *jobs.Handle, +) (Result, error) { + externalID, err := d.Delegate(ctx, req) + if err != nil { + return Result{}, fmt.Errorf("delegate request: %w", err) + } + + if err := m.store.SetItemExternalID(ctx, item.ID, externalID); err != nil { + m.logger.Warn("could not record external id", "error", err) + } + + m.setStates(ctx, req.ID, item.ID, StateGrabbing) + + if job != nil { + job.SetPhase("Waiting on external manager") + job.Logf(jobs.LevelInfo, "Handed request to "+string(item.Candidate.Kind)) + } + + ctx, cancel := context.WithTimeout(ctx, delegateTimeout) + defer cancel() + + ticker := time.NewTicker(m.delegatePoll) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + _ = d.Withdraw(context.WithoutCancel(ctx), externalID) + + return Result{}, fmt.Errorf("delegate timed out: %w", ctx.Err()) + case <-ticker.C: + } + + status, err := d.Poll(ctx, externalID) + if err != nil { + m.logger.Warn("delegate poll failed", "error", err) + + continue + } + + if job != nil && status.Progress >= 0 { + job.SetProgress(int64(status.Progress*100), 100) + } + + switch status.State { + case StateComplete: + // The manager placed the files itself, so there is nothing + // in staging and nothing for us to move. Report its paths + // so the item records what landed where. + return Result{ + Files: status.ImportedPaths, + Delegated: true, + }, nil + case StateFailed, StateCancelled: + return Result{}, fmt.Errorf( + "%w: %s: %s", ErrDelegateFailed, status.State, status.Message, + ) + case StateSearching, StateFound, StateQueued, StateGrabbing, + StateVerifying, StateTagging, StateImporting: + // Still working. + } + } +} + +// progressReporter returns a throttled ProgressFunc that updates both +// the job and the stored item. Transports call this per chunk, so it +// must be cheap: the job registry already coalesces, but a database +// write per chunk would not survive contact with a fast transfer. +func (m *Manager) progressReporter( + ctx context.Context, + itemID string, + job *jobs.Handle, +) ProgressFunc { + const dbInterval = 3 * time.Second + + var ( + mu sync.Mutex + lastSave time.Time + ) + + return func(p Progress) { + if job != nil { + job.SetProgress(p.Current, p.Total) + + if p.Phase != "" { + job.SetPhase(p.Phase) + } + } + + mu.Lock() + + if time.Since(lastSave) < dbInterval { + mu.Unlock() + + return + } + + lastSave = time.Now() + mu.Unlock() + + if err := m.store.SetItemProgress( + ctx, itemID, p.Current, p.Total, + ); err != nil { + m.logger.Debug("could not save item progress", "error", err) + } + } +} + +// Candidates returns the ranked candidates for a live request. +func (m *Manager) Candidates(requestID string) []Candidate { + m.resMu.RLock() + defer m.resMu.RUnlock() + + out := make([]Candidate, len(m.results[requestID])) + copy(out, m.results[requestID]) + + return out +} + +// setStates advances a request and its item together. +func (m *Manager) setStates( + ctx context.Context, + requestID, itemID string, + state State, +) { + if err := m.store.SetRequestState(ctx, requestID, state, ""); err != nil { + m.logger.Warn("could not set request state", "error", err) + } + + if err := m.store.SetItemState(ctx, itemID, state, ""); err != nil { + m.logger.Warn("could not set item state", "error", err) + } +} + +// failRequest records a request-level failure. +func (m *Manager) failRequest( + ctx context.Context, + job *jobs.Handle, + requestID string, + err error, +) { + m.logger.Warn("download request failed", "request", requestID, "error", err) + + if serr := m.store.SetRequestState( + ctx, requestID, StateFailed, err.Error(), + ); serr != nil { + m.logger.Warn("could not record failure", "error", serr) + } + + if job != nil { + job.Fail(err) + } +} + +// failItem records an item-level failure and fails its request. +func (m *Manager) failItem( + ctx context.Context, + job *jobs.Handle, + item Item, + requestID string, + err error, +) { + if serr := m.store.SetItemState( + ctx, item.ID, StateFailed, err.Error(), + ); serr != nil { + m.logger.Warn("could not record item failure", "error", serr) + } + + m.failRequest(ctx, job, requestID, err) +} + +// startJob registers the request in the background jobs panel. +func (m *Manager) startJob(req Request) *jobs.Handle { + if m.jobsReg == nil { + return nil + } + + title := req.Album + if title == "" { + title = req.SearchText() + } + + return m.jobsReg.Start(jobs.Spec{ + ID: "download-" + req.ID, + Kind: jobs.KindDownload, + Title: "Downloading " + title, + Subtitle: req.Artist, + State: jobs.StateRunning, + Caps: jobs.Caps{Cancellable: true}, + Controls: jobs.Controls{ + Cancel: func() { + if err := m.Cancel(context.Background(), req.ID); err != nil { + m.logger.Warn("cancel failed", "error", err) + } + }, + }, + }) +} + +// importStages renders the pipeline tail as job stages. +func importStages(done int) []jobs.Stage { + names := []string{"Search", "Download", "Import"} + out := make([]jobs.Stage, 0, len(names)) + + for i, n := range names { + state := "pending" + + switch { + case i < done: + state = "complete" + case i == done: + state = "running" + } + + out = append(out, jobs.Stage{Name: n, State: state}) + } + + return out +} + +// newID returns a random identifier for a request or item. +func newID() string { + var b [12]byte + + if _, err := rand.Read(b[:]); err != nil { + // crypto/rand failing means the system is in a state where a + // timestamp fallback is the least of anyone's problems, but a + // collision here would silently merge two downloads. + return "dl-" + time.Now().Format("20060102150405.000000000") + } + + return hex.EncodeToString(b[:]) +} + +// itoa formats an int for job stats. +func itoa(n int) string { + return strconv.Itoa(n) +} diff --git a/backend/download/manager_test.go b/backend/download/manager_test.go new file mode 100644 index 0000000..be27c4b --- /dev/null +++ b/backend/download/manager_test.go @@ -0,0 +1,507 @@ +package download + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + "time" + + "yellowjacket/backend/database" +) + +// managerFixture wires a manager over a real (in-memory) database, a +// temp staging area and a temp library root, with no network anywhere. +type managerFixture struct { + manager *Manager + store *Store + staging *Staging + lib *stubLibrary + tags *recordingTagWriter + root string +} + +func newManagerFixture(t *testing.T) managerFixture { + t.Helper() + + db := database.NewTestDB(t) + seedLibrary(t, db) + + store := NewStore(db) + staging := newTestStaging(t) + tags := newRecordingTagWriter() + lib := &stubLibrary{} + root := t.TempDir() + + imp := NewImporter(slogDiscard(), staging, tags, lib) + + m := NewManager( + slogDiscard(), store, NewMemSecretStore(), staging, imp, lib, + ) + m.SetImportOptions(ImportOptions{LibraryRoot: root}) + + return managerFixture{ + manager: m, + store: store, + staging: staging, + lib: lib, + tags: tags, + root: root, + } +} + +// seedLibrary inserts the library row download_requests references. +func seedLibrary(t *testing.T, db *database.DB) { + t.Helper() + + if _, err := db.ExecContext( + `INSERT INTO libraries (id, name, path) VALUES (1, 'Test', '/music')`, + ); err != nil { + t.Fatalf("seed library: %v", err) + } +} + +// fakeWithAlbum returns a fake provider that finds and can deliver the +// four-track reference album. +func fakeWithAlbum(id int64, name string, ext string) *FakeProvider { + f := NewFakeProvider(id, name, Caps{CanSearch: true, CanTransport: true}) + + titles := allTitles() + c := candidateFor(name+"-cand", titles, ext, 30_000_000) + c.ProviderID = id + + f.Candidates = []Candidate{c} + + for i, tt := range titles { + f.Written[trackToken(i+1)+" - "+tt+ext] = []byte("audio-data") + } + + return f +} + +func TestManagerSearchRanksAcrossProviders(t *testing.T) { + t.Parallel() + + f := newManagerFixture(t) + + mp3 := fakeWithAlbum(1, "mp3-source", ".mp3") + flac := fakeWithAlbum(2, "flac-source", ".flac") + + f.manager.installProvider(Config{ID: 1, Priority: 50}, mp3) + f.manager.installProvider(Config{ID: 2, Priority: 50}, flac) + + ranked, err := f.manager.Search(context.Background(), fourTrackRequest()) + if err != nil { + t.Fatalf("Search: %v", err) + } + + if len(ranked) != 2 { + t.Fatalf("got %d candidates, want 2", len(ranked)) + } + + if ranked[0].ProviderID != 2 { + t.Errorf( + "winner from provider %d, want 2 (FLAC)", ranked[0].ProviderID, + ) + } + + if mp3.SearchCalls != 1 || flac.SearchCalls != 1 { + t.Errorf( + "search calls: mp3=%d flac=%d, want 1 each", + mp3.SearchCalls, flac.SearchCalls, + ) + } +} + +// One broken provider must not take the others down with it. +func TestManagerSearchToleratesProviderFailure(t *testing.T) { + t.Parallel() + + f := newManagerFixture(t) + + broken := NewFakeProvider(1, "broken", Caps{CanSearch: true}) + broken.SearchErr = errors.New("connection refused") //nolint:err113 // test + + working := fakeWithAlbum(2, "working", ".flac") + + f.manager.installProvider(Config{ID: 1, Priority: 50}, broken) + f.manager.installProvider(Config{ID: 2, Priority: 50}, working) + + ranked, err := f.manager.Search(context.Background(), fourTrackRequest()) + if err != nil { + t.Fatalf("Search: %v", err) + } + + if len(ranked) != 1 { + t.Fatalf("got %d candidates, want 1 from the working provider", len(ranked)) + } +} + +func TestManagerSearchNoProviders(t *testing.T) { + t.Parallel() + + f := newManagerFixture(t) + + _, err := f.manager.Search(context.Background(), fourTrackRequest()) + if !errors.Is(err, ErrNoProviders) { + t.Fatalf("error = %v, want ErrNoProviders", err) + } +} + +func TestManagerSearchNoCandidates(t *testing.T) { + t.Parallel() + + f := newManagerFixture(t) + + empty := NewFakeProvider(1, "empty", Caps{CanSearch: true}) + f.manager.installProvider(Config{ID: 1, Priority: 50}, empty) + + _, err := f.manager.Search(context.Background(), fourTrackRequest()) + if !errors.Is(err, ErrNoCandidates) { + t.Fatalf("error = %v, want ErrNoCandidates", err) + } +} + +// The whole pipeline: request → search → auto-pick → grab → verify → +// tag → import → library scan. +func TestManagerEndToEndAutoPick(t *testing.T) { + t.Parallel() + + f := newManagerFixture(t) + + provider := fakeWithAlbum(1, "flac-source", ".flac") + f.manager.installProvider(Config{ID: 1, Priority: 50}, provider) + + req := fourTrackRequest() + + ranked, err := f.manager.Start(context.Background(), req) + if err != nil { + t.Fatalf("Start: %v", err) + } + + if !AutoPickable(req, ranked) { + t.Fatalf( + "expected a clear winner to auto-pick; best match %f quality %f", + ranked[0].Match.Overall, ranked[0].Quality.Overall, + ) + } + + waitForRequestState(t, f.store, req.ID, StateComplete) + + if provider.GrabCalls != 1 { + t.Errorf("grab calls = %d, want 1", provider.GrabCalls) + } + + // Files landed in the library, laid out by the template. + want := filepath.Join(f.root, "Radiohead", "OK Computer", "01 Airbag.flac") + if _, err := os.Stat(want); err != nil { + t.Errorf("expected imported file at %s: %v", want, err) + } + + // Staging was released only after a successful import. + entries, err := os.ReadDir(f.staging.Root()) + if err != nil { + t.Fatalf("read staging root: %v", err) + } + + if len(entries) != 0 { + t.Errorf("staging not released: %d dirs remain", len(entries)) + } + + // The library was told to rescan. + f.lib.mu.Lock() + scanned := len(f.lib.scanned) + f.lib.mu.Unlock() + + if scanned != 1 { + t.Errorf("library scans = %d, want 1", scanned) + } +} + +// An ambiguous result set must park for the user rather than guess. +func TestManagerWaitsWhenAmbiguous(t *testing.T) { + t.Parallel() + + f := newManagerFixture(t) + + a := fakeWithAlbum(1, "source-a", ".flac") + b := fakeWithAlbum(2, "source-b", ".flac") + + f.manager.installProvider(Config{ID: 1, Priority: 50}, a) + f.manager.installProvider(Config{ID: 2, Priority: 50}, b) + + req := fourTrackRequest() + + ranked, err := f.manager.Start(context.Background(), req) + if err != nil { + t.Fatalf("Start: %v", err) + } + + if AutoPickable(req, ranked) { + t.Fatal("two equivalent candidates must not auto-pick") + } + + // Nothing was grabbed while waiting for the user. + if a.GrabCalls != 0 || b.GrabCalls != 0 { + t.Errorf( + "grabs happened without a pick: a=%d b=%d", + a.GrabCalls, b.GrabCalls, + ) + } + + stored, err := f.store.GetRequest(context.Background(), req.ID) + if err != nil { + t.Fatalf("GetRequest: %v", err) + } + + if stored.ID != req.ID { + t.Errorf("stored request id = %s, want %s", stored.ID, req.ID) + } + + // The user picks the second one explicitly. + if err := f.manager.Pick( + context.Background(), req.ID, ranked[1].ID, + ); err != nil { + t.Fatalf("Pick: %v", err) + } + + waitForRequestState(t, f.store, req.ID, StateComplete) +} + +func TestManagerPickUnknownCandidate(t *testing.T) { + t.Parallel() + + f := newManagerFixture(t) + + provider := fakeWithAlbum(1, "source", ".flac") + f.manager.installProvider(Config{ID: 1, Priority: 50}, provider) + + req := fourTrackRequest() + req.Expected = nil // free text: never auto-picks + + if _, err := f.manager.Start(context.Background(), req); err != nil { + t.Fatalf("Start: %v", err) + } + + err := f.manager.Pick(context.Background(), req.ID, "no-such-candidate") + if !errors.Is(err, ErrCandidateGone) { + t.Fatalf("error = %v, want ErrCandidateGone", err) + } +} + +// A failed grab must leave staging intact for retry and must not put +// anything in the library. +func TestManagerFailedGrabLeavesLibraryClean(t *testing.T) { + t.Parallel() + + f := newManagerFixture(t) + + provider := fakeWithAlbum(1, "source", ".flac") + provider.GrabErr = errors.New("peer went offline") //nolint:err113 // test + + f.manager.installProvider(Config{ID: 1, Priority: 50}, provider) + + req := fourTrackRequest() + + if _, err := f.manager.Start(context.Background(), req); err != nil { + t.Fatalf("Start: %v", err) + } + + waitForRequestState(t, f.store, req.ID, StateFailed) + + entries, err := os.ReadDir(f.root) + if err != nil { + t.Fatalf("read library root: %v", err) + } + + if len(entries) != 0 { + t.Errorf("failed grab wrote %d entries into the library", len(entries)) + } + + staged, err := os.ReadDir(f.staging.Root()) + if err != nil { + t.Fatalf("read staging root: %v", err) + } + + if len(staged) == 0 { + t.Error("staging released after a failure; nothing left to retry") + } +} + +// A search-only provider's candidate is fetched by whichever enabled +// transport handles its protocol. +func TestManagerPairsSearcherWithTransport(t *testing.T) { + t.Parallel() + + f := newManagerFixture(t) + + searcher := NewFakeProvider(1, "indexer", Caps{CanSearch: true}) + + titles := allTitles() + c := candidateFor("torrent-cand", titles, ".flac", 30_000_000) + c.Protocol = ProtocolTorrent + searcher.Candidates = []Candidate{c} + + transport := NewFakeProvider(2, "torrent-client", Caps{ + CanTransport: true, + Transports: []Protocol{ProtocolTorrent}, + }) + + for i, tt := range titles { + transport.Written[trackToken(i+1)+" - "+tt+".flac"] = []byte("audio-data") + } + + f.manager.installProvider(Config{ID: 1, Priority: 50}, searcher) + f.manager.installProvider(Config{ID: 2, Priority: 50}, transport) + + req := fourTrackRequest() + + if _, err := f.manager.Start(context.Background(), req); err != nil { + t.Fatalf("Start: %v", err) + } + + waitForRequestState(t, f.store, req.ID, StateComplete) + + if transport.GrabCalls != 1 { + t.Errorf("transport grabs = %d, want 1", transport.GrabCalls) + } + + if searcher.GrabCalls != 0 { + t.Errorf("searcher should not have grabbed, got %d", searcher.GrabCalls) + } +} + +// A candidate whose protocol nothing handles must fail loudly rather +// than being silently dropped. +func TestManagerNoTransportForProtocol(t *testing.T) { + t.Parallel() + + f := newManagerFixture(t) + + searcher := NewFakeProvider(1, "indexer", Caps{CanSearch: true}) + + c := candidateFor("usenet-cand", allTitles(), ".flac", 30_000_000) + c.Protocol = ProtocolUsenet + searcher.Candidates = []Candidate{c} + + f.manager.installProvider(Config{ID: 1, Priority: 50}, searcher) + + req := fourTrackRequest() + + if _, err := f.manager.Start(context.Background(), req); err != nil { + t.Fatalf("Start: %v", err) + } + + waitForRequestState(t, f.store, req.ID, StateFailed) +} + +// waitForRequestState polls until a request reaches the wanted state. +// The pipeline runs on its own goroutine, so tests observe it through +// the store rather than by reaching into the manager. +func waitForRequestState( + t *testing.T, + store *Store, + requestID string, + want State, +) { + t.Helper() + + deadline := time.Now().Add(5 * time.Second) + + var last State + + for time.Now().Before(deadline) { + state, _, err := store.GetRequestState(context.Background(), requestID) + if err == nil { + last = state + if last == want { + return + } + } + + time.Sleep(10 * time.Millisecond) + } + + t.Fatalf("request state = %q after 5s, want %q", last, want) +} + +// A delegate's files are already in the external manager's library, +// tagged by it and at paths it chose. The pipeline must record them in +// place rather than moving them out from under a system that is still +// managing them. +func TestManagerDelegateReconcilesInPlace(t *testing.T) { + t.Parallel() + + f := newManagerFixture(t) + f.manager.delegatePoll = time.Millisecond + + delegate := NewFakeProvider(1, "lidarr", Caps{ + CanSearch: true, + CanDelegate: true, + }) + + c := candidateFor("delegate-cand", allTitles(), ".flac", 30_000_000) + c.ProviderID = 1 + delegate.Candidates = []Candidate{c} + + external := []string{ + "/external/library/Radiohead/OK Computer/01 Airbag.flac", + "/external/library/Radiohead/OK Computer/02 Paranoid Android.flac", + } + + delegate.DelegateStatuses = []DelegateStatus{ + {State: StateGrabbing, Progress: 0.5}, + {State: StateComplete, Progress: 1, ImportedPaths: external}, + } + + f.manager.installProvider(Config{ID: 1, Priority: 50}, delegate) + + req := fourTrackRequest() + + if _, err := f.manager.Start(context.Background(), req); err != nil { + t.Fatalf("Start: %v", err) + } + + waitForRequestState(t, f.store, req.ID, StateComplete) + + if delegate.DelegateCalls != 1 { + t.Errorf("delegate calls = %d, want 1", delegate.DelegateCalls) + } + + // Nothing was tagged or written into our library root — the files + // belong to the external manager. + entries, err := os.ReadDir(f.root) + if err != nil { + t.Fatalf("read library root: %v", err) + } + + if len(entries) != 0 { + t.Errorf("delegate import wrote %d entries into our library", len(entries)) + } + + f.tags.mu.Lock() + writes := len(f.tags.writes) + f.tags.mu.Unlock() + + if writes != 0 { + t.Errorf("tagged %d files, want 0 for a delegated import", writes) + } + + // The external paths were recorded against the item. + items, err := f.store.ListItemsForRequest(context.Background(), req.ID) + if err != nil { + t.Fatalf("ListItemsForRequest: %v", err) + } + + if len(items) != 1 { + t.Fatalf("got %d items, want 1", len(items)) + } + + if len(items[0].Imported) != len(external) { + t.Errorf( + "recorded %v, want the external manager's paths %v", + items[0].Imported, external, + ) + } +} diff --git a/backend/download/pathmatch.go b/backend/download/pathmatch.go new file mode 100644 index 0000000..78a2249 --- /dev/null +++ b/backend/download/pathmatch.go @@ -0,0 +1,314 @@ +package download + +import ( + "path" + "regexp" + "strconv" + "strings" + + "yellowjacket/backend/autotag" +) + +// Candidate files arrive as paths, not tags — a Soulseek result is +// `@@abc\Music\Pink Floyd - The Wall (1979) [FLAC]\1-01 In The Flesh.flac` +// and nothing more. Everything the ranker knows about whether a +// candidate is the right album comes from parsing that string, so the +// heuristics here carry real weight. + +// audioExtensions maps a lowercase file extension to its format. +var audioExtensions = map[string]Format{ + ".flac": FormatFLAC, + ".mp3": FormatMP3, + ".ogg": FormatOGG, + ".oga": FormatOGG, + ".opus": FormatOpus, + ".wav": FormatWAV, + ".m4a": FormatAAC, + ".aac": FormatAAC, + ".alac": FormatALAC, + ".wma": FormatWMA, + ".ape": FormatUnknown, + ".wv": FormatUnknown, +} + +var ( + // trackNumPattern matches a leading track number in the common + // shapes: "01 - Title", "1. Title", "1-01 Title" (disc-track), + // "[01] Title". The disc group is optional. + trackNumPattern = regexp.MustCompile( + `^\s*\[?(?:(\d{1,2})\s*[-_.]\s*)?(\d{1,3})\]?\s*[-_.)\]]?\s+`, + ) + + // bareTrackNumPattern matches a number with no separator at all + // ("01Title" is rare, but "01 Title" with a single space is not). + bareTrackNumPattern = regexp.MustCompile(`^\s*(\d{1,3})\s+`) + + // bitratePattern finds a bitrate hint in a folder or file name: + // "[320]", "V0", "320kbps", "(V2)". + bitratePattern = regexp.MustCompile( + `(?i)\b(\d{2,4})\s*k(?:bps|b/s)?\b|\[(\d{2,4})\]`, + ) + + // vbrPattern finds LAME VBR preset names, which imply a bitrate + // band rather than a number. + vbrPattern = regexp.MustCompile(`(?i)\b(V[0-2])\b`) + + // yearPattern finds a 4-digit year in parentheses or brackets. + yearPattern = regexp.MustCompile(`[(\[](19|20)\d{2}[)\]]`) + + // junkSuffixPattern strips scene/rip tags from a folder name before + // comparing it to an album title. + junkSuffixPattern = regexp.MustCompile( + `(?i)[\[(]\s*(flac|mp3|web|cd|vinyl|24bit|16bit|lossless|` + + `v0|v2|320|256|192|128|kbps|reissue|remaster(ed)?|` + + `\d{2,3}\s*k(bps)?)\s*[^\])]*[\])]`, + ) + + // separatorPattern splits "Artist - Album" style folder names. + separatorPattern = regexp.MustCompile(`\s+[-–—]\s+`) +) + +// FormatForPath returns the audio format implied by a path's extension, +// and whether the path is audio at all. Cue sheets, logs, playlists +// and cover images are not. +func FormatForPath(p string) (Format, bool) { + ext := strings.ToLower(path.Ext(strings.ReplaceAll(p, `\`, "/"))) + + f, ok := audioExtensions[ext] + + return f, ok +} + +// TrackHint is what a single candidate file's path reveals about the +// track it holds. Every field is best-effort and may be zero. +type TrackHint struct { + Disc int + Track int + + // Title is the filename with the extension, track number and any + // leading artist credit removed. + Title string + + // Folder is the immediate parent directory name, cleaned of scene + // tags — the best available proxy for the album title. + Folder string +} + +// ParsePath extracts what it can from one candidate file path. +func ParsePath(p string) TrackHint { + // Soulseek paths are Windows-style; normalize before splitting. + norm := strings.ReplaceAll(p, `\`, "/") + base := path.Base(norm) + folder := path.Base(path.Dir(norm)) + + name := strings.TrimSuffix(base, path.Ext(base)) + + hint := TrackHint{Folder: cleanAlbumName(folder)} + + if m := trackNumPattern.FindStringSubmatch(name); m != nil { + if m[1] != "" { + hint.Disc, _ = strconv.Atoi(m[1]) + } + + hint.Track, _ = strconv.Atoi(m[2]) + name = name[len(m[0]):] + } else if m := bareTrackNumPattern.FindStringSubmatch(name); m != nil { + hint.Track, _ = strconv.Atoi(m[1]) + name = name[len(m[0]):] + } + + // "Artist - Title" inside the filename: drop the leading credit + // when what follows is substantial. Guessing wrong here costs a + // little title similarity; not doing it costs a lot, because most + // Soulseek folders name the artist in every file. + if parts := separatorPattern.Split(name, 2); len(parts) == 2 { + if len(strings.TrimSpace(parts[1])) >= 3 { + name = parts[1] + } + } + + hint.Title = strings.TrimSpace(name) + + return hint +} + +// cleanAlbumName strips year markers and scene tags from a folder name +// so it can be compared against a release title. +func cleanAlbumName(folder string) string { + s := junkSuffixPattern.ReplaceAllString(folder, " ") + s = yearPattern.ReplaceAllString(s, " ") + + // A folder is often "Artist - Album"; keep the right-hand side when + // there is one, since the album is what we compare against. + if parts := separatorPattern.Split(s, 2); len(parts) == 2 { + if len(strings.TrimSpace(parts[1])) >= 2 { + s = parts[1] + } + } + + return strings.TrimSpace(strings.Join(strings.Fields(s), " ")) +} + +// BitrateForPath infers a bitrate in kbps from path text. Returns 0 +// when nothing is stated. VBR presets map to their nominal average. +func BitrateForPath(p string) int { + if m := vbrPattern.FindStringSubmatch(p); m != nil { + switch strings.ToUpper(m[1]) { + case "V0": + return 245 + case "V1": + return 225 + case "V2": + return 190 + } + } + + if m := bitratePattern.FindStringSubmatch(p); m != nil { + raw := m[1] + if raw == "" { + raw = m[2] + } + + if n, err := strconv.Atoi(raw); err == nil && n >= 32 && n <= 3000 { + return n + } + } + + return 0 +} + +// AnnotateFiles fills in Format, IsAudio and Bitrate for a candidate's +// files. Providers call this so each adapter does not re-derive the +// same things from the same paths. +func AnnotateFiles(files []CandidateFile) []CandidateFile { + out := make([]CandidateFile, len(files)) + + for i, f := range files { + format, isAudio := FormatForPath(f.Path) + + f.IsAudio = isAudio + if f.Format == FormatUnknown { + f.Format = format + } + + if f.Bitrate == 0 { + f.Bitrate = BitrateForPath(f.Path) + } + + out[i] = f + } + + return out +} + +// matchFiles aligns a candidate's audio files to the expected tracklist +// and returns the per-file assignment plus the mean title similarity of +// the aligned pairs. +// +// Alignment is greedy by score rather than optimal: candidate folders +// are small (a few dozen files at most) and the common cases — correct +// track numbers, or clean "NN Title" names — are unambiguous, so the +// extra machinery of Hungarian assignment buys nothing here. +func matchFiles( + files []CandidateFile, + expected []ExpectedTrack, +) ([]CandidateFile, float64) { + annotated := make([]CandidateFile, len(files)) + copy(annotated, files) + + if len(expected) == 0 { + return annotated, 0 + } + + hints := make([]TrackHint, len(annotated)) + for i, f := range annotated { + hints[i] = ParsePath(f.Path) + } + + takenExpected := make(map[int]bool, len(expected)) + + var ( + total float64 + matched int + ) + + // Pass 1: trust explicit track numbers when they are unique and in + // range. A folder that numbers its files correctly is the strong + // case, and title comparison only adds noise there. + for i := range annotated { + if !annotated[i].IsAudio || hints[i].Track == 0 { + continue + } + + idx := indexForPosition(expected, hints[i].Disc, hints[i].Track) + if idx < 0 || takenExpected[idx] { + continue + } + + takenExpected[idx] = true + annotated[i].MatchedTo = expected[idx].Position + + total += autotag.TitleSimilarity(hints[i].Title, expected[idx].Title) + matched++ + } + + // Pass 2: title similarity for whatever is left. + for i := range annotated { + if !annotated[i].IsAudio || annotated[i].MatchedTo != 0 { + continue + } + + bestIdx, bestSim := -1, 0.0 + + for j := range expected { + if takenExpected[j] { + continue + } + + sim := autotag.TitleSimilarity(hints[i].Title, expected[j].Title) + if sim > bestSim { + bestIdx, bestSim = j, sim + } + } + + // Below this the "match" is two unrelated strings sharing a few + // characters, and counting it drags the mean toward noise. + const minTitleSim = 0.55 + + if bestIdx < 0 || bestSim < minTitleSim { + continue + } + + takenExpected[bestIdx] = true + annotated[i].MatchedTo = expected[bestIdx].Position + + total += bestSim + matched++ + } + + if matched == 0 { + return annotated, 0 + } + + return annotated, total / float64(matched) +} + +// indexForPosition finds the expected track at a disc/track position. +// A zero disc hint matches on track number alone, which is right for +// single-disc releases and the best guess for multi-disc folders that +// do not encode the disc. +func indexForPosition(expected []ExpectedTrack, disc, track int) int { + for i, e := range expected { + if e.Position != track { + continue + } + + if disc != 0 && e.DiscNumber != 0 && e.DiscNumber != disc { + continue + } + + return i + } + + return -1 +} diff --git a/backend/download/pathmatch_test.go b/backend/download/pathmatch_test.go new file mode 100644 index 0000000..22e1ec3 --- /dev/null +++ b/backend/download/pathmatch_test.go @@ -0,0 +1,242 @@ +package download + +import "testing" + +func TestParsePath(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + path string + wantDisc int + wantTrack int + wantTitle string + }{ + { + name: "soulseek windows path with disc and track", + path: `@@abc\Music\Pink Floyd - The Wall (1979) [FLAC]\1-05 Another Brick In The Wall.flac`, + wantDisc: 1, + wantTrack: 5, + wantTitle: "Another Brick In The Wall", + }, + { + name: "dash separated track number", + path: "Radiohead - OK Computer/03 - Subterranean Homesick Alien.mp3", + wantTrack: 3, + wantTitle: "Subterranean Homesick Alien", + }, + { + name: "dotted track number", + path: "Album/7. Karma Police.flac", + wantTrack: 7, + wantTitle: "Karma Police", + }, + { + name: "bracketed track number", + path: "Album/[02] Paranoid Android.mp3", + wantTrack: 2, + wantTitle: "Paranoid Android", + }, + { + name: "bare number and space", + path: "Album/11 Lucky.ogg", + wantTrack: 11, + wantTitle: "Lucky", + }, + { + name: "artist credit inside filename is dropped", + path: "VA - Comp/04 - Aphex Twin - Xtal.flac", + wantTrack: 4, + wantTitle: "Xtal", + }, + { + name: "no track number", + path: "Album/Introduction.flac", + wantTrack: 0, + wantTitle: "Introduction", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := ParsePath(tt.path) + + if got.Disc != tt.wantDisc { + t.Errorf("Disc = %d, want %d", got.Disc, tt.wantDisc) + } + + if got.Track != tt.wantTrack { + t.Errorf("Track = %d, want %d", got.Track, tt.wantTrack) + } + + if got.Title != tt.wantTitle { + t.Errorf("Title = %q, want %q", got.Title, tt.wantTitle) + } + }) + } +} + +func TestCleanAlbumName(t *testing.T) { + t.Parallel() + + tests := []struct { + in string + want string + }{ + {"Pink Floyd - The Wall (1979) [FLAC]", "The Wall"}, + {"Radiohead - OK Computer [V0]", "OK Computer"}, + {"In Rainbows", "In Rainbows"}, + {"Artist - Album [320kbps]", "Album"}, + {"Kid A (2000)", "Kid A"}, + } + + for _, tt := range tests { + t.Run(tt.in, func(t *testing.T) { + t.Parallel() + + if got := cleanAlbumName(tt.in); got != tt.want { + t.Errorf("cleanAlbumName(%q) = %q, want %q", tt.in, got, tt.want) + } + }) + } +} + +func TestBitrateForPath(t *testing.T) { + t.Parallel() + + tests := []struct { + in string + want int + }{ + {"Album [320]/01 Track.mp3", 320}, + {"Album [V0]/01 Track.mp3", 245}, + {"Album (V2)/01 Track.mp3", 190}, + {"Album 192kbps/01 Track.mp3", 192}, + {"Album/01 Track.flac", 0}, + {"Album [9999]/01 Track.mp3", 0}, // out of plausible range + } + + for _, tt := range tests { + t.Run(tt.in, func(t *testing.T) { + t.Parallel() + + if got := BitrateForPath(tt.in); got != tt.want { + t.Errorf("BitrateForPath(%q) = %d, want %d", tt.in, got, tt.want) + } + }) + } +} + +func TestFormatForPath(t *testing.T) { + t.Parallel() + + tests := []struct { + path string + want Format + wantAudio bool + }{ + {"a/b.flac", FormatFLAC, true}, + {"a/b.MP3", FormatMP3, true}, + {`a\b.ogg`, FormatOGG, true}, + {"a/cover.jpg", FormatUnknown, false}, + {"a/rip.log", FormatUnknown, false}, + {"a/playlist.m3u", FormatUnknown, false}, + } + + for _, tt := range tests { + t.Run(tt.path, func(t *testing.T) { + t.Parallel() + + got, isAudio := FormatForPath(tt.path) + + if isAudio != tt.wantAudio { + t.Errorf("isAudio = %v, want %v", isAudio, tt.wantAudio) + } + + if isAudio && got != tt.want { + t.Errorf("format = %q, want %q", got, tt.want) + } + }) + } +} + +func TestMatchFilesByTrackNumber(t *testing.T) { + t.Parallel() + + expected := []ExpectedTrack{ + {Position: 1, Title: "Airbag"}, + {Position: 2, Title: "Paranoid Android"}, + {Position: 3, Title: "Subterranean Homesick Alien"}, + } + + files := []CandidateFile{ + {Path: "OK Computer/01 - Airbag.flac", IsAudio: true}, + {Path: "OK Computer/02 - Paranoid Android.flac", IsAudio: true}, + {Path: "OK Computer/03 - Subterranean Homesick Alien.flac", IsAudio: true}, + } + + matched, sim := matchFiles(files, expected) + + for i, m := range matched { + if m.MatchedTo != i+1 { + t.Errorf("file %d matched to %d, want %d", i, m.MatchedTo, i+1) + } + } + + if sim < 0.99 { + t.Errorf("similarity = %f, want ~1.0", sim) + } +} + +// Track numbers that lie are the common Soulseek failure: a folder +// numbered 1..N whose contents are a different album entirely. Title +// matching has to be what catches it. +func TestMatchFilesFallsBackToTitles(t *testing.T) { + t.Parallel() + + expected := []ExpectedTrack{ + {Position: 1, Title: "Airbag"}, + {Position: 2, Title: "Paranoid Android"}, + } + + files := []CandidateFile{ + {Path: "Album/Paranoid Android.flac", IsAudio: true}, + {Path: "Album/Airbag.flac", IsAudio: true}, + } + + matched, sim := matchFiles(files, expected) + + if matched[0].MatchedTo != 2 { + t.Errorf("first file matched to %d, want 2", matched[0].MatchedTo) + } + + if matched[1].MatchedTo != 1 { + t.Errorf("second file matched to %d, want 1", matched[1].MatchedTo) + } + + if sim < 0.9 { + t.Errorf("similarity = %f, want high", sim) + } +} + +func TestMatchFilesUnrelatedScoresLow(t *testing.T) { + t.Parallel() + + expected := []ExpectedTrack{ + {Position: 1, Title: "Airbag"}, + {Position: 2, Title: "Paranoid Android"}, + } + + files := []CandidateFile{ + {Path: "Other/Enter Sandman.flac", IsAudio: true}, + {Path: "Other/Master Of Puppets.flac", IsAudio: true}, + } + + _, sim := matchFiles(files, expected) + + if sim > 0.5 { + t.Errorf("similarity = %f, want low for unrelated tracks", sim) + } +} diff --git a/backend/download/provider.go b/backend/download/provider.go new file mode 100644 index 0000000..f753735 --- /dev/null +++ b/backend/download/provider.go @@ -0,0 +1,350 @@ +package download + +import ( + "context" + "errors" + "fmt" + "log/slog" + "sort" + "strconv" + "sync" +) + +// Provider errors. +var ( + // ErrUnknownKind is returned when a stored provider row names a kind + // no constructor is registered for — an old row after a provider + // was removed, or a config file from a newer build. + ErrUnknownKind = errors.New("unknown provider kind") + + // ErrNotConfigured means the provider exists but is missing + // required settings (host, API key) and cannot be used yet. + ErrNotConfigured = errors.New("provider is not configured") + + // ErrUnsupported is returned when a caller asks a provider for a + // role it does not fill. + ErrUnsupported = errors.New("provider does not support this operation") + + // ErrNoTransport means a search-only provider produced a candidate + // whose protocol no enabled transport can fetch. + ErrNoTransport = errors.New("no enabled transport handles this protocol") +) + +// Provider is the common surface every adapter implements. The three +// role interfaces below are optional and discovered by type assertion, +// gated on what Caps declares. +type Provider interface { + // Info returns the provider's identity and declared capabilities. + Info() ProviderInfo + + // Check verifies the provider is reachable and configured + // correctly. It backs the "test connection" button, and the + // pipeline calls it before first use in a session. + Check(ctx context.Context) error + + // Close releases any long-lived resources (sessions, cookies). + Close() error +} + +// Searcher turns a request into candidates. Implementations must +// respect ctx deadlines: the pipeline searches providers concurrently +// with a per-provider timeout and takes whatever came back in time. +type Searcher interface { + Search(ctx context.Context, req Request) ([]Candidate, error) +} + +// Transporter moves a candidate's bytes into dst, which the pipeline +// has already created and which the transport owns for the duration. +// +// Implementations report progress through onProgress (best-effort, may +// be nil) and must return promptly when ctx is cancelled, leaving +// partial files in place — the pipeline sweeps them. +type Transporter interface { + Grab( + ctx context.Context, + c Candidate, + dst string, + onProgress ProgressFunc, + ) (Result, error) +} + +// Delegator hands the whole request to an external manager. Unlike a +// Transporter we do not own the transfer, so the pipeline polls until +// the manager reports terminal state. +type Delegator interface { + // Delegate submits the request and returns the manager's own ID. + Delegate(ctx context.Context, req Request) (string, error) + + // Poll reports on a previously delegated request. + Poll(ctx context.Context, externalID string) (DelegateStatus, error) + + // Withdraw asks the manager to drop the request. Best-effort. + Withdraw(ctx context.Context, externalID string) error +} + +// Lister is a provider that keeps a persistent wanted list of its own — +// Lidarr monitoring an artist, say. It is the fourth role, and it +// exists because for those systems "I want this" is a durable statement +// they already model, and mirroring it there means the user's intent +// survives in the place they will look for it. +// +// Sync through this interface is one-directional in the loop: this app +// pushes, the external system receives. Pulling happens only when the +// user explicitly imports. +type Lister interface { + // PushWant records a want in the provider's own list and returns + // the provider's identifier for it. Implementations must be + // idempotent: pushing a want the provider already has returns the + // existing identifier rather than duplicating it. + PushWant(ctx context.Context, w Want) (string, error) + + // RemoveWant drops a previously pushed want. Best-effort. + RemoveWant(ctx context.Context, externalID string) error + + // ListWants reads the provider's list back, for the deliberate + // import path. LibraryID is filled in by the caller. + ListWants(ctx context.Context) ([]Want, error) +} + +// ProviderInfo is a provider's identity as the frontend sees it. +type ProviderInfo struct { + ID int64 `json:"id"` + Kind Kind `json:"kind"` + Name string `json:"name"` + Enabled bool `json:"enabled"` + + // Priority breaks ties between providers that found equally good + // candidates. Higher wins; default 50. + Priority int `json:"priority"` + + Caps Caps `json:"caps"` +} + +// Config is a provider's stored settings. Secret values are not held +// here — they live in the secrets store keyed by provider ID, so a +// config blob can be logged or shown in the UI without redaction. +type Config struct { + ID int64 `json:"id"` + Kind Kind `json:"kind"` + Name string `json:"name"` + Enabled bool `json:"enabled"` + Priority int `json:"priority"` + Settings map[string]string `json:"settings"` +} + +// Setting returns a config value, or fallback when unset. +func (c Config) Setting(key, fallback string) string { + if v, ok := c.Settings[key]; ok && v != "" { + return v + } + + return fallback +} + +// Constructor builds a provider from its stored config. The secret +// lookup is passed in rather than the secret itself so a provider can +// fetch several (username and password, say) and so nothing forces the +// secret into a struct field that might get logged. +type Constructor func( + cfg Config, + secrets SecretLookup, + logger *slog.Logger, +) (Provider, error) + +// SecretLookup retrieves a named secret for a provider. +type SecretLookup func(name string) (string, error) + +// registry maps provider kinds to their constructors. Adapters +// register themselves in an init function, so adding a provider does +// not require editing this file. +var ( + registryMu sync.RWMutex + constructors = map[Kind]Constructor{} + descriptors = map[Kind]Descriptor{} +) + +// Descriptor is the static, instance-independent description of a +// provider kind: what it is called, what it can do, and which settings +// it needs. The settings page renders its form from this, so a new +// provider gets a config UI without any frontend work. +type Descriptor struct { + Kind Kind `json:"kind"` + Name string `json:"name"` + + // Summary is one line explaining what connecting this gets you. + Summary string `json:"summary"` + + // Caps are the kind's inherent capabilities, before configuration. + Caps Caps `json:"caps"` + + // Fields are the settings the user must supply. + Fields []Field `json:"fields"` + + // RequiresExternal names the software the user must run themselves + // (a slskd daemon, a Lidarr instance), or is empty for providers + // that need nothing but a binary on PATH. + RequiresExternal string `json:"requiresExternal,omitempty"` +} + +// Field describes one provider setting for the settings form. +type Field struct { + Key string `json:"key"` + Label string `json:"label"` + Placeholder string `json:"placeholder,omitempty"` + Help string `json:"help,omitempty"` + + // Secret marks a value stored in the secrets store rather than the + // provider config row, and rendered as a password input. + Secret bool `json:"secret"` + + Required bool `json:"required"` + Default string `json:"default,omitempty"` +} + +// Register makes a provider kind available. Called from adapter init +// functions; panics on a duplicate kind because that is a build-time +// programming error, not a runtime condition. +func Register(d Descriptor, c Constructor) { + registryMu.Lock() + defer registryMu.Unlock() + + if _, exists := constructors[d.Kind]; exists { + panic("download: duplicate provider kind " + string(d.Kind)) + } + + // Every provider that moves bytes gets a transfer limit it can be + // tuned with, appended here rather than repeated in each adapter's + // descriptor: the setting means the same thing everywhere, only the + // sensible default differs. + if d.Caps.CanTransport { + d.Fields = append(d.Fields, concurrencyField(d.Kind)) + } + + constructors[d.Kind] = c + descriptors[d.Kind] = d +} + +// concurrencyField describes the per-provider transfer limit, with help +// text explaining why the default is what it is — a user who raises +// slskd from 1 to 8 and gets themselves queued behind every other +// Soulseek user deserves to have been warned. +func concurrencyField(k Kind) Field { + help := "Maximum simultaneous transfers from this client." + + if k == KindSlskd { + help = "Maximum simultaneous transfers. Soulseek peers serve " + + "one file at a time and queue or ban clients that ask for " + + "more, so 1 is both the polite setting and usually the " + + "fastest." + } + + return Field{ + Key: concurrencyKey, + Label: "Simultaneous transfers", + Help: help, + Default: strconv.Itoa(kindConcurrency[k]), + } +} + +// New builds a provider instance from stored config. +func New( + cfg Config, + secrets SecretLookup, + logger *slog.Logger, +) (Provider, error) { + registryMu.RLock() + + ctor, ok := constructors[cfg.Kind] + + registryMu.RUnlock() + + if !ok { + return nil, fmt.Errorf("%w: %s", ErrUnknownKind, cfg.Kind) + } + + p, err := ctor(cfg, secrets, logger) + if err != nil { + return nil, fmt.Errorf("build %s provider: %w", cfg.Kind, err) + } + + return p, nil +} + +// Descriptors returns every registered provider kind, name-ordered, for +// the "add a download client" picker. +func Descriptors() []Descriptor { + registryMu.RLock() + defer registryMu.RUnlock() + + out := make([]Descriptor, 0, len(descriptors)) + + for _, d := range descriptors { + if d.Kind == KindFake { + continue + } + + out = append(out, d) + } + + sort.Slice(out, func(i, j int) bool { + return out[i].Name < out[j].Name + }) + + return out +} + +// DescriptorFor returns the descriptor for a kind. +func DescriptorFor(k Kind) (Descriptor, bool) { + registryMu.RLock() + defer registryMu.RUnlock() + + d, ok := descriptors[k] + + return d, ok +} + +// asSearcher returns the provider's Searcher role, gated on Caps so a +// type that implements the method but declares it unsupported (because +// it is misconfigured) is not used. +func asSearcher(p Provider) (Searcher, bool) { + if !p.Info().Caps.CanSearch { + return nil, false + } + + s, ok := p.(Searcher) + + return s, ok +} + +// asTransporter returns the provider's Transporter role. +func asTransporter(p Provider) (Transporter, bool) { + if !p.Info().Caps.CanTransport { + return nil, false + } + + t, ok := p.(Transporter) + + return t, ok +} + +// asDelegator returns the provider's Delegator role. +func asDelegator(p Provider) (Delegator, bool) { + if !p.Info().Caps.CanDelegate { + return nil, false + } + + d, ok := p.(Delegator) + + return d, ok +} + +// asLister returns the provider's Lister role. +func asLister(p Provider) (Lister, bool) { + if !p.Info().Caps.CanList { + return nil, false + } + + l, ok := p.(Lister) + + return l, ok +} diff --git a/backend/download/provider_lidarr.go b/backend/download/provider_lidarr.go new file mode 100644 index 0000000..146d9ac --- /dev/null +++ b/backend/download/provider_lidarr.go @@ -0,0 +1,548 @@ +package download + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/url" + "strconv" + "strings" + "time" +) + +// Lidarr is the delegate shape, and it is genuinely different from the +// other two: we do not search, we do not move bytes, and we do not own +// the import. We hand Lidarr an album and ask periodically whether it +// is done. +// +// The consequence that shapes this adapter: when Lidarr finishes, the +// files are already in Lidarr's library, tagged by Lidarr, at paths +// Lidarr chose. Re-importing them would mean moving files out from +// under a system that is actively managing them. So a completed +// delegate returns the paths Lidarr reports and the pipeline skips +// tagging and moving — it reconciles rather than imports. + +// Lidarr provider errors. +var ( + // ErrLidarrUnreachable means the instance did not answer. + ErrLidarrUnreachable = errors.New("lidarr is unreachable") + + // ErrLidarrAuth means the API key was rejected. + ErrLidarrAuth = errors.New("lidarr rejected the API key") + + // ErrLidarrNoMatch means Lidarr could not find the release. + ErrLidarrNoMatch = errors.New("lidarr could not find this release") + + // ErrLidarrNoRootFolder means no root folder is configured, so + // Lidarr has nowhere to put anything it finds. + ErrLidarrNoRootFolder = errors.New("lidarr has no root folder configured") +) + +// lidarrHTTPTimeout bounds one API call. +const lidarrHTTPTimeout = 30 * time.Second + +func init() { + Register( + Descriptor{ + Kind: KindLidarr, + Name: "Lidarr", + Summary: "Hand album requests to an existing Lidarr instance " + + "and let it do the searching and importing.", + RequiresExternal: "Lidarr", + Caps: Caps{ + CanDelegate: true, + CanCancel: true, + CanList: true, + }, + Fields: []Field{ + { + Key: "url", + Label: "Lidarr URL", + Placeholder: "http://localhost:8686", + Required: true, + Default: "http://localhost:8686", + }, + { + Key: "apiKey", + Label: "API key", + Secret: true, + Required: true, + Help: "Lidarr → Settings → General → API Key.", + }, + { + Key: "qualityProfileId", + Label: "Quality profile ID", + Help: "Numeric ID of the Lidarr quality profile to use. " + + "Leave blank to use the first one.", + }, + { + Key: "metadataProfileId", + Label: "Metadata profile ID", + Help: "Leave blank to use the first one.", + }, + { + Key: "rootFolderPath", + Label: "Root folder", + Help: "Leave blank to use Lidarr's first configured " + + "root folder.", + }, + }, + }, + newLidarr, + ) +} + +// lidarr is the Lidarr delegate provider. +type lidarr struct { + info ProviderInfo + logger *slog.Logger + client *apiClient + + qualityProfileID int + metadataProfileID int + rootFolderPath string +} + +// newLidarr builds the provider from config. +func newLidarr( + cfg Config, + secrets SecretLookup, + logger *slog.Logger, +) (Provider, error) { + base := strings.TrimRight(cfg.Setting("url", ""), "/") + if base == "" { + return nil, fmt.Errorf("%w: Lidarr URL is required", ErrNotConfigured) + } + + apiKey := "" + + if secrets != nil { + key, err := secrets("apiKey") + if err != nil { + return nil, fmt.Errorf("%w: no API key stored", ErrNotConfigured) + } + + apiKey = key + } + + quality, _ := strconv.Atoi(cfg.Setting("qualityProfileId", "0")) + metadata, _ := strconv.Atoi(cfg.Setting("metadataProfileId", "0")) + + return &lidarr{ + info: ProviderInfo{ + ID: cfg.ID, + Kind: KindLidarr, + Name: cfg.Name, + Enabled: cfg.Enabled, + Priority: cfg.Priority, + Caps: Caps{ + CanDelegate: true, + CanCancel: true, + CanList: true, + }, + }, + logger: logger.With("provider", "lidarr"), + client: newAPIClient( + base, "X-Api-Key", apiKey, lidarrHTTPTimeout, + ErrLidarrUnreachable, ErrLidarrAuth, + ), + qualityProfileID: quality, + metadataProfileID: metadata, + rootFolderPath: cfg.Setting("rootFolderPath", ""), + }, nil +} + +// Info returns the provider's identity. +func (l *lidarr) Info() ProviderInfo { + return l.info +} + +// Close is a no-op. +func (l *lidarr) Close() error { + return nil +} + +// Check verifies the instance answers and has somewhere to put music. +func (l *lidarr) Check(ctx context.Context) error { + var status struct { + Version string `json:"version"` + } + + if err := l.client.get(ctx, "/api/v1/system/status", &status); err != nil { + return err + } + + folders, err := l.rootFolders(ctx) + if err != nil { + return err + } + + if len(folders) == 0 { + return ErrLidarrNoRootFolder + } + + return nil +} + +// --------------------------------------------------------------------------- +// API types +// --------------------------------------------------------------------------- + +// lidarrAlbum is the subset of Lidarr's album resource used here. +type lidarrAlbum struct { + ID int `json:"id"` + Title string `json:"title"` + ForeignAlbum string `json:"foreignAlbumId"` + Monitored bool `json:"monitored"` + ArtistID int `json:"artistId"` + + Artist lidarrArtist `json:"artist"` + Statistics lidarrAlbumStats `json:"statistics"` +} + +// lidarrArtist is the artist an album belongs to. Lidarr models albums +// as children of artists, so an album it does not yet know about cannot +// be monitored until its artist exists. +type lidarrArtist struct { + ID int `json:"id"` + ArtistName string `json:"artistName"` + ForeignArtistID string `json:"foreignArtistId"` +} + +// lidarrAlbumStats is how Lidarr reports import progress. +type lidarrAlbumStats struct { + TrackFileCount int `json:"trackFileCount"` + TrackCount int `json:"trackCount"` + PercentOfTracks float64 `json:"percentOfTracks"` +} + +// lidarrRootFolder is a configured library root. +type lidarrRootFolder struct { + ID int `json:"id"` + Path string `json:"path"` +} + +// lidarrTrackFile is one imported file. +type lidarrTrackFile struct { + ID int `json:"id"` + Path string `json:"path"` + AlbumID int `json:"albumId"` +} + +// --------------------------------------------------------------------------- +// Delegate +// --------------------------------------------------------------------------- + +// Delegate adds the album to Lidarr, monitors it and triggers a search. +// The external ID returned is Lidarr's album ID, which is what Poll +// needs and what survives a restart. +func (l *lidarr) Delegate(ctx context.Context, req Request) (string, error) { + album, err := l.findAlbum(ctx, req) + if err != nil { + return "", err + } + + // An album Lidarr already knows about only needs monitoring turned + // on; one it does not needs its artist added first, because Lidarr + // models albums as children of artists. + if album.ID == 0 { + added, err := l.addArtistForAlbum(ctx, album) + if err != nil { + return "", err + } + + album = added + } + + if !album.Monitored { + if err := l.monitorAlbum(ctx, album.ID); err != nil { + return "", err + } + } + + if err := l.command(ctx, map[string]any{ + "name": "AlbumSearch", + "albumIds": []int{album.ID}, + }); err != nil { + return "", err + } + + return strconv.Itoa(album.ID), nil +} + +// findAlbum looks for the requested album, preferring the MusicBrainz +// release-group ID because that is unambiguous where a title search is +// not. +func (l *lidarr) findAlbum( + ctx context.Context, + req Request, +) (lidarrAlbum, error) { + term := req.SearchText() + + if req.ReleaseGroupMBID != "" { + term = "lidarr:" + req.ReleaseGroupMBID + } + + var results []struct { + Album lidarrAlbum `json:"album"` + } + + endpoint := "/api/v1/search?term=" + url.QueryEscape(term) + + if err := l.client.get(ctx, endpoint, &results); err != nil { + return lidarrAlbum{}, err + } + + for _, r := range results { + if r.Album.Title != "" { + return r.Album, nil + } + } + + return lidarrAlbum{}, fmt.Errorf("%w: %s", ErrLidarrNoMatch, req.SearchText()) +} + +// addArtistForAlbum adds the album's artist so the album becomes a real +// record Lidarr can monitor. +func (l *lidarr) addArtistForAlbum( + ctx context.Context, + album lidarrAlbum, +) (lidarrAlbum, error) { + root := l.rootFolderPath + + if root == "" { + folders, err := l.rootFolders(ctx) + if err != nil { + return lidarrAlbum{}, err + } + + if len(folders) == 0 { + return lidarrAlbum{}, ErrLidarrNoRootFolder + } + + root = folders[0].Path + } + + quality, metadata, err := l.profiles(ctx) + if err != nil { + return lidarrAlbum{}, err + } + + body := map[string]any{ + "foreignArtistId": album.Artist.ForeignArtistID, + "artistName": album.Artist.ArtistName, + "qualityProfileId": quality, + "metadataProfileId": metadata, + "rootFolderPath": root, + "monitored": true, + "addOptions": map[string]any{ + // Monitor nothing by default and turn on just the requested + // album below. Adding an artist with everything monitored + // would kick off downloads of their entire discography, + // which is emphatically not what the user asked for. + "monitor": "none", + "searchForMissingAlbums": false, + }, + } + + var created struct { + ID int `json:"id"` + } + + if err := l.client.post(ctx, "/api/v1/artist", body, &created); err != nil { + return lidarrAlbum{}, err + } + + // Re-resolve the album now that its artist exists. + var albums []lidarrAlbum + + endpoint := "/api/v1/album?artistId=" + strconv.Itoa(created.ID) + + if err := l.client.get(ctx, endpoint, &albums); err != nil { + return lidarrAlbum{}, err + } + + for _, a := range albums { + if a.ForeignAlbum == album.ForeignAlbum { + return a, nil + } + } + + return lidarrAlbum{}, fmt.Errorf( + "%w: album not present after adding artist", ErrLidarrNoMatch, + ) +} + +// monitorAlbum turns on monitoring for one album. +func (l *lidarr) monitorAlbum(ctx context.Context, albumID int) error { + body := map[string]any{ + "albumIds": []int{albumID}, + "monitored": true, + } + + return l.client.put(ctx, "/api/v1/album/monitor", body, nil) +} + +// Poll reports whether Lidarr has finished with the album. +// +// Completion is judged by imported track files rather than by queue +// state: the queue empties when a download finishes, which is before +// the import happens, and reporting success then would have the +// pipeline reconcile files that are not there yet. +func (l *lidarr) Poll( + ctx context.Context, + externalID string, +) (DelegateStatus, error) { + albumID, err := strconv.Atoi(externalID) + if err != nil { + return DelegateStatus{}, fmt.Errorf( + "%w: bad album id %q", ErrLidarrNoMatch, externalID, + ) + } + + var album lidarrAlbum + + endpoint := "/api/v1/album/" + strconv.Itoa(albumID) + + if err := l.client.get(ctx, endpoint, &album); err != nil { + return DelegateStatus{}, err + } + + total := album.Statistics.TrackCount + got := album.Statistics.TrackFileCount + + progress := -1.0 + if total > 0 { + progress = float64(got) / float64(total) + } + + if total > 0 && got >= total { + paths, err := l.trackFilePaths(ctx, albumID) + if err != nil { + return DelegateStatus{}, err + } + + return DelegateStatus{ + State: StateComplete, + Progress: 1, + ImportedPaths: paths, + Message: fmt.Sprintf( + "Lidarr imported %d of %d tracks", got, total, + ), + }, nil + } + + return DelegateStatus{ + State: StateGrabbing, + Progress: progress, + Message: fmt.Sprintf( + "Lidarr has %d of %d tracks", got, total, + ), + }, nil +} + +// trackFilePaths returns the on-disk paths Lidarr imported. +func (l *lidarr) trackFilePaths( + ctx context.Context, + albumID int, +) ([]string, error) { + var files []lidarrTrackFile + + endpoint := "/api/v1/trackfile?albumId=" + strconv.Itoa(albumID) + + if err := l.client.get(ctx, endpoint, &files); err != nil { + return nil, err + } + + out := make([]string, 0, len(files)) + + for _, f := range files { + if f.Path != "" { + out = append(out, f.Path) + } + } + + return out, nil +} + +// Withdraw stops monitoring the album so Lidarr gives up on it. The +// album record is left in place: deleting it would be a bigger action +// than the user asked for, and it may predate this request. +func (l *lidarr) Withdraw(ctx context.Context, externalID string) error { + albumID, err := strconv.Atoi(externalID) + if err != nil { + return fmt.Errorf("%w: bad album id %q", ErrLidarrNoMatch, externalID) + } + + body := map[string]any{ + "albumIds": []int{albumID}, + "monitored": false, + } + + return l.client.put(ctx, "/api/v1/album/monitor", body, nil) +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +// rootFolders lists Lidarr's configured library roots. +func (l *lidarr) rootFolders(ctx context.Context) ([]lidarrRootFolder, error) { + var folders []lidarrRootFolder + + if err := l.client.get(ctx, "/api/v1/rootfolder", &folders); err != nil { + return nil, err + } + + return folders, nil +} + +// profiles resolves the quality and metadata profile IDs to use, +// falling back to the first of each when unconfigured. +func (l *lidarr) profiles(ctx context.Context) (quality, metadata int, err error) { + quality, metadata = l.qualityProfileID, l.metadataProfileID + + if quality == 0 { + var profiles []struct { + ID int `json:"id"` + } + + if err := l.client.get(ctx, "/api/v1/qualityprofile", &profiles); err != nil { + return 0, 0, err + } + + if len(profiles) == 0 { + return 0, 0, fmt.Errorf( + "%w: no quality profiles configured", ErrNotConfigured, + ) + } + + quality = profiles[0].ID + } + + if metadata == 0 { + var profiles []struct { + ID int `json:"id"` + } + + if err := l.client.get(ctx, "/api/v1/metadataprofile", &profiles); err != nil { + return 0, 0, err + } + + if len(profiles) == 0 { + return 0, 0, fmt.Errorf( + "%w: no metadata profiles configured", ErrNotConfigured, + ) + } + + metadata = profiles[0].ID + } + + return quality, metadata, nil +} + +// command posts to Lidarr's command endpoint. +func (l *lidarr) command(ctx context.Context, body map[string]any) error { + return l.client.post(ctx, "/api/v1/command", body, nil) +} diff --git a/backend/download/provider_lidarr_list.go b/backend/download/provider_lidarr_list.go new file mode 100644 index 0000000..769ab84 --- /dev/null +++ b/backend/download/provider_lidarr_list.go @@ -0,0 +1,262 @@ +package download + +import ( + "context" + "fmt" + "net/url" + "strconv" +) + +// Lidarr's Lister role: mirroring this app's wanted list into Lidarr's +// own monitoring. +// +// Lidarr already models exactly what a want is — a monitored artist or +// a monitored album — so the mapping is direct and, more usefully, it +// means an artist subscription made here keeps working through Lidarr's +// own release-checking even while this app is closed. That is the +// whole reason the Lister role exists: a desktop player is not running +// most of the time, and a always-on system that already watches for new +// releases is a better place for a subscription to live than a loop +// that only ticks when someone opens the app. +// +// The mapping is deliberately lossy in one direction only: +// +// artist want -> Lidarr artist, monitored +// release-group want -> Lidarr album, monitored (artist added if new) +// release want -> same, at release-group granularity +// recording want -> not pushed; Lidarr has no concept of wanting +// one track, and monitoring the whole album to +// get it would download far more than asked. +// +// Nothing here searches. Pushing a want expresses intent; Lidarr +// decides when to act on it, which is the point of delegating. + +// PushWant records a want in Lidarr's own monitoring. +// +// It is idempotent because Lidarr is: adding an artist that already +// exists returns the existing record, and monitoring an already- +// monitored album is a no-op. Callers rely on that — the reconciler +// pushes on every pass until it gets an ID back. +func (l *lidarr) PushWant(ctx context.Context, w Want) (string, error) { + switch w.Entity { + case EntityArtist: + return l.pushArtistWant(ctx, w) + case EntityReleaseGroup, EntityRelease: + return l.pushAlbumWant(ctx, w) + case EntityRecording: + // Deliberately unsupported rather than approximated. See the + // mapping note above. + return "", nil + default: + return "", fmt.Errorf("%w: entity %q", ErrUnsupported, w.Entity) + } +} + +// pushArtistWant makes Lidarr monitor an artist. +// +// Scope is honoured through Lidarr's own monitor option rather than by +// pushing each album separately: "future" maps to monitoring new +// releases only, "all" to monitoring everything missing. Letting +// Lidarr apply the policy means it stays applied to albums released +// after this push, which is what a subscription is for. +func (l *lidarr) pushArtistWant(ctx context.Context, w Want) (string, error) { + existing, err := l.findArtistByMBID(ctx, w.MBID) + if err != nil { + return "", err + } + + monitor := "future" + if w.Scope == ScopeAll { + monitor = "missing" + } + + if existing.ID != 0 { + return strconv.Itoa(existing.ID), nil + } + + root, err := l.resolveRootFolder(ctx) + if err != nil { + return "", err + } + + quality, metadata, err := l.profiles(ctx) + if err != nil { + return "", err + } + + name := w.Artist + if name == "" { + name = w.Title + } + + body := map[string]any{ + "foreignArtistId": w.MBID, + "artistName": name, + "qualityProfileId": quality, + "metadataProfileId": metadata, + "rootFolderPath": root, + "monitored": true, + "addOptions": map[string]any{ + "monitor": monitor, + // Searching is left off even for a full-discography + // subscription: adding an artist should not launch forty + // simultaneous searches on a system the user shares with + // their own queue. Lidarr picks the albums up on its next + // scheduled search. + "searchForMissingAlbums": false, + }, + } + + var created struct { + ID int `json:"id"` + } + + if err := l.client.post(ctx, "/api/v1/artist", body, &created); err != nil { + return "", err + } + + return strconv.Itoa(created.ID), nil +} + +// pushAlbumWant makes Lidarr monitor one album, adding its artist if +// Lidarr has never heard of them. +func (l *lidarr) pushAlbumWant(ctx context.Context, w Want) (string, error) { + album, err := l.findAlbum(ctx, Request{ + ReleaseGroupMBID: w.MBID, + Artist: w.Artist, + Album: w.Title, + }) + if err != nil { + return "", err + } + + if album.ID == 0 { + album, err = l.addArtistForAlbum(ctx, album) + if err != nil { + return "", err + } + } + + if !album.Monitored { + if err := l.monitorAlbum(ctx, album.ID); err != nil { + return "", err + } + } + + return strconv.Itoa(album.ID), nil +} + +// RemoveWant stops Lidarr monitoring something. +// +// It unmonitors rather than deletes: the user's Lidarr may have been +// monitoring that artist long before this app existed, and removing a +// want here is not permission to tear down their setup. An unmonitored +// artist stays in their library with its files intact. +func (l *lidarr) RemoveWant(ctx context.Context, externalID string) error { + id, err := strconv.Atoi(externalID) + if err != nil { + return fmt.Errorf("%w: bad lidarr id %q", ErrLidarrNoMatch, externalID) + } + + // The ID may name an album or an artist and the caller does not + // track which, so try the album endpoint first and fall back. + if err := l.client.put(ctx, "/api/v1/album/monitor", map[string]any{ + "albumIds": []int{id}, + "monitored": false, + }, nil); err == nil { + return nil + } + + var artist map[string]any + + endpoint := "/api/v1/artist/" + strconv.Itoa(id) + + if err := l.client.get(ctx, endpoint, &artist); err != nil { + return err + } + + artist["monitored"] = false + + return l.client.put(ctx, endpoint, artist, nil) +} + +// ListWants reads Lidarr's monitored artists back, for the deliberate +// "import what Lidarr is already watching" action. +// +// Only artists are imported, not their individual monitored albums: an +// artist is the durable statement of intent, and importing every +// monitored album alongside it would produce a wanted list that is +// mostly redundant with the subscription that generated it. +func (l *lidarr) ListWants(ctx context.Context) ([]Want, error) { + var artists []struct { + lidarrArtist + + Monitored bool `json:"monitored"` + } + + if err := l.client.get(ctx, "/api/v1/artist", &artists); err != nil { + return nil, err + } + + out := make([]Want, 0, len(artists)) + + for _, a := range artists { + if !a.Monitored || a.ForeignArtistID == "" { + continue + } + + out = append(out, Want{ + MBID: a.ForeignArtistID, + Entity: EntityArtist, + Artist: a.ArtistName, + Title: a.ArtistName, + // Imported subscriptions take the conservative scope: the + // user can widen it, but silently queueing a back catalogue + // on import would be a nasty surprise. + Scope: ScopeFuture, + }) + } + + return out, nil +} + +// findArtistByMBID looks up an artist Lidarr already has. +func (l *lidarr) findArtistByMBID( + ctx context.Context, + mbid string, +) (lidarrArtist, error) { + var artists []lidarrArtist + + endpoint := "/api/v1/artist?mbId=" + url.QueryEscape(mbid) + + if err := l.client.get(ctx, endpoint, &artists); err != nil { + return lidarrArtist{}, err + } + + for _, a := range artists { + if a.ForeignArtistID == mbid { + return a, nil + } + } + + return lidarrArtist{}, nil +} + +// resolveRootFolder returns the configured root folder, or Lidarr's +// first if none is configured. +func (l *lidarr) resolveRootFolder(ctx context.Context) (string, error) { + if l.rootFolderPath != "" { + return l.rootFolderPath, nil + } + + folders, err := l.rootFolders(ctx) + if err != nil { + return "", err + } + + if len(folders) == 0 { + return "", ErrLidarrNoRootFolder + } + + return folders[0].Path, nil +} diff --git a/backend/download/provider_lidarr_list_test.go b/backend/download/provider_lidarr_list_test.go new file mode 100644 index 0000000..a2147c0 --- /dev/null +++ b/backend/download/provider_lidarr_list_test.go @@ -0,0 +1,240 @@ +package download + +import ( + "context" + "testing" +) + +// An artist subscription becomes a monitored Lidarr artist, with the +// scope translated into Lidarr's own monitor option — so the policy +// keeps applying to albums released after the push, which is the whole +// reason to mirror a subscription rather than a list of albums. +func TestLidarrPushArtistWantMapsScope(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + scope WantScope + wantMonitor string + }{ + {name: "future", scope: ScopeFuture, wantMonitor: "future"}, + {name: "all", scope: ScopeAll, wantMonitor: "missing"}, + } + + for _, tt := range tests { + stub := newLidarrStub(t) + l := newStubLidarr(t, stub) + + id, err := l.PushWant(context.Background(), Want{ + MBID: "artist-mbid", + Entity: EntityArtist, + Artist: "Radiohead", + Scope: tt.scope, + }) + if err != nil { + t.Fatalf("%s: PushWant: %v", tt.name, err) + } + + if id != "42" { + t.Errorf("%s: external id = %q, want 42", tt.name, id) + } + + stub.mu.Lock() + added := append([]map[string]any(nil), stub.addedArtists...) + stub.mu.Unlock() + + if len(added) != 1 { + t.Fatalf("%s: added %d artists, want 1", tt.name, len(added)) + } + + opts, ok := added[0]["addOptions"].(map[string]any) + if !ok { + t.Fatalf("%s: no addOptions in the artist body", tt.name) + } + + if opts["monitor"] != tt.wantMonitor { + t.Errorf( + "%s: monitor = %v, want %q", + tt.name, opts["monitor"], tt.wantMonitor, + ) + } + + // Adding an artist must never kick off a discography-wide + // search on a system the user shares with their own queue. + if opts["searchForMissingAlbums"] != false { + t.Errorf("%s: push triggered a search", tt.name) + } + } +} + +// Pushing a want Lidarr already has returns the existing ID instead of +// adding a second copy — the reconciler pushes on every pass, so this +// is load-bearing rather than tidy. +func TestLidarrPushArtistWantIsIdempotent(t *testing.T) { + t.Parallel() + + stub := newLidarrStub(t) + stub.artists = []map[string]any{{ + "id": 7, + "artistName": "Radiohead", + "foreignArtistId": "artist-mbid", + "monitored": true, + }} + + l := newStubLidarr(t, stub) + + id, err := l.PushWant(context.Background(), Want{ + MBID: "artist-mbid", + Entity: EntityArtist, + Artist: "Radiohead", + }) + if err != nil { + t.Fatalf("PushWant: %v", err) + } + + if id != "7" { + t.Errorf("external id = %q, want the existing artist's 7", id) + } + + stub.mu.Lock() + added := len(stub.addedArtists) + stub.mu.Unlock() + + if added != 0 { + t.Errorf("added %d artists, want 0 — it already existed", added) + } +} + +// Lidarr cannot express "I want one track", and monitoring the whole +// album to get it would download far more than was asked for. +func TestLidarrPushRecordingWantIsSkipped(t *testing.T) { + t.Parallel() + + stub := newLidarrStub(t) + l := newStubLidarr(t, stub) + + id, err := l.PushWant(context.Background(), Want{ + MBID: "recording-mbid", + Entity: EntityRecording, + Title: "Paranoid Android", + }) + if err != nil { + t.Fatalf("PushWant: %v", err) + } + + if id != "" { + t.Errorf("external id = %q, want empty (not pushed)", id) + } + + stub.mu.Lock() + added := len(stub.addedArtists) + monitors := len(stub.monitorCalls) + stub.mu.Unlock() + + if added != 0 || monitors != 0 { + t.Errorf( + "track want touched Lidarr: %d artists, %d monitors", + added, monitors, + ) + } +} + +// Importing adopts monitored artists conservatively: a subscription +// pulled in from elsewhere must not queue a back catalogue. +func TestLidarrListWantsImportsMonitoredArtistsOnly(t *testing.T) { + t.Parallel() + + stub := newLidarrStub(t) + stub.artists = []map[string]any{ + { + "id": 1, + "artistName": "Radiohead", + "foreignArtistId": "artist-1", + "monitored": true, + }, + { + "id": 2, + "artistName": "Unmonitored Band", + "foreignArtistId": "artist-2", + "monitored": false, + }, + { + "id": 3, + "artistName": "No MBID", + "foreignArtistId": "", + "monitored": true, + }, + } + + l := newStubLidarr(t, stub) + + wants, err := l.ListWants(context.Background()) + if err != nil { + t.Fatalf("ListWants: %v", err) + } + + if len(wants) != 1 { + t.Fatalf("imported %d wants, want 1", len(wants)) + } + + w := wants[0] + + if w.MBID != "artist-1" { + t.Errorf("mbid = %q, want artist-1", w.MBID) + } + + if w.Entity != EntityArtist { + t.Errorf("entity = %q, want artist", w.Entity) + } + + if w.Scope != ScopeFuture { + t.Errorf("scope = %q, want the conservative future", w.Scope) + } +} + +// Removing a want must not tear down a Lidarr setup that may predate +// this app: it unmonitors, it does not delete. +func TestLidarrRemoveWantUnmonitorsOnly(t *testing.T) { + t.Parallel() + + stub := newLidarrStub(t) + l := newStubLidarr(t, stub) + + if err := l.RemoveWant(context.Background(), "55"); err != nil { + t.Fatalf("RemoveWant: %v", err) + } + + stub.mu.Lock() + monitors := append([]map[string]any(nil), stub.monitorCalls...) + stub.mu.Unlock() + + if len(monitors) != 1 { + t.Fatalf("got %d monitor calls, want 1", len(monitors)) + } + + if monitors[0]["monitored"] != false { + t.Errorf("monitored = %v, want false", monitors[0]["monitored"]) + } +} + +// The Lister role has to be declared, not just implemented, or the +// reconciler never finds it. +func TestLidarrDeclaresListerCapability(t *testing.T) { + t.Parallel() + + stub := newLidarrStub(t) + l := newStubLidarr(t, stub) + + if _, ok := asLister(l); !ok { + t.Error("lidarr does not present as a Lister") + } + + desc, ok := DescriptorFor(KindLidarr) + if !ok { + t.Fatal("no descriptor registered for lidarr") + } + + if !desc.Caps.CanList { + t.Error("lidarr's descriptor does not declare CanList") + } +} diff --git a/backend/download/provider_lidarr_test.go b/backend/download/provider_lidarr_test.go new file mode 100644 index 0000000..7be6077 --- /dev/null +++ b/backend/download/provider_lidarr_test.go @@ -0,0 +1,562 @@ +package download + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" +) + +// lidarrStub is a fake Lidarr instance. +type lidarrStub struct { + server *httptest.Server + + mu sync.Mutex + + // searchAlbum is what /api/v1/search returns. + searchAlbum lidarrAlbum + + // album is what /api/v1/album/{id} returns, in order; the last + // entry repeats. + albumStates []lidarrAlbum + pollCount int + + // trackFiles is what /api/v1/trackfile returns. + trackFiles []lidarrTrackFile + + // rootFolders is what /api/v1/rootfolder returns. + rootFolders []lidarrRootFolder + + // commands records the commands that were issued. + commands []string + + // monitorCalls records album-monitor toggles. + monitorCalls []map[string]any + + // addedArtists records artist additions. + addedArtists []map[string]any + + // artists is what a GET of /api/v1/artist returns, which is how the + // Lister role looks up and enumerates monitored artists. + artists []map[string]any + + unauthorized bool +} + +func newLidarrStub(t *testing.T) *lidarrStub { + t.Helper() + + s := &lidarrStub{ + rootFolders: []lidarrRootFolder{{ID: 1, Path: "/music"}}, + } + + mux := http.NewServeMux() + + mux.HandleFunc("/api/v1/system/status", func(w http.ResponseWriter, r *http.Request) { + if s.reject(w, r) { + return + } + + writeJSON(t, w, map[string]any{"version": "2.0.0"}) + }) + + mux.HandleFunc("/api/v1/rootfolder", func(w http.ResponseWriter, r *http.Request) { + if s.reject(w, r) { + return + } + + s.mu.Lock() + folders := s.rootFolders + s.mu.Unlock() + + writeJSON(t, w, folders) + }) + + mux.HandleFunc("/api/v1/qualityprofile", func(w http.ResponseWriter, r *http.Request) { + if s.reject(w, r) { + return + } + + writeJSON(t, w, []map[string]any{{"id": 7}}) + }) + + mux.HandleFunc("/api/v1/metadataprofile", func(w http.ResponseWriter, r *http.Request) { + if s.reject(w, r) { + return + } + + writeJSON(t, w, []map[string]any{{"id": 3}}) + }) + + mux.HandleFunc("/api/v1/search", func(w http.ResponseWriter, r *http.Request) { + if s.reject(w, r) { + return + } + + s.mu.Lock() + album := s.searchAlbum + s.mu.Unlock() + + writeJSON(t, w, []map[string]any{{"album": album}}) + }) + + mux.HandleFunc("/api/v1/artist", func(w http.ResponseWriter, r *http.Request) { + if s.reject(w, r) { + return + } + + if r.Method == http.MethodGet { + s.mu.Lock() + artists := s.artists + s.mu.Unlock() + + if artists == nil { + artists = []map[string]any{} + } + + writeJSON(t, w, artists) + + return + } + + var body map[string]any + + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Errorf("decode artist body: %v", err) + } + + s.mu.Lock() + s.addedArtists = append(s.addedArtists, body) + s.mu.Unlock() + + writeJSON(t, w, map[string]any{"id": 42}) + }) + + mux.HandleFunc("/api/v1/album/monitor", func(w http.ResponseWriter, r *http.Request) { + if s.reject(w, r) { + return + } + + var body map[string]any + + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Errorf("decode monitor body: %v", err) + } + + s.mu.Lock() + s.monitorCalls = append(s.monitorCalls, body) + s.mu.Unlock() + + w.WriteHeader(http.StatusAccepted) + }) + + mux.HandleFunc("/api/v1/album", func(w http.ResponseWriter, r *http.Request) { + if s.reject(w, r) { + return + } + + s.mu.Lock() + album := s.searchAlbum + s.mu.Unlock() + + album.ID = 99 + + writeJSON(t, w, []lidarrAlbum{album}) + }) + + mux.HandleFunc("/api/v1/album/", func(w http.ResponseWriter, r *http.Request) { + if s.reject(w, r) { + return + } + + s.mu.Lock() + + idx := s.pollCount + if idx >= len(s.albumStates) { + idx = len(s.albumStates) - 1 + } else { + s.pollCount++ + } + + var album lidarrAlbum + if idx >= 0 && len(s.albumStates) > 0 { + album = s.albumStates[idx] + } + + s.mu.Unlock() + + writeJSON(t, w, album) + }) + + mux.HandleFunc("/api/v1/trackfile", func(w http.ResponseWriter, r *http.Request) { + if s.reject(w, r) { + return + } + + s.mu.Lock() + files := s.trackFiles + s.mu.Unlock() + + writeJSON(t, w, files) + }) + + mux.HandleFunc("/api/v1/command", func(w http.ResponseWriter, r *http.Request) { + if s.reject(w, r) { + return + } + + var body map[string]any + + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Errorf("decode command body: %v", err) + } + + name, _ := body["name"].(string) + + s.mu.Lock() + s.commands = append(s.commands, name) + s.mu.Unlock() + + writeJSON(t, w, map[string]any{"id": 1}) + }) + + s.server = httptest.NewServer(mux) + t.Cleanup(s.server.Close) + + return s +} + +func (s *lidarrStub) reject(w http.ResponseWriter, r *http.Request) bool { + s.mu.Lock() + unauthorized := s.unauthorized + s.mu.Unlock() + + if unauthorized || r.Header.Get("X-Api-Key") != "test-key" { + w.WriteHeader(http.StatusUnauthorized) + + return true + } + + return false +} + +func newStubLidarr(t *testing.T, stub *lidarrStub) *lidarr { + t.Helper() + + p, err := newLidarr( + Config{ + ID: 1, + Kind: KindLidarr, + Name: "lidarr", + Enabled: true, + Settings: map[string]string{ + "url": stub.server.URL, + }, + }, + func(string) (string, error) { return "test-key", nil }, + slogDiscard(), + ) + if err != nil { + t.Fatalf("newLidarr: %v", err) + } + + l, ok := p.(*lidarr) + if !ok { + t.Fatalf("provider is %T, want *lidarr", p) + } + + return l +} + +func TestLidarrCheck(t *testing.T) { + t.Parallel() + + stub := newLidarrStub(t) + l := newStubLidarr(t, stub) + + if err := l.Check(context.Background()); err != nil { + t.Errorf("Check: %v", err) + } +} + +func TestLidarrCheckRejectsBadKey(t *testing.T) { + t.Parallel() + + stub := newLidarrStub(t) + stub.unauthorized = true + + l := newStubLidarr(t, stub) + + if err := l.Check(context.Background()); !errors.Is(err, ErrLidarrAuth) { + t.Errorf("error = %v, want ErrLidarrAuth", err) + } +} + +// Lidarr with nowhere to put music cannot fulfil anything, and that +// should be visible at configuration time. +func TestLidarrCheckRequiresRootFolder(t *testing.T) { + t.Parallel() + + stub := newLidarrStub(t) + stub.rootFolders = nil + + l := newStubLidarr(t, stub) + + if err := l.Check(context.Background()); !errors.Is( + err, ErrLidarrNoRootFolder, + ) { + t.Errorf("error = %v, want ErrLidarrNoRootFolder", err) + } +} + +// An album Lidarr already tracks only needs monitoring and a search. +func TestLidarrDelegateExistingAlbum(t *testing.T) { + t.Parallel() + + stub := newLidarrStub(t) + stub.searchAlbum = lidarrAlbum{ + ID: 55, + Title: "OK Computer", + ForeignAlbum: "rg-mbid", + Monitored: false, + } + + l := newStubLidarr(t, stub) + + externalID, err := l.Delegate(context.Background(), Request{ + ReleaseGroupMBID: "rg-mbid", + Artist: "Radiohead", + Album: "OK Computer", + }) + if err != nil { + t.Fatalf("Delegate: %v", err) + } + + if externalID != "55" { + t.Errorf("external id = %q, want 55", externalID) + } + + stub.mu.Lock() + commands := append([]string(nil), stub.commands...) + monitors := len(stub.monitorCalls) + added := len(stub.addedArtists) + stub.mu.Unlock() + + if added != 0 { + t.Errorf("added %d artists, want 0 for an album Lidarr already has", added) + } + + if monitors != 1 { + t.Errorf("monitor calls = %d, want 1", monitors) + } + + if len(commands) != 1 || commands[0] != "AlbumSearch" { + t.Errorf("commands = %v, want [AlbumSearch]", commands) + } +} + +// Adding an artist must not kick off their entire discography. +func TestLidarrDelegateNewArtistMonitorsNothingByDefault(t *testing.T) { + t.Parallel() + + stub := newLidarrStub(t) + stub.searchAlbum = lidarrAlbum{ + Title: "OK Computer", + ForeignAlbum: "rg-mbid", + } + stub.searchAlbum.Artist.ForeignArtistID = "artist-mbid" + stub.searchAlbum.Artist.ArtistName = "Radiohead" + + l := newStubLidarr(t, stub) + + if _, err := l.Delegate(context.Background(), Request{ + ReleaseGroupMBID: "rg-mbid", + Artist: "Radiohead", + Album: "OK Computer", + }); err != nil { + t.Fatalf("Delegate: %v", err) + } + + stub.mu.Lock() + added := append([]map[string]any(nil), stub.addedArtists...) + stub.mu.Unlock() + + if len(added) != 1 { + t.Fatalf("added %d artists, want 1", len(added)) + } + + opts, ok := added[0]["addOptions"].(map[string]any) + if !ok { + t.Fatalf("addOptions missing from %v", added[0]) + } + + if opts["monitor"] != "none" { + t.Errorf("monitor = %v, want none", opts["monitor"]) + } + + if opts["searchForMissingAlbums"] != false { + t.Errorf( + "searchForMissingAlbums = %v, want false", + opts["searchForMissingAlbums"], + ) + } +} + +func TestLidarrDelegateNoMatch(t *testing.T) { + t.Parallel() + + stub := newLidarrStub(t) + // searchAlbum stays zero-valued: no title means no match. + + l := newStubLidarr(t, stub) + + _, err := l.Delegate(context.Background(), Request{ + Artist: "Nobody", + Album: "Nothing", + }) + + if !errors.Is(err, ErrLidarrNoMatch) { + t.Errorf("error = %v, want ErrLidarrNoMatch", err) + } +} + +// Completion is judged by imported files, not by an empty queue: the +// queue drains when the download finishes, which is before the import. +func TestLidarrPollWaitsForImportedFiles(t *testing.T) { + t.Parallel() + + stub := newLidarrStub(t) + stub.albumStates = []lidarrAlbum{ + {ID: 55, Statistics: lidarrAlbumStats{TrackCount: 12}}, + {ID: 55, Statistics: lidarrAlbumStats{TrackFileCount: 6, TrackCount: 12}}, + {ID: 55, Statistics: lidarrAlbumStats{TrackFileCount: 12, TrackCount: 12}}, + } + stub.trackFiles = []lidarrTrackFile{ + {ID: 1, Path: "/music/Radiohead/OK Computer/01 Airbag.flac"}, + {ID: 2, Path: "/music/Radiohead/OK Computer/02 Paranoid Android.flac"}, + } + + l := newStubLidarr(t, stub) + ctx := context.Background() + + // Nothing imported yet. + first, err := l.Poll(ctx, "55") + if err != nil { + t.Fatalf("Poll: %v", err) + } + + if first.State != StateGrabbing { + t.Errorf("state = %q, want grabbing", first.State) + } + + // Half done. + second, err := l.Poll(ctx, "55") + if err != nil { + t.Fatalf("Poll: %v", err) + } + + if second.State != StateGrabbing { + t.Errorf("state = %q, want grabbing", second.State) + } + + if second.Progress <= first.Progress { + t.Errorf( + "progress did not advance: %f then %f", + first.Progress, second.Progress, + ) + } + + // Complete, with the paths Lidarr imported to. + third, err := l.Poll(ctx, "55") + if err != nil { + t.Fatalf("Poll: %v", err) + } + + if third.State != StateComplete { + t.Fatalf("state = %q, want complete", third.State) + } + + if len(third.ImportedPaths) != 2 { + t.Errorf("imported paths = %v, want 2", third.ImportedPaths) + } + + for _, p := range third.ImportedPaths { + if !strings.HasPrefix(p, "/music/") { + t.Errorf("path %q is not in Lidarr's library", p) + } + } +} + +func TestLidarrPollRejectsBadExternalID(t *testing.T) { + t.Parallel() + + stub := newLidarrStub(t) + l := newStubLidarr(t, stub) + + if _, err := l.Poll(context.Background(), "not-a-number"); !errors.Is( + err, ErrLidarrNoMatch, + ) { + t.Errorf("error = %v, want ErrLidarrNoMatch", err) + } +} + +// Withdrawing stops monitoring; it must not delete the album, which may +// predate this request. +func TestLidarrWithdrawUnmonitorsOnly(t *testing.T) { + t.Parallel() + + stub := newLidarrStub(t) + l := newStubLidarr(t, stub) + + if err := l.Withdraw(context.Background(), "55"); err != nil { + t.Fatalf("Withdraw: %v", err) + } + + stub.mu.Lock() + calls := append([]map[string]any(nil), stub.monitorCalls...) + stub.mu.Unlock() + + if len(calls) != 1 { + t.Fatalf("monitor calls = %d, want 1", len(calls)) + } + + if calls[0]["monitored"] != false { + t.Errorf("monitored = %v, want false", calls[0]["monitored"]) + } +} + +func TestLidarrRequiresConfiguration(t *testing.T) { + t.Parallel() + + t.Run("no url", func(t *testing.T) { + t.Parallel() + + _, err := newLidarr( + Config{}, + func(string) (string, error) { return "k", nil }, + slogDiscard(), + ) + + if !errors.Is(err, ErrNotConfigured) { + t.Errorf("error = %v, want ErrNotConfigured", err) + } + }) + + t.Run("no api key", func(t *testing.T) { + t.Parallel() + + _, err := newLidarr( + Config{Settings: map[string]string{"url": "http://localhost:8686"}}, + func(string) (string, error) { return "", ErrSecretNotFound }, + slogDiscard(), + ) + + if !errors.Is(err, ErrNotConfigured) { + t.Errorf("error = %v, want ErrNotConfigured", err) + } + }) +} diff --git a/backend/download/provider_prowlarr.go b/backend/download/provider_prowlarr.go new file mode 100644 index 0000000..a205645 --- /dev/null +++ b/backend/download/provider_prowlarr.go @@ -0,0 +1,318 @@ +package download + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/url" + "strconv" + "strings" + "time" +) + +// Prowlarr is the one provider that cannot finish a job by itself: it +// searches dozens of indexers and hands back magnet links and NZB URLs, +// which some other client has to actually fetch. That is the whole +// reason the role interfaces are separate — Prowlarr implements +// Searcher and nothing else, and the pipeline pairs its results with +// whichever enabled transport handles the protocol. +// +// Its search results also carry no file list. A torrent is one opaque +// blob until it is fetched, so match scoring here has only the release +// title to work with, and candidates are marked accordingly rather than +// pretending to know what is inside. + +// Prowlarr provider errors. +var ( + // ErrProwlarrUnreachable means the instance did not answer. + ErrProwlarrUnreachable = errors.New("prowlarr is unreachable") + + // ErrProwlarrAuth means the API key was rejected. + ErrProwlarrAuth = errors.New("prowlarr rejected the API key") + + // ErrProwlarrNoIndexers means nothing is configured to search. + ErrProwlarrNoIndexers = errors.New("prowlarr has no enabled indexers") +) + +// prowlarrHTTPTimeout bounds one API call. Indexer fan-out is slow, so +// this is longer than the other adapters'. +const prowlarrHTTPTimeout = 45 * time.Second + +// prowlarrMusicCategory is Newznab's music category. Searching without +// it returns every match across film and software too. +const prowlarrMusicCategory = "3000" + +// prowlarrMaxResults caps how many results are turned into candidates. +const prowlarrMaxResults = 40 + +func init() { + Register( + Descriptor{ + Kind: KindProwlarr, + Name: "Prowlarr", + Summary: "Search many torrent and usenet indexers at once. " + + "Needs a download client (qBittorrent or SABnzbd) to fetch results.", + RequiresExternal: "Prowlarr", + Caps: Caps{ + CanSearch: true, + }, + Fields: []Field{ + { + Key: "url", + Label: "Prowlarr URL", + Placeholder: "http://localhost:9696", + Required: true, + Default: "http://localhost:9696", + }, + { + Key: "apiKey", + Label: "API key", + Secret: true, + Required: true, + Help: "Prowlarr → Settings → General → API Key.", + }, + { + Key: "indexerIds", + Label: "Indexer IDs", + Help: "Comma-separated numeric IDs to restrict the search to. " + + "Leave blank to search all enabled indexers.", + }, + { + Key: "minSeeders", + Label: "Minimum seeders", + Help: "Torrent results below this are hidden. " + + "Defaults to 1.", + Default: "1", + }, + }, + }, + newProwlarr, + ) +} + +// prowlarr is the Prowlarr search provider. +type prowlarr struct { + info ProviderInfo + logger *slog.Logger + client *apiClient + + indexerIDs []string + minSeeders int +} + +// newProwlarr builds the provider from config. +func newProwlarr( + cfg Config, + secrets SecretLookup, + logger *slog.Logger, +) (Provider, error) { + base := strings.TrimRight(cfg.Setting("url", ""), "/") + if base == "" { + return nil, fmt.Errorf("%w: Prowlarr URL is required", ErrNotConfigured) + } + + apiKey := "" + + if secrets != nil { + key, err := secrets("apiKey") + if err != nil { + return nil, fmt.Errorf("%w: no API key stored", ErrNotConfigured) + } + + apiKey = key + } + + minSeeders, err := strconv.Atoi(cfg.Setting("minSeeders", "1")) + if err != nil { + minSeeders = 1 + } + + var indexers []string + + for _, id := range strings.Split(cfg.Setting("indexerIds", ""), ",") { + if trimmed := strings.TrimSpace(id); trimmed != "" { + indexers = append(indexers, trimmed) + } + } + + return &prowlarr{ + info: ProviderInfo{ + ID: cfg.ID, + Kind: KindProwlarr, + Name: cfg.Name, + Enabled: cfg.Enabled, + Priority: cfg.Priority, + Caps: Caps{CanSearch: true}, + }, + logger: logger.With("provider", "prowlarr"), + client: newAPIClient( + base, "X-Api-Key", apiKey, prowlarrHTTPTimeout, + ErrProwlarrUnreachable, ErrProwlarrAuth, + ), + indexerIDs: indexers, + minSeeders: minSeeders, + }, nil +} + +// Info returns the provider's identity. +func (p *prowlarr) Info() ProviderInfo { + return p.info +} + +// Close is a no-op. +func (p *prowlarr) Close() error { + return nil +} + +// Check verifies the instance answers and has something to search. +func (p *prowlarr) Check(ctx context.Context) error { + var status struct { + Version string `json:"version"` + } + + if err := p.client.get(ctx, "/api/v1/system/status", &status); err != nil { + return err + } + + var indexers []struct { + ID int `json:"id"` + Enable bool `json:"enable"` + } + + if err := p.client.get(ctx, "/api/v1/indexer", &indexers); err != nil { + return err + } + + for _, i := range indexers { + if i.Enable { + return nil + } + } + + return ErrProwlarrNoIndexers +} + +// prowlarrResult is one indexer hit. +type prowlarrResult struct { + GUID string `json:"guid"` + Title string `json:"title"` + Indexer string `json:"indexer"` + Size int64 `json:"size"` + Seeders int `json:"seeders"` + Leechers int `json:"leechers"` + Protocol string `json:"protocol"` // "torrent" or "usenet" + DownloadURL string `json:"downloadUrl"` + MagnetURL string `json:"magnetUrl"` + InfoHash string `json:"infoHash"` +} + +// Search queries every configured indexer through Prowlarr. +func (p *prowlarr) Search( + ctx context.Context, + req Request, +) ([]Candidate, error) { + query := url.Values{} + query.Set("query", req.SearchText()) + query.Set("categories", prowlarrMusicCategory) + query.Set("type", "search") + + for _, id := range p.indexerIDs { + query.Add("indexerIds", id) + } + + var results []prowlarrResult + + endpoint := "/api/v1/search?" + query.Encode() + + if err := p.client.get(ctx, endpoint, &results); err != nil { + return nil, err + } + + out := make([]Candidate, 0, len(results)) + + for _, r := range results { + if len(out) >= prowlarrMaxResults { + break + } + + protocol := protocolFor(r.Protocol) + if protocol == ProtocolDirect { + continue + } + + // A torrent with no seeders will never finish. Offering it + // wastes the user's pick on something that cannot complete. + if protocol == ProtocolTorrent && r.Seeders < p.minSeeders { + continue + } + + link := r.MagnetURL + if link == "" { + link = r.DownloadURL + } + + if link == "" { + continue + } + + out = append(out, Candidate{ + ID: "prowlarr:" + r.GUID, + Kind: KindProwlarr, + Protocol: protocol, + Title: r.Title, + Origin: r.Indexer, + // Indexer results are opaque before they are fetched: there + // is no file list, so no per-file scoring is possible and + // the ranker works from the release title alone. + Files: nil, + TotalSize: r.Size, + Health: swarmHealth(protocol, r.Seeders), + Payload: map[string]string{ + "link": link, + "indexer": r.Indexer, + "infoHash": r.InfoHash, + }, + }) + } + + return out, nil +} + +// protocolFor maps Prowlarr's protocol string onto ours. +func protocolFor(s string) Protocol { + switch strings.ToLower(s) { + case "torrent": + return ProtocolTorrent + case "usenet": + return ProtocolUsenet + default: + return ProtocolDirect + } +} + +// swarmHealth scores availability, in 0..1. Usenet has no swarm: a +// retained article either downloads at full speed or is gone, so it +// gets a flat, confident score. +func swarmHealth(protocol Protocol, seeders int) float64 { + if protocol == ProtocolUsenet { + return 0.85 + } + + // Seeder counts have sharply diminishing returns — the difference + // between 1 and 10 is enormous, between 100 and 500 irrelevant. + switch { + case seeders <= 0: + return 0.05 + case seeders == 1: + return 0.3 + case seeders < 5: + return 0.5 + case seeders < 20: + return 0.75 + case seeders < 100: + return 0.9 + default: + return 1.0 + } +} diff --git a/backend/download/provider_prowlarr_test.go b/backend/download/provider_prowlarr_test.go new file mode 100644 index 0000000..7bc94b9 --- /dev/null +++ b/backend/download/provider_prowlarr_test.go @@ -0,0 +1,313 @@ +package download + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" +) + +// prowlarrStub is a fake Prowlarr instance. +type prowlarrStub struct { + server *httptest.Server + + mu sync.Mutex + + results []prowlarrResult + indexers []map[string]any + + // lastQuery records the search query string for assertions. + lastQuery string + + unauthorized bool +} + +func newProwlarrStub(t *testing.T) *prowlarrStub { + t.Helper() + + s := &prowlarrStub{ + indexers: []map[string]any{{"id": 1, "enable": true}}, + } + + mux := http.NewServeMux() + + mux.HandleFunc("/api/v1/system/status", func(w http.ResponseWriter, r *http.Request) { + if s.reject(w, r) { + return + } + + writeJSON(t, w, map[string]any{"version": "1.0.0"}) + }) + + mux.HandleFunc("/api/v1/indexer", func(w http.ResponseWriter, r *http.Request) { + if s.reject(w, r) { + return + } + + s.mu.Lock() + indexers := s.indexers + s.mu.Unlock() + + writeJSON(t, w, indexers) + }) + + mux.HandleFunc("/api/v1/search", func(w http.ResponseWriter, r *http.Request) { + if s.reject(w, r) { + return + } + + s.mu.Lock() + s.lastQuery = r.URL.RawQuery + results := s.results + s.mu.Unlock() + + writeJSON(t, w, results) + }) + + s.server = httptest.NewServer(mux) + t.Cleanup(s.server.Close) + + return s +} + +func (s *prowlarrStub) reject(w http.ResponseWriter, r *http.Request) bool { + s.mu.Lock() + unauthorized := s.unauthorized + s.mu.Unlock() + + if unauthorized || r.Header.Get("X-Api-Key") != "test-key" { + w.WriteHeader(http.StatusUnauthorized) + + return true + } + + return false +} + +func newStubProwlarr(t *testing.T, stub *prowlarrStub, settings map[string]string) *prowlarr { + t.Helper() + + if settings == nil { + settings = map[string]string{} + } + + settings["url"] = stub.server.URL + + p, err := newProwlarr( + Config{ID: 1, Kind: KindProwlarr, Name: "prowlarr", Settings: settings}, + func(string) (string, error) { return "test-key", nil }, + slogDiscard(), + ) + if err != nil { + t.Fatalf("newProwlarr: %v", err) + } + + pr, ok := p.(*prowlarr) + if !ok { + t.Fatalf("provider is %T, want *prowlarr", p) + } + + return pr +} + +func TestProwlarrCheck(t *testing.T) { + t.Parallel() + + stub := newProwlarrStub(t) + p := newStubProwlarr(t, stub, nil) + + if err := p.Check(context.Background()); err != nil { + t.Errorf("Check: %v", err) + } +} + +// Prowlarr with every indexer disabled will silently return nothing +// forever, which is worth surfacing at configuration time. +func TestProwlarrCheckRequiresEnabledIndexer(t *testing.T) { + t.Parallel() + + stub := newProwlarrStub(t) + stub.indexers = []map[string]any{{"id": 1, "enable": false}} + + p := newStubProwlarr(t, stub, nil) + + if err := p.Check(context.Background()); !errors.Is( + err, ErrProwlarrNoIndexers, + ) { + t.Errorf("error = %v, want ErrProwlarrNoIndexers", err) + } +} + +// Prowlarr fills only the Searcher role, so its candidates must carry a +// protocol the pipeline can pair with a transport. +func TestProwlarrSearchMarksProtocols(t *testing.T) { + t.Parallel() + + stub := newProwlarrStub(t) + stub.results = []prowlarrResult{ + { + GUID: "a", + Title: "Radiohead - OK Computer [FLAC]", + Indexer: "SomeTracker", + Protocol: "torrent", + Seeders: 50, + Size: 400_000_000, + MagnetURL: "magnet:?xt=urn:btih:abc123", + }, + { + GUID: "b", + Title: "Radiohead - OK Computer [MP3]", + Indexer: "SomeUsenet", + Protocol: "usenet", + Size: 90_000_000, + DownloadURL: "https://example.com/x.nzb", + }, + } + + p := newStubProwlarr(t, stub, nil) + + got, err := p.Search(context.Background(), Request{ + Artist: "Radiohead", + Album: "OK Computer", + }) + if err != nil { + t.Fatalf("Search: %v", err) + } + + if len(got) != 2 { + t.Fatalf("got %d candidates, want 2", len(got)) + } + + byProtocol := map[Protocol]Candidate{} + for _, c := range got { + byProtocol[c.Protocol] = c + } + + torrent, ok := byProtocol[ProtocolTorrent] + if !ok { + t.Fatal("no torrent candidate") + } + + if torrent.Payload["link"] != "magnet:?xt=urn:btih:abc123" { + t.Errorf("torrent link = %q, want the magnet", torrent.Payload["link"]) + } + + usenet, ok := byProtocol[ProtocolUsenet] + if !ok { + t.Fatal("no usenet candidate") + } + + if usenet.Payload["link"] != "https://example.com/x.nzb" { + t.Errorf("usenet link = %q, want the NZB URL", usenet.Payload["link"]) + } + + // Indexer results are opaque before fetching; claiming a file list + // would be inventing information. + if len(torrent.Files) != 0 { + t.Errorf("torrent candidate has %d files, want none", len(torrent.Files)) + } +} + +// A torrent nobody is seeding will never finish, so offering it wastes +// the user's choice. +func TestProwlarrFiltersDeadTorrents(t *testing.T) { + t.Parallel() + + stub := newProwlarrStub(t) + stub.results = []prowlarrResult{ + { + GUID: "dead", Title: "Dead", Protocol: "torrent", + Seeders: 0, MagnetURL: "magnet:?xt=urn:btih:dead", + }, + { + GUID: "alive", Title: "Alive", Protocol: "torrent", + Seeders: 10, MagnetURL: "magnet:?xt=urn:btih:alive", + }, + } + + p := newStubProwlarr(t, stub, map[string]string{"minSeeders": "1"}) + + got, err := p.Search(context.Background(), Request{Query: "x"}) + if err != nil { + t.Fatalf("Search: %v", err) + } + + if len(got) != 1 { + t.Fatalf("got %d candidates, want 1", len(got)) + } + + if got[0].Title != "Alive" { + t.Errorf("kept %q, want the seeded torrent", got[0].Title) + } +} + +// Searching without a category constraint returns films and software +// alongside music. +func TestProwlarrSearchesMusicCategory(t *testing.T) { + t.Parallel() + + stub := newProwlarrStub(t) + p := newStubProwlarr(t, stub, nil) + + if _, err := p.Search( + context.Background(), Request{Query: "radiohead"}, + ); err != nil { + t.Fatalf("Search: %v", err) + } + + stub.mu.Lock() + query := stub.lastQuery + stub.mu.Unlock() + + if !strings.Contains(query, "categories="+prowlarrMusicCategory) { + t.Errorf("query %q does not constrain to the music category", query) + } +} + +func TestSwarmHealth(t *testing.T) { + t.Parallel() + + // More seeders is never worse. + prev := -1.0 + + for _, seeders := range []int{0, 1, 3, 10, 50, 500} { + got := swarmHealth(ProtocolTorrent, seeders) + + if got < prev { + t.Errorf("health fell at %d seeders: %f after %f", seeders, got, prev) + } + + if got < 0 || got > 1 { + t.Errorf("health %f out of range at %d seeders", got, seeders) + } + + prev = got + } + + // Usenet has no swarm, so seeder count is meaningless there. + if a, b := swarmHealth(ProtocolUsenet, 0), swarmHealth(ProtocolUsenet, 99); a != b { + t.Errorf("usenet health varied with seeders: %f vs %f", a, b) + } +} + +func TestProtocolFor(t *testing.T) { + t.Parallel() + + tests := map[string]Protocol{ + "torrent": ProtocolTorrent, + "Torrent": ProtocolTorrent, + "usenet": ProtocolUsenet, + "USENET": ProtocolUsenet, + "weird": ProtocolDirect, + "": ProtocolDirect, + } + + for in, want := range tests { + if got := protocolFor(in); got != want { + t.Errorf("protocolFor(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/backend/download/provider_qbittorrent.go b/backend/download/provider_qbittorrent.go new file mode 100644 index 0000000..d9459b8 --- /dev/null +++ b/backend/download/provider_qbittorrent.go @@ -0,0 +1,595 @@ +package download + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/cookiejar" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +// qBittorrent is a pure transport: it never searches, it just takes a +// magnet link Prowlarr found and moves the bytes. That is the point of +// splitting Transporter out — this adapter knows nothing about music. +// +// Two things make it different from the others. Its auth is a session +// cookie rather than a header, so it needs its own client. And it can +// be told where to save, which means the transfer lands directly in our +// staging directory instead of needing to be collected afterwards — +// provided qBittorrent sees the same filesystem we do. + +// qBittorrent provider errors. +var ( + // ErrQbitUnreachable means the instance did not answer. + ErrQbitUnreachable = errors.New("qbittorrent is unreachable") + + // ErrQbitAuth means the credentials were rejected. + ErrQbitAuth = errors.New("qbittorrent rejected the credentials") + + // ErrQbitNoHash means the candidate carried no usable torrent + // identifier, so the transfer could not be tracked. + ErrQbitNoHash = errors.New("candidate has no torrent hash") + + // ErrQbitTransferFailed means the torrent errored or stalled out. + ErrQbitTransferFailed = errors.New("qbittorrent transfer failed") +) + +// qBittorrent tuning. +const ( + // qbitHTTPTimeout bounds one API call. + qbitHTTPTimeout = 30 * time.Second + + // qbitPollInterval is how often torrent state is checked. + qbitPollInterval = 5 * time.Second + + // qbitAppearWait is how long to wait for a just-added torrent to + // show up in the torrent list before concluding it was rejected. + qbitAppearWait = 60 * time.Second +) + +func init() { + Register( + Descriptor{ + Kind: KindQBittorrent, + Name: "qBittorrent", + Summary: "Download torrents found by an indexer. " + + "Pairs with Prowlarr; does not search on its own.", + RequiresExternal: "qBittorrent", + Caps: Caps{ + CanTransport: true, + CanCancel: true, + CanResume: true, + ReportsSize: true, + Transports: []Protocol{ProtocolTorrent}, + }, + Fields: []Field{ + { + Key: "url", + Label: "qBittorrent URL", + Placeholder: "http://localhost:8080", + Required: true, + Default: "http://localhost:8080", + }, + { + Key: "username", + Label: "Username", + Required: true, + Default: "admin", + }, + { + Key: "password", + Label: "Password", + Secret: true, + Required: true, + }, + { + Key: "category", + Label: "Category", + Help: "qBittorrent category to tag these downloads with. " + + "Useful for keeping them out of your other rules.", + Default: "yellowjacket", + }, + }, + }, + newQBittorrent, + ) +} + +// qbittorrent is the qBittorrent transport. +type qbittorrent struct { + info ProviderInfo + logger *slog.Logger + client *http.Client + + baseURL string + username string + password string + category string + + // authMu guards the lazy login, so concurrent grabs share one + // session instead of racing to create several. + authMu sync.Mutex + authenticated bool + + pollInterval time.Duration +} + +// newQBittorrent builds the transport from config. +func newQBittorrent( + cfg Config, + secrets SecretLookup, + logger *slog.Logger, +) (Provider, error) { + base := strings.TrimRight(cfg.Setting("url", ""), "/") + if base == "" { + return nil, fmt.Errorf( + "%w: qBittorrent URL is required", ErrNotConfigured, + ) + } + + password := "" + + if secrets != nil { + pw, err := secrets("password") + if err != nil { + return nil, fmt.Errorf("%w: no password stored", ErrNotConfigured) + } + + password = pw + } + + jar, err := cookiejar.New(nil) + if err != nil { + return nil, fmt.Errorf("create cookie jar: %w", err) + } + + return &qbittorrent{ + info: ProviderInfo{ + ID: cfg.ID, + Kind: KindQBittorrent, + Name: cfg.Name, + Enabled: cfg.Enabled, + Priority: cfg.Priority, + Caps: Caps{ + CanTransport: true, + CanCancel: true, + CanResume: true, + ReportsSize: true, + Transports: []Protocol{ProtocolTorrent}, + }, + }, + logger: logger.With("provider", "qbittorrent"), + client: &http.Client{ + Timeout: qbitHTTPTimeout, + Jar: jar, + }, + baseURL: base, + username: cfg.Setting("username", "admin"), + password: password, + category: cfg.Setting("category", "yellowjacket"), + pollInterval: qbitPollInterval, + }, nil +} + +// Info returns the provider's identity. +func (q *qbittorrent) Info() ProviderInfo { + return q.info +} + +// Close is a no-op; the session expires on its own. +func (q *qbittorrent) Close() error { + return nil +} + +// Check logs in and asks for the version. +func (q *qbittorrent) Check(ctx context.Context) error { + if err := q.login(ctx); err != nil { + return err + } + + _, err := q.call(ctx, "/api/v2/app/version", nil) + + return err +} + +// login establishes a session cookie. qBittorrent answers a bad login +// with 200 and the body "Fails.", not a 401, so the body is what has to +// be checked. +func (q *qbittorrent) login(ctx context.Context) error { + q.authMu.Lock() + defer q.authMu.Unlock() + + form := url.Values{} + form.Set("username", q.username) + form.Set("password", q.password) + + body, err := q.post(ctx, "/api/v2/auth/login", form) + if err != nil { + return err + } + + if !strings.Contains(strings.ToLower(body), "ok") { + q.authenticated = false + + return ErrQbitAuth + } + + q.authenticated = true + + return nil +} + +// ensureAuth logs in if this client has not yet done so. +func (q *qbittorrent) ensureAuth(ctx context.Context) error { + q.authMu.Lock() + done := q.authenticated + q.authMu.Unlock() + + if done { + return nil + } + + return q.login(ctx) +} + +// qbitTorrent is the subset of qBittorrent's torrent list used here. +type qbitTorrent struct { + Hash string `json:"hash"` + Name string `json:"name"` + State string `json:"state"` + Progress float64 `json:"progress"` + Size int64 `json:"size"` + Completed int64 `json:"completed"` + ContentPath string `json:"content_path"` + SavePath string `json:"save_path"` +} + +// finished reports whether the torrent has all its data. +func (t qbitTorrent) finished() bool { + switch t.State { + case "uploading", "stalledUP", "queuedUP", "pausedUP", "forcedUP", + "checkingUP": + return true + default: + return t.Progress >= 1.0 + } +} + +// failed reports whether the torrent is in an unrecoverable state. +func (t qbitTorrent) failed() bool { + return t.State == "error" || t.State == "missingFiles" +} + +// Grab adds the torrent, waits for it to complete, and collects its +// files into dst. +func (q *qbittorrent) Grab( + ctx context.Context, + c Candidate, + dst string, + onProgress ProgressFunc, +) (Result, error) { + if err := q.ensureAuth(ctx); err != nil { + return Result{}, err + } + + link := c.Payload["link"] + if link == "" { + return Result{}, fmt.Errorf( + "%w: candidate has no magnet or torrent URL", ErrQbitNoHash, + ) + } + + form := url.Values{} + form.Set("urls", link) + form.Set("savepath", dst) + form.Set("category", q.category) + // Skip qBittorrent's own "move on completion" rules: the file must + // stay where we put it until the import step decides otherwise. + form.Set("autoTMM", "false") + + if _, err := q.post(ctx, "/api/v2/torrents/add", form); err != nil { + return Result{}, err + } + + hash, err := q.resolveHash(ctx, c, link) + if err != nil { + return Result{}, err + } + + torrent, err := q.await(ctx, hash, c, onProgress) + if err != nil { + return Result{}, err + } + + return collectTree(torrent.ContentPath, dst) +} + +// resolveHash finds the torrent's hash, preferring the one the indexer +// supplied and falling back to matching the newest torrent in our +// category — qBittorrent's add endpoint returns nothing useful. +func (q *qbittorrent) resolveHash( + ctx context.Context, + c Candidate, + link string, +) (string, error) { + if h := c.Payload["infoHash"]; h != "" { + return strings.ToLower(h), nil + } + + if h := infoHashFromMagnet(link); h != "" { + return h, nil + } + + // Poll briefly for a torrent in our category that was not there + // before; the add is asynchronous. + deadline := time.Now().Add(qbitAppearWait) + + for time.Now().Before(deadline) { + select { + case <-ctx.Done(): + return "", fmt.Errorf("%w: cancelled", ErrQbitTransferFailed) + case <-time.After(q.pollInterval): + } + + torrents, err := q.list(ctx, "") + if err != nil { + continue + } + + for _, t := range torrents { + if strings.EqualFold(t.Name, c.Title) { + return t.Hash, nil + } + } + } + + return "", fmt.Errorf( + "%w: torrent never appeared in qBittorrent", ErrQbitNoHash, + ) +} + +// await polls until the torrent finishes or fails. +func (q *qbittorrent) await( + ctx context.Context, + hash string, + c Candidate, + onProgress ProgressFunc, +) (qbitTorrent, error) { + for { + select { + case <-ctx.Done(): + return qbitTorrent{}, fmt.Errorf( + "%w: cancelled", ErrQbitTransferFailed, + ) + case <-time.After(q.pollInterval): + } + + torrents, err := q.list(ctx, hash) + if err != nil { + q.logger.Debug("qbittorrent poll failed", "error", err) + + continue + } + + if len(torrents) == 0 { + continue + } + + t := torrents[0] + + if onProgress != nil { + total := t.Size + if total == 0 { + total = c.TotalSize + } + + onProgress(Progress{ + Current: t.Completed, + Total: total, + Phase: "Downloading torrent (" + t.State + ")", + }) + } + + if t.failed() { + return qbitTorrent{}, fmt.Errorf( + "%w: state %s", ErrQbitTransferFailed, t.State, + ) + } + + if t.finished() { + return t, nil + } + } +} + +// list returns torrents, optionally filtered to one hash. +func (q *qbittorrent) list( + ctx context.Context, + hash string, +) ([]qbitTorrent, error) { + params := url.Values{} + if hash != "" { + params.Set("hashes", hash) + } + + var out []qbitTorrent + + if err := q.getJSON(ctx, "/api/v2/torrents/info", params, &out); err != nil { + return nil, err + } + + return out, nil +} + +// infoHashFromMagnet extracts the btih hash from a magnet URI. +func infoHashFromMagnet(magnet string) string { + if !strings.HasPrefix(magnet, "magnet:") { + return "" + } + + u, err := url.Parse(magnet) + if err != nil { + return "" + } + + for _, xt := range u.Query()["xt"] { + if after, ok := strings.CutPrefix(xt, "urn:btih:"); ok { + return strings.ToLower(after) + } + } + + return "" +} + +// collectTree gathers every file under root into dst. A torrent may be +// a single file or a directory tree; either way the importer wants a +// flat set of paths inside the staging directory. +func collectTree(root, dst string) (Result, error) { + result := Result{Dir: dst, Files: []string{}} + + info, err := os.Stat(root) + if err != nil { + return Result{}, fmt.Errorf("stat downloaded content: %w", err) + } + + if !info.IsDir() { + target := filepath.Join(dst, filepath.Base(root)) + + if root != target { + if err := movePath(root, target); err != nil { + return Result{}, err + } + } + + result.Files = append(result.Files, target) + result.BytesTransferred = info.Size() + + return result, nil + } + + err = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err //nolint:wrapcheck // walk error passthrough + } + + fi, err := d.Info() + if err != nil || fi.Size() == 0 { + return nil //nolint:nilerr // skip unreadable entries + } + + target := filepath.Join(dst, filepath.Base(path)) + + if path != target { + if err := movePath(path, target); err != nil { + return err + } + } + + result.Files = append(result.Files, target) + result.BytesTransferred += fi.Size() + + return nil + }) + if err != nil { + return Result{}, fmt.Errorf("collect downloaded files: %w", err) + } + + return result, nil +} + +// --------------------------------------------------------------------------- +// HTTP +// --------------------------------------------------------------------------- + +// call performs a GET and returns the raw body. +func (q *qbittorrent) call( + ctx context.Context, + endpoint string, + params url.Values, +) (string, error) { + target := q.baseURL + endpoint + if len(params) > 0 { + target += "?" + params.Encode() + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil) + if err != nil { + return "", fmt.Errorf("build qbittorrent request: %w", err) + } + + return q.send(req) +} + +// getJSON performs a GET and decodes JSON. +func (q *qbittorrent) getJSON( + ctx context.Context, + endpoint string, + params url.Values, + out any, +) error { + body, err := q.call(ctx, endpoint, params) + if err != nil { + return err + } + + if err := decodeJSON(body, out); err != nil { + return fmt.Errorf("decode qbittorrent response: %w", err) + } + + return nil +} + +// post performs a form POST and returns the raw body. +func (q *qbittorrent) post( + ctx context.Context, + endpoint string, + form url.Values, +) (string, error) { + req, err := http.NewRequestWithContext( + ctx, + http.MethodPost, + q.baseURL+endpoint, + strings.NewReader(form.Encode()), + ) + if err != nil { + return "", fmt.Errorf("build qbittorrent request: %w", err) + } + + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + // qBittorrent rejects cross-origin requests unless Referer matches. + req.Header.Set("Referer", q.baseURL) + + return q.send(req) +} + +// send executes a request and normalizes failures. +func (q *qbittorrent) send(req *http.Request) (string, error) { + resp, err := q.client.Do(req) + if err != nil { + return "", fmt.Errorf("%w: %w", ErrQbitUnreachable, err) + } + + defer func() { _ = resp.Body.Close() }() + + body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + + switch { + case resp.StatusCode == http.StatusForbidden: + return "", ErrQbitAuth + case resp.StatusCode >= 400: + return "", fmt.Errorf( + "%w: HTTP %d: %s", + ErrQbitUnreachable, resp.StatusCode, strings.TrimSpace(string(body)), + ) + } + + return string(body), nil +} diff --git a/backend/download/provider_sabnzbd.go b/backend/download/provider_sabnzbd.go new file mode 100644 index 0000000..3a25f2f --- /dev/null +++ b/backend/download/provider_sabnzbd.go @@ -0,0 +1,388 @@ +package download + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/url" + "strings" + "time" +) + +// SABnzbd is the usenet half of the split-role pair: Prowlarr finds an +// NZB, SABnzbd fetches and unpacks it. +// +// Like slskd, it writes to its own completed-downloads directory rather +// than one we choose, and unlike qBittorrent there is no per-job save +// path we can set reliably across versions. It does, however, report +// the final storage path in its history, so the collect step reads that +// rather than guessing. + +// SABnzbd provider errors. +var ( + // ErrSabUnreachable means the instance did not answer. + ErrSabUnreachable = errors.New("sabnzbd is unreachable") + + // ErrSabAuth means the API key was rejected. + ErrSabAuth = errors.New("sabnzbd rejected the API key") + + // ErrSabTransferFailed means the job failed or was removed. + ErrSabTransferFailed = errors.New("sabnzbd job failed") + + // ErrSabNoJob means the queued job vanished from both queue and + // history without completing. + ErrSabNoJob = errors.New("sabnzbd job disappeared") +) + +// SABnzbd tuning. +const ( + // sabHTTPTimeout bounds one API call. + sabHTTPTimeout = 30 * time.Second + + // sabPollInterval is how often job state is checked. + sabPollInterval = 5 * time.Second +) + +func init() { + Register( + Descriptor{ + Kind: KindSABnzbd, + Name: "SABnzbd", + Summary: "Download usenet releases found by an indexer. " + + "Pairs with Prowlarr; does not search on its own.", + RequiresExternal: "SABnzbd", + Caps: Caps{ + CanTransport: true, + CanCancel: true, + ReportsSize: true, + Transports: []Protocol{ProtocolUsenet}, + }, + Fields: []Field{ + { + Key: "url", + Label: "SABnzbd URL", + Placeholder: "http://localhost:8080", + Required: true, + Default: "http://localhost:8080", + }, + { + Key: "apiKey", + Label: "API key", + Secret: true, + Required: true, + Help: "SABnzbd → Config → General → API Key.", + }, + { + Key: "category", + Label: "Category", + Help: "SABnzbd category for these downloads. " + + "Its folder must be readable from this machine.", + Default: "music", + }, + }, + }, + newSABnzbd, + ) +} + +// sabnzbd is the SABnzbd transport. +type sabnzbd struct { + info ProviderInfo + logger *slog.Logger + client *apiClient + + apiKey string + category string + + pollInterval time.Duration +} + +// newSABnzbd builds the transport from config. +func newSABnzbd( + cfg Config, + secrets SecretLookup, + logger *slog.Logger, +) (Provider, error) { + base := strings.TrimRight(cfg.Setting("url", ""), "/") + if base == "" { + return nil, fmt.Errorf("%w: SABnzbd URL is required", ErrNotConfigured) + } + + apiKey := "" + + if secrets != nil { + key, err := secrets("apiKey") + if err != nil { + return nil, fmt.Errorf("%w: no API key stored", ErrNotConfigured) + } + + apiKey = key + } + + return &sabnzbd{ + info: ProviderInfo{ + ID: cfg.ID, + Kind: KindSABnzbd, + Name: cfg.Name, + Enabled: cfg.Enabled, + Priority: cfg.Priority, + Caps: Caps{ + CanTransport: true, + CanCancel: true, + ReportsSize: true, + Transports: []Protocol{ProtocolUsenet}, + }, + }, + logger: logger.With("provider", "sabnzbd"), + // SABnzbd authenticates by query parameter, not header, so the + // shared client carries no auth header here. + client: newAPIClient( + base, "", "", sabHTTPTimeout, ErrSabUnreachable, ErrSabAuth, + ), + apiKey: apiKey, + category: cfg.Setting("category", "music"), + pollInterval: sabPollInterval, + }, nil +} + +// Info returns the provider's identity. +func (s *sabnzbd) Info() ProviderInfo { + return s.info +} + +// Close is a no-op. +func (s *sabnzbd) Close() error { + return nil +} + +// sabResponse is SABnzbd's common envelope. It answers a bad API key +// with HTTP 200 and status:false, so the body has to be inspected. +type sabResponse struct { + Status bool `json:"status"` + Error string `json:"error"` + NzoIDs []string `json:"nzo_ids"` +} + +// sabQueue is the queue view. +type sabQueue struct { + Queue struct { + Slots []sabQueueSlot `json:"slots"` + } `json:"queue"` +} + +// sabQueueSlot is one in-flight job. +type sabQueueSlot struct { + NzoID string `json:"nzo_id"` + Filename string `json:"filename"` + Status string `json:"status"` + Percentage string `json:"percentage"` + MB string `json:"mb"` + MBLeft string `json:"mbleft"` +} + +// sabHistory is the history view. +type sabHistory struct { + History struct { + Slots []sabHistorySlot `json:"slots"` + } `json:"history"` +} + +// sabHistorySlot is one finished job. Storage is the unpacked path, +// which is the only reliable way to find what SABnzbd produced. +type sabHistorySlot struct { + NzoID string `json:"nzo_id"` + Name string `json:"name"` + Status string `json:"status"` + Storage string `json:"storage"` + FailMsg string `json:"fail_message"` + Bytes int64 `json:"bytes"` +} + +// Check verifies the instance answers and the key is accepted. +func (s *sabnzbd) Check(ctx context.Context) error { + var resp struct { + Version string `json:"version"` + } + + if err := s.call(ctx, url.Values{"mode": {"version"}}, &resp); err != nil { + return err + } + + // version answers without auth, so make one authenticated call too. + var queue sabQueue + + return s.call(ctx, url.Values{"mode": {"queue"}}, &queue) +} + +// Grab adds the NZB, waits for SABnzbd to finish, and moves the +// unpacked files into dst. +func (s *sabnzbd) Grab( + ctx context.Context, + c Candidate, + dst string, + onProgress ProgressFunc, +) (Result, error) { + link := c.Payload["link"] + if link == "" { + return Result{}, fmt.Errorf( + "%w: candidate has no NZB URL", ErrSabTransferFailed, + ) + } + + if err := validateHTTPURL(link); err != nil { + return Result{}, err + } + + var added sabResponse + + if err := s.call(ctx, url.Values{ + "mode": {"addurl"}, + "name": {link}, + "cat": {s.category}, + "nzbname": {c.Title}, + "priority": {"0"}, + }, &added); err != nil { + return Result{}, err + } + + if !added.Status || len(added.NzoIDs) == 0 { + return Result{}, fmt.Errorf( + "%w: %s", ErrSabTransferFailed, added.Error, + ) + } + + nzoID := added.NzoIDs[0] + + slot, err := s.await(ctx, nzoID, onProgress) + if err != nil { + return Result{}, err + } + + if !strings.EqualFold(slot.Status, "Completed") { + return Result{}, fmt.Errorf( + "%w: %s: %s", ErrSabTransferFailed, slot.Status, slot.FailMsg, + ) + } + + return collectTree(slot.Storage, dst) +} + +// await polls the queue until the job leaves it, then reads history for +// the outcome. SABnzbd moves a job from queue to history when it +// finishes post-processing, so history is where completion is truthful. +func (s *sabnzbd) await( + ctx context.Context, + nzoID string, + onProgress ProgressFunc, +) (sabHistorySlot, error) { + for { + select { + case <-ctx.Done(): + return sabHistorySlot{}, fmt.Errorf( + "%w: cancelled", ErrSabTransferFailed, + ) + case <-time.After(s.pollInterval): + } + + inQueue, slot, err := s.queueSlot(ctx, nzoID) + if err != nil { + s.logger.Debug("sabnzbd queue poll failed", "error", err) + + continue + } + + if inQueue { + if onProgress != nil { + onProgress(Progress{ + Current: parseMB(slot.MB) - parseMB(slot.MBLeft), + Total: parseMB(slot.MB), + Phase: "Downloading from usenet (" + slot.Status + ")", + }) + } + + continue + } + + found, hist, err := s.historySlot(ctx, nzoID) + if err != nil { + s.logger.Debug("sabnzbd history poll failed", "error", err) + + continue + } + + if found { + return hist, nil + } + + // Not in the queue and not in history: the job was removed out + // from under us. + return sabHistorySlot{}, ErrSabNoJob + } +} + +// queueSlot looks for a job in the queue. +func (s *sabnzbd) queueSlot( + ctx context.Context, + nzoID string, +) (bool, sabQueueSlot, error) { + var queue sabQueue + + if err := s.call(ctx, url.Values{"mode": {"queue"}}, &queue); err != nil { + return false, sabQueueSlot{}, err + } + + for _, slot := range queue.Queue.Slots { + if slot.NzoID == nzoID { + return true, slot, nil + } + } + + return false, sabQueueSlot{}, nil +} + +// historySlot looks for a job in history. +func (s *sabnzbd) historySlot( + ctx context.Context, + nzoID string, +) (bool, sabHistorySlot, error) { + var history sabHistory + + if err := s.call( + ctx, url.Values{"mode": {"history"}}, &history, + ); err != nil { + return false, sabHistorySlot{}, err + } + + for _, slot := range history.History.Slots { + if slot.NzoID == nzoID { + return true, slot, nil + } + } + + return false, sabHistorySlot{}, nil +} + +// call performs one API request. SABnzbd puts everything on the query +// string of a single endpoint. +func (s *sabnzbd) call(ctx context.Context, params url.Values, out any) error { + params.Set("apikey", s.apiKey) + params.Set("output", "json") + + return s.client.get(ctx, "/api?"+params.Encode(), out) +} + +// parseMB converts SABnzbd's megabyte strings to bytes. Values are +// decimal strings like "1024.5"; a malformed one yields 0 rather than +// failing a transfer that is otherwise fine. +func parseMB(s string) int64 { + const bytesPerMB = 1024 * 1024 + + var mb float64 + + if _, err := fmt.Sscanf(strings.TrimSpace(s), "%f", &mb); err != nil { + return 0 + } + + return int64(mb * bytesPerMB) +} diff --git a/backend/download/provider_slskd.go b/backend/download/provider_slskd.go new file mode 100644 index 0000000..c07ad36 --- /dev/null +++ b/backend/download/provider_slskd.go @@ -0,0 +1,624 @@ +package download + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + "path" + "path/filepath" + "strings" + "time" +) + +// Soulseek is reached through a user-run slskd daemon rather than the +// wire protocol. That trades a setup step for not having to implement +// peer connections, distributed search, and upload obligations — and +// keeps the user's Soulseek credentials in their daemon instead of in +// this process. +// +// One wrinkle shapes this adapter: slskd downloads into its own +// configured directory, not one we hand it. There is no API to stream +// a finished file back. So the user tells us where that directory is, +// and Grab waits for the transfer, then moves the files into staging. +// When slskd runs on another machine, that path has to be a mount — +// which is why Check verifies it exists rather than discovering the +// problem after a two-hour transfer. + +// slskd provider errors. +var ( + // ErrSlskdUnreachable means the daemon did not answer. + ErrSlskdUnreachable = errors.New("slskd is unreachable") + + // ErrSlskdAuth means the API key was rejected. + ErrSlskdAuth = errors.New("slskd rejected the API key") + + // ErrSlskdDownloadsPath means the configured downloads directory is + // missing or unreadable from this machine. + ErrSlskdDownloadsPath = errors.New( + "slskd downloads directory is not readable from here", + ) + + // ErrSlskdTransferFailed means a peer transfer ended badly. + ErrSlskdTransferFailed = errors.New("slskd transfer failed") + + // ErrSlskdTimeout means a search or transfer outlived its budget. + ErrSlskdTimeout = errors.New("slskd timed out") +) + +// slskd tuning. +const ( + // slskdSearchPoll is how often an in-flight search is polled. + slskdSearchPoll = 1 * time.Second + + // slskdSearchWait bounds a single search. Soulseek searches return + // results progressively; waiting the full budget gets noticeably + // more peers than bailing at the first response. + slskdSearchWait = 12 * time.Second + + // slskdTransferPoll is how often transfer state is polled. + slskdTransferPoll = 3 * time.Second + + // slskdMinFiles is the fewest audio files a folder needs before it + // is offered as a candidate. Soulseek returns a lot of one-file + // noise for common queries. + slskdMinFiles = 2 + + // slskdHTTPTimeout bounds one API call. + slskdHTTPTimeout = 20 * time.Second +) + +func init() { + Register( + Descriptor{ + Kind: KindSlskd, + Name: "Soulseek (slskd)", + Summary: "Search and download from the Soulseek network " + + "through your own slskd daemon.", + RequiresExternal: "slskd", + Caps: Caps{ + CanSearch: true, + CanTransport: true, + CanCancel: true, + ReportsSize: true, + }, + Fields: []Field{ + { + Key: "url", + Label: "slskd URL", + Placeholder: "http://localhost:5030", + Required: true, + Default: "http://localhost:5030", + }, + { + Key: "apiKey", + Label: "API key", + Secret: true, + Required: true, + Help: "From your slskd configuration under web.authentication.", + }, + { + Key: "downloadsPath", + Label: "slskd downloads folder", + Placeholder: "/var/lib/slskd/downloads", + Required: true, + Help: "The folder slskd saves to, as this machine sees it. " + + "If slskd runs elsewhere, this must be a mounted share.", + }, + }, + }, + newSlskd, + ) +} + +// slskd is the Soulseek provider. +type slskd struct { + info ProviderInfo + logger *slog.Logger + client *apiClient + + downloadsPath string + + // Poll intervals are fields rather than constants so tests can run + // the full search-and-transfer flow without sleeping through it. + searchPoll time.Duration + searchWait time.Duration + transferPoll time.Duration +} + +// newSlskd builds the provider from config. +func newSlskd( + cfg Config, + secrets SecretLookup, + logger *slog.Logger, +) (Provider, error) { + base := strings.TrimRight(cfg.Setting("url", ""), "/") + if base == "" { + return nil, fmt.Errorf("%w: slskd URL is required", ErrNotConfigured) + } + + downloads := cfg.Setting("downloadsPath", "") + if downloads == "" { + return nil, fmt.Errorf( + "%w: slskd downloads folder is required", ErrNotConfigured, + ) + } + + apiKey := "" + + if secrets != nil { + key, err := secrets("apiKey") + if err != nil { + return nil, fmt.Errorf("%w: no API key stored", ErrNotConfigured) + } + + apiKey = key + } + + return &slskd{ + info: ProviderInfo{ + ID: cfg.ID, + Kind: KindSlskd, + Name: cfg.Name, + Enabled: cfg.Enabled, + Priority: cfg.Priority, + Caps: Caps{ + CanSearch: true, + CanTransport: true, + CanCancel: true, + ReportsSize: true, + }, + }, + logger: logger.With("provider", "slskd"), + client: newAPIClient( + base, "X-Api-Key", apiKey, slskdHTTPTimeout, + ErrSlskdUnreachable, ErrSlskdAuth, + ), + downloadsPath: downloads, + searchPoll: slskdSearchPoll, + searchWait: slskdSearchWait, + transferPoll: slskdTransferPoll, + }, nil +} + +// Info returns the provider's identity. +func (s *slskd) Info() ProviderInfo { + return s.info +} + +// Close is a no-op; the HTTP client holds no session. +func (s *slskd) Close() error { + return nil +} + +// Check verifies the daemon answers, the key is accepted, and the +// downloads directory is readable from this machine. +func (s *slskd) Check(ctx context.Context) error { + var app map[string]any + + if err := s.client.get(ctx, "/api/v0/application", &app); err != nil { + return err + } + + info, err := os.Stat(s.downloadsPath) + if err != nil || !info.IsDir() { + return fmt.Errorf("%w: %s", ErrSlskdDownloadsPath, s.downloadsPath) + } + + return nil +} + +// --------------------------------------------------------------------------- +// API types +// --------------------------------------------------------------------------- + +// slskdSearch is a search as slskd reports it. +type slskdSearch struct { + ID string `json:"id"` + IsComplete bool `json:"isComplete"` + Responses []slskdResponse `json:"responses"` +} + +// slskdResponse is one peer's answer to a search. +type slskdResponse struct { + Username string `json:"username"` + HasFreeUploadSlot bool `json:"hasFreeUploadSlot"` + QueueLength int `json:"queueLength"` + UploadSpeed int64 `json:"uploadSpeed"` + Files []slskdFile `json:"files"` + LockedFileCount int `json:"lockedFileCount"` + FileCount int `json:"fileCount"` + FreeUploadSlotFlag bool `json:"freeUploadSlots"` +} + +// slskdFile is one file a peer is offering. +type slskdFile struct { + Filename string `json:"filename"` + Size int64 `json:"size"` + BitRate int `json:"bitRate"` + Length int `json:"length"` +} + +// slskdTransfer is one download's state. +type slskdTransfer struct { + ID string `json:"id"` + Username string `json:"username"` + Filename string `json:"filename"` + State string `json:"state"` + Size int64 `json:"size"` + BytesTransferred int64 `json:"bytesTransferred"` +} + +// done reports whether the transfer reached a terminal state, and +// whether it succeeded. slskd reports compound states such as +// "Completed, Succeeded" and "Completed, Errored". +func (t slskdTransfer) done() (finished, ok bool) { + if !strings.Contains(t.State, "Completed") { + return false, false + } + + return true, strings.Contains(t.State, "Succeeded") +} + +// --------------------------------------------------------------------------- +// Search +// --------------------------------------------------------------------------- + +// Search runs a Soulseek search and groups the results into per-peer, +// per-folder candidates. A folder from one peer is the unit a user +// actually wants: Soulseek has no album concept, but people organise +// their shares by album directory. +func (s *slskd) Search(ctx context.Context, req Request) ([]Candidate, error) { + searchID := newID() + + body := map[string]any{ + "id": searchID, + "searchText": req.SearchText(), + } + + if err := s.client.post(ctx, "/api/v0/searches", body, nil); err != nil { + return nil, err + } + + search, err := s.awaitSearch(ctx, searchID) + if err != nil { + return nil, err + } + + // Best effort cleanup; a left-behind search is harmless but clutters + // the slskd UI. + defer func() { + _ = s.client.delete( + context.WithoutCancel(ctx), "/api/v0/searches/"+searchID, + ) + }() + + return s.candidatesFrom(search), nil +} + +// awaitSearch polls until the search completes or the budget runs out. +// A timeout is not an error: partial Soulseek results are normal and +// often good enough. +func (s *slskd) awaitSearch( + ctx context.Context, + searchID string, +) (slskdSearch, error) { + deadline := time.Now().Add(s.searchWait) + + var last slskdSearch + + for time.Now().Before(deadline) { + select { + case <-ctx.Done(): + return last, fmt.Errorf("%w: search cancelled", ErrSlskdTimeout) + case <-time.After(s.searchPoll): + } + + var search slskdSearch + + if err := s.client.get( + ctx, + "/api/v0/searches/"+searchID+"?includeResponses=true", + &search, + ); err != nil { + return last, err + } + + last = search + + if search.IsComplete { + return search, nil + } + } + + return last, nil +} + +// candidatesFrom groups a search's responses into candidates. +func (s *slskd) candidatesFrom(search slskdSearch) []Candidate { + out := make([]Candidate, 0, len(search.Responses)) + + for _, resp := range search.Responses { + for folder, files := range groupByFolder(resp.Files) { + audio := 0 + + cfiles := make([]CandidateFile, 0, len(files)) + + var total int64 + + for _, f := range files { + format, isAudio := FormatForPath(f.Filename) + if isAudio { + audio++ + } + + cfiles = append(cfiles, CandidateFile{ + Path: f.Filename, + Size: f.Size, + Format: format, + Bitrate: f.BitRate, + IsAudio: isAudio, + }) + + total += f.Size + } + + if audio < slskdMinFiles { + continue + } + + out = append(out, Candidate{ + ID: "slskd:" + resp.Username + ":" + folder, + Kind: KindSlskd, + Protocol: ProtocolDirect, + Title: path.Base(strings.ReplaceAll(folder, `\`, "/")), + Origin: resp.Username, + Files: cfiles, + TotalSize: total, + Health: peerHealth(resp), + Payload: map[string]string{"username": resp.Username}, + }) + } + } + + return out +} + +// groupByFolder buckets a peer's files by their containing directory. +func groupByFolder(files []slskdFile) map[string][]slskdFile { + out := map[string][]slskdFile{} + + for _, f := range files { + norm := strings.ReplaceAll(f.Filename, `\`, "/") + out[path.Dir(norm)] = append(out[path.Dir(norm)], f) + } + + return out +} + +// peerHealth scores how likely a peer is to actually deliver, in 0..1. +// On Soulseek this matters more than it does for torrents: a queue of +// 40 behind a single upload slot means the transfer starts tomorrow, +// and that is the difference between a good candidate and a bad one no +// matter how good the files look. +func peerHealth(r slskdResponse) float64 { + score := 0.35 + + if r.HasFreeUploadSlot || r.FreeUploadSlotFlag { + score += 0.4 + } + + switch { + case r.QueueLength == 0: + score += 0.15 + case r.QueueLength <= 3: + score += 0.08 + case r.QueueLength > 20: + score -= 0.2 + } + + // Anything above roughly 1 MB/s is fast enough that more speed does + // not change the experience. + const fastEnough = 1_000_000 + + if r.UploadSpeed > 0 { + ratio := float64(r.UploadSpeed) / fastEnough + if ratio > 1 { + ratio = 1 + } + + score += 0.1 * ratio + } + + return clamp01(score) +} + +// --------------------------------------------------------------------------- +// Transfer +// --------------------------------------------------------------------------- + +// Grab enqueues a candidate's files with slskd, waits for the peer to +// send them, then moves them out of slskd's download directory into the +// staging directory. +func (s *slskd) Grab( + ctx context.Context, + c Candidate, + dst string, + onProgress ProgressFunc, +) (Result, error) { + username := c.Payload["username"] + if username == "" { + return Result{}, fmt.Errorf( + "%w: candidate has no peer username", ErrSlskdTransferFailed, + ) + } + + wanted := make([]map[string]any, 0, len(c.Files)) + for _, f := range c.Files { + wanted = append(wanted, map[string]any{ + "filename": f.Path, + "size": f.Size, + }) + } + + if err := s.client.post( + ctx, "/api/v0/transfers/downloads/"+username, wanted, nil, + ); err != nil { + return Result{}, err + } + + if err := s.awaitTransfers(ctx, username, c, onProgress); err != nil { + return Result{}, err + } + + return s.collect(c, dst) +} + +// awaitTransfers polls until every requested file reaches a terminal +// state. Soulseek queues are measured in hours, so the only deadline +// is the caller's context. +func (s *slskd) awaitTransfers( + ctx context.Context, + username string, + c Candidate, + onProgress ProgressFunc, +) error { + wanted := make(map[string]bool, len(c.Files)) + for _, f := range c.Files { + wanted[f.Path] = true + } + + for { + select { + case <-ctx.Done(): + return fmt.Errorf("%w: transfer cancelled", ErrSlskdTimeout) + case <-time.After(s.transferPoll): + } + + transfers, err := s.transfersFor(ctx, username) + if err != nil { + // A blip talking to the daemon should not abandon a + // transfer that may be hours in. + s.logger.Debug("slskd transfer poll failed", "error", err) + + continue + } + + var ( + done, failed int + current int64 + ) + + for _, t := range transfers { + if !wanted[t.Filename] { + continue + } + + current += t.BytesTransferred + + finished, ok := t.done() + if !finished { + continue + } + + if ok { + done++ + } else { + failed++ + } + } + + if onProgress != nil { + onProgress(Progress{ + Current: current, + Total: c.TotalSize, + Phase: fmt.Sprintf( + "Transferring from %s (%d/%d)", username, done, len(wanted), + ), + }) + } + + if done+failed < len(wanted) { + continue + } + + // Some files failing is normal — a peer goes offline mid-folder. + // Let the importer's completeness check decide whether what + // arrived is enough, rather than discarding it here. + if done == 0 { + return fmt.Errorf( + "%w: all %d files failed", ErrSlskdTransferFailed, failed, + ) + } + + return nil + } +} + +// transfersFor returns a peer's current downloads. slskd nests +// transfers under directories, so this flattens them. +func (s *slskd) transfersFor( + ctx context.Context, + username string, +) ([]slskdTransfer, error) { + var raw struct { + Directories []struct { + Files []slskdTransfer `json:"files"` + } `json:"directories"` + } + + if err := s.client.get( + ctx, "/api/v0/transfers/downloads/"+username, &raw, + ); err != nil { + return nil, err + } + + out := make([]slskdTransfer, 0, len(raw.Directories)) + for _, d := range raw.Directories { + out = append(out, d.Files...) + } + + return out, nil +} + +// collect moves finished files out of slskd's download directory into +// staging. slskd lays them out as //, so each +// wanted file is looked up by its base name under the folder slskd +// derived from the remote path. +func (s *slskd) collect(c Candidate, dst string) (Result, error) { + result := Result{Dir: dst, Files: make([]string, 0, len(c.Files))} + + for _, f := range c.Files { + norm := strings.ReplaceAll(f.Path, `\`, "/") + folder := path.Base(path.Dir(norm)) + base := path.Base(norm) + + src := filepath.Join(s.downloadsPath, folder, base) + + info, err := os.Stat(src) + if err != nil || info.Size() == 0 { + // Not every requested file arrives; that is expected and + // handled by completeness scoring downstream. + continue + } + + target := filepath.Join(dst, base) + + if err := movePath(src, target); err != nil { + return Result{}, fmt.Errorf("collect %s: %w", base, err) + } + + result.Files = append(result.Files, target) + result.BytesTransferred += info.Size() + } + + if len(result.Files) == 0 { + return Result{}, fmt.Errorf( + "%w: nothing found under %s", + ErrSlskdDownloadsPath, s.downloadsPath, + ) + } + + return result, nil +} diff --git a/backend/download/provider_slskd_test.go b/backend/download/provider_slskd_test.go new file mode 100644 index 0000000..37d8a47 --- /dev/null +++ b/backend/download/provider_slskd_test.go @@ -0,0 +1,567 @@ +package download + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +// slskd tests run against an httptest server shaped like the real API. +// No daemon, no Soulseek account, no network. + +// slskdStub is a fake slskd daemon. +type slskdStub struct { + server *httptest.Server + + mu sync.Mutex + + // responses is what a search returns. + responses []slskdResponse + + // transfers is what the downloads endpoint reports, in order; the + // last entry repeats. + transfers [][]slskdTransfer + pollCount int + + // enqueued records what was requested for download. + enqueued []map[string]any + + // unauthorized makes every call return 401. + unauthorized bool +} + +func newSlskdStub(t *testing.T) *slskdStub { + t.Helper() + + s := &slskdStub{} + mux := http.NewServeMux() + + mux.HandleFunc("/api/v0/application", func(w http.ResponseWriter, r *http.Request) { + if s.reject(w, r) { + return + } + + writeJSON(t, w, map[string]any{"version": "0.21.0"}) + }) + + mux.HandleFunc("/api/v0/searches", func(w http.ResponseWriter, r *http.Request) { + if s.reject(w, r) { + return + } + + w.WriteHeader(http.StatusCreated) + }) + + mux.HandleFunc("/api/v0/searches/", func(w http.ResponseWriter, r *http.Request) { + if s.reject(w, r) { + return + } + + if r.Method == http.MethodDelete { + w.WriteHeader(http.StatusNoContent) + + return + } + + s.mu.Lock() + responses := s.responses + s.mu.Unlock() + + writeJSON(t, w, slskdSearch{ + ID: "search-1", + IsComplete: true, + Responses: responses, + }) + }) + + mux.HandleFunc("/api/v0/transfers/downloads/", func(w http.ResponseWriter, r *http.Request) { + if s.reject(w, r) { + return + } + + if r.Method == http.MethodPost { + var body []map[string]any + + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Errorf("decode enqueue body: %v", err) + } + + s.mu.Lock() + s.enqueued = body + s.mu.Unlock() + + w.WriteHeader(http.StatusCreated) + + return + } + + s.mu.Lock() + + idx := s.pollCount + if idx >= len(s.transfers) { + idx = len(s.transfers) - 1 + } else { + s.pollCount++ + } + + var batch []slskdTransfer + if idx >= 0 && len(s.transfers) > 0 { + batch = s.transfers[idx] + } + + s.mu.Unlock() + + writeJSON(t, w, map[string]any{ + "directories": []map[string]any{{"files": batch}}, + }) + }) + + s.server = httptest.NewServer(mux) + t.Cleanup(s.server.Close) + + return s +} + +// reject enforces API-key auth like the real daemon. +func (s *slskdStub) reject(w http.ResponseWriter, r *http.Request) bool { + s.mu.Lock() + unauthorized := s.unauthorized + s.mu.Unlock() + + if unauthorized || r.Header.Get("X-Api-Key") != "test-key" { + w.WriteHeader(http.StatusUnauthorized) + + return true + } + + return false +} + +func writeJSON(t *testing.T, w http.ResponseWriter, v any) { + t.Helper() + + w.Header().Set("Content-Type", "application/json") + + if err := json.NewEncoder(w).Encode(v); err != nil { + t.Errorf("encode response: %v", err) + } +} + +// newStubSlskd builds the provider pointed at the stub, with a real +// temp directory standing in for slskd's downloads folder. +func newStubSlskd(t *testing.T, stub *slskdStub) (*slskd, string) { + t.Helper() + + downloads := t.TempDir() + + p, err := newSlskd( + Config{ + ID: 1, + Kind: KindSlskd, + Name: "slskd", + Enabled: true, + Settings: map[string]string{ + "url": stub.server.URL, + "downloadsPath": downloads, + }, + }, + func(string) (string, error) { return "test-key", nil }, + slogDiscard(), + ) + if err != nil { + t.Fatalf("newSlskd: %v", err) + } + + s, ok := p.(*slskd) + if !ok { + t.Fatalf("provider is %T, want *slskd", p) + } + + // Real intervals are tuned for Soulseek's pace; tests only care + // about the state machine, so run it at full speed. + s.searchPoll = time.Millisecond + s.searchWait = 200 * time.Millisecond + s.transferPoll = time.Millisecond + + return s, downloads +} + +func TestSlskdCheck(t *testing.T) { + t.Parallel() + + stub := newSlskdStub(t) + s, _ := newStubSlskd(t, stub) + + if err := s.Check(context.Background()); err != nil { + t.Errorf("Check: %v", err) + } +} + +func TestSlskdCheckRejectsBadKey(t *testing.T) { + t.Parallel() + + stub := newSlskdStub(t) + stub.unauthorized = true + + s, _ := newStubSlskd(t, stub) + + if err := s.Check(context.Background()); !errors.Is(err, ErrSlskdAuth) { + t.Errorf("error = %v, want ErrSlskdAuth", err) + } +} + +// A downloads folder that is not readable from this machine is the +// classic slskd-on-a-NAS misconfiguration, and must surface at +// configuration time rather than after a long transfer. +func TestSlskdCheckRejectsUnreadableDownloadsPath(t *testing.T) { + t.Parallel() + + stub := newSlskdStub(t) + + p, err := newSlskd( + Config{ + ID: 1, + Settings: map[string]string{ + "url": stub.server.URL, + "downloadsPath": "/definitely/not/a/real/path", + }, + }, + func(string) (string, error) { return "test-key", nil }, + slogDiscard(), + ) + if err != nil { + t.Fatalf("newSlskd: %v", err) + } + + if err := p.Check(context.Background()); !errors.Is( + err, ErrSlskdDownloadsPath, + ) { + t.Errorf("error = %v, want ErrSlskdDownloadsPath", err) + } +} + +// Soulseek has no album concept, so candidates are built by grouping a +// peer's files into the folders they live in. +func TestSlskdGroupsResultsByPeerAndFolder(t *testing.T) { + t.Parallel() + + stub := newSlskdStub(t) + stub.responses = []slskdResponse{ + { + Username: "peer-one", + HasFreeUploadSlot: true, + QueueLength: 0, + UploadSpeed: 2_000_000, + Files: []slskdFile{ + {Filename: `@@x\Music\OK Computer\01 Airbag.flac`, Size: 30_000_000}, + {Filename: `@@x\Music\OK Computer\02 Paranoid Android.flac`, Size: 40_000_000}, + {Filename: `@@x\Music\Kid A\01 Everything.flac`, Size: 30_000_000}, + {Filename: `@@x\Music\Kid A\02 Kid A.flac`, Size: 30_000_000}, + }, + }, + { + Username: "peer-two", + HasFreeUploadSlot: false, + QueueLength: 40, + Files: []slskdFile{ + { + Filename: `\share\OK Computer [320]\01 - Airbag.mp3`, + Size: 8_000_000, + BitRate: 320, + }, + { + Filename: `\share\OK Computer [320]\02 - Paranoid Android.mp3`, + Size: 9_000_000, + BitRate: 320, + }, + }, + }, + } + + s, _ := newStubSlskd(t, stub) + + got, err := s.Search(context.Background(), Request{ + Artist: "Radiohead", + Album: "OK Computer", + }) + if err != nil { + t.Fatalf("Search: %v", err) + } + + // Two folders from peer-one, one from peer-two. + if len(got) != 3 { + t.Fatalf("got %d candidates, want 3", len(got)) + } + + byOrigin := map[string]int{} + for _, c := range got { + byOrigin[c.Origin]++ + } + + if byOrigin["peer-one"] != 2 { + t.Errorf("peer-one folders = %d, want 2", byOrigin["peer-one"]) + } + + if byOrigin["peer-two"] != 1 { + t.Errorf("peer-two folders = %d, want 1", byOrigin["peer-two"]) + } +} + +// A busy peer behind a long queue is a worse bet than a free one, no +// matter how good the files look. +func TestSlskdPeerHealthReflectsAvailability(t *testing.T) { + t.Parallel() + + free := peerHealth(slskdResponse{ + HasFreeUploadSlot: true, + QueueLength: 0, + UploadSpeed: 2_000_000, + }) + + busy := peerHealth(slskdResponse{ + HasFreeUploadSlot: false, + QueueLength: 40, + }) + + if free <= busy { + t.Errorf("free peer health %f should exceed busy peer %f", free, busy) + } + + if free > 1 || busy < 0 { + t.Errorf("health out of range: free=%f busy=%f", free, busy) + } +} + +// Folders with almost nothing in them are Soulseek noise, not albums. +func TestSlskdSkipsTinyFolders(t *testing.T) { + t.Parallel() + + stub := newSlskdStub(t) + stub.responses = []slskdResponse{{ + Username: "peer", + Files: []slskdFile{ + {Filename: `\share\Random\one.mp3`, Size: 5_000_000}, + }, + }} + + s, _ := newStubSlskd(t, stub) + + got, err := s.Search(context.Background(), Request{Query: "x"}) + if err != nil { + t.Fatalf("Search: %v", err) + } + + if len(got) != 0 { + t.Errorf("got %d candidates, want 0 for a single-file folder", len(got)) + } +} + +func TestSlskdGrabCollectsFromDownloadsFolder(t *testing.T) { + t.Parallel() + + stub := newSlskdStub(t) + stub.transfers = [][]slskdTransfer{ + { + { + Filename: `\share\OK Computer\01 Airbag.flac`, + State: "InProgress", + BytesTransferred: 100, + }, + { + Filename: `\share\OK Computer\02 Paranoid Android.flac`, + State: "InProgress", + }, + }, + { + { + Filename: `\share\OK Computer\01 Airbag.flac`, + State: "Completed, Succeeded", + BytesTransferred: 500, + }, + { + Filename: `\share\OK Computer\02 Paranoid Android.flac`, + State: "Completed, Succeeded", + BytesTransferred: 500, + }, + }, + } + + s, downloads := newStubSlskd(t, stub) + + // slskd writes into //. + folder := filepath.Join(downloads, "OK Computer") + if err := os.MkdirAll(folder, 0o750); err != nil { + t.Fatalf("mkdir: %v", err) + } + + for _, name := range []string{ + "01 Airbag.flac", + "02 Paranoid Android.flac", + } { + if err := os.WriteFile( + filepath.Join(folder, name), []byte("audio"), 0o600, + ); err != nil { + t.Fatalf("write: %v", err) + } + } + + c := Candidate{ + ID: "slskd:peer:OK Computer", + Protocol: ProtocolDirect, + Files: []CandidateFile{ + {Path: `\share\OK Computer\01 Airbag.flac`, Size: 500, IsAudio: true}, + {Path: `\share\OK Computer\02 Paranoid Android.flac`, Size: 500, IsAudio: true}, + }, + TotalSize: 1000, + Payload: map[string]string{"username": "peer"}, + } + + dst := t.TempDir() + + got, err := s.Grab(context.Background(), c, dst, nil) + if err != nil { + t.Fatalf("Grab: %v", err) + } + + if len(got.Files) != 2 { + t.Fatalf("collected %d files, want 2", len(got.Files)) + } + + for _, f := range got.Files { + if !strings.HasPrefix(f, dst) { + t.Errorf("file %s is not inside the staging dir %s", f, dst) + } + + if _, err := os.Stat(f); err != nil { + t.Errorf("collected file missing: %v", err) + } + } + + // The enqueue request named the files the candidate listed. + stub.mu.Lock() + enqueued := len(stub.enqueued) + stub.mu.Unlock() + + if enqueued != 2 { + t.Errorf("enqueued %d files, want 2", enqueued) + } +} + +// A peer that drops mid-folder is normal; partial results go forward +// and the importer's completeness check decides. +func TestSlskdGrabToleratesPartialFailure(t *testing.T) { + t.Parallel() + + stub := newSlskdStub(t) + stub.transfers = [][]slskdTransfer{{ + { + Filename: `\s\Album\01 A.flac`, + State: "Completed, Succeeded", + BytesTransferred: 500, + }, + {Filename: `\s\Album\02 B.flac`, State: "Completed, Errored"}, + }} + + s, downloads := newStubSlskd(t, stub) + + folder := filepath.Join(downloads, "Album") + if err := os.MkdirAll(folder, 0o750); err != nil { + t.Fatalf("mkdir: %v", err) + } + + if err := os.WriteFile( + filepath.Join(folder, "01 A.flac"), []byte("audio"), 0o600, + ); err != nil { + t.Fatalf("write: %v", err) + } + + c := Candidate{ + Files: []CandidateFile{ + {Path: `\s\Album\01 A.flac`, Size: 500, IsAudio: true}, + {Path: `\s\Album\02 B.flac`, Size: 500, IsAudio: true}, + }, + Payload: map[string]string{"username": "peer"}, + } + + got, err := s.Grab(context.Background(), c, t.TempDir(), nil) + if err != nil { + t.Fatalf("Grab: %v", err) + } + + if len(got.Files) != 1 { + t.Errorf("collected %d files, want the 1 that succeeded", len(got.Files)) + } +} + +func TestSlskdGrabFailsWhenEverythingFails(t *testing.T) { + t.Parallel() + + stub := newSlskdStub(t) + stub.transfers = [][]slskdTransfer{{ + {Filename: `\s\Album\01 A.flac`, State: "Completed, Errored"}, + }} + + s, _ := newStubSlskd(t, stub) + + c := Candidate{ + Files: []CandidateFile{{Path: `\s\Album\01 A.flac`, IsAudio: true}}, + Payload: map[string]string{"username": "peer"}, + } + + _, err := s.Grab(context.Background(), c, t.TempDir(), nil) + if !errors.Is(err, ErrSlskdTransferFailed) { + t.Errorf("error = %v, want ErrSlskdTransferFailed", err) + } +} + +func TestSlskdRequiresConfiguration(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + settings map[string]string + secrets SecretLookup + }{ + { + name: "no url", + settings: map[string]string{"downloadsPath": "/tmp"}, + secrets: func(string) (string, error) { return "k", nil }, + }, + { + name: "no downloads path", + settings: map[string]string{"url": "http://localhost:5030"}, + secrets: func(string) (string, error) { return "k", nil }, + }, + { + name: "no api key", + settings: map[string]string{ + "url": "http://localhost:5030", "downloadsPath": "/tmp", + }, + secrets: func(string) (string, error) { + return "", ErrSecretNotFound + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + _, err := newSlskd( + Config{Settings: tt.settings}, tt.secrets, slogDiscard(), + ) + + if !errors.Is(err, ErrNotConfigured) { + t.Errorf("error = %v, want ErrNotConfigured", err) + } + }) + } +} diff --git a/backend/download/provider_ytdlp.go b/backend/download/provider_ytdlp.go new file mode 100644 index 0000000..08cc2a6 --- /dev/null +++ b/backend/download/provider_ytdlp.go @@ -0,0 +1,532 @@ +package download + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/url" + "os/exec" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + + "golang.org/x/sync/errgroup" +) + +// yt-dlp is the local-subprocess shape: no server for the user to run, +// no credentials, but everything comes back as text from a binary whose +// output format is not a stable contract. The defences are: pin a +// minimum version and check it before use, ask for JSON rather than +// parsing human output, and never build a shell command — every +// invocation is an argv slice, so a track title containing `; rm -rf` +// is an argument and not a command. + +// yt-dlp provider errors. +var ( + // ErrYtDlpMissing means the binary was not found. + ErrYtDlpMissing = errors.New("yt-dlp was not found") + + // ErrYtDlpTooOld means the installed version predates the output + // format this adapter relies on. + ErrYtDlpTooOld = errors.New("yt-dlp is too old") + + // ErrYtDlpFailed wraps a non-zero exit. + ErrYtDlpFailed = errors.New("yt-dlp failed") + + // ErrUnsafeURL rejects a URL that is not plain http(s). yt-dlp + // accepts things like file:// that must never come from a search + // result. + ErrUnsafeURL = errors.New("refusing to fetch a non-http URL") +) + +// minYtDlpVersion is the oldest release known to support the +// --progress-template and --dump-json output this adapter parses. +// yt-dlp versions are date-stamped, so this compares lexically. +const minYtDlpVersion = "2023.01.01" + +// ytSearchCount is how many results to ask for per query. +const ytSearchCount = 5 + +// ytTrackConcurrency bounds parallel per-track searches when assembling +// an album. YouTube throttles aggressively; three is fast enough to +// finish inside the search timeout without tripping it. +const ytTrackConcurrency = 3 + +func init() { + Register( + Descriptor{ + Kind: KindYtDlp, + Name: "yt-dlp", + Summary: "Download audio from YouTube, SoundCloud, Bandcamp and other sites yt-dlp supports.", + Caps: Caps{ + CanSearch: true, + CanTransport: true, + CanCancel: true, + ReportsSize: true, + }, + Fields: []Field{ + { + Key: "binary", + Label: "yt-dlp path", + Placeholder: "yt-dlp", + Help: "Leave blank to find yt-dlp on your PATH.", + Default: "yt-dlp", + }, + { + Key: "audioFormat", + Label: "Audio format", + Help: "flac, mp3, opus, m4a, or 'best' to keep the source format.", + Default: "flac", + }, + { + Key: "searchPrefix", + Label: "Search source", + Help: "ytsearch for YouTube, ytmsearch for YouTube Music. " + + "Defaults to ytsearch.", + Default: "ytsearch", + }, + }, + }, + newYtDlp, + ) +} + +// ytDlp is the yt-dlp provider. +type ytDlp struct { + info ProviderInfo + logger *slog.Logger + + binary string + audioFormat string + searchPrefix string +} + +// newYtDlp builds a yt-dlp provider from config. +func newYtDlp( + cfg Config, + _ SecretLookup, + logger *slog.Logger, +) (Provider, error) { + binary := cfg.Setting("binary", "yt-dlp") + + resolved, err := exec.LookPath(binary) + if err != nil { + return nil, fmt.Errorf("%w: %s", ErrYtDlpMissing, binary) + } + + return &ytDlp{ + info: ProviderInfo{ + ID: cfg.ID, + Kind: KindYtDlp, + Name: cfg.Name, + Enabled: cfg.Enabled, + Priority: cfg.Priority, + Caps: Caps{ + CanSearch: true, + CanTransport: true, + CanCancel: true, + ReportsSize: true, + }, + }, + logger: logger.With("provider", "yt-dlp"), + binary: resolved, + audioFormat: cfg.Setting("audioFormat", "flac"), + searchPrefix: cfg.Setting("searchPrefix", "ytsearch"), + }, nil +} + +// Info returns the provider's identity. +func (y *ytDlp) Info() ProviderInfo { + return y.info +} + +// Close is a no-op; each invocation is its own process. +func (y *ytDlp) Close() error { + return nil +} + +// Check verifies the binary runs and is new enough. Version drift is +// yt-dlp's defining trait, so this is the difference between a clear +// error at configuration time and garbled output at download time. +func (y *ytDlp) Check(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + + out, err := exec.CommandContext(ctx, y.binary, "--version").Output() + if err != nil { + return fmt.Errorf("%w: %w", ErrYtDlpFailed, err) + } + + version := strings.TrimSpace(string(out)) + if version < minYtDlpVersion { + return fmt.Errorf( + "%w: found %s, need %s or newer", + ErrYtDlpTooOld, version, minYtDlpVersion, + ) + } + + return nil +} + +// ytEntry is the subset of yt-dlp's --dump-json output this adapter +// uses. yt-dlp emits far more; naming only what is needed means a +// field being added or reordered upstream cannot break parsing. +type ytEntry struct { + ID string `json:"id"` + Title string `json:"title"` + URL string `json:"url"` + WebURL string `json:"webpage_url"` + Uploader string `json:"uploader"` + Duration float64 `json:"duration"` + Filesize int64 `json:"filesize_approx"` +} + +// link returns the entry's best usable URL. +func (e ytEntry) link() string { + if e.WebURL != "" { + return e.WebURL + } + + return e.URL +} + +// Search assembles candidates. With an expected tracklist it searches +// per track and offers the assembled album as one candidate, which is +// how yt-dlp is actually useful for albums — a single "full album" +// video is one file and cannot be imported as tracks. Without a +// tracklist it falls back to returning the top individual results. +func (y *ytDlp) Search(ctx context.Context, req Request) ([]Candidate, error) { + if len(req.Expected) > 0 { + c, err := y.assembleAlbum(ctx, req) + if err != nil { + return nil, err + } + + if len(c.Files) > 0 { + return []Candidate{c}, nil + } + } + + entries, err := y.search(ctx, req.SearchText(), ytSearchCount) + if err != nil { + return nil, err + } + + out := make([]Candidate, 0, len(entries)) + + for _, e := range entries { + link := e.link() + if link == "" { + continue + } + + name := sanitizePathPart(e.Title) + "." + y.extension() + + out = append(out, Candidate{ + ID: "ytdlp:" + e.ID, + Kind: KindYtDlp, + Protocol: ProtocolDirect, + Title: e.Title, + Artist: e.Uploader, + Origin: "yt-dlp", + Files: []CandidateFile{{ + Path: name, + Size: e.Filesize, + IsAudio: true, + }}, + TotalSize: e.Filesize, + // yt-dlp results are always available; there is no peer to + // be offline, so health carries no information here. + Health: 0.75, + Payload: map[string]string{name: link}, + }) + } + + return out, nil +} + +// assembleAlbum searches once per expected track and builds a single +// multi-file candidate. Tracks that find no result are left out; the +// completeness score then reflects the gap, and the importer's +// threshold decides whether what arrived is enough. +func (y *ytDlp) assembleAlbum( + ctx context.Context, + req Request, +) (Candidate, error) { + type hit struct { + index int + entry ytEntry + } + + var ( + mu sync.Mutex + hits []hit + ) + + group, gctx := errgroup.WithContext(ctx) + group.SetLimit(ytTrackConcurrency) + + for i, track := range req.Expected { + group.Go(func() error { + query := strings.TrimSpace( + req.Artist + " " + track.Title, + ) + + entries, err := y.search(gctx, query, 1) + if err != nil || len(entries) == 0 { + // One missing track is not a failed search. Recording + // nothing lets completeness scoring speak for it. + return nil //nolint:nilerr // partial results are expected + } + + mu.Lock() + + hits = append(hits, hit{index: i, entry: entries[0]}) + + mu.Unlock() + + return nil + }) + } + + if err := group.Wait(); err != nil { + return Candidate{}, fmt.Errorf("assemble album: %w", err) + } + + c := Candidate{ + ID: "ytdlp:album:" + req.ID, + Kind: KindYtDlp, + Protocol: ProtocolDirect, + Title: req.Album, + Artist: req.Artist, + Origin: "yt-dlp (assembled per track)", + Health: 0.75, + Payload: map[string]string{}, + Files: make([]CandidateFile, 0, len(hits)), + } + + for _, h := range hits { + track := req.Expected[h.index] + + link := h.entry.link() + if link == "" { + continue + } + + // Name the staged file after the expected track, not the video + // title: the video is called whatever the uploader felt like, + // and the import step matches on filename. + name := trackToken(track.Position) + " - " + + sanitizePathPart(track.Title) + "." + y.extension() + + c.Files = append(c.Files, CandidateFile{ + Path: name, + Size: h.entry.Filesize, + IsAudio: true, + MatchedTo: track.Position, + }) + + c.TotalSize += h.entry.Filesize + c.Payload[name] = link + } + + return c, nil +} + +// search runs one yt-dlp search and decodes its JSON lines. +func (y *ytDlp) search( + ctx context.Context, + query string, + count int, +) ([]ytEntry, error) { + if strings.TrimSpace(query) == "" { + return nil, nil + } + + // The search term is one argv element; yt-dlp parses the + // "ytsearchN:" prefix itself. No shell is involved at any point. + target := y.searchPrefix + strconv.Itoa(count) + ":" + query + + args := []string{ + "--dump-json", + "--flat-playlist", + "--no-warnings", + "--no-playlist", + "--ignore-config", + "--socket-timeout", "15", + target, + } + + cmd := exec.CommandContext(ctx, y.binary, args...) + + out, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("%w: search: %w", ErrYtDlpFailed, err) + } + + return decodeYtEntries(strings.NewReader(string(out))), nil +} + +// decodeYtEntries reads newline-delimited JSON, skipping lines that do +// not parse. yt-dlp mixes warnings into stdout in some versions, and +// one bad line must not discard the rest of the results. +func decodeYtEntries(r io.Reader) []ytEntry { + var out []ytEntry + + scanner := bufio.NewScanner(r) + scanner.Buffer(make([]byte, 0, 64*1024), 4*1024*1024) + + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || !strings.HasPrefix(line, "{") { + continue + } + + var e ytEntry + if err := json.Unmarshal([]byte(line), &e); err != nil { + continue + } + + if e.ID == "" { + continue + } + + out = append(out, e) + } + + return out +} + +// Grab downloads each of the candidate's files into dst. +func (y *ytDlp) Grab( + ctx context.Context, + c Candidate, + dst string, + onProgress ProgressFunc, +) (Result, error) { + result := Result{Dir: dst, Files: make([]string, 0, len(c.Files))} + + for i, f := range c.Files { + link, ok := c.Payload[f.Path] + if !ok { + continue + } + + if err := validateHTTPURL(link); err != nil { + return Result{}, err + } + + if onProgress != nil { + onProgress(Progress{ + Current: int64(i), + Total: int64(len(c.Files)), + Phase: fmt.Sprintf( + "Downloading %d of %d", i+1, len(c.Files), + ), + }) + } + + path, err := y.fetchOne(ctx, link, dst, f.Path) + if err != nil { + return Result{}, err + } + + result.Files = append(result.Files, path) + } + + return result, nil +} + +// fetchOne downloads a single URL to a known filename inside dst. +func (y *ytDlp) fetchOne( + ctx context.Context, + link, dst, name string, +) (string, error) { + // Strip the extension from the output template: yt-dlp appends the + // real one after extraction, and forcing it here produces + // double-extensioned files. + stem := strings.TrimSuffix(name, filepath.Ext(name)) + template := filepath.Join(dst, stem) + ".%(ext)s" + + args := []string{ + "--extract-audio", + "--no-playlist", + "--no-warnings", + "--ignore-config", + "--newline", + "--no-part", + "--socket-timeout", "30", + "--output", template, + } + + if y.audioFormat != "" && y.audioFormat != "best" { + args = append(args, "--audio-format", y.audioFormat) + } + + args = append(args, "--", link) + + cmd := exec.CommandContext(ctx, y.binary, args...) + + if out, err := cmd.CombinedOutput(); err != nil { + return "", fmt.Errorf( + "%w: download: %w: %s", + ErrYtDlpFailed, err, lastLine(string(out)), + ) + } + + // The extracted extension is whatever yt-dlp produced, so find the + // file by stem rather than assuming. + matches, err := filepath.Glob(filepath.Join(dst, stem) + ".*") + if err != nil || len(matches) == 0 { + return "", fmt.Errorf( + "%w: no output file for %s", ErrYtDlpFailed, stem, + ) + } + + return matches[0], nil +} + +// extension returns the file extension downloads will have. +func (y *ytDlp) extension() string { + if y.audioFormat == "" || y.audioFormat == "best" { + return "opus" + } + + return y.audioFormat +} + +// validateHTTPURL rejects anything that is not plain http(s). yt-dlp +// happily accepts file:// and other schemes, and a search result is +// untrusted input. +func validateHTTPURL(raw string) error { + u, err := url.Parse(raw) + if err != nil { + return fmt.Errorf("%w: %s", ErrUnsafeURL, raw) + } + + if u.Scheme != "http" && u.Scheme != "https" { + return fmt.Errorf("%w: %s", ErrUnsafeURL, raw) + } + + return nil +} + +// lastLine returns the final non-empty line of output, which is where +// yt-dlp puts its error message. +func lastLine(s string) string { + lines := strings.Split(strings.TrimSpace(s), "\n") + + for i := len(lines) - 1; i >= 0; i-- { + if line := strings.TrimSpace(lines[i]); line != "" { + return line + } + } + + return "" +} diff --git a/backend/download/provider_ytdlp_test.go b/backend/download/provider_ytdlp_test.go new file mode 100644 index 0000000..0ea4ed6 --- /dev/null +++ b/backend/download/provider_ytdlp_test.go @@ -0,0 +1,382 @@ +package download + +import ( + "context" + "errors" + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +// yt-dlp tests drive a stub shell script rather than the real binary: +// the adapter's contract is "what argv do we build and what do we do +// with the output", and a stub tests exactly that without a network, +// a YouTube account, or a 40MB dependency. + +// stubYtDlp writes an executable script that echoes the given stdout +// and returns it as a provider config binary path. +func stubYtDlp(t *testing.T, script string) string { + t.Helper() + + if runtime.GOOS == "windows" { + t.Skip("stub binary test uses a shell script") + } + + path := filepath.Join(t.TempDir(), "yt-dlp") + + if err := os.WriteFile( + path, []byte("#!/bin/sh\n"+script), 0o700, + ); err != nil { + t.Fatalf("write stub: %v", err) + } + + return path +} + +// newStubYtDlp builds the provider over a stub binary. +func newStubYtDlp(t *testing.T, script string) *ytDlp { + t.Helper() + + p, err := newYtDlp( + Config{ + ID: 1, + Kind: KindYtDlp, + Name: "yt-dlp", + Enabled: true, + Priority: 50, + Settings: map[string]string{ + "binary": stubYtDlp(t, script), + "audioFormat": "flac", + }, + }, + nil, + slogDiscard(), + ) + if err != nil { + t.Fatalf("newYtDlp: %v", err) + } + + y, ok := p.(*ytDlp) + if !ok { + t.Fatalf("provider is %T, want *ytDlp", p) + } + + return y +} + +func TestYtDlpCheckVersion(t *testing.T) { + t.Parallel() + + t.Run("recent version passes", func(t *testing.T) { + t.Parallel() + + y := newStubYtDlp(t, `echo "2024.08.06"`) + + if err := y.Check(context.Background()); err != nil { + t.Errorf("Check: %v", err) + } + }) + + t.Run("old version is rejected", func(t *testing.T) { + t.Parallel() + + y := newStubYtDlp(t, `echo "2021.01.01"`) + + if err := y.Check(context.Background()); !errors.Is( + err, ErrYtDlpTooOld, + ) { + t.Errorf("error = %v, want ErrYtDlpTooOld", err) + } + }) + + t.Run("non-zero exit is reported", func(t *testing.T) { + t.Parallel() + + y := newStubYtDlp(t, `exit 1`) + + if err := y.Check(context.Background()); !errors.Is( + err, ErrYtDlpFailed, + ) { + t.Errorf("error = %v, want ErrYtDlpFailed", err) + } + }) +} + +func TestYtDlpMissingBinary(t *testing.T) { + t.Parallel() + + _, err := newYtDlp( + Config{Settings: map[string]string{ + "binary": "definitely-not-a-real-binary-xyzzy", + }}, + nil, + slogDiscard(), + ) + + if !errors.Is(err, ErrYtDlpMissing) { + t.Errorf("error = %v, want ErrYtDlpMissing", err) + } +} + +func TestYtDlpSearchParsesJSONLines(t *testing.T) { + t.Parallel() + + y := newStubYtDlp(t, ` +cat <<'EOF' +{"id":"aaa","title":"Airbag","webpage_url":"https://example.com/aaa","uploader":"Radiohead","duration":284,"filesize_approx":5000000} +{"id":"bbb","title":"Paranoid Android","webpage_url":"https://example.com/bbb","uploader":"Radiohead","duration":383} +EOF +`) + + got, err := y.Search(context.Background(), Request{ + Artist: "Radiohead", + Album: "OK Computer", + }) + if err != nil { + t.Fatalf("Search: %v", err) + } + + if len(got) != 2 { + t.Fatalf("got %d candidates, want 2", len(got)) + } + + if got[0].Title != "Airbag" { + t.Errorf("title = %q, want Airbag", got[0].Title) + } + + if got[0].Protocol != ProtocolDirect { + t.Errorf("protocol = %q, want direct", got[0].Protocol) + } + + if len(got[0].Files) != 1 { + t.Fatalf("got %d files, want 1", len(got[0].Files)) + } + + link := got[0].Payload[got[0].Files[0].Path] + if link != "https://example.com/aaa" { + t.Errorf("payload url = %q, want the webpage_url", link) + } +} + +// Warning text mixed into stdout must not discard valid results. +func TestYtDlpSearchSkipsUnparseableLines(t *testing.T) { + t.Parallel() + + y := newStubYtDlp(t, ` +cat <<'EOF' +WARNING: something happened +{"id":"aaa","title":"Airbag","webpage_url":"https://example.com/aaa"} +not json at all +{"id":"bbb","title":"Lucky","webpage_url":"https://example.com/bbb"} +EOF +`) + + got, err := y.Search(context.Background(), Request{Query: "radiohead"}) + if err != nil { + t.Fatalf("Search: %v", err) + } + + if len(got) != 2 { + t.Fatalf("got %d candidates, want 2 valid ones", len(got)) + } +} + +// With a tracklist the adapter assembles an album from per-track +// searches, because a single "full album" video cannot be imported as +// separate tracks. +func TestYtDlpAssemblesAlbumFromTracklist(t *testing.T) { + t.Parallel() + + y := newStubYtDlp(t, ` +echo '{"id":"x","title":"whatever the uploader called it","webpage_url":"https://example.com/x","filesize_approx":4000000}' +`) + + req := Request{ + ID: "req-1", + ReleaseMBID: "mbid-1", + Artist: "Radiohead", + Album: "OK Computer", + Expected: []ExpectedTrack{ + {Position: 1, Title: "Airbag"}, + {Position: 2, Title: "Paranoid Android"}, + {Position: 3, Title: "Subterranean Homesick Alien"}, + }, + } + + got, err := y.Search(context.Background(), req) + if err != nil { + t.Fatalf("Search: %v", err) + } + + if len(got) != 1 { + t.Fatalf("got %d candidates, want 1 assembled album", len(got)) + } + + album := got[0] + + if len(album.Files) != 3 { + t.Fatalf("got %d files, want 3", len(album.Files)) + } + + // Files are named after the expected tracks, not the video titles, + // because the import step matches on filename. + names := make([]string, 0, len(album.Files)) + for _, f := range album.Files { + names = append(names, f.Path) + } + + for _, want := range []string{ + "01 - Airbag.flac", + "02 - Paranoid Android.flac", + "03 - Subterranean Homesick Alien.flac", + } { + if !containsString(names, want) { + t.Errorf("missing %q in %v", want, names) + } + } + + if album.TotalSize != 12_000_000 { + t.Errorf("total size = %d, want 12000000", album.TotalSize) + } +} + +// A track with no search result is left out, so completeness scoring +// can speak for the gap instead of the search failing outright. +func TestYtDlpAssembleToleratesMissingTracks(t *testing.T) { + t.Parallel() + + y := newStubYtDlp(t, ` +case "$*" in + *Airbag*) echo '{"id":"a","title":"Airbag","webpage_url":"https://example.com/a"}' ;; + *) exit 1 ;; +esac +`) + + got, err := y.Search(context.Background(), Request{ + ID: "req-1", + ReleaseMBID: "mbid-1", + Artist: "Radiohead", + Album: "OK Computer", + Expected: []ExpectedTrack{ + {Position: 1, Title: "Airbag"}, + {Position: 2, Title: "Paranoid Android"}, + }, + }) + if err != nil { + t.Fatalf("Search: %v", err) + } + + if len(got) != 1 { + t.Fatalf("got %d candidates, want 1", len(got)) + } + + if len(got[0].Files) != 1 { + t.Errorf("got %d files, want just the one that was found", len(got[0].Files)) + } +} + +func TestYtDlpGrabWritesIntoStaging(t *testing.T) { + t.Parallel() + + // The stub writes a file at whatever --output stem it is given, + // mimicking yt-dlp's post-extraction naming. + y := newStubYtDlp(t, ` +out="" +while [ $# -gt 0 ]; do + case "$1" in + --output) out="$2"; shift 2 ;; + *) shift ;; + esac +done +target=$(printf '%s' "$out" | sed 's/%(ext)s/flac/') +printf 'audio' > "$target" +`) + + dst := t.TempDir() + + c := Candidate{ + ID: "ytdlp:album:req-1", + Protocol: ProtocolDirect, + Files: []CandidateFile{ + {Path: "01 - Airbag.flac", IsAudio: true}, + }, + Payload: map[string]string{ + "01 - Airbag.flac": "https://example.com/a", + }, + } + + got, err := y.Grab(context.Background(), c, dst, nil) + if err != nil { + t.Fatalf("Grab: %v", err) + } + + if len(got.Files) != 1 { + t.Fatalf("got %d files, want 1", len(got.Files)) + } + + if _, err := os.Stat(got.Files[0]); err != nil { + t.Errorf("downloaded file missing: %v", err) + } + + if !strings.HasSuffix(got.Files[0], ".flac") { + t.Errorf("file = %s, want a .flac", got.Files[0]) + } +} + +// A search result is untrusted input, and yt-dlp accepts schemes that +// would read the local filesystem. +func TestYtDlpGrabRejectsNonHTTPURL(t *testing.T) { + t.Parallel() + + y := newStubYtDlp(t, `exit 0`) + + c := Candidate{ + Files: []CandidateFile{{Path: "x.flac", IsAudio: true}}, + Payload: map[string]string{"x.flac": "file:///etc/passwd"}, + } + + _, err := y.Grab(context.Background(), c, t.TempDir(), nil) + if !errors.Is(err, ErrUnsafeURL) { + t.Errorf("error = %v, want ErrUnsafeURL", err) + } +} + +func TestValidateHTTPURL(t *testing.T) { + t.Parallel() + + tests := []struct { + url string + wantErr bool + }{ + {"https://example.com/a", false}, + {"http://example.com/a", false}, + {"file:///etc/passwd", true}, + {"ftp://example.com/a", true}, + {"javascript:alert(1)", true}, + {"://nonsense", true}, + } + + for _, tt := range tests { + t.Run(tt.url, func(t *testing.T) { + t.Parallel() + + err := validateHTTPURL(tt.url) + if (err != nil) != tt.wantErr { + t.Errorf("validateHTTPURL(%q) error = %v, wantErr %v", + tt.url, err, tt.wantErr) + } + }) + } +} + +func containsString(haystack []string, needle string) bool { + for _, h := range haystack { + if h == needle { + return true + } + } + + return false +} diff --git a/backend/download/rank.go b/backend/download/rank.go new file mode 100644 index 0000000..324706d --- /dev/null +++ b/backend/download/rank.go @@ -0,0 +1,415 @@ +package download + +import ( + "math" + "sort" + "strings" + + "yellowjacket/backend/autotag" +) + +// Ranking keeps two questions apart: +// +// match — is this the release the user asked for? +// quality — is it a good copy of it? +// +// They are reported separately because they fail differently and trade +// off against each other: a flawless FLAC of the wrong album is useless, +// a 128kbps rip of the right one is merely disappointing, and only the +// user knows which they will accept. A single blended number cannot be +// explained, and the review UI has to explain itself. + +// Ranking weights. Match dominates, because a wrong album at any +// bitrate is a failed download. +const ( + weightMatch = 0.72 + weightQuality = 0.28 +) + +// Match sub-weights. +const ( + weightTitleFit = 0.40 + weightCompleteness = 0.30 + weightAlbumFit = 0.18 + weightArtistFit = 0.12 +) + +// Quality sub-weights. +const ( + weightFormat = 0.45 + weightBitrate = 0.25 + weightHealth = 0.20 + weightPriority = 0.10 +) + +// unanchoredCap bounds the match score of a free-text request. Without +// an MBID there is no tracklist to be right about, so a confident- +// looking score would be a lie — and auto-pick keys off this. +const unanchoredCap = 0.65 + +// Score fills a candidate's Match, Quality and Score fields. +func Score(req Request, c Candidate, priority int) Candidate { + c.Files = AnnotateFiles(c.Files) + + audio := c.AudioFiles() + + matched, titleFit := matchFiles(audio, req.Expected) + + // Write the alignment back so the picker can show which file maps + // to which track. + c.Files = mergeMatched(c.Files, matched) + + c.Match = scoreMatch(req, c, audio, titleFit) + c.Quality = scoreQuality(c, audio, priority) + + c.Score = weightMatch*c.Match.Overall + weightQuality*c.Quality.Overall + + return c +} + +// scoreMatch answers whether this candidate is the requested release. +func scoreMatch( + req Request, + c Candidate, + audio []CandidateFile, + titleFit float64, +) MatchScore { + m := MatchScore{ + Anchored: req.Anchored(), + TitleFit: titleFit, + } + + m.Completeness = completeness(len(audio), len(req.Expected)) + + // The candidate's own title, and the folder its files sit in, are + // two independent guesses at the album name. Take the better one: + // providers vary in which is meaningful. + folder := "" + if len(audio) > 0 { + folder = ParsePath(audio[0].Path).Folder + } + + m.AlbumFit = math.Max( + autotag.TitleSimilarity(req.Album, c.Title), + autotag.TitleSimilarity(req.Album, folder), + ) + + m.ArtistFit = artistFit(req.Artist, c) + + // With no expected tracklist there is no title signal at all, so + // redistribute its weight onto the album/artist evidence rather + // than scoring every free-text result as half-wrong. + if len(req.Expected) == 0 { + m.Overall = 0.55*m.AlbumFit + 0.45*m.ArtistFit + } else { + m.Overall = weightTitleFit*m.TitleFit + + weightCompleteness*m.Completeness + + weightAlbumFit*m.AlbumFit + + weightArtistFit*m.ArtistFit + } + + if !m.Anchored { + m.Overall = math.Min(m.Overall, unanchoredCap) + } + + return m +} + +// artistFit compares the requested artist against the candidate's +// artist field, its title, and the path of its first audio file, taking +// the best. Providers disagree about where the artist name lands. +func artistFit(want string, c Candidate) float64 { + if strings.TrimSpace(want) == "" { + return 0.5 + } + + best := autotag.TitleSimilarity(want, c.Artist) + + if s := autotag.TitleSimilarity(want, c.Title); s > best { + best = s + } + + // A path containing the artist name anywhere is weak but real + // evidence — most folders are "Artist - Album". + norm := autotag.Normalize(want) + if norm != "" { + for _, f := range c.Files { + if strings.Contains(autotag.Normalize(f.Path), norm) { + if best < 0.8 { + best = 0.8 + } + + break + } + } + } + + return best +} + +// completeness scores audio file count against the expected track +// count. Extra files are penalized far more gently than missing ones: +// a folder with bonus tracks or a stray intro is still the album, while +// a folder missing half the tracks is not. +func completeness(got, want int) float64 { + if want == 0 { + if got > 0 { + return 0.5 + } + + return 0 + } + + if got == 0 { + return 0 + } + + if got >= want { + extra := float64(got-want) / float64(want) + + return math.Max(0.75, 1.0-0.25*extra) + } + + return float64(got) / float64(want) +} + +// scoreQuality answers whether this is a good copy. +func scoreQuality( + c Candidate, + audio []CandidateFile, + priority int, +) QualityScore { + q := QualityScore{ + Health: clamp01(c.Health), + Priority: clamp01(float64(priority) / 100.0), + } + + if len(audio) == 0 { + return q + } + + // Format: score the worst file, not the average. A folder that is + // mostly FLAC with three MP3s transcoded in is a worse copy than + // its average suggests, and that is exactly what the user would + // want flagged. + worst := 1.0 + first := audio[0].Format + + for _, f := range audio { + if r := formatRank(f.Format); r < worst { + worst = r + } + + if f.Format != first { + q.Mixed = true + } + } + + q.FormatRank = worst + q.Bitrate = bitrateScore(audio) + + q.Overall = weightFormat*q.FormatRank + + weightBitrate*q.Bitrate + + weightHealth*q.Health + + weightPriority*q.Priority + + if q.Mixed { + q.Overall *= 0.9 + } + + return q +} + +// formatRank scores a format on its own terms, in 0..1. Lossless +// formats top out; lossy formats sit below and are further separated by +// bitrate. Formats the player cannot decode are penalized but not +// zeroed — the user may be acquiring them deliberately. +func formatRank(f Format) float64 { + base := 0.0 + + switch f { + case FormatFLAC: + base = 1.0 + case FormatALAC: + base = 0.95 + case FormatWAV: + base = 0.85 // lossless, but untaggable and huge + case FormatMP3: + base = 0.6 + case FormatAAC, FormatOpus: + base = 0.6 + case FormatOGG: + base = 0.55 + case FormatWMA: + base = 0.3 + case FormatUnknown: + base = 0.2 + default: + base = 0.2 + } + + if !f.Supported() && f != FormatUnknown { + base *= 0.8 + } + + return base +} + +// bitrateScore maps the mean stated bitrate of lossy files onto 0..1. +// Lossless files score 1.0 and are excluded from the mean. Returns a +// neutral 0.5 when nothing states a bitrate, which is the common case +// for Soulseek results. +func bitrateScore(audio []CandidateFile) float64 { + var ( + sum float64 + count int + ) + + for _, f := range audio { + if f.Format.Lossless() { + sum += 1.0 + count++ + + continue + } + + if f.Bitrate == 0 { + continue + } + + sum += lossyBitrateScore(f.Bitrate) + count++ + } + + if count == 0 { + return 0.5 + } + + return sum / float64(count) +} + +// lossyBitrateScore maps kbps onto 0..1 with the knee where it belongs +// perceptually: the gap between 128 and 192 matters much more than the +// gap between 256 and 320. +func lossyBitrateScore(kbps int) float64 { + switch { + case kbps >= 320: + return 1.0 + case kbps >= 256: + return 0.9 + case kbps >= 224: + return 0.82 + case kbps >= 192: + return 0.72 + case kbps >= 160: + return 0.55 + case kbps >= 128: + return 0.4 + case kbps >= 96: + return 0.2 + default: + return 0.1 + } +} + +// Rank scores every candidate and returns them best-first. Ties break +// on match, then on provider priority, then on file count, so the order +// is stable across runs rather than map-iteration dependent. +func Rank( + req Request, + candidates []Candidate, + priority func(providerID int64) int, +) []Candidate { + out := make([]Candidate, 0, len(candidates)) + + for _, c := range candidates { + p := 50 + if priority != nil { + p = priority(c.ProviderID) + } + + out = append(out, Score(req, c, p)) + } + + sort.SliceStable(out, func(i, j int) bool { + if out[i].Score != out[j].Score { + return out[i].Score > out[j].Score + } + + if out[i].Match.Overall != out[j].Match.Overall { + return out[i].Match.Overall > out[j].Match.Overall + } + + if out[i].Quality.Priority != out[j].Quality.Priority { + return out[i].Quality.Priority > out[j].Quality.Priority + } + + return len(out[i].Files) > len(out[j].Files) + }) + + return out +} + +// AutoPickable reports whether a ranked list has a clear enough winner +// to grab without asking. It demands an anchored request, a high match, +// decent quality, and daylight between first and second place — if two +// candidates are close, the choice is the user's. +func AutoPickable(req Request, ranked []Candidate) bool { + const ( + minMatch = 0.85 + minQuality = 0.5 + minLead = 0.08 + ) + + if !req.Anchored() || len(ranked) == 0 { + return false + } + + // An anchor with no tracklist behind it is an anchor in name only: + // the match score then rests on album and artist text alone, which + // is exactly the evidence a wrong-album candidate also has. This + // matters most for the wanted list, where nobody is watching. + if len(req.Expected) == 0 { + return false + } + + best := ranked[0] + if best.Match.Overall < minMatch || best.Quality.Overall < minQuality { + return false + } + + if len(ranked) > 1 && best.Score-ranked[1].Score < minLead { + return false + } + + return true +} + +// mergeMatched copies MatchedTo assignments from the audio-only slice +// back onto the full file list. +func mergeMatched(all, matched []CandidateFile) []CandidateFile { + if len(matched) == 0 { + return all + } + + byPath := make(map[string]int, len(matched)) + for _, m := range matched { + byPath[m.Path] = m.MatchedTo + } + + out := make([]CandidateFile, len(all)) + copy(out, all) + + for i := range out { + if pos, ok := byPath[out[i].Path]; ok { + out[i].MatchedTo = pos + } + } + + return out +} + +// clamp01 bounds a value to 0..1. +func clamp01(v float64) float64 { + return math.Max(0, math.Min(1, v)) +} diff --git a/backend/download/rank_test.go b/backend/download/rank_test.go new file mode 100644 index 0000000..639ef3c --- /dev/null +++ b/backend/download/rank_test.go @@ -0,0 +1,300 @@ +package download + +import "testing" + +// okComputer is the reference request used across ranking tests. +func okComputer() Request { + return Request{ + ReleaseMBID: "mbid-ok-computer", + Artist: "Radiohead", + Album: "OK Computer", + Expected: []ExpectedTrack{ + {Position: 1, Title: "Airbag"}, + {Position: 2, Title: "Paranoid Android"}, + {Position: 3, Title: "Subterranean Homesick Alien"}, + {Position: 4, Title: "Exit Music (For a Film)"}, + }, + } +} + +// candidateFor builds a candidate whose files follow "NN - Title.ext". +func candidateFor(id string, titles []string, ext string, size int64) Candidate { + files := make([]CandidateFile, 0, len(titles)) + + for i, tt := range titles { + files = append(files, CandidateFile{ + Path: "Radiohead - OK Computer/" + + trackToken(i+1) + " - " + tt + ext, + Size: size, + }) + } + + return Candidate{ + ID: id, + Protocol: ProtocolDirect, + Title: "Radiohead - OK Computer", + Artist: "Radiohead", + Files: files, + Health: 0.5, + } +} + +func allTitles() []string { + return []string{ + "Airbag", + "Paranoid Android", + "Subterranean Homesick Alien", + "Exit Music (For a Film)", + } +} + +// The headline behaviour: a well-matched FLAC beats a well-matched +// 128kbps MP3, but a mismatched FLAC loses to both. +func TestRankPrefersQualityAtEqualMatch(t *testing.T) { + t.Parallel() + + req := okComputer() + + flac := candidateFor("flac", allTitles(), ".flac", 30_000_000) + mp3 := candidateFor("mp3", allTitles(), ".mp3", 3_000_000) + + for i := range mp3.Files { + mp3.Files[i].Bitrate = 128 + } + + ranked := Rank(req, []Candidate{mp3, flac}, nil) + + if ranked[0].ID != "flac" { + t.Fatalf("winner = %s, want flac", ranked[0].ID) + } + + if ranked[0].Match.Overall < 0.9 { + t.Errorf("flac match = %f, want high", ranked[0].Match.Overall) + } + + if ranked[0].Quality.Overall <= ranked[1].Quality.Overall { + t.Errorf( + "flac quality %f should exceed mp3 %f", + ranked[0].Quality.Overall, ranked[1].Quality.Overall, + ) + } +} + +func TestRankMatchDominatesQuality(t *testing.T) { + t.Parallel() + + req := okComputer() + + // Right album, poor bitrate. + right := candidateFor("right", allTitles(), ".mp3", 2_000_000) + for i := range right.Files { + right.Files[i].Bitrate = 128 + } + + // Wrong album, pristine FLAC. + wrong := candidateFor("wrong", []string{ + "Enter Sandman", "Sad But True", "Holier Than Thou", "The Unforgiven", + }, ".flac", 30_000_000) + wrong.Title = "Metallica - Metallica" + wrong.Artist = "Metallica" + + for i := range wrong.Files { + wrong.Files[i].Path = "Metallica - Metallica/" + + trackToken(i+1) + " - x.flac" + } + + ranked := Rank(req, []Candidate{wrong, right}, nil) + + if ranked[0].ID != "right" { + t.Fatalf( + "winner = %s (score %f vs %f), want the correctly matched album", + ranked[0].ID, ranked[0].Score, ranked[1].Score, + ) + } +} + +func TestIncompleteCandidateScoresLower(t *testing.T) { + t.Parallel() + + req := okComputer() + + full := candidateFor("full", allTitles(), ".flac", 30_000_000) + partial := candidateFor("partial", allTitles()[:2], ".flac", 30_000_000) + + ranked := Rank(req, []Candidate{partial, full}, nil) + + if ranked[0].ID != "full" { + t.Fatalf("winner = %s, want full", ranked[0].ID) + } + + if ranked[1].Match.Completeness >= ranked[0].Match.Completeness { + t.Errorf( + "partial completeness %f should be below full %f", + ranked[1].Match.Completeness, ranked[0].Match.Completeness, + ) + } +} + +func TestMixedFormatIsPenalized(t *testing.T) { + t.Parallel() + + req := okComputer() + + clean := candidateFor("clean", allTitles(), ".flac", 30_000_000) + + mixed := candidateFor("mixed", allTitles(), ".flac", 30_000_000) + mixed.Files[2].Path = "Radiohead - OK Computer/03 - x.mp3" + mixed.Files[2].Format = FormatUnknown + + ranked := Rank(req, []Candidate{mixed, clean}, nil) + + var mixedScore QualityScore + + for _, c := range ranked { + if c.ID == "mixed" { + mixedScore = c.Quality + } + } + + if !mixedScore.Mixed { + t.Error("mixed-format candidate not flagged") + } + + if ranked[0].ID != "clean" { + t.Errorf("winner = %s, want clean", ranked[0].ID) + } +} + +// Without an MBID there is no tracklist to be right about, so the match +// score must not look confident regardless of how good the strings are. +func TestUnanchoredMatchIsCapped(t *testing.T) { + t.Parallel() + + req := Request{Artist: "Radiohead", Album: "OK Computer"} + c := candidateFor("c", allTitles(), ".flac", 30_000_000) + + scored := Score(req, c, 50) + + if scored.Match.Anchored { + t.Error("free-text request reported as anchored") + } + + if scored.Match.Overall > unanchoredCap { + t.Errorf( + "unanchored match = %f, want <= %f", + scored.Match.Overall, unanchoredCap, + ) + } +} + +func TestAutoPickableRequiresAnchorAndLead(t *testing.T) { + t.Parallel() + + req := okComputer() + best := Score(req, candidateFor("a", allTitles(), ".flac", 30_000_000), 50) + + t.Run("clear winner is auto-pickable", func(t *testing.T) { + t.Parallel() + + weak := Score( + req, + candidateFor("b", allTitles()[:2], ".mp3", 1_000_000), + 50, + ) + + if !AutoPickable(req, []Candidate{best, weak}) { + t.Errorf( + "want auto-pickable: match %f quality %f lead %f", + best.Match.Overall, best.Quality.Overall, best.Score-weak.Score, + ) + } + }) + + t.Run("two close candidates are not", func(t *testing.T) { + t.Parallel() + + twin := best + twin.ID = "twin" + + if AutoPickable(req, []Candidate{best, twin}) { + t.Error("identical candidates must not auto-pick") + } + }) + + t.Run("free text is never auto-pickable", func(t *testing.T) { + t.Parallel() + + free := Request{Artist: "Radiohead", Album: "OK Computer"} + + if AutoPickable(free, []Candidate{best}) { + t.Error("unanchored request must not auto-pick") + } + }) + + t.Run("empty list is not", func(t *testing.T) { + t.Parallel() + + if AutoPickable(req, nil) { + t.Error("empty candidate list must not auto-pick") + } + }) +} + +func TestCompleteness(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + got int + want int + minScore float64 + maxScore float64 + }{ + {"exact", 10, 10, 1.0, 1.0}, + {"half missing", 5, 10, 0.49, 0.51}, + {"one bonus track", 11, 10, 0.95, 1.0}, + {"double", 20, 10, 0.74, 0.76}, + {"nothing", 0, 10, 0, 0}, + {"no expectation", 5, 0, 0.5, 0.5}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := completeness(tt.got, tt.want) + if got < tt.minScore || got > tt.maxScore { + t.Errorf( + "completeness(%d, %d) = %f, want in [%f, %f]", + tt.got, tt.want, got, tt.minScore, tt.maxScore, + ) + } + }) + } +} + +func TestProviderPriorityBreaksTies(t *testing.T) { + t.Parallel() + + req := okComputer() + + a := candidateFor("a", allTitles(), ".flac", 30_000_000) + a.ProviderID = 1 + + b := candidateFor("b", allTitles(), ".flac", 30_000_000) + b.ProviderID = 2 + + priority := func(id int64) int { + if id == 2 { + return 90 + } + + return 10 + } + + ranked := Rank(req, []Candidate{a, b}, priority) + + if ranked[0].ID != "b" { + t.Errorf("winner = %s, want b (higher provider priority)", ranked[0].ID) + } +} diff --git a/backend/download/reconcile.go b/backend/download/reconcile.go new file mode 100644 index 0000000..e40a1ab --- /dev/null +++ b/backend/download/reconcile.go @@ -0,0 +1,746 @@ +package download + +import ( + "context" + "errors" + "fmt" + "log/slog" + "strconv" + "strings" + "sync" + "time" +) + +// The reconciler is the only thing that turns wants into downloads. +// +// It runs on a slow loop rather than reacting to events, because +// everything it cares about changes slowly: a release the user wants +// appears on a source days or weeks after they asked, an artist puts +// out an album once a year, and a library gains files by scan rather +// than by notification. A loop that wakes a few times a day is +// sufficient for all of it and costs nothing, where an event-driven +// design here would mean subscribing to three subsystems to learn the +// same facts later anyway. +// +// Each pass does four things, in this order and for this reason: +// +// 1. Expand artist subscriptions into per-album wants, so step 2 sees +// them this pass rather than next. +// 2. Retire wants the library already owns — including ones the user +// satisfied by other means, which is why ownership is checked +// rather than assumed from our own downloads. +// 3. Push the list to clients that keep their own (Lidarr), so the +// user's intent is expressed in both places. +// 4. Attempt a bounded batch of due wants. +// +// Nothing here fails a want. A want that cannot be found gets an +// attempt recorded and a longer backoff, and stays exactly as wanted as +// it was. + +// CatalogPort is what the reconciler needs to know about the world of +// music, kept narrow so the download package does not depend on the +// explore package (and so tests can answer these four questions from a +// map). +type CatalogPort interface { + // ReleaseGroupsForArtist returns an artist's discography. An empty + // result is not an error: the explore index fetches discographies + // lazily, so the honest answer is often "not yet". + ReleaseGroupsForArtist( + ctx context.Context, + artistMBID string, + ) ([]CatalogItem, error) + + // Tracklist resolves a release group or release to the tracks it + // should contain. This is what makes a want's download safe to + // complete unattended, so a want with no tracklist is never + // auto-grabbed. + Tracklist( + ctx context.Context, + entity Entity, + mbid string, + ) ([]ExpectedTrack, error) + + // Owns reports whether the library already has the thing an MBID + // names. + Owns(ctx context.Context, entity Entity, mbid string) (bool, error) + + // Describe fills in display text for a want the user added by MBID + // alone. Best-effort: an unknown MBID returns false and the want + // is still perfectly valid. + Describe( + ctx context.Context, + entity Entity, + mbid string, + ) (CatalogItem, bool) +} + +// CatalogItem is one thing the catalog knows about, in the download +// package's own terms. +type CatalogItem struct { + MBID string + Title string + Artist string + ArtistMBID string + + // PrimaryType is the MusicBrainz release-group type ("Album", + // "Single", "EP"). + PrimaryType string + + // SecondaryTypes carries "Compilation", "Live", "Remix" and + // friends. Their presence is what an artist want's default scope + // filters out. + SecondaryTypes []string + + // FirstReleaseDate is a MusicBrainz partial date: "1997", + // "1997-04", or "1997-04-22". + FirstReleaseDate string + + InLibrary bool +} + +// Reconciler defaults. +const ( + // defaultReconcileInterval is how often the wanted list is worked. + // Four times a day is far more often than new music appears and far + // less often than any provider would object to. + defaultReconcileInterval = 6 * time.Hour + + // startupDelay lets the app finish starting — library scan, explore + // index, provider construction — before the first pass. A wanted + // list worked against an index that has not loaded yet would record + // a pile of pointless attempts. + startupDelay = 3 * time.Minute + + // maxExpandPerArtist bounds how many child wants one artist + // subscription creates in a single pass, so switching an artist to + // full-discography scope does not enqueue four hundred albums at + // once. The remainder is picked up next pass. + maxExpandPerArtist = 40 +) + +// Reconciler works the wanted list. +type Reconciler struct { + logger *slog.Logger + store *Store + manager *Manager + catalog CatalogPort + + interval time.Duration + batch int + + // now is injectable so tests can drive backoff without waiting. + now func() time.Time + + // trigger is a nudge for an out-of-band pass, buffered to one + // because more than one pending "run now" is the same as one. + trigger chan struct{} + + // onChange fires after any pass that altered the list, so the UI + // can refresh without polling. + onChange func() + + stopOnce sync.Once + stop chan struct{} + + // runMu serializes passes: two reconcilers racing would search for + // the same want twice. + runMu sync.Mutex +} + +// NewReconciler builds a reconciler. catalog may be nil, in which case +// the wanted list still stores and lists wants but never acts on them — +// which is the right behaviour when the explore index is unavailable. +func NewReconciler( + logger *slog.Logger, + store *Store, + manager *Manager, + catalog CatalogPort, +) *Reconciler { + return &Reconciler{ + logger: logger, + store: store, + manager: manager, + catalog: catalog, + interval: defaultReconcileInterval, + batch: defaultDueBatch, + now: time.Now, + trigger: make(chan struct{}, 1), + stop: make(chan struct{}), + } +} + +// SetInterval overrides the pass interval. +func (r *Reconciler) SetInterval(d time.Duration) { + if d > 0 { + r.interval = d + } +} + +// SetBatch overrides how many wants one pass attempts. +func (r *Reconciler) SetBatch(n int) { + if n > 0 { + r.batch = n + } +} + +// SetOnChange registers a callback fired after a pass that changed the +// list. +func (r *Reconciler) SetOnChange(fn func()) { + r.onChange = fn +} + +// Start runs the reconcile loop until ctx is done or Stop is called. +func (r *Reconciler) Start(ctx context.Context) { + go r.loop(ctx) +} + +// Stop ends the loop. +func (r *Reconciler) Stop() { + r.stopOnce.Do(func() { close(r.stop) }) +} + +// Trigger asks for a pass as soon as possible without blocking the +// caller. Used when the user adds a want and expects something to +// happen. +func (r *Reconciler) Trigger() { + select { + case r.trigger <- struct{}{}: + default: + } +} + +// loop is the reconcile timer. +func (r *Reconciler) loop(ctx context.Context) { + first := time.NewTimer(startupDelay) + defer first.Stop() + + ticker := time.NewTicker(r.interval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-r.stop: + return + case <-first.C: + case <-ticker.C: + case <-r.trigger: + } + + if _, err := r.RunOnce(ctx); err != nil { + r.logger.Warn("wanted list reconcile failed", "error", err) + } + } +} + +// Summary reports what a pass did, for logging and for the UI. +type Summary struct { + // Expanded is how many child wants artist subscriptions produced. + Expanded int `json:"expanded"` + + // Satisfied is how many wants the library turned out to own. + Satisfied int `json:"satisfied"` + + // Attempted is how many wants were searched for. + Attempted int `json:"attempted"` + + // Started is how many of those found a clear enough winner to + // download unattended. + Started int `json:"started"` + + // Synced is how many wants were pushed to an external list. + Synced int `json:"synced"` +} + +// changed reports whether the pass altered anything worth refreshing +// the UI for. +func (s Summary) changed() bool { + return s.Expanded > 0 || s.Satisfied > 0 || s.Started > 0 +} + +// RunOnce works the wanted list once. It is safe to call directly, and +// the "search now" button does. +func (r *Reconciler) RunOnce(ctx context.Context) (Summary, error) { + r.runMu.Lock() + defer r.runMu.Unlock() + + var summary Summary + + if r.catalog == nil { + return summary, nil + } + + expanded, err := r.expandArtists(ctx) + if err != nil { + return summary, err + } + + summary.Expanded = expanded + + satisfied, err := r.retireOwned(ctx) + if err != nil { + return summary, err + } + + summary.Satisfied = satisfied + + summary.Synced = r.syncExternalLists(ctx) + + attempted, started, err := r.attemptDue(ctx) + if err != nil { + return summary, err + } + + summary.Attempted = attempted + summary.Started = started + + r.logger.Info( + "reconciled wanted list", + "expanded", summary.Expanded, + "satisfied", summary.Satisfied, + "attempted", summary.Attempted, + "started", summary.Started, + "synced", summary.Synced, + ) + + if summary.changed() && r.onChange != nil { + r.onChange() + } + + return summary, nil +} + +// --------------------------------------------------------------------------- +// Artist expansion +// --------------------------------------------------------------------------- + +// expandArtists turns artist subscriptions into per-album wants. +// +// The expansion is idempotent: child wants are upserted on (mbid, +// library), so re-running adds only what is genuinely new. That is +// what makes an artist want a standing subscription rather than a +// one-time queue-filling operation — an album released next year gets +// picked up by the same code path that ran today. +func (r *Reconciler) expandArtists(ctx context.Context) (int, error) { + artists, err := r.store.ListArtistWants(ctx) + if err != nil { + return 0, err + } + + created := 0 + + for _, artist := range artists { + n, err := r.expandArtist(ctx, artist) + if err != nil { + // One artist whose discography will not resolve must not + // stop the rest of the list. + r.logger.Warn( + "could not expand artist want", + "artist", artist.Label(), + "mbid", artist.MBID, + "error", err, + ) + + continue + } + + created += n + } + + return created, nil +} + +// expandArtist expands one subscription. +func (r *Reconciler) expandArtist(ctx context.Context, artist Want) (int, error) { + groups, err := r.catalog.ReleaseGroupsForArtist(ctx, artist.MBID) + if err != nil { + return 0, err + } + + created := 0 + + for _, rg := range groups { + if created >= maxExpandPerArtist { + break + } + + if !wantsReleaseGroup(artist, rg) { + continue + } + + // Existence is checked before inserting rather than relying on + // the upsert, because "how many albums are new this pass" is + // the number the UI reports and an upsert cannot tell an insert + // from a no-op. It also means a want the user pinned by hand + // is never quietly reparented under the artist. + if _, exists, err := r.store.FindWant( + ctx, rg.MBID, artist.LibraryID, + ); err != nil || exists { + continue + } + + credit := rg.Artist + if credit == "" { + credit = artist.Artist + } + + if _, err := r.store.AddWant(ctx, Want{ + MBID: rg.MBID, + Entity: EntityReleaseGroup, + LibraryID: artist.LibraryID, + Artist: credit, + Title: rg.Title, + ParentID: artist.ID, + }); err != nil { + r.logger.Warn( + "could not add derived want", + "release_group", rg.MBID, + "error", err, + ) + + continue + } + + created++ + } + + return created, nil +} + +// wantsReleaseGroup applies an artist subscription's filters to one +// release group. +func wantsReleaseGroup(artist Want, rg CatalogItem) bool { + if rg.MBID == "" || rg.InLibrary { + return false + } + + if !artist.Secondary && len(rg.SecondaryTypes) > 0 { + return false + } + + if artist.Scope == ScopeAll { + return true + } + + // ScopeFuture: only releases the artist put out after the user + // subscribed. A partial MusicBrainz date is compared as a string, + // which sorts correctly for ISO dates and treats a bare year as the + // first of January — the conservative reading, since a release + // dated only "2026" against a subscription made in March 2026 + // should not be assumed to be new. + return releaseDateAfter(rg.FirstReleaseDate, artist.CreatedAt) +} + +// releaseDateAfter compares a MusicBrainz partial date against a +// timestamp. An unknown or unparseable date is treated as not-after, +// because a release with no date is almost always an old one. +func releaseDateAfter(date string, since time.Time) bool { + date = strings.TrimSpace(date) + if date == "" { + return false + } + + // Pad a partial date to a full one so string comparison works: + // "1997" becomes "1997-01-01", "1997-04" becomes "1997-04-01". + switch len(date) { + case 4: + date += "-01-01" + case 7: //nolint:mnd // length of "YYYY-MM" + date += "-01" + } + + return date > since.Format(time.DateOnly) +} + +// --------------------------------------------------------------------------- +// Retiring what the library already has +// --------------------------------------------------------------------------- + +// retireOwned satisfies wants the library turns out to own. +// +// Ownership is asked of the library rather than inferred from our own +// completed downloads on purpose: the user may have bought the album, +// ripped their CD, or copied it in from another machine, and a wanted +// list that keeps hunting for music already sitting on disk is worse +// than no wanted list at all. +func (r *Reconciler) retireOwned(ctx context.Context) (int, error) { + wants, err := r.store.ListWants(ctx) + if err != nil { + return 0, err + } + + satisfied := 0 + + for _, w := range wants { + if w.State != WantStateWanted || w.Entity.Expands() { + continue + } + + owned, err := r.catalog.Owns(ctx, w.Entity, w.MBID) + if err != nil { + r.logger.Debug( + "ownership check failed", "want", w.MBID, "error", err, + ) + + continue + } + + if !owned { + continue + } + + if err := r.store.SatisfyWant(ctx, w.ID); err != nil { + r.logger.Warn("could not satisfy want", "want", w.ID, "error", err) + + continue + } + + satisfied++ + } + + return satisfied, nil +} + +// --------------------------------------------------------------------------- +// Attempting downloads +// --------------------------------------------------------------------------- + +// attemptDue searches for a bounded batch of due wants and grabs the +// ones with a clear winner. +func (r *Reconciler) attemptDue(ctx context.Context) (attempted, started int, err error) { + due, err := r.store.ListDueWants(ctx, r.batch) + if err != nil { + return 0, 0, err + } + + for _, w := range due { + select { + case <-ctx.Done(): + return attempted, started, nil + default: + } + + attempted++ + + ok, reason := r.attempt(ctx, w) + if ok { + started++ + + continue + } + + if err := r.store.RecordAttempt( + ctx, w.ID, w.Attempts, reason, + ); err != nil { + r.logger.Warn( + "could not record want attempt", "want", w.ID, "error", err, + ) + } + } + + return attempted, started, nil +} + +// tracklistFor resolves what a want should contain, which is the +// evidence an unattended download is checked against. +// +// A track want is its own tracklist: one entry, built from the title +// the want already carries. That single expected title is what lets +// filename matching score a track download at all — without it a +// request for one song would be scored as an album with no tracks and +// could never clear the auto-pick bar. +func (r *Reconciler) tracklistFor( + ctx context.Context, + w Want, +) ([]ExpectedTrack, error) { + if w.Entity != EntityRecording { + return r.catalog.Tracklist(ctx, w.Entity, w.MBID) + } + + if w.Title == "" { + return nil, nil + } + + return []ExpectedTrack{{ + Position: 1, + Title: w.Title, + Artist: w.Artist, + }}, nil +} + +// attempt tries one want. It returns false with a human-readable +// reason rather than an error, because none of the ways this does not +// work out are failures: no providers configured yet, nothing on any +// source, or nothing good enough to take without asking are all just +// "not today". +func (r *Reconciler) attempt(ctx context.Context, w Want) (bool, string) { + expected, err := r.tracklistFor(ctx, w) + if err != nil || len(expected) == 0 { + // Without a tracklist an unattended grab has nothing to verify + // itself against, so this want waits rather than guessing. The + // tracklist usually arrives on its own once the explore index + // fetches the release. + return false, "waiting for the tracklist to resolve" + } + + req := w.ToRequest(newID()) + req.Expected = expected + + if req.Artist == "" || req.Album == "" { + if item, ok := r.catalog.Describe(ctx, w.Entity, w.MBID); ok { + if req.Artist == "" { + req.Artist = item.Artist + } + + if req.Album == "" { + req.Album = item.Title + } + } + } + + started, reason, err := r.manager.Attempt(ctx, req) + if err != nil { + if errors.Is(err, ErrNoProviders) { + return false, "no download clients are enabled" + } + + if errors.Is(err, ErrNoCandidates) { + return false, "no source has it yet" + } + + return false, err.Error() + } + + return started, reason +} + +// --------------------------------------------------------------------------- +// External list sync +// --------------------------------------------------------------------------- + +// syncExternalLists pushes wants to providers that keep a persistent +// list of their own. +// +// The sync is one-directional by design. Two systems that both accept +// edits to the same list need conflict resolution, and the honest +// version of that here is "whichever the user touched last", which is +// not something we can observe. So this app's list is the source of +// truth and the external one is a projection of it — with the single +// exception of ImportExternal below, which the user runs deliberately. +func (r *Reconciler) syncExternalLists(ctx context.Context) int { + listers := r.manager.listers() + if len(listers) == 0 { + return 0 + } + + wants, err := r.store.ListWants(ctx) + if err != nil { + r.logger.Warn("could not list wants for sync", "error", err) + + return 0 + } + + synced := 0 + + for _, w := range wants { + if w.State != WantStateWanted { + continue + } + + external := w.ExternalIDs + if external == nil { + external = map[string]string{} + } + + changed := false + + for id, l := range listers { + key := strconv.FormatInt(id, 10) + if _, done := external[key]; done { + continue + } + + externalID, err := l.PushWant(ctx, w) + if err != nil { + r.logger.Debug( + "could not push want to external list", + "want", w.MBID, + "provider", id, + "error", err, + ) + + continue + } + + if externalID == "" { + continue + } + + external[key] = externalID + changed = true + synced++ + } + + if !changed { + continue + } + + if err := r.store.SetWantExternalIDs(ctx, w.ID, external); err != nil { + r.logger.Warn( + "could not record external want ids", "want", w.ID, "error", err, + ) + } + } + + return synced +} + +// ImportExternal pulls an external manager's own list into the wanted +// list. This is the one place data flows the other way, and it is a +// deliberate user action ("import my monitored Lidarr artists") rather +// than part of the loop, because silently adopting whatever another +// system is monitoring is not something to do behind the user's back. +func (r *Reconciler) ImportExternal( + ctx context.Context, + providerID int64, + libraryID int64, +) (int, error) { + listers := r.manager.listers() + + l, ok := listers[providerID] + if !ok { + return 0, fmt.Errorf( + "%w: provider %d keeps no list", ErrUnsupported, providerID, + ) + } + + external, err := l.ListWants(ctx) + if err != nil { + return 0, fmt.Errorf("list external wants: %w", err) + } + + imported := 0 + + for _, w := range external { + w.LibraryID = libraryID + + if _, err := r.store.AddWant(ctx, w); err != nil { + r.logger.Warn( + "could not import external want", "mbid", w.MBID, "error", err, + ) + + continue + } + + imported++ + } + + if imported > 0 && r.onChange != nil { + r.onChange() + } + + r.Trigger() + + return imported, nil +} diff --git a/backend/download/reconcile_test.go b/backend/download/reconcile_test.go new file mode 100644 index 0000000..0eb11e4 --- /dev/null +++ b/backend/download/reconcile_test.go @@ -0,0 +1,497 @@ +package download + +import ( + "context" + "errors" + "sync" + "testing" + "time" +) + +// fakeCatalog answers the reconciler's four questions from maps, so the +// wanted list can be tested without an explore index. +type fakeCatalog struct { + mu sync.Mutex + + // discographies maps artist MBID to release groups. + discographies map[string][]CatalogItem + + // tracklists maps an MBID to what it should contain. + tracklists map[string][]ExpectedTrack + + // owned is the set of MBIDs the library has. + owned map[string]bool + + // discographyErr is returned by ReleaseGroupsForArtist when set. + discographyErr error +} + +func newFakeCatalog() *fakeCatalog { + return &fakeCatalog{ + discographies: map[string][]CatalogItem{}, + tracklists: map[string][]ExpectedTrack{}, + owned: map[string]bool{}, + } +} + +func (c *fakeCatalog) ReleaseGroupsForArtist( + _ context.Context, + artistMBID string, +) ([]CatalogItem, error) { + c.mu.Lock() + defer c.mu.Unlock() + + if c.discographyErr != nil { + return nil, c.discographyErr + } + + return c.discographies[artistMBID], nil +} + +func (c *fakeCatalog) Tracklist( + _ context.Context, + _ Entity, + mbid string, +) ([]ExpectedTrack, error) { + c.mu.Lock() + defer c.mu.Unlock() + + return c.tracklists[mbid], nil +} + +func (c *fakeCatalog) Owns( + _ context.Context, + _ Entity, + mbid string, +) (bool, error) { + c.mu.Lock() + defer c.mu.Unlock() + + return c.owned[mbid], nil +} + +func (c *fakeCatalog) Describe( + _ context.Context, + _ Entity, + _ string, +) (CatalogItem, bool) { + return CatalogItem{}, false +} + +// reconcileFixture is a manager fixture plus a wanted list over it. +type reconcileFixture struct { + managerFixture + + catalog *fakeCatalog + reconciler *Reconciler +} + +func newReconcileFixture(t *testing.T) reconcileFixture { + t.Helper() + + mf := newManagerFixture(t) + cat := newFakeCatalog() + + r := NewReconciler(slogDiscard(), mf.store, mf.manager, cat) + + return reconcileFixture{managerFixture: mf, catalog: cat, reconciler: r} +} + +// An artist subscription becomes one want per album, and re-running +// adds nothing — which is what makes it a subscription rather than a +// one-time queue fill. +func TestExpandArtistIsIdempotent(t *testing.T) { + t.Parallel() + + f := newReconcileFixture(t) + ctx := context.Background() + + f.catalog.discographies["artist-1"] = []CatalogItem{ + {MBID: "rg-1", Title: "First", FirstReleaseDate: "2030-01-01"}, + {MBID: "rg-2", Title: "Second", FirstReleaseDate: "2030-06-01"}, + } + + if _, err := f.store.AddWant(ctx, Want{ + MBID: "artist-1", + Entity: EntityArtist, + LibraryID: 1, + Artist: "Radiohead", + Scope: ScopeAll, + }); err != nil { + t.Fatalf("AddWant: %v", err) + } + + first, err := f.reconciler.expandArtists(ctx) + if err != nil { + t.Fatalf("expandArtists: %v", err) + } + + if first != 2 { + t.Fatalf("first pass created %d wants, want 2", first) + } + + second, err := f.reconciler.expandArtists(ctx) + if err != nil { + t.Fatalf("expandArtists again: %v", err) + } + + if second != 0 { + t.Errorf("second pass created %d wants, want 0", second) + } + + // A new album appearing later is picked up by the same pass. + f.catalog.mu.Lock() + f.catalog.discographies["artist-1"] = append( + f.catalog.discographies["artist-1"], + CatalogItem{MBID: "rg-3", Title: "Third", FirstReleaseDate: "2031-01-01"}, + ) + f.catalog.mu.Unlock() + + third, err := f.reconciler.expandArtists(ctx) + if err != nil { + t.Fatalf("expandArtists third: %v", err) + } + + if third != 1 { + t.Errorf("third pass created %d wants, want 1", third) + } +} + +// A default artist subscription takes new releases only, so subscribing +// does not silently queue a back catalogue. +func TestExpandArtistFutureScopeSkipsBackCatalogue(t *testing.T) { + t.Parallel() + + f := newReconcileFixture(t) + ctx := context.Background() + + f.catalog.discographies["artist-1"] = []CatalogItem{ + {MBID: "rg-old", Title: "Old", FirstReleaseDate: "1997-04-22"}, + {MBID: "rg-new", Title: "New", FirstReleaseDate: "2099-01-01"}, + } + + if _, err := f.store.AddWant(ctx, Want{ + MBID: "artist-1", + Entity: EntityArtist, + LibraryID: 1, + Scope: ScopeFuture, + }); err != nil { + t.Fatalf("AddWant: %v", err) + } + + if _, err := f.reconciler.expandArtists(ctx); err != nil { + t.Fatalf("expandArtists: %v", err) + } + + wants, err := f.store.ListWants(ctx) + if err != nil { + t.Fatalf("ListWants: %v", err) + } + + for _, w := range wants { + if w.MBID == "rg-old" { + t.Error("future scope queued a back-catalogue album") + } + } + + if len(wants) != 2 { + t.Errorf("got %d wants (artist + new album), want 2", len(wants)) + } +} + +// One artist whose discography will not resolve must not stop the rest +// of the list being expanded. +func TestExpandArtistToleratesCatalogFailure(t *testing.T) { + t.Parallel() + + f := newReconcileFixture(t) + ctx := context.Background() + + f.catalog.discographyErr = errors.New("index not ready") //nolint:err113 // test + + if _, err := f.store.AddWant(ctx, Want{ + MBID: "artist-1", + Entity: EntityArtist, + LibraryID: 1, + }); err != nil { + t.Fatalf("AddWant: %v", err) + } + + created, err := f.reconciler.expandArtists(ctx) + if err != nil { + t.Fatalf("expandArtists returned an error for one bad artist: %v", err) + } + + if created != 0 { + t.Errorf("created %d wants from a failing catalog, want 0", created) + } +} + +// Something the library already owns is retired, however it got there. +func TestRetireOwnedSatisfiesWants(t *testing.T) { + t.Parallel() + + f := newReconcileFixture(t) + ctx := context.Background() + + id, err := f.store.AddWant(ctx, Want{ + MBID: "rg-1", + Entity: EntityReleaseGroup, + LibraryID: 1, + }) + if err != nil { + t.Fatalf("AddWant: %v", err) + } + + f.catalog.owned["rg-1"] = true + + n, err := f.reconciler.retireOwned(ctx) + if err != nil { + t.Fatalf("retireOwned: %v", err) + } + + if n != 1 { + t.Fatalf("retired %d wants, want 1", n) + } + + w, err := f.store.GetWant(ctx, id) + if err != nil { + t.Fatalf("GetWant: %v", err) + } + + if w.State != WantStateSatisfied { + t.Errorf("state = %q, want satisfied", w.State) + } +} + +// The end-to-end case: a want with a clear winner downloads without +// anyone watching, and is satisfied when the files land. +func TestReconcileDownloadsAndSatisfies(t *testing.T) { + t.Parallel() + + f := newReconcileFixture(t) + ctx := context.Background() + + provider := fakeWithAlbum(1, "source", ".flac") + f.manager.installProvider(Config{ID: 1, Priority: 50}, provider) + + id, err := f.store.AddWant(ctx, Want{ + MBID: "rg-1", + Entity: EntityReleaseGroup, + LibraryID: 1, + Artist: "Radiohead", + Title: "OK Computer", + }) + if err != nil { + t.Fatalf("AddWant: %v", err) + } + + f.catalog.tracklists["rg-1"] = fourTrackRequest().Expected + + summary, err := f.reconciler.RunOnce(ctx) + if err != nil { + t.Fatalf("RunOnce: %v", err) + } + + if summary.Attempted != 1 || summary.Started != 1 { + t.Fatalf( + "attempted=%d started=%d, want 1 and 1 (last error: see want)", + summary.Attempted, summary.Started, + ) + } + + waitFor(t, func() bool { + w, err := f.store.GetWant(ctx, id) + + return err == nil && w.State == WantStateSatisfied + }, "want was never satisfied after its download completed") + + if provider.GrabCalls != 1 { + t.Errorf("grab calls = %d, want 1", provider.GrabCalls) + } +} + +// Nothing good enough is not a failure. The want stays wanted, gains +// an attempt and a reason, and leaves no request row behind. +func TestReconcileKeepsWantingWhenNothingIsGoodEnough(t *testing.T) { + t.Parallel() + + f := newReconcileFixture(t) + ctx := context.Background() + + // A provider that finds only an unrelated album: enough to return + // candidates, nowhere near enough to auto-pick. + provider := NewFakeProvider(1, "weak", Caps{CanSearch: true, CanTransport: true}) + provider.Candidates = []Candidate{candidateFor( + "weak-1", []string{"Something Else Entirely"}, ".mp3", 3_000_000, + )} + + f.manager.installProvider(Config{ID: 1, Priority: 50}, provider) + + id, err := f.store.AddWant(ctx, Want{ + MBID: "rg-1", + Entity: EntityReleaseGroup, + LibraryID: 1, + Artist: "Radiohead", + Title: "OK Computer", + }) + if err != nil { + t.Fatalf("AddWant: %v", err) + } + + f.catalog.tracklists["rg-1"] = fourTrackRequest().Expected + + summary, err := f.reconciler.RunOnce(ctx) + if err != nil { + t.Fatalf("RunOnce: %v", err) + } + + if summary.Started != 0 { + t.Fatalf("started %d downloads, want 0", summary.Started) + } + + w, err := f.store.GetWant(ctx, id) + if err != nil { + t.Fatalf("GetWant: %v", err) + } + + if w.State != WantStateWanted { + t.Errorf("state = %q, want it still wanted", w.State) + } + + if w.Attempts != 1 { + t.Errorf("attempts = %d, want 1", w.Attempts) + } + + if w.LastError == "" { + t.Error("no reason was recorded for the user") + } + + if !w.NextTryAt.After(time.Now()) { + t.Errorf("next try at %v, want it in the future", w.NextTryAt) + } + + // The whole point of Attempt over Start: an unsuccessful pass + // leaves no request row to clutter the downloads list. + requests, err := f.store.ListRequests(ctx, 50) + if err != nil { + t.Fatalf("ListRequests: %v", err) + } + + if len(requests) != 0 { + t.Errorf("got %d request rows from a fruitless pass, want 0", len(requests)) + } +} + +// A want with no resolvable tracklist waits rather than guessing. +func TestReconcileWaitsWithoutTracklist(t *testing.T) { + t.Parallel() + + f := newReconcileFixture(t) + ctx := context.Background() + + provider := fakeWithAlbum(1, "source", ".flac") + f.manager.installProvider(Config{ID: 1, Priority: 50}, provider) + + if _, err := f.store.AddWant(ctx, Want{ + MBID: "rg-1", + Entity: EntityReleaseGroup, + LibraryID: 1, + Artist: "Radiohead", + Title: "OK Computer", + }); err != nil { + t.Fatalf("AddWant: %v", err) + } + + summary, err := f.reconciler.RunOnce(ctx) + if err != nil { + t.Fatalf("RunOnce: %v", err) + } + + if summary.Started != 0 { + t.Errorf("started %d downloads with no tracklist, want 0", summary.Started) + } + + if provider.SearchCalls != 0 { + t.Errorf( + "searched %d times with no tracklist to verify against, want 0", + provider.SearchCalls, + ) + } +} + +// Artist subscriptions are never attempted as downloads: they expand. +func TestArtistWantsAreNeverDue(t *testing.T) { + t.Parallel() + + f := newReconcileFixture(t) + ctx := context.Background() + + if _, err := f.store.AddWant(ctx, Want{ + MBID: "artist-1", + Entity: EntityArtist, + LibraryID: 1, + }); err != nil { + t.Fatalf("AddWant: %v", err) + } + + due, err := f.store.ListDueWants(ctx, 10) + if err != nil { + t.Fatalf("ListDueWants: %v", err) + } + + if len(due) != 0 { + t.Errorf("got %d due wants, want 0 — artists expand, not download", len(due)) + } +} + +// A pass attempts at most its batch size, so a large list is worked +// through steadily rather than in one flood. +func TestReconcileRespectsBatchSize(t *testing.T) { + t.Parallel() + + f := newReconcileFixture(t) + ctx := context.Background() + + f.reconciler.SetBatch(2) + + for _, mbid := range []string{"rg-1", "rg-2", "rg-3", "rg-4"} { + if _, err := f.store.AddWant(ctx, Want{ + MBID: mbid, + Entity: EntityReleaseGroup, + LibraryID: 1, + Title: "Album " + mbid, + }); err != nil { + t.Fatalf("AddWant: %v", err) + } + + f.catalog.tracklists[mbid] = fourTrackRequest().Expected + } + + summary, err := f.reconciler.RunOnce(ctx) + if err != nil { + t.Fatalf("RunOnce: %v", err) + } + + if summary.Attempted != 2 { + t.Errorf("attempted %d wants, want 2 (the batch size)", summary.Attempted) + } +} + +// waitFor polls a condition, failing the test if it never holds. Used +// where the pipeline hands work to a goroutine. +func waitFor(t *testing.T, cond func() bool, msg string) { + t.Helper() + + deadline := time.Now().Add(5 * time.Second) + + for time.Now().Before(deadline) { + if cond() { + return + } + + time.Sleep(10 * time.Millisecond) + } + + t.Fatal(msg) +} diff --git a/backend/download/secrets.go b/backend/download/secrets.go new file mode 100644 index 0000000..7af25d2 --- /dev/null +++ b/backend/download/secrets.go @@ -0,0 +1,269 @@ +package download + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "sync" + + "yellowjacket/backend/system" +) + +// Provider credentials — slskd API keys, Lidarr tokens, qBittorrent +// passwords — must not sit in the TOML config, which is world-readable +// by default and gets pasted into bug reports. +// +// They live in a separate 0600 JSON file in the user data directory. +// That is deliberately not encryption: a key stored beside the data it +// unlocks protects nothing, and pretending otherwise is worse than +// being clear about it. What the file mode does buy is protection from +// other local users and from the config file being shared casually. +// +// An OS keyring backend (libsecret / Keychain / DPAPI) is the right +// long-term answer and the Store interface exists so it can be added +// without touching any provider. + +// ErrSecretNotFound is returned when a named secret has never been set. +var ErrSecretNotFound = errors.New("secret not found") + +// secretsFileName is the store's file inside the user data directory. +const secretsFileName = "download-secrets.json" + +// secretsFileMode is owner read/write only. +const secretsFileMode = 0o600 + +// SecretStore holds provider credentials. +type SecretStore interface { + // Get returns the secret for a provider's named field. + Get(providerID int64, name string) (string, error) + + // Set stores a secret. An empty value deletes it. + Set(providerID int64, name, value string) error + + // DeleteProvider removes every secret belonging to a provider. + DeleteProvider(providerID int64) error +} + +// fileSecretStore is the default SecretStore: a 0600 JSON file. +type fileSecretStore struct { + path string + + mu sync.RWMutex + loaded bool + data map[string]string +} + +// NewFileSecretStore returns a SecretStore backed by a 0600 file in the +// user data directory. +func NewFileSecretStore() (SecretStore, error) { + dir, err := system.GetUserDataDirPath() + if err != nil { + return nil, fmt.Errorf("resolve user data dir: %w", err) + } + + return &fileSecretStore{ + path: filepath.Join(dir, secretsFileName), + data: map[string]string{}, + }, nil +} + +// NewFileSecretStoreAt returns a store backed by an explicit path. +// Tests use this; production goes through NewFileSecretStore. +func NewFileSecretStoreAt(path string) SecretStore { + return &fileSecretStore{path: path, data: map[string]string{}} +} + +// secretKey namespaces a secret by provider so two slskd instances do +// not share credentials. +func secretKey(providerID int64, name string) string { + return strconv.FormatInt(providerID, 10) + ":" + name +} + +// load reads the file once. A missing file is an empty store, not an +// error — nothing has been configured yet. +func (s *fileSecretStore) load() error { + if s.loaded { + return nil + } + + raw, err := os.ReadFile(s.path) + if err != nil { + if os.IsNotExist(err) { + s.data = map[string]string{} + s.loaded = true + + return nil + } + + return fmt.Errorf("read secrets file: %w", err) + } + + data := map[string]string{} + if err := json.Unmarshal(raw, &data); err != nil { + return fmt.Errorf("parse secrets file: %w", err) + } + + s.data = data + s.loaded = true + + return nil +} + +// save writes the file atomically with restrictive permissions. +func (s *fileSecretStore) save() error { + raw, err := json.Marshal(s.data) + if err != nil { + return fmt.Errorf("encode secrets: %w", err) + } + + if err := os.MkdirAll(filepath.Dir(s.path), 0o700); err != nil { + return fmt.Errorf("create secrets dir: %w", err) + } + + // Write to a temp file in the same directory, chmod before the + // rename, so the secret is never briefly world-readable. + tmp := s.path + ".tmp" + + if err := os.WriteFile(tmp, raw, secretsFileMode); err != nil { + return fmt.Errorf("write secrets file: %w", err) + } + + if err := os.Chmod(tmp, secretsFileMode); err != nil { + _ = os.Remove(tmp) + + return fmt.Errorf("chmod secrets file: %w", err) + } + + if err := os.Rename(tmp, s.path); err != nil { + _ = os.Remove(tmp) + + return fmt.Errorf("replace secrets file: %w", err) + } + + return nil +} + +// Get returns a stored secret. +func (s *fileSecretStore) Get(providerID int64, name string) (string, error) { + s.mu.Lock() + defer s.mu.Unlock() + + if err := s.load(); err != nil { + return "", err + } + + v, ok := s.data[secretKey(providerID, name)] + if !ok { + return "", fmt.Errorf("%w: %s", ErrSecretNotFound, name) + } + + return v, nil +} + +// Set stores or clears a secret. +func (s *fileSecretStore) Set(providerID int64, name, value string) error { + s.mu.Lock() + defer s.mu.Unlock() + + if err := s.load(); err != nil { + return err + } + + key := secretKey(providerID, name) + + if value == "" { + delete(s.data, key) + } else { + s.data[key] = value + } + + return s.save() +} + +// DeleteProvider drops every secret for a provider. +func (s *fileSecretStore) DeleteProvider(providerID int64) error { + s.mu.Lock() + defer s.mu.Unlock() + + if err := s.load(); err != nil { + return err + } + + prefix := strconv.FormatInt(providerID, 10) + ":" + + for k := range s.data { + if len(k) > len(prefix) && k[:len(prefix)] == prefix { + delete(s.data, k) + } + } + + return s.save() +} + +// lookupFor binds a store to one provider, producing the SecretLookup +// handed to constructors. +func lookupFor(store SecretStore, providerID int64) SecretLookup { + return func(name string) (string, error) { + if store == nil { + return "", fmt.Errorf("%w: %s", ErrSecretNotFound, name) + } + + return store.Get(providerID, name) + } +} + +// memSecretStore is an in-memory SecretStore for tests. +type memSecretStore struct { + mu sync.RWMutex + data map[string]string +} + +// NewMemSecretStore returns an in-memory SecretStore. +func NewMemSecretStore() SecretStore { + return &memSecretStore{data: map[string]string{}} +} + +func (s *memSecretStore) Get(providerID int64, name string) (string, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + v, ok := s.data[secretKey(providerID, name)] + if !ok { + return "", fmt.Errorf("%w: %s", ErrSecretNotFound, name) + } + + return v, nil +} + +func (s *memSecretStore) Set(providerID int64, name, value string) error { + s.mu.Lock() + defer s.mu.Unlock() + + if value == "" { + delete(s.data, secretKey(providerID, name)) + + return nil + } + + s.data[secretKey(providerID, name)] = value + + return nil +} + +func (s *memSecretStore) DeleteProvider(providerID int64) error { + s.mu.Lock() + defer s.mu.Unlock() + + prefix := strconv.FormatInt(providerID, 10) + ":" + + for k := range s.data { + if len(k) > len(prefix) && k[:len(prefix)] == prefix { + delete(s.data, k) + } + } + + return nil +} diff --git a/backend/download/secrets_test.go b/backend/download/secrets_test.go new file mode 100644 index 0000000..63a1e8a --- /dev/null +++ b/backend/download/secrets_test.go @@ -0,0 +1,142 @@ +package download + +import ( + "errors" + "os" + "path/filepath" + "testing" +) + +func TestFileSecretStoreRoundTrip(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "secrets.json") + store := NewFileSecretStoreAt(path) + + if err := store.Set(1, "apiKey", "hunter2"); err != nil { + t.Fatalf("Set: %v", err) + } + + got, err := store.Get(1, "apiKey") + if err != nil { + t.Fatalf("Get: %v", err) + } + + if got != "hunter2" { + t.Errorf("Get = %q, want hunter2", got) + } + + // A fresh store over the same file must see the value. + if got, err := NewFileSecretStoreAt(path).Get(1, "apiKey"); err != nil || + got != "hunter2" { + t.Errorf("reload got %q, err %v; want hunter2", got, err) + } +} + +// Credentials must not be readable by other local users. +func TestFileSecretStorePermissions(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "secrets.json") + store := NewFileSecretStoreAt(path) + + if err := store.Set(1, "apiKey", "hunter2"); err != nil { + t.Fatalf("Set: %v", err) + } + + info, err := os.Stat(path) + if err != nil { + t.Fatalf("Stat: %v", err) + } + + if perm := info.Mode().Perm(); perm != secretsFileMode { + t.Errorf("mode = %o, want %o", perm, secretsFileMode) + } +} + +// Two configured instances of the same provider kind must not share +// credentials. +func TestFileSecretStoreNamespacesByProvider(t *testing.T) { + t.Parallel() + + store := NewFileSecretStoreAt(filepath.Join(t.TempDir(), "secrets.json")) + + if err := store.Set(1, "apiKey", "first"); err != nil { + t.Fatalf("Set: %v", err) + } + + if err := store.Set(2, "apiKey", "second"); err != nil { + t.Fatalf("Set: %v", err) + } + + for id, want := range map[int64]string{1: "first", 2: "second"} { + got, err := store.Get(id, "apiKey") + if err != nil { + t.Fatalf("Get(%d): %v", id, err) + } + + if got != want { + t.Errorf("Get(%d) = %q, want %q", id, got, want) + } + } +} + +func TestFileSecretStoreDeleteProvider(t *testing.T) { + t.Parallel() + + store := NewFileSecretStoreAt(filepath.Join(t.TempDir(), "secrets.json")) + + if err := store.Set(1, "apiKey", "a"); err != nil { + t.Fatalf("Set: %v", err) + } + + if err := store.Set(1, "password", "b"); err != nil { + t.Fatalf("Set: %v", err) + } + + if err := store.Set(2, "apiKey", "keep"); err != nil { + t.Fatalf("Set: %v", err) + } + + if err := store.DeleteProvider(1); err != nil { + t.Fatalf("DeleteProvider: %v", err) + } + + for _, name := range []string{"apiKey", "password"} { + if _, err := store.Get(1, name); !errors.Is(err, ErrSecretNotFound) { + t.Errorf("Get(1, %q) error = %v, want ErrSecretNotFound", name, err) + } + } + + if got, err := store.Get(2, "apiKey"); err != nil || got != "keep" { + t.Errorf("other provider's secret was removed: %q, %v", got, err) + } +} + +func TestFileSecretStoreMissingFileIsEmpty(t *testing.T) { + t.Parallel() + + store := NewFileSecretStoreAt(filepath.Join(t.TempDir(), "nope.json")) + + if _, err := store.Get(1, "apiKey"); !errors.Is(err, ErrSecretNotFound) { + t.Errorf("error = %v, want ErrSecretNotFound", err) + } +} + +func TestSetEmptyValueDeletes(t *testing.T) { + t.Parallel() + + store := NewFileSecretStoreAt(filepath.Join(t.TempDir(), "secrets.json")) + + if err := store.Set(1, "apiKey", "x"); err != nil { + t.Fatalf("Set: %v", err) + } + + if err := store.Set(1, "apiKey", ""); err != nil { + t.Fatalf("Set empty: %v", err) + } + + if _, err := store.Get(1, "apiKey"); !errors.Is(err, ErrSecretNotFound) { + t.Errorf("error = %v, want ErrSecretNotFound", err) + } +} diff --git a/backend/download/service.go b/backend/download/service.go new file mode 100644 index 0000000..ff89344 --- /dev/null +++ b/backend/download/service.go @@ -0,0 +1,598 @@ +package download + +import ( + "context" + "fmt" + "log/slog" + "strconv" + + "github.com/wailsapp/wails/v2/pkg/runtime" + + "yellowjacket/backend/events" +) + +// Service is the frontend-facing surface of the download subsystem. +// Its methods are bound into Wails and called from TypeScript, so +// signatures use plain types and return errors the UI can render. +type Service struct { + logger *slog.Logger + manager *Manager + store *Store + secrets SecretStore + + // reconciler works the wanted list. Optional; nil means wants are + // stored but never acted on. + reconciler *Reconciler + + ctx context.Context +} + +// NewService builds the bound service. +func NewService( + logger *slog.Logger, + manager *Manager, + store *Store, + secrets SecretStore, +) *Service { + return &Service{ + logger: logger, + manager: manager, + store: store, + secrets: secrets, + } +} + +// SetContext injects the Wails runtime context for event emission. +func (s *Service) SetContext(ctx context.Context) { + s.ctx = ctx +} + +// emit publishes an event, tolerating a service that has no runtime +// context yet. Emitting on a non-runtime context is fatal in Wails, so +// the nil check is load-bearing rather than defensive. +func (s *Service) emit(name string, data ...any) { + if s.ctx == nil { + return + } + + runtime.EventsEmit(s.ctx, name, data...) +} + +// --------------------------------------------------------------------------- +// Provider configuration +// --------------------------------------------------------------------------- + +// ProviderKinds returns every provider type that can be added, with the +// settings each one needs. The settings page renders its forms from +// this, so a new adapter needs no frontend change. +func (s *Service) ProviderKinds() []Descriptor { + return Descriptors() +} + +// ListProviders returns the user's configured download clients. +func (s *Service) ListProviders() ([]Config, error) { + return s.store.ListProviders(context.Background()) +} + +// AddProvider creates a provider and stores any secret settings +// separately. Secrets arrive in the same map as ordinary settings +// because that is what the form submits; they are split out here and +// never written to the provider row. +func (s *Service) AddProvider( + kind string, + name string, + settings map[string]string, +) (int64, error) { + desc, ok := DescriptorFor(Kind(kind)) + if !ok { + return 0, fmt.Errorf("%w: %s", ErrUnknownKind, kind) + } + + plain, secret := splitSecrets(desc, settings) + + id, err := s.store.CreateProvider(context.Background(), Config{ + Kind: Kind(kind), + Name: name, + Enabled: true, + Priority: 50, + Settings: plain, + }) + if err != nil { + return 0, err + } + + for k, v := range secret { + if err := s.secrets.Set(id, k, v); err != nil { + return 0, err + } + } + + if err := s.manager.Reload(context.Background()); err != nil { + return 0, err + } + + s.emit(events.DownloadProvidersChanged) + + return id, nil +} + +// UpdateProvider saves changes to a provider. A secret field left +// blank keeps its stored value rather than clearing it — the form does +// not echo secrets back, so an empty box means "unchanged", not +// "delete". +func (s *Service) UpdateProvider( + id int64, + name string, + enabled bool, + priority int, + settings map[string]string, +) error { + ctx := context.Background() + + existing, err := s.store.GetProvider(ctx, id) + if err != nil { + return err + } + + desc, ok := DescriptorFor(existing.Kind) + if !ok { + return fmt.Errorf("%w: %s", ErrUnknownKind, existing.Kind) + } + + plain, secret := splitSecrets(desc, settings) + + if err := s.store.UpdateProvider(ctx, Config{ + ID: id, + Kind: existing.Kind, + Name: name, + Enabled: enabled, + Priority: priority, + Settings: plain, + }); err != nil { + return err + } + + for k, v := range secret { + if v == "" { + continue + } + + if err := s.secrets.Set(id, k, v); err != nil { + return err + } + } + + if err := s.manager.Reload(ctx); err != nil { + return err + } + + s.emit(events.DownloadProvidersChanged) + + return nil +} + +// DeleteProvider removes a provider and its credentials. +func (s *Service) DeleteProvider(id int64) error { + ctx := context.Background() + + if err := s.store.DeleteProvider(ctx, id); err != nil { + return err + } + + if err := s.secrets.DeleteProvider(id); err != nil { + s.logger.Warn( + "could not delete provider secrets", "provider", id, "error", err, + ) + } + + if err := s.manager.Reload(ctx); err != nil { + return err + } + + s.emit(events.DownloadProvidersChanged) + + return nil +} + +// TestProvider backs the "test connection" button. It builds the +// provider from its stored config and asks it to check itself, so the +// result reflects exactly what a real search would use. +func (s *Service) TestProvider(id int64) error { + ctx := context.Background() + + cfg, err := s.store.GetProvider(ctx, id) + if err != nil { + return err + } + + p, err := New(cfg, lookupFor(s.secrets, id), s.logger) + if err != nil { + return err + } + + defer func() { _ = p.Close() }() + + if err := p.Check(ctx); err != nil { + return fmt.Errorf("%s: %w", cfg.Name, err) + } + + return nil +} + +// --------------------------------------------------------------------------- +// Requests +// --------------------------------------------------------------------------- + +// SearchRequest is what the frontend submits to start a download. +type SearchRequest struct { + LibraryID int64 `json:"libraryId"` + ReleaseMBID string `json:"releaseMbid"` + ReleaseGroupMBID string `json:"releaseGroupMbid"` + Artist string `json:"artist"` + Album string `json:"album"` + Query string `json:"query"` + Expected []ExpectedTrack `json:"expected"` +} + +// StartResult is what the picker needs after a search. +type StartResult struct { + RequestID string `json:"requestId"` + Candidates []Candidate `json:"candidates"` + + // AutoPicked reports that the pipeline already chose and is + // downloading, so the picker should show progress rather than a + // list of choices. + AutoPicked bool `json:"autoPicked"` +} + +// Start searches for a release and either auto-picks a clear winner or +// returns ranked candidates for the user to choose from. +func (s *Service) Start(req SearchRequest) (StartResult, error) { + r := Request{ + ID: newID(), + LibraryID: req.LibraryID, + ReleaseMBID: req.ReleaseMBID, + ReleaseGroupMBID: req.ReleaseGroupMBID, + Artist: req.Artist, + Album: req.Album, + Query: req.Query, + Expected: req.Expected, + } + + candidates, err := s.manager.Start(context.Background(), r) + if err != nil { + return StartResult{}, err + } + + result := StartResult{ + RequestID: r.ID, + Candidates: candidates, + AutoPicked: AutoPickable(r, candidates), + } + + s.emit(events.DownloadsChanged) + + return result, nil +} + +// Pick starts the transfer for the candidate the user chose. +func (s *Service) Pick(requestID, candidateID string) error { + if err := s.manager.Pick( + context.Background(), requestID, candidateID, + ); err != nil { + return err + } + + s.emit(events.DownloadsChanged) + + return nil +} + +// Cancel aborts a live request. +func (s *Service) Cancel(requestID string) error { + if err := s.manager.Cancel(context.Background(), requestID); err != nil { + return err + } + + s.emit(events.DownloadsChanged) + + return nil +} + +// Candidates returns the ranked candidates of a live request, so the +// picker can be reopened without searching again. +func (s *Service) Candidates(requestID string) []Candidate { + return s.manager.Candidates(requestID) +} + +// RequestView is one row of the downloads list. +type RequestView struct { + Request + + State State `json:"state"` + Error string `json:"error,omitempty"` + Items []Item `json:"items"` +} + +// ListRequests returns recent download requests, newest first. +func (s *Service) ListRequests(limit int) ([]RequestView, error) { + const defaultLimit = 50 + + if limit <= 0 { + limit = defaultLimit + } + + ctx := context.Background() + + requests, err := s.store.ListRequests(ctx, limit) + if err != nil { + return nil, err + } + + out := make([]RequestView, 0, len(requests)) + + for _, r := range requests { + state, errText, err := s.store.GetRequestState(ctx, r.ID) + if err != nil { + return nil, err + } + + items, err := s.store.ListItemsForRequest(ctx, r.ID) + if err != nil { + return nil, err + } + + out = append(out, RequestView{ + Request: r, + State: state, + Error: errText, + Items: items, + }) + } + + return out, nil +} + +// ClearFinished removes terminal requests from the list. +func (s *Service) ClearFinished() error { + if err := s.store.ClearFinished(context.Background()); err != nil { + return err + } + + s.emit(events.DownloadsChanged) + + return nil +} + +// --------------------------------------------------------------------------- +// Wanted list +// --------------------------------------------------------------------------- + +// SetReconciler wires the wanted-list loop. Optional: without it the +// wanted list still stores and lists wants, it just never acts on them. +func (s *Service) SetReconciler(r *Reconciler) { + s.reconciler = r +} + +// WantRequest is what the frontend submits to want something. It is +// one MBID and the type of thing it names, because that is genuinely +// all a want is. +type WantRequest struct { + MBID string `json:"mbid"` + Entity string `json:"entity"` + LibraryID int64 `json:"libraryId"` + + // Artist and Title are display text only, and optional: the + // reconciler fills them in from the catalog when the caller has + // nothing but an MBID. + Artist string `json:"artist"` + Title string `json:"title"` + + // Scope and Secondary apply to artist wants. + Scope string `json:"scope"` + Secondary bool `json:"secondary"` +} + +// AddWant puts something on the wanted list and asks for a reconcile +// pass, so the user sees something happen rather than waiting six hours +// for the next scheduled one. +func (s *Service) AddWant(req WantRequest) (int64, error) { + entity := Entity(req.Entity) + if !entity.Valid() { + return 0, fmt.Errorf("%w: entity %q", ErrUnsupported, req.Entity) + } + + scope := WantScope(req.Scope) + if scope != ScopeAll { + scope = ScopeFuture + } + + id, err := s.store.AddWant(context.Background(), Want{ + MBID: req.MBID, + Entity: entity, + LibraryID: req.LibraryID, + Artist: req.Artist, + Title: req.Title, + Scope: scope, + Secondary: req.Secondary, + }) + if err != nil { + return 0, err + } + + s.emit(events.WantedListChanged) + + if s.reconciler != nil { + s.reconciler.Trigger() + } + + return id, nil +} + +// ListWants returns the whole wanted list. +func (s *Service) ListWants() ([]Want, error) { + return s.store.ListWants(context.Background()) +} + +// IsWanted answers the Explore pages' question — should this album show +// "want" or "wanted?" — without making them load the whole list. +func (s *Service) IsWanted(mbid string, libraryID int64) (bool, error) { + _, found, err := s.store.FindWant(context.Background(), mbid, libraryID) + + return found, err +} + +// RemoveWant takes something off the list. Removing an artist takes +// its derived albums with it, by cascade; an album the user pinned +// themselves has no parent and survives. +func (s *Service) RemoveWant(id int64) error { + ctx := context.Background() + + // Tell any external list first, while the row is still readable. + s.withdrawExternal(ctx, id) + + if err := s.store.DeleteWant(ctx, id); err != nil { + return err + } + + s.emit(events.WantedListChanged) + + return nil +} + +// PauseWant stops attempts without forgetting the want. +func (s *Service) PauseWant(id int64, paused bool) error { + state := WantStateWanted + if paused { + state = WantStatePaused + } + + if err := s.store.SetWantState( + context.Background(), id, state, "", + ); err != nil { + return err + } + + s.emit(events.WantedListChanged) + + return nil +} + +// ClearSatisfiedWants drops everything already owned. +func (s *Service) ClearSatisfiedWants() error { + if err := s.store.ClearSatisfiedWants(context.Background()); err != nil { + return err + } + + s.emit(events.WantedListChanged) + + return nil +} + +// ReconcileWanted runs a pass now and reports what it did. This backs +// the "check now" button, so it runs synchronously: the user pressed it +// and is waiting for an answer. +func (s *Service) ReconcileWanted() (Summary, error) { + if s.reconciler == nil { + return Summary{}, fmt.Errorf( + "%w: the wanted list is not running", ErrUnsupported, + ) + } + + summary, err := s.reconciler.RunOnce(context.Background()) + if err != nil { + return summary, err + } + + s.emit(events.WantedListChanged) + + return summary, nil +} + +// ImportExternalWants adopts a provider's own list — "import the +// artists Lidarr is already monitoring". +func (s *Service) ImportExternalWants( + providerID int64, + libraryID int64, +) (int, error) { + if s.reconciler == nil { + return 0, fmt.Errorf( + "%w: the wanted list is not running", ErrUnsupported, + ) + } + + n, err := s.reconciler.ImportExternal( + context.Background(), providerID, libraryID, + ) + if err != nil { + return 0, err + } + + s.emit(events.WantedListChanged) + + return n, nil +} + +// withdrawExternal best-effort unmonitors a want in the external lists +// it was pushed to. Failures are logged and ignored: the user asked to +// remove it from *this* list, and an unreachable Lidarr is not a reason +// to refuse. +func (s *Service) withdrawExternal(ctx context.Context, id int64) { + w, err := s.store.GetWant(ctx, id) + if err != nil || len(w.ExternalIDs) == 0 { + return + } + + for key, externalID := range w.ExternalIDs { + providerID, err := strconv.ParseInt(key, 10, 64) + if err != nil { + continue + } + + l, ok := s.manager.listers()[providerID] + if !ok { + continue + } + + if err := l.RemoveWant(ctx, externalID); err != nil { + s.logger.Debug( + "could not withdraw want from external list", + "want", id, + "provider", providerID, + "error", err, + ) + } + } +} + +// splitSecrets partitions a submitted settings map into plain values, +// which go in the provider row, and secret values, which go in the +// secret store. The split is driven by the descriptor so a provider +// declaring a field secret is enough to keep it out of the database. +func splitSecrets( + desc Descriptor, + settings map[string]string, +) (plain, secret map[string]string) { + plain = make(map[string]string, len(settings)) + secret = make(map[string]string) + + secretKeys := make(map[string]bool, len(desc.Fields)) + + for _, f := range desc.Fields { + if f.Secret { + secretKeys[f.Key] = true + } + } + + for k, v := range settings { + if secretKeys[k] { + secret[k] = v + + continue + } + + plain[k] = v + } + + return plain, secret +} diff --git a/backend/download/staging.go b/backend/download/staging.go new file mode 100644 index 0000000..593c900 --- /dev/null +++ b/backend/download/staging.go @@ -0,0 +1,272 @@ +package download + +import ( + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "strings" + "time" + + "yellowjacket/backend/system" +) + +// Downloads land here first and only move into the library once they +// have been verified and tagged. The reason is not tidiness: the +// library scanner watches library paths, and a half-written file or a +// mislabelled Soulseek folder that lands there gets ingested, indexed +// and surfaced to the user before anyone can check it. Staging makes +// the import step the single writer into library paths. + +// stagingDirName is the staging root inside the user data directory. +const stagingDirName = "downloads" + +// staleAge is how long an abandoned staging directory survives before +// the startup sweep removes it. Long enough that a download +// interrupted by a crash can still be inspected; short enough that a +// failed grab does not sit on disk forever. +const staleAge = 48 * time.Hour + +// ErrEscapesStaging is returned when a provider reports a file path +// outside the directory it was given. +var ErrEscapesStaging = errors.New("path escapes the staging directory") + +// Staging owns the download staging area. +type Staging struct { + root string + logger *slog.Logger +} + +// NewStaging creates the staging area under the user data directory. +func NewStaging(logger *slog.Logger) (*Staging, error) { + dir, err := system.GetUserDataDirPath() + if err != nil { + return nil, fmt.Errorf("resolve user data dir: %w", err) + } + + return NewStagingAt(filepath.Join(dir, stagingDirName), logger) +} + +// NewStagingAt creates a staging area at an explicit root. +func NewStagingAt(root string, logger *slog.Logger) (*Staging, error) { + if err := os.MkdirAll(root, 0o750); err != nil { + return nil, fmt.Errorf("create staging root: %w", err) + } + + return &Staging{root: root, logger: logger}, nil +} + +// Root returns the staging root directory. +func (s *Staging) Root() string { + return s.root +} + +// Reserve creates and returns a directory for one download item. +func (s *Staging) Reserve(itemID string) (string, error) { + dir := filepath.Join(s.root, sanitizeSegment(itemID)) + + if err := os.MkdirAll(dir, 0o750); err != nil { + return "", fmt.Errorf("create staging dir: %w", err) + } + + return dir, nil +} + +// Release removes a download item's staging directory and everything +// in it. Called after a successful import and after a failed grab. +func (s *Staging) Release(dir string) error { + if !s.contains(dir) { + return fmt.Errorf("%w: %s", ErrEscapesStaging, dir) + } + + if err := os.RemoveAll(dir); err != nil { + return fmt.Errorf("remove staging dir: %w", err) + } + + return nil +} + +// contains reports whether dir is inside the staging root. Guards +// every destructive operation, because dir ultimately comes from a +// database row a provider wrote. +func (s *Staging) contains(dir string) bool { + absRoot, err := filepath.Abs(s.root) + if err != nil { + return false + } + + absDir, err := filepath.Abs(dir) + if err != nil { + return false + } + + rel, err := filepath.Rel(absRoot, absDir) + if err != nil { + return false + } + + return rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) +} + +// Verify checks that every path a provider reported is a real file +// inside dir, and returns them cleaned. A transport that reports a +// path outside its directory is either buggy or hostile; either way the +// import must not follow it. +func (s *Staging) Verify(dir string, files []string) ([]string, error) { + if !s.contains(dir) { + return nil, fmt.Errorf("%w: %s", ErrEscapesStaging, dir) + } + + absDir, err := filepath.Abs(dir) + if err != nil { + return nil, fmt.Errorf("resolve staging dir: %w", err) + } + + out := make([]string, 0, len(files)) + + for _, f := range files { + abs := f + if !filepath.IsAbs(abs) { + abs = filepath.Join(absDir, f) + } + + abs = filepath.Clean(abs) + + rel, err := filepath.Rel(absDir, abs) + if err != nil || + rel == ".." || + strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return nil, fmt.Errorf("%w: %s", ErrEscapesStaging, f) + } + + info, err := os.Stat(abs) + if err != nil { + return nil, fmt.Errorf("stat downloaded file %s: %w", rel, err) + } + + if info.IsDir() || info.Size() == 0 { + continue + } + + out = append(out, abs) + } + + return out, nil +} + +// Sweep removes staging directories left behind by a previous run. +// Anything still present at startup belongs to a download that did not +// finish, since a completed import releases its directory. +// +// Directories younger than staleAge are kept: a grab may legitimately +// be resumed, and deleting a partial transfer the user is waiting on +// would be worse than leaving a few megabytes on disk. +func (s *Staging) Sweep() (removed int, err error) { + entries, err := os.ReadDir(s.root) + if err != nil { + if os.IsNotExist(err) { + return 0, nil + } + + return 0, fmt.Errorf("read staging root: %w", err) + } + + cutoff := time.Now().Add(-staleAge) + + for _, e := range entries { + if !e.IsDir() { + continue + } + + info, err := e.Info() + if err != nil { + continue + } + + if info.ModTime().After(cutoff) { + continue + } + + dir := filepath.Join(s.root, e.Name()) + + if err := os.RemoveAll(dir); err != nil { + s.logger.Warn( + "could not remove stale staging directory", + "dir", dir, + "error", err, + ) + + continue + } + + removed++ + } + + if removed > 0 { + s.logger.Info("removed stale staging directories", "count", removed) + } + + return removed, nil +} + +// SweepOrphans removes staging directories whose item IDs are not in +// the live set. Called after the item store is loaded, so a directory +// belonging to a download the database no longer knows about goes away +// even if it is recent. +func (s *Staging) SweepOrphans(live map[string]bool) (removed int, err error) { + entries, err := os.ReadDir(s.root) + if err != nil { + if os.IsNotExist(err) { + return 0, nil + } + + return 0, fmt.Errorf("read staging root: %w", err) + } + + for _, e := range entries { + if !e.IsDir() || live[e.Name()] { + continue + } + + if err := os.RemoveAll(filepath.Join(s.root, e.Name())); err != nil { + s.logger.Warn( + "could not remove orphaned staging directory", + "dir", e.Name(), + "error", err, + ) + + continue + } + + removed++ + } + + return removed, nil +} + +// sanitizeSegment reduces a string to a safe single path segment. +func sanitizeSegment(s string) string { + var b strings.Builder + + b.Grow(len(s)) + + for _, r := range s { + switch { + case r >= 'a' && r <= 'z', + r >= 'A' && r <= 'Z', + r >= '0' && r <= '9', + r == '-', r == '_': + b.WriteRune(r) + default: + b.WriteByte('_') + } + } + + out := b.String() + if out == "" { + return "item" + } + + return out +} diff --git a/backend/download/staging_test.go b/backend/download/staging_test.go new file mode 100644 index 0000000..de78b21 --- /dev/null +++ b/backend/download/staging_test.go @@ -0,0 +1,198 @@ +package download + +import ( + "errors" + "os" + "path/filepath" + "testing" + "time" +) + +func newTestStaging(t *testing.T) *Staging { + t.Helper() + + s, err := NewStagingAt(t.TempDir(), slogDiscard()) + if err != nil { + t.Fatalf("NewStagingAt: %v", err) + } + + return s +} + +func TestStagingReserveAndRelease(t *testing.T) { + t.Parallel() + + s := newTestStaging(t) + + dir, err := s.Reserve("item-1") + if err != nil { + t.Fatalf("Reserve: %v", err) + } + + if _, err := os.Stat(dir); err != nil { + t.Fatalf("staging dir not created: %v", err) + } + + if err := s.Release(dir); err != nil { + t.Fatalf("Release: %v", err) + } + + if _, err := os.Stat(dir); !os.IsNotExist(err) { + t.Error("staging dir still exists after Release") + } +} + +// A provider reporting a path outside its staging directory is either +// buggy or hostile. Either way the import must refuse to follow it, +// because the next step moves those paths into the library. +func TestStagingVerifyRejectsEscape(t *testing.T) { + t.Parallel() + + s := newTestStaging(t) + + dir, err := s.Reserve("item-1") + if err != nil { + t.Fatalf("Reserve: %v", err) + } + + outside := filepath.Join(s.Root(), "elsewhere.flac") + if err := os.WriteFile(outside, []byte("x"), 0o600); err != nil { + t.Fatalf("write: %v", err) + } + + tests := []string{ + "../elsewhere.flac", + outside, + filepath.Join(dir, "..", "elsewhere.flac"), + } + + for _, path := range tests { + t.Run(path, func(t *testing.T) { + t.Parallel() + + if _, err := s.Verify(dir, []string{path}); !errors.Is( + err, ErrEscapesStaging, + ) { + t.Errorf("Verify(%q) error = %v, want ErrEscapesStaging", path, err) + } + }) + } +} + +func TestStagingReleaseRejectsOutsideRoot(t *testing.T) { + t.Parallel() + + s := newTestStaging(t) + + other := t.TempDir() + + if err := s.Release(other); !errors.Is(err, ErrEscapesStaging) { + t.Errorf("Release outside root error = %v, want ErrEscapesStaging", err) + } + + if _, err := os.Stat(other); err != nil { + t.Error("Release removed a directory outside the staging root") + } +} + +func TestStagingVerifySkipsEmptyFiles(t *testing.T) { + t.Parallel() + + s := newTestStaging(t) + + dir, err := s.Reserve("item-1") + if err != nil { + t.Fatalf("Reserve: %v", err) + } + + good := filepath.Join(dir, "good.flac") + empty := filepath.Join(dir, "empty.flac") + + if err := os.WriteFile(good, []byte("data"), 0o600); err != nil { + t.Fatalf("write: %v", err) + } + + if err := os.WriteFile(empty, nil, 0o600); err != nil { + t.Fatalf("write: %v", err) + } + + files, err := s.Verify(dir, []string{good, empty}) + if err != nil { + t.Fatalf("Verify: %v", err) + } + + if len(files) != 1 || files[0] != good { + t.Errorf("Verify = %v, want just %s", files, good) + } +} + +func TestStagingSweepKeepsRecentDirs(t *testing.T) { + t.Parallel() + + s := newTestStaging(t) + + recent, err := s.Reserve("recent") + if err != nil { + t.Fatalf("Reserve: %v", err) + } + + stale, err := s.Reserve("stale") + if err != nil { + t.Fatalf("Reserve: %v", err) + } + + old := time.Now().Add(-staleAge - time.Hour) + if err := os.Chtimes(stale, old, old); err != nil { + t.Fatalf("Chtimes: %v", err) + } + + removed, err := s.Sweep() + if err != nil { + t.Fatalf("Sweep: %v", err) + } + + if removed != 1 { + t.Errorf("removed = %d, want 1", removed) + } + + if _, err := os.Stat(recent); err != nil { + t.Error("sweep removed a recent staging dir") + } + + if _, err := os.Stat(stale); !os.IsNotExist(err) { + t.Error("sweep kept a stale staging dir") + } +} + +func TestStagingSweepOrphans(t *testing.T) { + t.Parallel() + + s := newTestStaging(t) + + live, err := s.Reserve("live") + if err != nil { + t.Fatalf("Reserve: %v", err) + } + + orphan, err := s.Reserve("orphan") + if err != nil { + t.Fatalf("Reserve: %v", err) + } + + removed, err := s.SweepOrphans(map[string]bool{"live": true}) + if err != nil { + t.Fatalf("SweepOrphans: %v", err) + } + + if removed != 1 { + t.Errorf("removed = %d, want 1", removed) + } + + if _, err := os.Stat(live); err != nil { + t.Error("sweep removed a live staging dir") + } + + if _, err := os.Stat(orphan); !os.IsNotExist(err) { + t.Error("sweep kept an orphaned staging dir") + } +} diff --git a/backend/download/store.go b/backend/download/store.go new file mode 100644 index 0000000..ac8e714 --- /dev/null +++ b/backend/download/store.go @@ -0,0 +1,521 @@ +package download + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "time" + + "yellowjacket/backend/database" + "yellowjacket/backend/database/sql/sqlcgen" +) + +// ErrNotFound is returned when a request or item ID is unknown. +var ErrNotFound = errors.New("not found") + +// Store is the download subsystem's persistence layer. It owns the +// JSON encoding of the blob columns so nothing above it has to know +// that candidates are stored as text. +type Store struct { + db *database.DB +} + +// NewStore returns a Store over the application database. +func NewStore(db *database.DB) *Store { + return &Store{db: db} +} + +// --------------------------------------------------------------------------- +// Providers +// --------------------------------------------------------------------------- + +// ListProviders returns every configured provider, best priority first. +func (s *Store) ListProviders(ctx context.Context) ([]Config, error) { + rows, err := s.db.ReadQueries.ListDownloadProviders(ctx) + if err != nil { + return nil, fmt.Errorf("list download providers: %w", err) + } + + out := make([]Config, 0, len(rows)) + + for _, r := range rows { + out = append(out, providerRowToConfig(r)) + } + + return out, nil +} + +// GetProvider returns one provider's config. +func (s *Store) GetProvider(ctx context.Context, id int64) (Config, error) { + row, err := s.db.ReadQueries.GetDownloadProvider(ctx, id) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return Config{}, fmt.Errorf("%w: provider %d", ErrNotFound, id) + } + + return Config{}, fmt.Errorf("get download provider: %w", err) + } + + return providerRowToConfig(row), nil +} + +// CreateProvider inserts a provider and returns its new ID. +func (s *Store) CreateProvider(ctx context.Context, cfg Config) (int64, error) { + settings, err := json.Marshal(cfg.Settings) + if err != nil { + return 0, fmt.Errorf("encode provider settings: %w", err) + } + + id, err := s.db.Queries.CreateDownloadProvider( + ctx, + sqlcgen.CreateDownloadProviderParams{ + Kind: string(cfg.Kind), + Name: cfg.Name, + Enabled: boolToInt(cfg.Enabled), + Priority: int64(cfg.Priority), + Settings: string(settings), + }, + ) + if err != nil { + return 0, fmt.Errorf("create download provider: %w", err) + } + + return id, nil +} + +// UpdateProvider saves changes to an existing provider. +func (s *Store) UpdateProvider(ctx context.Context, cfg Config) error { + settings, err := json.Marshal(cfg.Settings) + if err != nil { + return fmt.Errorf("encode provider settings: %w", err) + } + + if err := s.db.Queries.UpdateDownloadProvider( + ctx, + sqlcgen.UpdateDownloadProviderParams{ + Name: cfg.Name, + Enabled: boolToInt(cfg.Enabled), + Priority: int64(cfg.Priority), + Settings: string(settings), + ID: cfg.ID, + }, + ); err != nil { + return fmt.Errorf("update download provider: %w", err) + } + + return nil +} + +// DeleteProvider removes a provider row. Its secrets are removed +// separately by the manager, which owns the secret store. +func (s *Store) DeleteProvider(ctx context.Context, id int64) error { + if err := s.db.Queries.DeleteDownloadProvider(ctx, id); err != nil { + return fmt.Errorf("delete download provider: %w", err) + } + + return nil +} + +// providerRowToConfig decodes a stored provider row. A settings blob +// that fails to parse yields an empty map rather than an error: the +// provider will fail its own Check with a useful message, which beats +// making the whole settings page unloadable. +func providerRowToConfig(r sqlcgen.DownloadProvider) Config { + settings := map[string]string{} + _ = json.Unmarshal([]byte(r.Settings), &settings) + + return Config{ + ID: r.ID, + Kind: Kind(r.Kind), + Name: r.Name, + Enabled: r.Enabled != 0, + Priority: int(r.Priority), + Settings: settings, + } +} + +// --------------------------------------------------------------------------- +// Requests +// --------------------------------------------------------------------------- + +// CreateRequest persists a new request. +func (s *Store) CreateRequest(ctx context.Context, req Request) error { + expected, err := json.Marshal(req.Expected) + if err != nil { + return fmt.Errorf("encode expected tracks: %w", err) + } + + source := req.Source + if source == "" { + source = "manual" + } + + wantID := sql.NullInt64{} + if req.WantID != 0 { + wantID = sql.NullInt64{Int64: req.WantID, Valid: true} + } + + if err := s.db.Queries.CreateDownloadRequest( + ctx, + sqlcgen.CreateDownloadRequestParams{ + ID: req.ID, + LibraryID: req.LibraryID, + Source: source, + WantID: wantID, + ReleaseMbid: toNullString(req.ReleaseMBID), + ReleaseGroupMbid: toNullString(req.ReleaseGroupMBID), + RecordingMbid: toNullString(req.RecordingMBID), + Artist: req.Artist, + Album: req.Album, + Query: req.Query, + Expected: string(expected), + State: string(StateSearching), + }, + ); err != nil { + return fmt.Errorf("create download request: %w", err) + } + + return nil +} + +// GetRequest loads a request by ID. +func (s *Store) GetRequest(ctx context.Context, id string) (Request, error) { + row, err := s.db.ReadQueries.GetDownloadRequest(ctx, id) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return Request{}, fmt.Errorf("%w: request %s", ErrNotFound, id) + } + + return Request{}, fmt.Errorf("get download request: %w", err) + } + + return requestRowToRequest(row), nil +} + +// GetRequestState returns a request's current state and error text. +// Kept separate from GetRequest because state is the one field that +// changes constantly while the rest of the row is immutable. +func (s *Store) GetRequestState( + ctx context.Context, + id string, +) (State, string, error) { + row, err := s.db.ReadQueries.GetDownloadRequest(ctx, id) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return "", "", fmt.Errorf("%w: request %s", ErrNotFound, id) + } + + return "", "", fmt.Errorf("get download request state: %w", err) + } + + return State(row.State), row.Error, nil +} + +// ListRequests returns the most recent requests, newest first. +func (s *Store) ListRequests(ctx context.Context, limit int) ([]Request, error) { + rows, err := s.db.ReadQueries.ListDownloadRequests(ctx, int64(limit)) + if err != nil { + return nil, fmt.Errorf("list download requests: %w", err) + } + + out := make([]Request, 0, len(rows)) + + for _, r := range rows { + out = append(out, requestRowToRequest(r)) + } + + return out, nil +} + +// SetRequestState updates a request's state and error text. +func (s *Store) SetRequestState( + ctx context.Context, + id string, + state State, + errText string, +) error { + if err := s.db.Queries.SetDownloadRequestState( + ctx, + sqlcgen.SetDownloadRequestStateParams{ + State: string(state), + Error: errText, + ID: id, + }, + ); err != nil { + return fmt.Errorf("set download request state: %w", err) + } + + return nil +} + +// DeleteRequest removes a request and, by cascade, its items. +func (s *Store) DeleteRequest(ctx context.Context, id string) error { + if err := s.db.Queries.DeleteDownloadRequest(ctx, id); err != nil { + return fmt.Errorf("delete download request: %w", err) + } + + return nil +} + +// ClearFinished removes every terminal request. +func (s *Store) ClearFinished(ctx context.Context) error { + if err := s.db.Queries.DeleteFinishedDownloadRequests(ctx); err != nil { + return fmt.Errorf("clear finished download requests: %w", err) + } + + return nil +} + +// requestRowToRequest decodes a stored request row. +func requestRowToRequest(r sqlcgen.DownloadRequest) Request { + var expected []ExpectedTrack + + _ = json.Unmarshal([]byte(r.Expected), &expected) + + return Request{ + ID: r.ID, + LibraryID: r.LibraryID, + Source: r.Source, + WantID: r.WantID.Int64, + ReleaseMBID: r.ReleaseMbid.String, + ReleaseGroupMBID: r.ReleaseGroupMbid.String, + RecordingMBID: r.RecordingMbid.String, + Artist: r.Artist, + Album: r.Album, + Query: r.Query, + Expected: expected, + CreatedAt: r.CreatedAt, + } +} + +// --------------------------------------------------------------------------- +// Items +// --------------------------------------------------------------------------- + +// Item is one grab attempt, as stored. +type Item struct { + ID string `json:"id"` + RequestID string `json:"requestId"` + ProviderID int64 `json:"providerId"` + Transport int64 `json:"transportId,omitempty"` + ExternalID string `json:"externalId,omitempty"` + Candidate Candidate `json:"candidate"` + State State `json:"state"` + StagingDir string `json:"-"` + BytesDone int64 `json:"bytesDone"` + BytesTotal int64 `json:"bytesTotal"` + Imported []string `json:"-"` + Error string `json:"error,omitempty"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` +} + +// CreateItem persists a grab attempt. +func (s *Store) CreateItem(ctx context.Context, item Item) error { + candidate, err := json.Marshal(item.Candidate) + if err != nil { + return fmt.Errorf("encode candidate: %w", err) + } + + transport := sql.NullInt64{} + if item.Transport != 0 { + transport = sql.NullInt64{Int64: item.Transport, Valid: true} + } + + if err := s.db.Queries.CreateDownloadItem( + ctx, + sqlcgen.CreateDownloadItemParams{ + ID: item.ID, + RequestID: item.RequestID, + ProviderID: item.ProviderID, + TransportID: transport, + ExternalID: item.ExternalID, + Candidate: string(candidate), + State: string(item.State), + StagingDir: item.StagingDir, + BytesTotal: item.BytesTotal, + }, + ); err != nil { + return fmt.Errorf("create download item: %w", err) + } + + return nil +} + +// GetItem loads one item. +func (s *Store) GetItem(ctx context.Context, id string) (Item, error) { + row, err := s.db.ReadQueries.GetDownloadItem(ctx, id) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return Item{}, fmt.Errorf("%w: item %s", ErrNotFound, id) + } + + return Item{}, fmt.Errorf("get download item: %w", err) + } + + return itemRowToItem(row), nil +} + +// ListItemsForRequest returns a request's grab attempts, oldest first. +func (s *Store) ListItemsForRequest( + ctx context.Context, + requestID string, +) ([]Item, error) { + rows, err := s.db.ReadQueries.ListDownloadItemsForRequest(ctx, requestID) + if err != nil { + return nil, fmt.Errorf("list download items: %w", err) + } + + out := make([]Item, 0, len(rows)) + + for _, r := range rows { + out = append(out, itemRowToItem(r)) + } + + return out, nil +} + +// ListLiveItems returns every non-terminal item. Called at startup to +// decide what to resume, reconcile or abandon. +func (s *Store) ListLiveItems(ctx context.Context) ([]Item, error) { + rows, err := s.db.ReadQueries.ListLiveDownloadItems(ctx) + if err != nil { + return nil, fmt.Errorf("list live download items: %w", err) + } + + out := make([]Item, 0, len(rows)) + + for _, r := range rows { + out = append(out, itemRowToItem(r)) + } + + return out, nil +} + +// SetItemState updates an item's state and error text. +func (s *Store) SetItemState( + ctx context.Context, + id string, + state State, + errText string, +) error { + if err := s.db.Queries.SetDownloadItemState( + ctx, + sqlcgen.SetDownloadItemStateParams{ + State: string(state), + Error: errText, + ID: id, + }, + ); err != nil { + return fmt.Errorf("set download item state: %w", err) + } + + return nil +} + +// SetItemProgress records transfer progress. +func (s *Store) SetItemProgress( + ctx context.Context, + id string, + done, total int64, +) error { + if err := s.db.Queries.SetDownloadItemProgress( + ctx, + sqlcgen.SetDownloadItemProgressParams{ + BytesDone: done, + BytesTotal: total, + ID: id, + }, + ); err != nil { + return fmt.Errorf("set download item progress: %w", err) + } + + return nil +} + +// SetItemExternalID records a delegating manager's own identifier. +func (s *Store) SetItemExternalID( + ctx context.Context, + id, externalID string, +) error { + if err := s.db.Queries.SetDownloadItemExternalID( + ctx, + sqlcgen.SetDownloadItemExternalIDParams{ + ExternalID: externalID, + ID: id, + }, + ); err != nil { + return fmt.Errorf("set download item external id: %w", err) + } + + return nil +} + +// SetItemImported records the library paths files landed at and marks +// the item complete. +func (s *Store) SetItemImported( + ctx context.Context, + id string, + paths []string, +) error { + encoded, err := json.Marshal(paths) + if err != nil { + return fmt.Errorf("encode imported paths: %w", err) + } + + if err := s.db.Queries.SetDownloadItemImported( + ctx, + sqlcgen.SetDownloadItemImportedParams{ + ImportedPaths: string(encoded), + ID: id, + }, + ); err != nil { + return fmt.Errorf("set download item imported: %w", err) + } + + return nil +} + +// itemRowToItem decodes a stored item row. +func itemRowToItem(r sqlcgen.DownloadItem) Item { + var ( + candidate Candidate + imported []string + ) + + _ = json.Unmarshal([]byte(r.Candidate), &candidate) + _ = json.Unmarshal([]byte(r.ImportedPaths), &imported) + + return Item{ + ID: r.ID, + RequestID: r.RequestID, + ProviderID: r.ProviderID, + Transport: r.TransportID.Int64, + ExternalID: r.ExternalID, + Candidate: candidate, + State: State(r.State), + StagingDir: r.StagingDir, + BytesDone: r.BytesDone, + BytesTotal: r.BytesTotal, + Imported: imported, + Error: r.Error, + CreatedAt: r.CreatedAt, + UpdatedAt: r.UpdatedAt, + } +} + +// boolToInt converts a bool to SQLite's integer boolean. +func boolToInt(b bool) int64 { + if b { + return 1 + } + + return 0 +} + +// toNullString wraps a possibly-empty string for a nullable column. +func toNullString(s string) sql.NullString { + return sql.NullString{String: s, Valid: s != ""} +} diff --git a/backend/download/transports_test.go b/backend/download/transports_test.go new file mode 100644 index 0000000..7751869 --- /dev/null +++ b/backend/download/transports_test.go @@ -0,0 +1,667 @@ +package download + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +// --------------------------------------------------------------------------- +// qBittorrent +// --------------------------------------------------------------------------- + +// qbitStub is a fake qBittorrent Web API. +type qbitStub struct { + server *httptest.Server + + mu sync.Mutex + + // loginOK controls whether auth succeeds. + loginOK bool + + // torrentStates is what the info endpoint returns, in order; the + // last entry repeats. + torrentStates [][]qbitTorrent + pollCount int + + // addedForm records the add-torrent parameters. + addedForm url.Values +} + +func newQbitStub(t *testing.T) *qbitStub { + t.Helper() + + s := &qbitStub{loginOK: true} + mux := http.NewServeMux() + + mux.HandleFunc("/api/v2/auth/login", func(w http.ResponseWriter, _ *http.Request) { + s.mu.Lock() + ok := s.loginOK + s.mu.Unlock() + + if ok { + _, _ = w.Write([]byte("Ok.")) + + return + } + + // qBittorrent answers a bad login with 200 and "Fails.". + _, _ = w.Write([]byte("Fails.")) + }) + + mux.HandleFunc("/api/v2/app/version", func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("v4.6.0")) + }) + + mux.HandleFunc("/api/v2/torrents/add", func(w http.ResponseWriter, r *http.Request) { + if err := r.ParseForm(); err != nil { + t.Errorf("parse add form: %v", err) + } + + s.mu.Lock() + s.addedForm = r.PostForm + s.mu.Unlock() + + _, _ = w.Write([]byte("Ok.")) + }) + + mux.HandleFunc("/api/v2/torrents/info", func(w http.ResponseWriter, _ *http.Request) { + s.mu.Lock() + + idx := s.pollCount + if idx >= len(s.torrentStates) { + idx = len(s.torrentStates) - 1 + } else { + s.pollCount++ + } + + var batch []qbitTorrent + if idx >= 0 && len(s.torrentStates) > 0 { + batch = s.torrentStates[idx] + } + + s.mu.Unlock() + + writeJSON(t, w, batch) + }) + + s.server = httptest.NewServer(mux) + t.Cleanup(s.server.Close) + + return s +} + +func newStubQbit(t *testing.T, stub *qbitStub) *qbittorrent { + t.Helper() + + p, err := newQBittorrent( + Config{ + ID: 1, + Kind: KindQBittorrent, + Name: "qbittorrent", + Settings: map[string]string{ + "url": stub.server.URL, + "username": "admin", + }, + }, + func(string) (string, error) { return "secret", nil }, + slogDiscard(), + ) + if err != nil { + t.Fatalf("newQBittorrent: %v", err) + } + + q, ok := p.(*qbittorrent) + if !ok { + t.Fatalf("provider is %T, want *qbittorrent", p) + } + + q.pollInterval = time.Millisecond + + return q +} + +func TestQbitCheck(t *testing.T) { + t.Parallel() + + stub := newQbitStub(t) + q := newStubQbit(t, stub) + + if err := q.Check(context.Background()); err != nil { + t.Errorf("Check: %v", err) + } +} + +// qBittorrent returns HTTP 200 with "Fails." for a bad password, so a +// status-code-only check would report success. +func TestQbitRejectsBadPassword(t *testing.T) { + t.Parallel() + + stub := newQbitStub(t) + stub.loginOK = false + + q := newStubQbit(t, stub) + + if err := q.Check(context.Background()); !errors.Is(err, ErrQbitAuth) { + t.Errorf("error = %v, want ErrQbitAuth", err) + } +} + +// The transport must be a pure transport: no search role. +func TestQbitDeclaresTransportOnly(t *testing.T) { + t.Parallel() + + stub := newQbitStub(t) + q := newStubQbit(t, stub) + + caps := q.Info().Caps + + if caps.CanSearch || caps.CanDelegate { + t.Errorf("qBittorrent should transport only, got %+v", caps) + } + + if !caps.Handles(ProtocolTorrent) { + t.Error("qBittorrent should handle the torrent protocol") + } + + if caps.Handles(ProtocolUsenet) { + t.Error("qBittorrent should not claim usenet") + } +} + +func TestQbitGrabCollectsCompletedTorrent(t *testing.T) { + t.Parallel() + + stub := newQbitStub(t) + + // The torrent's content lands in a directory qBittorrent owns. + content := t.TempDir() + + for _, name := range []string{"01 Airbag.flac", "02 Paranoid Android.flac"} { + if err := os.WriteFile( + filepath.Join(content, name), []byte("audio"), 0o600, + ); err != nil { + t.Fatalf("write: %v", err) + } + } + + stub.torrentStates = [][]qbitTorrent{ + {{ + Hash: "abc123", State: "downloading", + Progress: 0.4, Size: 1000, Completed: 400, + }}, + {{ + Hash: "abc123", State: "uploading", + Progress: 1.0, Size: 1000, Completed: 1000, + ContentPath: content, + }}, + } + + q := newStubQbit(t, stub) + dst := t.TempDir() + + c := Candidate{ + ID: "prowlarr:x", + Protocol: ProtocolTorrent, + Title: "Radiohead - OK Computer", + Payload: map[string]string{ + "link": "magnet:?xt=urn:btih:abc123", + "infoHash": "abc123", + }, + } + + got, err := q.Grab(context.Background(), c, dst, nil) + if err != nil { + t.Fatalf("Grab: %v", err) + } + + if len(got.Files) != 2 { + t.Fatalf("collected %d files, want 2", len(got.Files)) + } + + for _, f := range got.Files { + if !strings.HasPrefix(f, dst) { + t.Errorf("file %s is outside the staging dir", f) + } + } + + // The torrent was saved into our staging directory and kept out of + // qBittorrent's own move-on-completion rules. + stub.mu.Lock() + form := stub.addedForm + stub.mu.Unlock() + + if form.Get("savepath") != dst { + t.Errorf("savepath = %q, want the staging dir %q", form.Get("savepath"), dst) + } + + if form.Get("autoTMM") != "false" { + t.Errorf("autoTMM = %q, want false", form.Get("autoTMM")) + } +} + +func TestQbitGrabFailsOnErrorState(t *testing.T) { + t.Parallel() + + stub := newQbitStub(t) + stub.torrentStates = [][]qbitTorrent{ + {{Hash: "abc123", State: "error"}}, + } + + q := newStubQbit(t, stub) + + c := Candidate{ + Protocol: ProtocolTorrent, + Payload: map[string]string{ + "link": "magnet:?xt=urn:btih:abc123", "infoHash": "abc123", + }, + } + + _, err := q.Grab(context.Background(), c, t.TempDir(), nil) + if !errors.Is(err, ErrQbitTransferFailed) { + t.Errorf("error = %v, want ErrQbitTransferFailed", err) + } +} + +func TestInfoHashFromMagnet(t *testing.T) { + t.Parallel() + + tests := []struct { + in string + want string + }{ + {"magnet:?xt=urn:btih:ABC123&dn=x", "abc123"}, + {"magnet:?dn=x&xt=urn:btih:def456", "def456"}, + {"magnet:?dn=no-hash", ""}, + {"https://example.com/x.torrent", ""}, + {"", ""}, + } + + for _, tt := range tests { + t.Run(tt.in, func(t *testing.T) { + t.Parallel() + + if got := infoHashFromMagnet(tt.in); got != tt.want { + t.Errorf("infoHashFromMagnet(%q) = %q, want %q", tt.in, got, tt.want) + } + }) + } +} + +// --------------------------------------------------------------------------- +// SABnzbd +// --------------------------------------------------------------------------- + +// sabStub is a fake SABnzbd API. +type sabStub struct { + server *httptest.Server + + mu sync.Mutex + + // queueSlots and historySlots are returned in order per mode; the + // last entry repeats. + queueSlots [][]sabQueueSlot + queuePolls int + historySlots []sabHistorySlot + + addStatus bool + addError string + + badKey bool +} + +func newSabStub(t *testing.T) *sabStub { + t.Helper() + + s := &sabStub{addStatus: true} + + s.server = httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + badKey := s.badKey + s.mu.Unlock() + + if badKey { + // SABnzbd reports a bad key with HTTP 200 and a body. + writeJSON(t, w, map[string]any{ + "status": false, "error": "API Key Incorrect", + }) + + return + } + + switch r.URL.Query().Get("mode") { + case "version": + writeJSON(t, w, map[string]any{"version": "4.1.0"}) + case "addurl": + s.mu.Lock() + status, errMsg := s.addStatus, s.addError + s.mu.Unlock() + + writeJSON(t, w, map[string]any{ + "status": status, "error": errMsg, + "nzo_ids": []string{"SABnzbd_nzo_1"}, + }) + case "queue": + s.mu.Lock() + + idx := s.queuePolls + if idx >= len(s.queueSlots) { + idx = len(s.queueSlots) - 1 + } else { + s.queuePolls++ + } + + var slots []sabQueueSlot + if idx >= 0 && len(s.queueSlots) > 0 { + slots = s.queueSlots[idx] + } + + s.mu.Unlock() + + writeJSON(t, w, map[string]any{ + "queue": map[string]any{"slots": slots}, + }) + case "history": + s.mu.Lock() + slots := s.historySlots + s.mu.Unlock() + + writeJSON(t, w, map[string]any{ + "history": map[string]any{"slots": slots}, + }) + default: + w.WriteHeader(http.StatusBadRequest) + } + }, + )) + + t.Cleanup(s.server.Close) + + return s +} + +func newStubSab(t *testing.T, stub *sabStub) *sabnzbd { + t.Helper() + + p, err := newSABnzbd( + Config{ + ID: 1, + Kind: KindSABnzbd, + Name: "sabnzbd", + Settings: map[string]string{"url": stub.server.URL}, + }, + func(string) (string, error) { return "test-key", nil }, + slogDiscard(), + ) + if err != nil { + t.Fatalf("newSABnzbd: %v", err) + } + + s, ok := p.(*sabnzbd) + if !ok { + t.Fatalf("provider is %T, want *sabnzbd", p) + } + + s.pollInterval = time.Millisecond + + return s +} + +func TestSabCheck(t *testing.T) { + t.Parallel() + + stub := newSabStub(t) + s := newStubSab(t, stub) + + if err := s.Check(context.Background()); err != nil { + t.Errorf("Check: %v", err) + } +} + +func TestSabDeclaresUsenetOnly(t *testing.T) { + t.Parallel() + + stub := newSabStub(t) + s := newStubSab(t, stub) + + caps := s.Info().Caps + + if caps.CanSearch || caps.CanDelegate { + t.Errorf("SABnzbd should transport only, got %+v", caps) + } + + if !caps.Handles(ProtocolUsenet) { + t.Error("SABnzbd should handle the usenet protocol") + } + + if caps.Handles(ProtocolTorrent) { + t.Error("SABnzbd should not claim torrents") + } +} + +// Completion is read from history, not from the queue emptying: a job +// leaves the queue before post-processing finishes. +func TestSabGrabWaitsForHistory(t *testing.T) { + t.Parallel() + + stub := newSabStub(t) + + storage := t.TempDir() + + for _, name := range []string{"01 Airbag.flac", "02 Paranoid Android.flac"} { + if err := os.WriteFile( + filepath.Join(storage, name), []byte("audio"), 0o600, + ); err != nil { + t.Fatalf("write: %v", err) + } + } + + stub.queueSlots = [][]sabQueueSlot{ + {{ + NzoID: "SABnzbd_nzo_1", Status: "Downloading", + MB: "100.0", MBLeft: "60.0", + }}, + // Second poll: gone from the queue. + {}, + } + stub.historySlots = []sabHistorySlot{{ + NzoID: "SABnzbd_nzo_1", Status: "Completed", Storage: storage, + }} + + s := newStubSab(t, stub) + dst := t.TempDir() + + c := Candidate{ + Protocol: ProtocolUsenet, + Title: "Radiohead - OK Computer", + Payload: map[string]string{"link": "https://example.com/x.nzb"}, + } + + got, err := s.Grab(context.Background(), c, dst, nil) + if err != nil { + t.Fatalf("Grab: %v", err) + } + + if len(got.Files) != 2 { + t.Fatalf("collected %d files, want 2", len(got.Files)) + } + + for _, f := range got.Files { + if !strings.HasPrefix(f, dst) { + t.Errorf("file %s is outside the staging dir", f) + } + } +} + +func TestSabGrabFailsOnFailedJob(t *testing.T) { + t.Parallel() + + stub := newSabStub(t) + stub.queueSlots = [][]sabQueueSlot{{}} + stub.historySlots = []sabHistorySlot{{ + NzoID: "SABnzbd_nzo_1", + Status: "Failed", + FailMsg: "Unpacking failed", + }} + + s := newStubSab(t, stub) + + c := Candidate{ + Payload: map[string]string{"link": "https://example.com/x.nzb"}, + } + + _, err := s.Grab(context.Background(), c, t.TempDir(), nil) + if !errors.Is(err, ErrSabTransferFailed) { + t.Errorf("error = %v, want ErrSabTransferFailed", err) + } +} + +// A job that vanishes from both queue and history was removed out from +// under us, which must not look like success. +func TestSabGrabDetectsVanishedJob(t *testing.T) { + t.Parallel() + + stub := newSabStub(t) + stub.queueSlots = [][]sabQueueSlot{{}} + stub.historySlots = nil + + s := newStubSab(t, stub) + + c := Candidate{ + Payload: map[string]string{"link": "https://example.com/x.nzb"}, + } + + _, err := s.Grab(context.Background(), c, t.TempDir(), nil) + if !errors.Is(err, ErrSabNoJob) { + t.Errorf("error = %v, want ErrSabNoJob", err) + } +} + +// An NZB URL is untrusted input from an indexer. +func TestSabGrabRejectsNonHTTPURL(t *testing.T) { + t.Parallel() + + stub := newSabStub(t) + s := newStubSab(t, stub) + + c := Candidate{Payload: map[string]string{"link": "file:///etc/passwd"}} + + _, err := s.Grab(context.Background(), c, t.TempDir(), nil) + if !errors.Is(err, ErrUnsafeURL) { + t.Errorf("error = %v, want ErrUnsafeURL", err) + } +} + +func TestParseMB(t *testing.T) { + t.Parallel() + + tests := []struct { + in string + want int64 + }{ + {"1.0", 1024 * 1024}, + {"0", 0}, + {" 2.5 ", int64(2.5 * 1024 * 1024)}, + {"garbage", 0}, + {"", 0}, + } + + for _, tt := range tests { + t.Run(tt.in, func(t *testing.T) { + t.Parallel() + + if got := parseMB(tt.in); got != tt.want { + t.Errorf("parseMB(%q) = %d, want %d", tt.in, got, tt.want) + } + }) + } +} + +// --------------------------------------------------------------------------- +// Shared +// --------------------------------------------------------------------------- + +// collectTree flattens whatever shape the transport produced, because +// the importer wants a flat set of paths inside staging. +func TestCollectTree(t *testing.T) { + t.Parallel() + + t.Run("directory tree is flattened", func(t *testing.T) { + t.Parallel() + + root := t.TempDir() + nested := filepath.Join(root, "CD1") + + if err := os.MkdirAll(nested, 0o750); err != nil { + t.Fatalf("mkdir: %v", err) + } + + if err := os.WriteFile( + filepath.Join(root, "a.flac"), []byte("x"), 0o600, + ); err != nil { + t.Fatalf("write: %v", err) + } + + if err := os.WriteFile( + filepath.Join(nested, "b.flac"), []byte("y"), 0o600, + ); err != nil { + t.Fatalf("write: %v", err) + } + + dst := t.TempDir() + + got, err := collectTree(root, dst) + if err != nil { + t.Fatalf("collectTree: %v", err) + } + + if len(got.Files) != 2 { + t.Fatalf("collected %d files, want 2", len(got.Files)) + } + }) + + t.Run("single file", func(t *testing.T) { + t.Parallel() + + root := t.TempDir() + file := filepath.Join(root, "single.flac") + + if err := os.WriteFile(file, []byte("x"), 0o600); err != nil { + t.Fatalf("write: %v", err) + } + + dst := t.TempDir() + + got, err := collectTree(file, dst) + if err != nil { + t.Fatalf("collectTree: %v", err) + } + + if len(got.Files) != 1 { + t.Fatalf("collected %d files, want 1", len(got.Files)) + } + + if filepath.Dir(got.Files[0]) != dst { + t.Errorf("file landed at %s, want inside %s", got.Files[0], dst) + } + }) + + t.Run("missing path errors", func(t *testing.T) { + t.Parallel() + + if _, err := collectTree( + filepath.Join(t.TempDir(), "nope"), t.TempDir(), + ); err == nil { + t.Error("want an error for a missing content path") + } + }) +} diff --git a/backend/download/types.go b/backend/download/types.go new file mode 100644 index 0000000..d43ea08 --- /dev/null +++ b/backend/download/types.go @@ -0,0 +1,351 @@ +// Package download acquires music from user-configured external +// services and imports it into the library. +// +// The services users connect are not the same kind of thing: some +// search, some move bytes, some are whole automation systems we hand a +// request to. Rather than one interface every adapter half-implements, +// a provider fills one or more of three roles — Searcher, Transporter, +// Delegator — and declares which in its Caps. The pipeline composes +// them: a search-only provider (Prowlarr) is paired with a transport +// (qBittorrent, SABnzbd) by protocol at grab time, while providers that +// do both (slskd, yt-dlp) pair with themselves. +// +// Nothing here downloads into the library. Grabs land in a staging +// directory, are verified and tagged against the release the user +// actually asked for, and only then move into library paths. +package download + +import ( + "slices" + "strings" + "time" +) + +// Kind identifies a provider implementation. It is stored in the +// database and used to look up the constructor in the registry, so +// values are stable strings and never renamed. +type Kind string + +// Provider kinds. +const ( + KindSlskd Kind = "slskd" + KindYtDlp Kind = "yt-dlp" + KindLidarr Kind = "lidarr" + KindProwlarr Kind = "prowlarr" + KindQBittorrent Kind = "qbittorrent" + KindSABnzbd Kind = "sabnzbd" + + // KindFake is an in-memory provider used by tests. It is never + // offered in the UI. + KindFake Kind = "fake" +) + +// Protocol is how a candidate's bytes are moved. Search-only providers +// report it so the pipeline can pick a compatible transport; providers +// that transport their own results use ProtocolDirect. +type Protocol string + +// Transport protocols. +const ( + // ProtocolDirect means the finding provider also does the fetch. + ProtocolDirect Protocol = "direct" + ProtocolTorrent Protocol = "torrent" + ProtocolUsenet Protocol = "usenet" +) + +// Caps declares which roles a provider fills and which optional +// behaviours it supports. The frontend renders controls from this +// rather than switching on Kind, so a provider that gains resume +// support later needs no frontend change. +type Caps struct { + // Roles. + CanSearch bool `json:"canSearch"` + CanTransport bool `json:"canTransport"` + CanDelegate bool `json:"canDelegate"` + + // CanList marks a provider that keeps a persistent wanted list of + // its own, which the reconciler mirrors this app's list into. + CanList bool `json:"canList"` + + // Optional behaviours. + CanResume bool `json:"canResume"` + CanCancel bool `json:"canCancel"` + ReportsSize bool `json:"reportsSize"` + + // Protocols this provider can transport. Empty for providers that + // only fetch their own search results. + Transports []Protocol `json:"transports"` +} + +// Handles reports whether the provider can transport the given protocol. +func (c Caps) Handles(p Protocol) bool { + return slices.Contains(c.Transports, p) +} + +// Request is what the user asked for. Requests that carry a MusicBrainz +// anchor are far more reliable than free-text ones, because the anchor +// gives the import step an expected tracklist to match against — so the +// pipeline records which it got and refuses to auto-pick without one. +type Request struct { + ID string `json:"id"` + + // Anchors. Any may be empty; all empty means free-text. + ReleaseMBID string `json:"releaseMbid,omitempty"` + ReleaseGroupMBID string `json:"releaseGroupMbid,omitempty"` + + // RecordingMBID anchors a single-track request. Its Expected holds + // exactly that one track, which is what lets a track request be + // scored — and therefore auto-picked — on the same footing as an + // album. + RecordingMBID string `json:"recordingMbid,omitempty"` + + // WantID links back to the wanted-list row this request was raised + // for, or 0 for a request the user started by hand. The reconciler + // writes the outcome back through it. + WantID int64 `json:"wantId,omitempty"` + + // Source records where the request came from, for the downloads + // list. Empty means "manual". + Source string `json:"source,omitempty"` + + // Display and query text. Artist/Album are what searches are built + // from; Query overrides them when the user typed something raw. + Artist string `json:"artist"` + Album string `json:"album"` + Query string `json:"query,omitempty"` + + // Expected is the tracklist the anchor resolves to, used for + // completeness scoring and for the autotag match at import. Empty + // for free-text requests. + Expected []ExpectedTrack `json:"expected,omitempty"` + + // LibraryID is the library imported files belong to. + LibraryID int64 `json:"libraryId"` + + CreatedAt time.Time `json:"createdAt"` +} + +// Anchored reports whether the request carries a MusicBrainz ID. Only +// anchored requests are eligible for auto-pick. +func (r Request) Anchored() bool { + return r.ReleaseMBID != "" || + r.ReleaseGroupMBID != "" || + r.RecordingMBID != "" +} + +// SearchText returns the string to hand a provider's search endpoint. +func (r Request) SearchText() string { + if r.Query != "" { + return r.Query + } + + return strings.TrimSpace(r.Artist + " " + r.Album) +} + +// ExpectedTrack is one track of the release the user asked for. +type ExpectedTrack struct { + Position int `json:"position"` + DiscNumber int `json:"discNumber"` + Title string `json:"title"` + Artist string `json:"artist"` + LengthMillis int64 `json:"lengthMillis"` +} + +// Candidate is one acquirable thing a provider found: a Soulseek user's +// folder, a torrent, a YouTube playlist. Providers fill the descriptive +// fields; the ranker fills Match, Quality and Score. +type Candidate struct { + // ID is unique within the provider that produced it, and is what + // gets handed back to Grab. + ID string `json:"id"` + ProviderID int64 `json:"providerId"` + Kind Kind `json:"kind"` + + // Protocol determines which transport can fetch this. + Protocol Protocol `json:"protocol"` + + // Descriptive. + Title string `json:"title"` + Artist string `json:"artist,omitempty"` + Origin string `json:"origin,omitempty"` // peer username, indexer name, channel + + Files []CandidateFile `json:"files"` + TotalSize int64 `json:"totalSize"` + + // Health is the provider's own availability signal, normalized to + // 0..1: seeder count for torrents, free upload slots and queue + // length for Soulseek. 0.5 when the provider has no signal. + Health float64 `json:"health"` + + // Scores, filled by the ranker. + Match MatchScore `json:"match"` + Quality QualityScore `json:"quality"` + Score float64 `json:"score"` + + // Payload is provider-private data needed to fetch this candidate + // (magnet URI, NZB URL, slskd file list). Never shown to the user. + Payload map[string]string `json:"-"` +} + +// CandidateFile is one file inside a candidate. Soulseek and torrent +// results give paths and sizes but no tags, so Format and duration are +// inferred from the path and size where possible. +type CandidateFile struct { + Path string `json:"path"` + Size int64 `json:"size"` + Format Format `json:"format"` + Bitrate int `json:"bitrate,omitempty"` // kbps, 0 when unknown + IsAudio bool `json:"isAudio"` + MatchedTo int `json:"matchedTo,omitempty"` // expected track position +} + +// Format is a normalized audio container/codec name. +type Format string + +// Audio formats, ordered by the quality ranking in formatRank. +const ( + FormatUnknown Format = "" + FormatFLAC Format = "flac" + FormatALAC Format = "alac" + FormatWAV Format = "wav" + FormatMP3 Format = "mp3" + FormatAAC Format = "aac" + FormatOGG Format = "ogg" + FormatOpus Format = "opus" + FormatWMA Format = "wma" +) + +// Lossless reports whether the format preserves the source exactly. +func (f Format) Lossless() bool { + return f == FormatFLAC || f == FormatALAC || f == FormatWAV +} + +// Supported reports whether the player can decode this format. Grabs +// of unsupported formats are still allowed — the user may want them — +// but they rank below playable ones. +func (f Format) Supported() bool { + switch f { + case FormatMP3, FormatFLAC, FormatOGG, FormatWAV: + return true + case FormatUnknown, FormatALAC, FormatAAC, FormatOpus, FormatWMA: + return false + default: + return false + } +} + +// MatchScore answers "is this the release the user asked for?" It is +// deliberately separate from QualityScore: a perfect match at 128kbps +// and a mediocre match in FLAC are different failures, and collapsing +// them into one number makes the ranking impossible to explain. +type MatchScore struct { + // Overall is 0..1. + Overall float64 `json:"overall"` + + TitleFit float64 `json:"titleFit"` // filenames vs expected titles + ArtistFit float64 `json:"artistFit"` // path/origin vs expected artist + AlbumFit float64 `json:"albumFit"` // folder name vs album title + Completeness float64 `json:"completeness"` // audio files vs expected count + + // Anchored records whether an MBID drove this score. Unanchored + // matches are capped, because there is nothing to be right about. + Anchored bool `json:"anchored"` +} + +// QualityScore answers "is this a good copy?". +type QualityScore struct { + // Overall is 0..1. + Overall float64 `json:"overall"` + + FormatRank float64 `json:"formatRank"` // FLAC > V0 > 320 > lower + Bitrate float64 `json:"bitrate"` + Health float64 `json:"health"` // seeders, free slots + Priority float64 `json:"priority"` // user's per-provider preference + + // Mixed marks a candidate whose files are not all the same format, + // which usually means a hand-assembled folder rather than a rip. + Mixed bool `json:"mixed"` +} + +// AudioFiles returns only the audio entries of a candidate. +func (c Candidate) AudioFiles() []CandidateFile { + out := make([]CandidateFile, 0, len(c.Files)) + + for _, f := range c.Files { + if f.IsAudio { + out = append(out, f) + } + } + + return out +} + +// State is the lifecycle position of a download item. +type State string + +// Download item states. Searching through Importing are live; +// Complete, Cancelled and Failed are terminal. +const ( + StateSearching State = "searching" + StateFound State = "found" + StateQueued State = "queued" + StateGrabbing State = "grabbing" + StateVerifying State = "verifying" + StateTagging State = "tagging" + StateImporting State = "importing" + StateComplete State = "complete" + StateCancelled State = "cancelled" + StateFailed State = "failed" +) + +// IsTerminal reports whether the state means no further progress will +// happen without a new attempt. +func (s State) IsTerminal() bool { + return s == StateComplete || s == StateCancelled || s == StateFailed +} + +// Progress is a transport's periodic report. Total is 0 when the +// provider cannot say how large the transfer is. +type Progress struct { + Current int64 + Total int64 + Phase string +} + +// ProgressFunc receives transport progress. Implementations must +// tolerate being called from any goroutine and at high frequency. +type ProgressFunc func(Progress) + +// Result is what a transport produced. +type Result struct { + // Dir is the staging directory the files landed in. + Dir string + + // Files are absolute paths, all under Dir. + Files []string + + // BytesTransferred is what actually moved, for reporting. + BytesTransferred int64 + + // Delegated marks a result produced by an external manager that has + // already imported the files into its own library. Files are then + // absolute paths outside staging, and the pipeline records them + // where they are instead of tagging and moving them. + Delegated bool +} + +// DelegateStatus is a delegating manager's answer to "are we there +// yet?". +type DelegateStatus struct { + State State + + // Progress is 0..1 when the manager reports it, -1 when it does not. + Progress float64 + + // ImportedPaths are files the manager has already placed on disk. + // A delegate that imports into its own library reports them here so + // the pipeline can reconcile rather than re-import. + ImportedPaths []string + + Message string +} diff --git a/backend/download/want.go b/backend/download/want.go new file mode 100644 index 0000000..cfd5b0c --- /dev/null +++ b/backend/download/want.go @@ -0,0 +1,236 @@ +package download + +import ( + "math" + "math/rand/v2" + "time" +) + +// A Want is a persistent "I want this", stored as a MusicBrainz ID and +// almost nothing else. +// +// The distinction from Request is the whole point of this file. A +// Request is one attempt: it searches, it grabs, it succeeds or fails, +// and then it is history. A Want outlives every attempt made on its +// behalf. Nothing being findable today is the normal case for obscure +// music, and the correct response is to try again next week, not to +// show the user a failed row they have to remember to retry. +// +// Because a Want is only an MBID, it stays true when everything around +// it changes: the explore index is rebuilt, a provider is swapped out, +// the release the user originally saw is superseded by a remaster. The +// display fields are a cache for the list view and are never consulted +// for matching. + +// Entity says what a want's MBID names, and is the only type +// distinction the wanted list makes. +type Entity string + +// Want entity types. +const ( + // EntityArtist is a subscription rather than a thing to fetch: it + // is never satisfied, and each reconcile expands the artist's + // discography into child wants. + EntityArtist Entity = "artist" + + // EntityReleaseGroup is an album in the abstract — any release of + // it satisfies the want, which is what a user means by "I want this + // album". + EntityReleaseGroup Entity = "release-group" + + // EntityRelease is one specific edition, used when the user picked + // a particular pressing. + EntityRelease Entity = "release" + + // EntityRecording is a single track. + EntityRecording Entity = "recording" +) + +// Valid reports whether e is a known entity type. +func (e Entity) Valid() bool { + switch e { + case EntityArtist, EntityReleaseGroup, EntityRelease, EntityRecording: + return true + default: + return false + } +} + +// Expands reports whether this entity produces child wants rather than +// being downloaded directly. +func (e Entity) Expands() bool { + return e == EntityArtist +} + +// WantState is where a want sits. There is deliberately no "failed": +// an attempt can fail, a want cannot. A want that has tried and not +// found anything is still wanted, with attempts and last_error +// recording why it is taking a while. +type WantState string + +// Want states. +const ( + // WantStateWanted is the active state: due for another attempt when + // its backoff elapses. + WantStateWanted WantState = "wanted" + + // WantStateSatisfied means the library owns it. How it got there — + // downloaded here, ripped, bought elsewhere — does not matter. + WantStateSatisfied WantState = "satisfied" + + // WantStatePaused is the user saying "keep this on the list but + // stop trying". + WantStatePaused WantState = "paused" +) + +// WantScope applies to artist wants only. +type WantScope string + +// Artist want scopes. +const ( + // ScopeFuture takes only releases first published after the artist + // was added. Default, because subscribing to an artist should not + // silently queue their entire back catalogue. + ScopeFuture WantScope = "future" + + // ScopeAll backfills the whole discography as well. + ScopeAll WantScope = "all" +) + +// Want is one row of the wanted list. +type Want struct { + ID int64 `json:"id"` + MBID string `json:"mbid"` + Entity Entity `json:"entity"` + LibraryID int64 `json:"libraryId"` + + // Artist and Title are display cache only. Matching always uses + // the MBID. + Artist string `json:"artist"` + Title string `json:"title"` + + Scope WantScope `json:"scope"` + + // Secondary includes compilations, live albums and remixes in an + // artist want's expansion. + Secondary bool `json:"secondary"` + + State WantState `json:"state"` + + // ParentID is set on wants the reconciler derived from an artist + // subscription. A want the user pinned directly has none, so + // removing the artist leaves it alone. + ParentID int64 `json:"parentId,omitempty"` + + Attempts int `json:"attempts"` + LastError string `json:"lastError,omitempty"` + LastTriedAt time.Time `json:"lastTriedAt,omitempty"` + NextTryAt time.Time `json:"nextTryAt,omitempty"` + + // ExternalIDs maps provider row ID (as a string, because JSON + // object keys are strings) to that provider's own identifier for + // this want. Only set for providers that keep a persistent list of + // their own. + ExternalIDs map[string]string `json:"externalIds,omitempty"` + + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` +} + +// Anchored is always true for a want: it is an MBID by construction. +// The method exists so wants and requests read the same at call sites. +func (w Want) Anchored() bool { return w.MBID != "" } + +// Label is the wanted list's one-line description of a want. +func (w Want) Label() string { + switch { + case w.Artist != "" && w.Title != "": + return w.Artist + " — " + w.Title + case w.Title != "": + return w.Title + case w.Artist != "": + return w.Artist + default: + return string(w.Entity) + " " + w.MBID + } +} + +// Retry backoff. A want that cannot be found is usually one that will +// not be findable for a while — a pre-release, something only ever on +// physical media, an artist no source indexes — so the schedule climbs +// fast and then sits at a weekly poll rather than hammering providers +// with the same fruitless search. +const ( + // wantRetryBase is the delay after the first unsuccessful attempt. + wantRetryBase = 6 * time.Hour + + // wantRetryMax caps the backoff. A weekly retry on a list of a few + // hundred wants is a handful of searches a day, which every + // provider tolerates. + wantRetryMax = 7 * 24 * time.Hour + + // wantRetryJitter spreads retries so a list added in one sitting + // does not come due in one burst. + wantRetryJitter = 0.2 +) + +// nextRetry returns when a want with the given attempt count should be +// tried again: exponential from wantRetryBase, capped at wantRetryMax, +// jittered so a batch added together does not stay in lockstep forever. +func nextRetry(now time.Time, attempts int) time.Time { + if attempts < 1 { + attempts = 1 + } + + // Cap the exponent before shifting so a long-lived want cannot + // overflow the duration into something negative. + const maxExp = 16 + + exp := min(attempts-1, maxExp) + + delay := float64(wantRetryBase) * math.Pow(2, float64(exp)) + if delay > float64(wantRetryMax) { + delay = float64(wantRetryMax) + } + + jitter := delay * wantRetryJitter * (rand.Float64()*2 - 1) //nolint:gosec // spreading retries, not a secret + + return now.Add(time.Duration(delay + jitter)) +} + +// wantSource is the request source recorded for reconciler-raised +// requests, so the downloads list can tell them apart from the ones a +// user started by hand. +const wantSource = "wanted" + +// ToRequest builds the download request that would satisfy this want. +// Expected is filled by the caller from the catalog, since resolving a +// tracklist is I/O and this is not. +func (w Want) ToRequest(id string) Request { + req := Request{ + ID: id, + LibraryID: w.LibraryID, + Artist: w.Artist, + Album: w.Title, + WantID: w.ID, + Source: wantSource, + } + + switch w.Entity { + case EntityRelease: + req.ReleaseMBID = w.MBID + case EntityReleaseGroup: + req.ReleaseGroupMBID = w.MBID + case EntityRecording: + // A recording has no release anchor, so ranking has only the + // title to go on and auto-pick stays off. The MBID is still + // carried in RecordingMBID so a provider that can use it does. + req.RecordingMBID = w.MBID + case EntityArtist: + // Artist wants expand into children and are never turned into + // a request directly; this case exists so the switch is + // exhaustive rather than because it can happen. + } + + return req +} diff --git a/backend/download/want_test.go b/backend/download/want_test.go new file mode 100644 index 0000000..c09b297 --- /dev/null +++ b/backend/download/want_test.go @@ -0,0 +1,378 @@ +package download + +import ( + "context" + "testing" + "time" +) + +func TestNextRetryClimbsAndCaps(t *testing.T) { + t.Parallel() + + now := time.Date(2026, 7, 29, 12, 0, 0, 0, time.UTC) + + // The jitter makes exact equality wrong to assert, so each step is + // checked as a band around its nominal delay. + tests := []struct { + attempts int + nominal time.Duration + }{ + {attempts: 1, nominal: wantRetryBase}, + {attempts: 2, nominal: 2 * wantRetryBase}, + {attempts: 3, nominal: 4 * wantRetryBase}, + {attempts: 20, nominal: wantRetryMax}, + {attempts: 500, nominal: wantRetryMax}, + } + + for _, tt := range tests { + got := nextRetry(now, tt.attempts).Sub(now) + + lo := time.Duration(float64(tt.nominal) * (1 - wantRetryJitter)) + hi := time.Duration(float64(tt.nominal) * (1 + wantRetryJitter)) + + if got < lo || got > hi { + t.Errorf( + "attempts=%d delay=%v, want within [%v, %v]", + tt.attempts, got, lo, hi, + ) + } + } +} + +// A want that has been retried for years must not overflow into a +// negative delay, which would make it due forever. +func TestNextRetryNeverGoesBackwards(t *testing.T) { + t.Parallel() + + now := time.Now() + + for _, attempts := range []int{0, 1, 64, 1000, 1 << 20} { + if next := nextRetry(now, attempts); !next.After(now) { + t.Errorf("attempts=%d scheduled %v, not after now", attempts, next) + } + } +} + +func TestReleaseDateAfter(t *testing.T) { + t.Parallel() + + since := time.Date(2026, 3, 15, 0, 0, 0, 0, time.UTC) + + tests := []struct { + name string + date string + want bool + }{ + {name: "later full date", date: "2026-06-01", want: true}, + {name: "earlier full date", date: "2026-01-01", want: false}, + {name: "same day is not after", date: "2026-03-15", want: false}, + {name: "later year", date: "2027", want: true}, + // A bare year is read as its 1 January, so the year of the + // subscription itself does not count as new. + {name: "same year, bare", date: "2026", want: false}, + {name: "later month, bare", date: "2026-08", want: true}, + {name: "earlier month, bare", date: "2026-02", want: false}, + {name: "unknown date is old", date: "", want: false}, + } + + for _, tt := range tests { + if got := releaseDateAfter(tt.date, since); got != tt.want { + t.Errorf("%s: releaseDateAfter(%q) = %v, want %v", + tt.name, tt.date, got, tt.want) + } + } +} + +func TestWantsReleaseGroupFilters(t *testing.T) { + t.Parallel() + + subscribed := time.Date(2026, 3, 15, 0, 0, 0, 0, time.UTC) + + future := Want{Scope: ScopeFuture, CreatedAt: subscribed} + all := Want{Scope: ScopeAll, CreatedAt: subscribed} + allSecondary := Want{ + Scope: ScopeAll, Secondary: true, CreatedAt: subscribed, + } + + newAlbum := CatalogItem{MBID: "a", FirstReleaseDate: "2026-09-01"} + oldAlbum := CatalogItem{MBID: "b", FirstReleaseDate: "1997-04-22"} + ownedAlbum := CatalogItem{ + MBID: "c", FirstReleaseDate: "2026-09-01", InLibrary: true, + } + liveAlbum := CatalogItem{ + MBID: "d", + FirstReleaseDate: "2026-09-01", + SecondaryTypes: []string{"Live"}, + } + + tests := []struct { + name string + artist Want + rg CatalogItem + want bool + }{ + {name: "future takes new", artist: future, rg: newAlbum, want: true}, + {name: "future skips old", artist: future, rg: oldAlbum, want: false}, + {name: "all takes old", artist: all, rg: oldAlbum, want: true}, + {name: "owned is never wanted", artist: all, rg: ownedAlbum, want: false}, + {name: "secondary off skips live", artist: all, rg: liveAlbum, want: false}, + { + name: "secondary on takes live", + artist: allSecondary, + rg: liveAlbum, + want: true, + }, + { + name: "no mbid is unusable", + artist: all, + rg: CatalogItem{FirstReleaseDate: "2026-09-01"}, + want: false, + }, + } + + for _, tt := range tests { + if got := wantsReleaseGroup(tt.artist, tt.rg); got != tt.want { + t.Errorf("%s: got %v, want %v", tt.name, got, tt.want) + } + } +} + +func TestWantToRequestAnchors(t *testing.T) { + t.Parallel() + + tests := []struct { + entity Entity + check func(Request) string + }{ + { + entity: EntityReleaseGroup, + check: func(r Request) string { + return r.ReleaseGroupMBID + }, + }, + { + entity: EntityRelease, + check: func(r Request) string { return r.ReleaseMBID }, + }, + { + entity: EntityRecording, + check: func(r Request) string { return r.RecordingMBID }, + }, + } + + for _, tt := range tests { + w := Want{ID: 7, MBID: "mbid-x", Entity: tt.entity, LibraryID: 1} + + req := w.ToRequest("req-1") + + if got := tt.check(req); got != "mbid-x" { + t.Errorf("%s: anchor = %q, want mbid-x", tt.entity, got) + } + + if !req.Anchored() { + t.Errorf("%s: request is not anchored", tt.entity) + } + + if req.WantID != 7 { + t.Errorf("%s: WantID = %d, want 7", tt.entity, req.WantID) + } + + if req.Source != wantSource { + t.Errorf("%s: Source = %q, want %q", tt.entity, req.Source, wantSource) + } + } +} + +// An anchored request with no tracklist has nothing to verify itself +// against, so it must not be auto-picked no matter how good the +// candidate looks. +func TestAutoPickableRequiresTracklist(t *testing.T) { + t.Parallel() + + req := Request{ReleaseGroupMBID: "rg-1", Artist: "A", Album: "B"} + + ranked := []Candidate{{ + Match: MatchScore{Overall: 0.99, Anchored: true}, + Quality: QualityScore{Overall: 0.9}, + Score: 0.95, + }} + + if AutoPickable(req, ranked) { + t.Error("auto-picked a request with no expected tracklist") + } + + req.Expected = []ExpectedTrack{{Position: 1, Title: "T"}} + + if !AutoPickable(req, ranked) { + t.Error("did not auto-pick a well-anchored, well-matched request") + } +} + +// The wanted list's identity is the MBID, so the same one arriving +// twice — in a different case, with whitespace — is one row. +func TestAddWantNormalizesAndDeduplicates(t *testing.T) { + t.Parallel() + + f := newManagerFixture(t) + ctx := context.Background() + + first, err := f.store.AddWant(ctx, Want{ + MBID: " ABC-123 ", + Entity: EntityReleaseGroup, + LibraryID: 1, + Title: "OK Computer", + }) + if err != nil { + t.Fatalf("AddWant: %v", err) + } + + second, err := f.store.AddWant(ctx, Want{ + MBID: "abc-123", + Entity: EntityReleaseGroup, + LibraryID: 1, + }) + if err != nil { + t.Fatalf("AddWant again: %v", err) + } + + if first != second { + t.Errorf("ids %d and %d, want the same row", first, second) + } + + // Re-adding with no title must not wipe the one we have. + w, err := f.store.GetWant(ctx, first) + if err != nil { + t.Fatalf("GetWant: %v", err) + } + + if w.Title != "OK Computer" { + t.Errorf("title = %q, want it preserved", w.Title) + } + + if w.MBID != "abc-123" { + t.Errorf("mbid = %q, want normalized", w.MBID) + } +} + +func TestWantStoreLifecycle(t *testing.T) { + t.Parallel() + + f := newManagerFixture(t) + ctx := context.Background() + + id, err := f.store.AddWant(ctx, Want{ + MBID: "rg-1", + Entity: EntityReleaseGroup, + LibraryID: 1, + Artist: "Radiohead", + Title: "OK Computer", + }) + if err != nil { + t.Fatalf("AddWant: %v", err) + } + + // A brand new want is due immediately. + due, err := f.store.ListDueWants(ctx, 10) + if err != nil { + t.Fatalf("ListDueWants: %v", err) + } + + if len(due) != 1 { + t.Fatalf("got %d due wants, want 1", len(due)) + } + + // Recording an attempt pushes it out of the due set without + // changing its state: a want that was not found is still wanted. + if err := f.store.RecordAttempt(ctx, id, 0, "no source has it yet"); err != nil { + t.Fatalf("RecordAttempt: %v", err) + } + + due, err = f.store.ListDueWants(ctx, 10) + if err != nil { + t.Fatalf("ListDueWants after attempt: %v", err) + } + + if len(due) != 0 { + t.Errorf("got %d due wants after an attempt, want 0", len(due)) + } + + w, err := f.store.GetWant(ctx, id) + if err != nil { + t.Fatalf("GetWant: %v", err) + } + + if w.State != WantStateWanted { + t.Errorf("state = %q, want it still wanted", w.State) + } + + if w.Attempts != 1 { + t.Errorf("attempts = %d, want 1", w.Attempts) + } + + if w.LastError == "" { + t.Error("last error was not recorded") + } + + if err := f.store.SatisfyWant(ctx, id); err != nil { + t.Fatalf("SatisfyWant: %v", err) + } + + w, err = f.store.GetWant(ctx, id) + if err != nil { + t.Fatalf("GetWant after satisfy: %v", err) + } + + if w.State != WantStateSatisfied { + t.Errorf("state = %q, want satisfied", w.State) + } +} + +// Removing an artist subscription takes the albums it derived with it, +// so a user who unsubscribes does not keep downloading that artist. +func TestDeleteArtistWantCascadesToChildren(t *testing.T) { + t.Parallel() + + f := newManagerFixture(t) + ctx := context.Background() + + artist, err := f.store.AddWant(ctx, Want{ + MBID: "artist-1", + Entity: EntityArtist, + LibraryID: 1, + Artist: "Radiohead", + }) + if err != nil { + t.Fatalf("AddWant artist: %v", err) + } + + if _, err := f.store.AddWant(ctx, Want{ + MBID: "rg-1", + Entity: EntityReleaseGroup, + LibraryID: 1, + ParentID: artist, + }); err != nil { + t.Fatalf("AddWant child: %v", err) + } + + children, err := f.store.ListChildWants(ctx, artist) + if err != nil { + t.Fatalf("ListChildWants: %v", err) + } + + if len(children) != 1 { + t.Fatalf("got %d children, want 1", len(children)) + } + + if err := f.store.DeleteWant(ctx, artist); err != nil { + t.Fatalf("DeleteWant: %v", err) + } + + all, err := f.store.ListWants(ctx) + if err != nil { + t.Fatalf("ListWants: %v", err) + } + + if len(all) != 0 { + t.Errorf("got %d wants after deleting the artist, want 0", len(all)) + } +} diff --git a/backend/download/wantstore.go b/backend/download/wantstore.go new file mode 100644 index 0000000..1eeb61f --- /dev/null +++ b/backend/download/wantstore.go @@ -0,0 +1,307 @@ +package download + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "yellowjacket/backend/database/sql/sqlcgen" +) + +// The wanted list's persistence. Kept apart from the request/item +// storage in store.go because the two have opposite lifetimes: items +// are written constantly and swept, wants are written rarely and kept. + +// defaultDueBatch bounds how many wants one reconcile pass picks up. +// The list can be thousands of rows after a discography backfill, and a +// pass that tried to search all of them would take a day and annoy +// every provider on the way. +const defaultDueBatch = 25 + +// AddWant inserts a want, or returns the existing row's ID if the same +// MBID is already wanted in this library. Asking twice is not two +// wants, and re-asking must not reset a backoff that is deliberately +// long. +func (s *Store) AddWant(ctx context.Context, w Want) (int64, error) { + if !w.Entity.Valid() { + return 0, fmt.Errorf("%w: entity %q", ErrUnsupported, w.Entity) + } + + if w.Scope == "" { + w.Scope = ScopeFuture + } + + // The MBID is the identity of a want, so it is normalized here + // rather than at each call site: the same identifier arriving from + // an Explore page and from a pasted URL must be one row, or the + // uniqueness constraint that makes artist expansion idempotent + // stops holding. + w.MBID = strings.ToLower(strings.TrimSpace(w.MBID)) + + if w.MBID == "" { + return 0, fmt.Errorf("%w: a want needs an MBID", ErrUnsupported) + } + + parent := sql.NullInt64{} + if w.ParentID != 0 { + parent = sql.NullInt64{Int64: w.ParentID, Valid: true} + } + + id, err := s.db.Queries.UpsertDownloadWant( + ctx, + sqlcgen.UpsertDownloadWantParams{ + Mbid: w.MBID, + Entity: string(w.Entity), + LibraryID: w.LibraryID, + Artist: w.Artist, + Title: w.Title, + Scope: string(w.Scope), + Secondary: boolToInt(w.Secondary), + ParentID: parent, + }, + ) + if err != nil { + return 0, fmt.Errorf("add download want: %w", err) + } + + return id, nil +} + +// GetWant loads one want. +func (s *Store) GetWant(ctx context.Context, id int64) (Want, error) { + row, err := s.db.ReadQueries.GetDownloadWant(ctx, id) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return Want{}, fmt.Errorf("%w: want %d", ErrNotFound, id) + } + + return Want{}, fmt.Errorf("get download want: %w", err) + } + + return wantRowToWant(row), nil +} + +// FindWant looks a want up by what it names rather than by row ID, +// which is how callers holding an MBID (the Explore pages, a provider +// sync) ask "is this already wanted?". +func (s *Store) FindWant( + ctx context.Context, + mbid string, + libraryID int64, +) (Want, bool, error) { + row, err := s.db.ReadQueries.GetDownloadWantByMBID( + ctx, + sqlcgen.GetDownloadWantByMBIDParams{Mbid: mbid, LibraryID: libraryID}, + ) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return Want{}, false, nil + } + + return Want{}, false, fmt.Errorf("find download want: %w", err) + } + + return wantRowToWant(row), true, nil +} + +// ListWants returns the whole wanted list, active first. +func (s *Store) ListWants(ctx context.Context) ([]Want, error) { + rows, err := s.db.ReadQueries.ListDownloadWants(ctx) + if err != nil { + return nil, fmt.Errorf("list download wants: %w", err) + } + + return wantRowsToWants(rows), nil +} + +// ListArtistWants returns active artist subscriptions, which are what +// the reconciler expands. +func (s *Store) ListArtistWants(ctx context.Context) ([]Want, error) { + rows, err := s.db.ReadQueries.ListDownloadWantsByEntity( + ctx, + sqlcgen.ListDownloadWantsByEntityParams{ + Entity: string(EntityArtist), + State: string(WantStateWanted), + }, + ) + if err != nil { + return nil, fmt.Errorf("list artist wants: %w", err) + } + + return wantRowsToWants(rows), nil +} + +// ListDueWants returns downloadable wants whose backoff has elapsed, +// least-attempted first so a new addition is not stuck behind a +// hundred long-shot retries. +func (s *Store) ListDueWants(ctx context.Context, limit int) ([]Want, error) { + if limit <= 0 { + limit = defaultDueBatch + } + + rows, err := s.db.ReadQueries.ListDueDownloadWants(ctx, int64(limit)) + if err != nil { + return nil, fmt.Errorf("list due download wants: %w", err) + } + + return wantRowsToWants(rows), nil +} + +// ListChildWants returns the wants an artist subscription produced. +func (s *Store) ListChildWants( + ctx context.Context, + parentID int64, +) ([]Want, error) { + rows, err := s.db.ReadQueries.ListChildDownloadWants( + ctx, + sql.NullInt64{Int64: parentID, Valid: true}, + ) + if err != nil { + return nil, fmt.Errorf("list child download wants: %w", err) + } + + return wantRowsToWants(rows), nil +} + +// SetWantState moves a want between wanted, paused and satisfied. +func (s *Store) SetWantState( + ctx context.Context, + id int64, + state WantState, + errText string, +) error { + if err := s.db.Queries.SetDownloadWantState( + ctx, + sqlcgen.SetDownloadWantStateParams{ + State: string(state), + LastError: errText, + ID: id, + }, + ); err != nil { + return fmt.Errorf("set download want state: %w", err) + } + + return nil +} + +// RecordAttempt notes an unsuccessful pass over a want and schedules +// the next one. The want stays wanted: not finding something is a fact +// about today's providers, not a verdict on the request. +func (s *Store) RecordAttempt( + ctx context.Context, + id int64, + attempts int, + reason string, +) error { + next := nextRetry(time.Now(), attempts+1) + + if err := s.db.Queries.RecordDownloadWantAttempt( + ctx, + sqlcgen.RecordDownloadWantAttemptParams{ + LastError: reason, + NextTryAt: sql.NullTime{Time: next, Valid: true}, + ID: id, + }, + ); err != nil { + return fmt.Errorf("record download want attempt: %w", err) + } + + return nil +} + +// SatisfyWant marks a want as owned. +func (s *Store) SatisfyWant(ctx context.Context, id int64) error { + if err := s.db.Queries.SatisfyDownloadWant(ctx, id); err != nil { + return fmt.Errorf("satisfy download want: %w", err) + } + + return nil +} + +// SetWantExternalIDs records the identifiers external managers gave +// this want in their own persistent lists. +func (s *Store) SetWantExternalIDs( + ctx context.Context, + id int64, + ids map[string]string, +) error { + encoded, err := json.Marshal(ids) + if err != nil { + return fmt.Errorf("encode want external ids: %w", err) + } + + if err := s.db.Queries.SetDownloadWantExternalIDs( + ctx, + sqlcgen.SetDownloadWantExternalIDsParams{ + ExternalIds: string(encoded), + ID: id, + }, + ); err != nil { + return fmt.Errorf("set want external ids: %w", err) + } + + return nil +} + +// DeleteWant removes a want and, by cascade, anything an artist want +// derived. +func (s *Store) DeleteWant(ctx context.Context, id int64) error { + if err := s.db.Queries.DeleteDownloadWant(ctx, id); err != nil { + return fmt.Errorf("delete download want: %w", err) + } + + return nil +} + +// ClearSatisfiedWants drops everything already owned. +func (s *Store) ClearSatisfiedWants(ctx context.Context) error { + if err := s.db.Queries.DeleteSatisfiedDownloadWants(ctx); err != nil { + return fmt.Errorf("clear satisfied download wants: %w", err) + } + + return nil +} + +// wantRowsToWants decodes a slice of stored rows. +func wantRowsToWants(rows []sqlcgen.DownloadWant) []Want { + out := make([]Want, 0, len(rows)) + + for _, r := range rows { + out = append(out, wantRowToWant(r)) + } + + return out +} + +// wantRowToWant decodes a stored want row. A malformed external-ID +// blob yields an empty map rather than an error: losing the link to a +// Lidarr row is recoverable on the next sync, making the wanted list +// unreadable is not. +func wantRowToWant(r sqlcgen.DownloadWant) Want { + external := map[string]string{} + _ = json.Unmarshal([]byte(r.ExternalIds), &external) + + return Want{ + ID: r.ID, + MBID: r.Mbid, + Entity: Entity(r.Entity), + LibraryID: r.LibraryID, + Artist: r.Artist, + Title: r.Title, + Scope: WantScope(r.Scope), + Secondary: r.Secondary != 0, + State: WantState(r.State), + ParentID: r.ParentID.Int64, + Attempts: int(r.Attempts), + LastError: r.LastError, + LastTriedAt: r.LastTriedAt.Time, + NextTryAt: r.NextTryAt.Time, + ExternalIDs: external, + CreatedAt: r.CreatedAt, + UpdatedAt: r.UpdatedAt, + } +} diff --git a/backend/downloadcatalog.go b/backend/downloadcatalog.go new file mode 100644 index 0000000..427df36 --- /dev/null +++ b/backend/downloadcatalog.go @@ -0,0 +1,217 @@ +package backend + +import ( + "context" + + "yellowjacket/backend/download" + "yellowjacket/backend/explore" +) + +// exploreCatalog adapts the explore service to the narrow view of the +// music world the wanted list needs. +// +// The adapter lives here, in the composition root, rather than in +// either package: explore should not know that downloads exist, and +// download should not pull in the whole explore index to ask four +// questions. Everything below is a translation, with no policy of its +// own — policy belongs to the reconciler. +type exploreCatalog struct { + explore *explore.Service +} + +// newExploreCatalog wires the wanted list to the explore index. +func newExploreCatalog(e *explore.Service) download.CatalogPort { + return &exploreCatalog{explore: e} +} + +// ReleaseGroupsForArtist returns an artist's discography. +// +// An empty result is not an error. The explore index fetches +// discographies lazily, so the first time an artist is subscribed to +// the honest answer is "not indexed yet" — and BrowseReleaseGroups has +// already kicked off the background fetch that makes the next pass +// useful. +func (c *exploreCatalog) ReleaseGroupsForArtist( + _ context.Context, + artistMBID string, +) ([]download.CatalogItem, error) { + groups, err := c.explore.BrowseReleaseGroups(artistMBID) + if err != nil { + return nil, err + } + + out := make([]download.CatalogItem, 0, len(groups)) + + for _, rg := range groups { + out = append(out, download.CatalogItem{ + MBID: rg.MBID, + Title: rg.Title, + Artist: rg.ArtistCredit, + ArtistMBID: rg.ArtistMBID, + PrimaryType: rg.PrimaryType, + SecondaryTypes: rg.SecondaryTypes, + FirstReleaseDate: rg.FirstReleaseDate, + InLibrary: rg.InLibrary, + }) + } + + return out, nil +} + +// Tracklist resolves what a want should contain. +// +// For an album this is the tracklist of its best release, which is what +// the download pipeline verifies an unattended grab against. For a +// single track it is that one track, built from the want's own cached +// title — there is no recording lookup on the index, and inventing one +// to learn a title the UI already passed in would be work for its own +// sake. +func (c *exploreCatalog) Tracklist( + _ context.Context, + entity download.Entity, + mbid string, +) ([]download.ExpectedTrack, error) { + if entity == download.EntityRecording { + // Handled by the reconciler from the want's own fields; see + // recordingTracklist below. + return nil, nil + } + + releases, err := c.explore.BrowseReleases(mbid) + if err != nil { + return nil, err + } + + best := bestRelease(releases, mbid) + if best == nil { + return nil, nil + } + + out := make([]download.ExpectedTrack, 0, len(best.Tracks)) + + for _, t := range best.Tracks { + out = append(out, download.ExpectedTrack{ + Position: t.Position, + DiscNumber: t.DiscNumber, + Title: t.Title, + Artist: best.ArtistCredit, + LengthMillis: int64(t.Length), + }) + } + + return out, nil +} + +// bestRelease picks which edition of an album to match a download +// against. +// +// When the want named a specific release, that one. Otherwise the one +// with the most tracks that still has a tracklist at all: a download +// scored against a truncated tracklist looks incomplete when it is +// fine, and a false "missing tracks" reading is what stops a good +// candidate from clearing the auto-pick bar. +func bestRelease(releases []explore.MBRelease, wantedMBID string) *explore.MBRelease { + var best *explore.MBRelease + + for i := range releases { + r := &releases[i] + + if r.MBID == wantedMBID && len(r.Tracks) > 0 { + return r + } + + if len(r.Tracks) == 0 { + continue + } + + if best == nil || len(r.Tracks) > len(best.Tracks) { + best = r + } + } + + return best +} + +// Owns reports whether the library already has what an MBID names. +// +// Releases are the gap: the library records release groups and +// recordings, not specific editions, so a want for one particular +// pressing is only retired when its own download completes. That is +// the right failure — quietly satisfying a "want the 1997 Japanese +// pressing" because some edition is owned would be answering a +// different question than the one asked. +func (c *exploreCatalog) Owns( + _ context.Context, + entity download.Entity, + mbid string, +) (bool, error) { + if mbid == "" || entity == download.EntityRelease { + return false, nil + } + + found := c.explore.CheckLibraryMBIDs([]string{mbid}) + + kind, ok := found[mbid] + if !ok { + return false, nil + } + + switch entity { + case download.EntityReleaseGroup: + return kind == "release_group", nil + case download.EntityRecording: + return kind == "recording", nil + case download.EntityArtist: + return kind == "artist", nil + case download.EntityRelease: + return false, nil + default: + return false, nil + } +} + +// Describe fills in display text for a want added as a bare MBID. +func (c *exploreCatalog) Describe( + _ context.Context, + entity download.Entity, + mbid string, +) (download.CatalogItem, bool) { + switch entity { + case download.EntityArtist: + artist, err := c.explore.LookupArtist(mbid) + if err != nil || artist == nil { + return download.CatalogItem{}, false + } + + return download.CatalogItem{ + MBID: mbid, + Artist: artist.Name, + Title: artist.Name, + }, true + + case download.EntityReleaseGroup, download.EntityRelease: + rg, err := c.explore.LookupReleaseGroup(mbid) + if err != nil || rg == nil { + return download.CatalogItem{}, false + } + + return download.CatalogItem{ + MBID: mbid, + Title: rg.Title, + Artist: rg.ArtistCredit, + ArtistMBID: rg.ArtistMBID, + PrimaryType: rg.PrimaryType, + SecondaryTypes: rg.SecondaryTypes, + FirstReleaseDate: rg.FirstReleaseDate, + InLibrary: rg.InLibrary, + }, true + + case download.EntityRecording: + // No recording lookup on the index; the caller keeps whatever + // title it was given. + return download.CatalogItem{}, false + + default: + return download.CatalogItem{}, false + } +} diff --git a/backend/events/events.go b/backend/events/events.go index 10944f5..b223c9e 100644 --- a/backend/events/events.go +++ b/backend/events/events.go @@ -117,4 +117,21 @@ const ( // the album detail page can re-fetch its versions / tracklist without // the initial request having blocked on a live MusicBrainz browse. AlbumReleasesReady = "AlbumReleasesReady" + + // DownloadProvidersChanged fires after a download client is added, + // edited, enabled/disabled or removed, so the settings page and any + // open download picker re-read the provider list. + DownloadProvidersChanged = "DownloadProvidersChanged" + + // DownloadsChanged fires when the set of download requests changes + // (started, picked, cancelled, cleared). Per-transfer progress does + // not use this — it flows through the jobs registry's JobsChanged, + // which already coalesces high-frequency updates. + DownloadsChanged = "DownloadsChanged" + + // WantedListChanged fires when the wanted list gains, loses or + // retires an entry — including from a background reconcile pass, + // which is why the list is event-driven rather than fetched once on + // mount. + WantedListChanged = "WantedListChanged" ) diff --git a/backend/explore/artifactbuild.go b/backend/explore/artifactbuild.go new file mode 100644 index 0000000..b592367 --- /dev/null +++ b/backend/explore/artifactbuild.go @@ -0,0 +1,87 @@ +package explore + +import ( + "context" + + "yellowjacket/backend/jobs" +) + +// Fetching and merging the prebuilt catalog artifact. +// +// This is the whole catalog build on a user's machine. The import that +// derives the catalog from the MetaBrainz dumps lives behind the +// `indexbuild` tag and runs only in CI (see dumpbuild_stub.go). + +// Artifact build stages, shown in the jobs panel. +const ( + artifactStageDownload = iota + artifactStageMerge +) + +var artifactStageNames = [...]string{ + "Downloading catalog", + "Merging catalog", +} + +// tryCoreArtifact fetches and merges the prebuilt catalog. Every +// failure path is non-fatal by design: the caller falls back, and a +// fresh install with no network still gets its own library in Explore. +func (si *SearchIndex) tryCoreArtifact(ctx context.Context) error { + si.mu.Lock() + si.buildStatus = IndexStatus{ + Building: true, + Tiers: []TierStatus{ + {Name: artifactStageNames[artifactStageDownload], State: "pending"}, + {Name: artifactStageNames[artifactStageMerge], State: "pending"}, + }, + } + si.mu.Unlock() + si.refreshStatusCounts() + + fetcher, err := newArtifactFetcher(si) + if err != nil { + return err + } + + si.setTierStatus(artifactStageNames[artifactStageDownload], "running", 0, 0) + si.logIndexJob(jobs.LevelInfo, "Fetching prebuilt catalog") + + path, err := fetcher.fetch(ctx) + if err != nil { + si.setTierStatus(artifactStageNames[artifactStageDownload], "error", 0, 0) + + return err + } + + si.setTierStatus(artifactStageNames[artifactStageDownload], "complete", 0, 0) + si.setTierStatus(artifactStageNames[artifactStageMerge], "running", 0, 0) + + if err := si.importCoreArtifact(ctx, path); err != nil { + si.setTierStatus(artifactStageNames[artifactStageMerge], "error", 0, 0) + si.removeArtifactFile(path) + + return err + } + + si.removeArtifactFile(path) + si.setTierStatus(artifactStageNames[artifactStageMerge], "complete", 0, 0) + + si.mu.Lock() + si.buildStatus.Building = false + si.mu.Unlock() + + // Fold the user's own library into the freshly-merged catalog: + // owned entities the artifact does not cover are inserted, and + // covered ones are flagged in_library. + si.PopulateLocalCrossReferences() + si.refreshStatusCounts() + si.scheduleChampionRebuild() + + return nil +} + +// artifactAlreadyMerged reports whether this index already carries a +// merged artifact, so a restart does not re-download one. +func (si *SearchIndex) artifactAlreadyMerged() bool { + return si.hasMeta(coreArtifactVersionKey) +} diff --git a/backend/explore/artifactfetch.go b/backend/explore/artifactfetch.go new file mode 100644 index 0000000..8970633 --- /dev/null +++ b/backend/explore/artifactfetch.go @@ -0,0 +1,415 @@ +package explore + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/klauspost/compress/zstd" + + "yellowjacket/backend/jobs" + "yellowjacket/backend/system" +) + +// Download of the prebuilt core index artifact. +// +// The artifact is published by .gitea/workflows/index-artifact.yml to a +// Gitea generic package registry under a fixed "latest" version, so the +// client needs no package-listing API (which requires a token) — only an +// anonymous GET of a predictable URL. +// +// Everything here degrades to "no artifact" rather than failing: an +// offline install, a 404 before the first artifact is published, or a +// corrupt download all leave the caller free to fall back to the normal +// build path. A missing artifact must never be the reason Explore is +// broken. + +const ( + // defaultCoreArtifactBaseURL is where the CI-built artifact lives. + // Overridable via YJ_CORE_INDEX_URL for testing and for anyone + // self-hosting their own index. + defaultCoreArtifactBaseURL = "https://git.ljones.me/api/packages/yonlu/" + + "generic/yellowjacket-core-index/latest/" + + // coreArtifactFile is the compressed artifact's filename, and + // coreArtifactChecksumFile its detached sha256. + coreArtifactFile = "core-index.db.zst" + coreArtifactChecksum = "core-index.db.zst.sha256" + + // artifactURLEnv overrides the base URL. + artifactURLEnv = "YJ_CORE_INDEX_URL" + + // artifactDiscoverTimeout bounds the checksum fetch, which doubles as + // the availability probe. Short: a first run should not sit for + // minutes deciding whether an artifact exists. + artifactDiscoverTimeout = 30 * time.Second + + // artifactMinFreeBytes is the free disk needed to fetch and unpack. + // The compressed artifact plus its expansion plus merge headroom — + // two orders of magnitude below the full dump import's 6GB floor, + // which is much of the point. + artifactMinFreeBytes = 1 << 30 + + // artifactMaxRetries bounds resume attempts for the body download. + artifactMaxRetries = 5 +) + +// ErrArtifactUnavailable means no artifact could be fetched. It is an +// expected outcome (offline, not yet published), not a failure. +var ErrArtifactUnavailable = errors.New("core index artifact unavailable") + +// coreArtifactBaseURL resolves the artifact location, honouring the +// environment override. +func coreArtifactBaseURL() string { + if v := strings.TrimSpace(os.Getenv(artifactURLEnv)); v != "" { + if !strings.HasSuffix(v, "/") { + return v + "/" + } + + return v + } + + return defaultCoreArtifactBaseURL +} + +// artifactFetcher downloads and unpacks the core index artifact. +type artifactFetcher struct { + si *SearchIndex + client *http.Client + baseURL string + stagingDir string +} + +func newArtifactFetcher(si *SearchIndex) (*artifactFetcher, error) { + dataDir, err := system.GetUserDataDirPath() + if err != nil { + return nil, fmt.Errorf("core artifact: data dir: %w", err) + } + + stagingDir := filepath.Join(dataDir, "explore-staging") + if err := os.MkdirAll(stagingDir, 0o755); err != nil { + return nil, fmt.Errorf("core artifact: staging dir: %w", err) + } + + return &artifactFetcher{ + si: si, + // No client-level timeout: a 70MB body over a slow link can take + // a while. Stalls are handled by resume rather than by killing + // the whole download. + client: &http.Client{}, + baseURL: coreArtifactBaseURL(), + stagingDir: stagingDir, + }, nil +} + +// compressedPath and unpackedPath are the two staging files. Both live +// beside the dump importer's own staging data and are removed on success. +func (f *artifactFetcher) compressedPath() string { + return filepath.Join(f.stagingDir, coreArtifactFile) +} + +func (f *artifactFetcher) unpackedPath() string { + return filepath.Join(f.stagingDir, "core-index.db") +} + +// fetch downloads, verifies and decompresses the artifact, returning the +// path to the ready-to-merge database. +func (f *artifactFetcher) fetch(ctx context.Context) (string, error) { + if err := checkFreeDisk(f.stagingDir, artifactMinFreeBytes); err != nil { + return "", err + } + + want, err := f.fetchChecksum(ctx) + if err != nil { + return "", err + } + + if err := f.download(ctx); err != nil { + return "", err + } + + got, err := fileSHA256(f.compressedPath()) + if err != nil { + return "", err + } + + if got != want { + // A partial file that resumed against a newer published artifact + // would fail here forever; discard it so the next attempt starts + // clean rather than re-resuming into the same mismatch. + _ = os.Remove(f.compressedPath()) + + return "", fmt.Errorf("%w: checksum mismatch (got %s, want %s)", + ErrArtifactUnusable, got, want) + } + + if err := f.decompress(ctx); err != nil { + return "", err + } + + // The compressed copy is dead weight once unpacked. + _ = os.Remove(f.compressedPath()) + + return f.unpackedPath(), nil +} + +// fetchChecksum retrieves the expected sha256. This doubles as the +// availability probe: it is a few bytes, so a missing or unreachable +// artifact is discovered without starting a large download. +func (f *artifactFetcher) fetchChecksum(ctx context.Context) (string, error) { + probeCtx, cancel := context.WithTimeout(ctx, artifactDiscoverTimeout) + defer cancel() + + req, err := http.NewRequestWithContext( + probeCtx, http.MethodGet, f.baseURL+coreArtifactChecksum, nil) + if err != nil { + return "", fmt.Errorf("%w: checksum request: %w", ErrArtifactUnavailable, err) + } + + req.Header.Set("User-Agent", lbUserAgent) + + resp, err := f.client.Do(req) + if err != nil { + return "", fmt.Errorf("%w: %w", ErrArtifactUnavailable, err) + } + + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("%w: HTTP %d fetching checksum", + ErrArtifactUnavailable, resp.StatusCode) + } + + // The file is `sha256sum` output: " ". + body, err := io.ReadAll(io.LimitReader(resp.Body, 4096)) + if err != nil { + return "", fmt.Errorf("%w: read checksum: %w", ErrArtifactUnavailable, err) + } + + sum := strings.TrimSpace(string(body)) + if i := strings.IndexAny(sum, " \t"); i > 0 { + sum = sum[:i] + } + + if len(sum) != sha256.Size*2 { + return "", fmt.Errorf("%w: malformed checksum %q", ErrArtifactUnusable, sum) + } + + return strings.ToLower(sum), nil +} + +// download fetches the artifact body, resuming a partial file with a +// Range request rather than restarting it. +func (f *artifactFetcher) download(ctx context.Context) error { + var lastErr error + + for attempt := range artifactMaxRetries { + if err := ctx.Err(); err != nil { + return err + } + + if attempt > 0 { + delay := min(streamRetryBaseDelay<<(attempt-1), streamRetryMaxDelay) + + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(delay): + } + } + + err := f.downloadOnce(ctx) + if err == nil { + return nil + } + + lastErr = err + + f.si.logger.Warn("core artifact: download attempt failed", + "attempt", attempt+1, "error", err, + ) + } + + return fmt.Errorf("%w: after %d attempts: %w", + ErrArtifactUnavailable, artifactMaxRetries, lastErr) +} + +func (f *artifactFetcher) downloadOnce(ctx context.Context) error { + // A file left by a previous attempt is resumed from its length. + var have int64 + + if fi, err := os.Stat(f.compressedPath()); err == nil { + have = fi.Size() + } + + req, err := http.NewRequestWithContext( + ctx, http.MethodGet, f.baseURL+coreArtifactFile, nil) + if err != nil { + return fmt.Errorf("artifact request: %w", err) + } + + req.Header.Set("User-Agent", lbUserAgent) + + if have > 0 { + req.Header.Set("Range", "bytes="+strconv.FormatInt(have, 10)+"-") + } + + resp, err := f.client.Do(req) + if err != nil { + return fmt.Errorf("artifact fetch: %w", err) + } + + defer func() { _ = resp.Body.Close() }() + + switch resp.StatusCode { + case http.StatusPartialContent: + // Resuming: append. + case http.StatusOK: + // The server ignored the Range header (or there was nothing to + // resume), so this is a whole-file body and the partial must go. + have = 0 + default: + return fmt.Errorf("%w: HTTP %d fetching artifact", + ErrArtifactUnavailable, resp.StatusCode) + } + + flags := os.O_CREATE | os.O_WRONLY + if have > 0 { + flags |= os.O_APPEND + } else { + flags |= os.O_TRUNC + } + + file, err := os.OpenFile(f.compressedPath(), flags, 0o644) + if err != nil { + return fmt.Errorf("open artifact file: %w", err) + } + + defer func() { _ = file.Close() }() + + total := have + resp.ContentLength + + if _, err := io.Copy(file, f.progressReader(resp.Body, have, total)); err != nil { + return fmt.Errorf("artifact download: %w", err) + } + + return file.Close() +} + +// progressReader wraps the body so download progress reaches the jobs +// panel, since this is the one visible wait on a fresh install. +func (f *artifactFetcher) progressReader(r io.Reader, done, total int64) io.Reader { + return &artifactProgress{ + inner: r, + done: done, + total: total, + si: f.si, + last: time.Now(), + } +} + +type artifactProgress struct { + inner io.Reader + done, total int64 + si *SearchIndex + last time.Time +} + +func (p *artifactProgress) Read(b []byte) (int, error) { + n, err := p.inner.Read(b) + p.done += int64(n) + + if time.Since(p.last) >= time.Second { + p.last = time.Now() + + detail := formatGB(p.done) + if p.total > 0 { + detail = fmt.Sprintf("%.0f%% of %s", + 100*float64(p.done)/float64(p.total), formatGB(p.total)) + } + + // Reported in KiB so a multi-hundred-MB artifact cannot overflow + // the int progress fields on a 32-bit build. + p.si.setTierDetail( + artifactStageNames[artifactStageDownload], "running", + int(p.done>>10), int(p.total>>10), detail, + ) + } + + if err != nil && !errors.Is(err, io.EOF) { + return n, fmt.Errorf("artifact body read: %w", err) + } + + return n, err //nolint:wrapcheck // io.EOF must reach the caller unwrapped. +} + +// decompress expands the zstd artifact into the staging directory. +func (f *artifactFetcher) decompress(ctx context.Context) error { + src, err := os.Open(f.compressedPath()) + if err != nil { + return fmt.Errorf("%w: open compressed artifact: %w", ErrArtifactUnusable, err) + } + + defer func() { _ = src.Close() }() + + zr, err := zstd.NewReader(src) + if err != nil { + return fmt.Errorf("%w: zstd reader: %w", ErrArtifactUnusable, err) + } + + defer zr.Close() + + dst, err := os.Create(f.unpackedPath()) + if err != nil { + return fmt.Errorf("%w: create artifact db: %w", ErrArtifactUnusable, err) + } + + defer func() { _ = dst.Close() }() + + f.si.logIndexJob(jobs.LevelInfo, "Unpacking prebuilt catalog") + + if _, err := io.Copy(dst, zr.IOReadCloser()); err != nil { + // A half-written database would be rejected by inspectArtifact, + // but leaving it around means the next run re-reads the same + // wreckage before rejecting it. + _ = os.Remove(f.unpackedPath()) + + return fmt.Errorf("%w: decompress: %w", ErrArtifactUnusable, err) + } + + if err := ctx.Err(); err != nil { + _ = os.Remove(f.unpackedPath()) + + return err + } + + return dst.Close() +} + +// fileSHA256 hashes a file. The whole file is hashed after the download +// completes rather than incrementally, because a resumed download never +// sees the bytes it skipped. +func fileSHA256(path string) (string, error) { + file, err := os.Open(path) + if err != nil { + return "", fmt.Errorf("%w: open for checksum: %w", ErrArtifactUnusable, err) + } + + defer func() { _ = file.Close() }() + + h := sha256.New() + if _, err := io.Copy(h, file); err != nil { + return "", fmt.Errorf("%w: checksum read: %w", ErrArtifactUnusable, err) + } + + return hex.EncodeToString(h.Sum(nil)), nil +} diff --git a/backend/explore/artifactfetch_test.go b/backend/explore/artifactfetch_test.go new file mode 100644 index 0000000..92dca13 --- /dev/null +++ b/backend/explore/artifactfetch_test.go @@ -0,0 +1,206 @@ +package explore + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/klauspost/compress/zstd" + + "yellowjacket/backend/database" +) + +// artifactServer serves a compressed artifact and its checksum the way +// the Gitea generic package registry does. +func artifactServer(t *testing.T, body []byte) *httptest.Server { + t.Helper() + + sum := sha256.Sum256(body) + checksum := hex.EncodeToString(sum[:]) + " " + coreArtifactFile + "\n" + + mux := http.NewServeMux() + + mux.HandleFunc("/"+coreArtifactChecksum, + func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(checksum)) + }) + + // ServeContent gives the stub real Range support, so the resume path + // is exercised against the same semantics as the package registry. + mux.HandleFunc("/"+coreArtifactFile, + func(w http.ResponseWriter, r *http.Request) { + http.ServeContent( + w, r, coreArtifactFile, time.Time{}, bytes.NewReader(body)) + }) + + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + + return srv +} + +// compressArtifact zstd-compresses a file the way CI publishes it. +func compressArtifact(t *testing.T, path string) []byte { + t.Helper() + + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read artifact: %v", err) + } + + enc, err := zstd.NewWriter(nil) + if err != nil { + t.Fatalf("zstd writer: %v", err) + } + + defer func() { _ = enc.Close() }() + + return enc.EncodeAll(raw, nil) +} + +// withArtifactEnv points the fetcher at a stub server and gives it an +// isolated data directory. +func withArtifactEnv(t *testing.T, baseURL string) { + t.Helper() + + t.Setenv(artifactURLEnv, baseURL) + t.Setenv("YJ_HOME", t.TempDir()) +} + +func TestFetchAndMergeArtifactEndToEnd(t *testing.T) { + src := writeTestArtifact(t, validMeta(), []artifactRow{ + {"artist", artA, "Artist A", "Artist A", artA, 5000}, + {"release_group", rgA, "Album A", "Artist A", artA, 3000}, + {"recording", recA, "Song A", "Artist A", artA, 2000}, + }) + + srv := artifactServer(t, compressArtifact(t, src)) + withArtifactEnv(t, srv.URL+"/") + + db := database.NewTestDB(t) + si := NewSearchIndex(db, nil, nil, testLogger()) + + if err := si.tryCoreArtifact(context.Background()); err != nil { + t.Fatalf("tryCoreArtifact: %v", err) + } + + var got int + if err := db.QueryRowWriter( + "SELECT COUNT(*) FROM explore_index", + ).Scan(&got); err != nil { + t.Fatalf("count rows: %v", err) + } + + if got != 3 { + t.Errorf("merged %d rows, want 3", got) + } + + if !si.artifactAlreadyMerged() { + t.Error("artifact merge not recorded; a restart would re-download it") + } + + // Staging must not keep a few hundred MB around after success. + staging := filepath.Join(os.Getenv("YJ_HOME"), "data", "explore-staging") + for _, name := range []string{coreArtifactFile, "core-index.db"} { + if _, err := os.Stat(filepath.Join(staging, name)); err == nil { + t.Errorf("%s left behind in staging after import", name) + } + } +} + +func TestFetchArtifactRejectsBadChecksum(t *testing.T) { + src := writeTestArtifact(t, validMeta(), []artifactRow{ + {"artist", artA, "Artist A", "Artist A", artA, 5000}, + }) + + body := compressArtifact(t, src) + + // Serve a checksum for different content. + mux := http.NewServeMux() + mux.HandleFunc("/"+coreArtifactChecksum, + func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(strings.Repeat("a", 64) + " " + coreArtifactFile)) + }) + mux.HandleFunc("/"+coreArtifactFile, + func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write(body) + }) + + srv := httptest.NewServer(mux) + defer srv.Close() + + withArtifactEnv(t, srv.URL+"/") + + db := database.NewTestDB(t) + si := NewSearchIndex(db, nil, nil, testLogger()) + + err := si.tryCoreArtifact(context.Background()) + if err == nil { + t.Fatal("expected checksum rejection") + } + + if !strings.Contains(err.Error(), "checksum mismatch") { + t.Errorf("error = %v, want a checksum mismatch", err) + } + + // A corrupt download must not leave the index claiming a catalog. + if si.hasMeta(dumpImportDoneKey) || si.artifactAlreadyMerged() { + t.Error("failed download still marked the catalog as imported") + } +} + +// A missing artifact is an ordinary outcome — the app has to keep +// working before the first one is ever published. +func TestFetchArtifactMissingIsUnavailable(t *testing.T) { + srv := httptest.NewServer(http.NotFoundHandler()) + defer srv.Close() + + withArtifactEnv(t, srv.URL+"/") + + db := database.NewTestDB(t) + si := NewSearchIndex(db, nil, nil, testLogger()) + + err := si.tryCoreArtifact(context.Background()) + if !errors.Is(err, ErrArtifactUnavailable) { + t.Errorf("error = %v, want ErrArtifactUnavailable", err) + } +} + +// The download resumes rather than restarting, which is what makes a +// large artifact survive a flaky connection. +func TestFetchArtifactResumesPartialDownload(t *testing.T) { + src := writeTestArtifact(t, validMeta(), []artifactRow{ + {"artist", artA, "Artist A", "Artist A", artA, 5000}, + }) + + body := compressArtifact(t, src) + srv := artifactServer(t, body) + withArtifactEnv(t, srv.URL+"/") + + db := database.NewTestDB(t) + si := NewSearchIndex(db, nil, nil, testLogger()) + + fetcher, err := newArtifactFetcher(si) + if err != nil { + t.Fatalf("newArtifactFetcher: %v", err) + } + + // Pre-seed a truncated download. + half := len(body) / 2 + if err := os.WriteFile(fetcher.compressedPath(), body[:half], 0o644); err != nil { + t.Fatalf("seed partial download: %v", err) + } + + if _, err := fetcher.fetch(context.Background()); err != nil { + t.Fatalf("fetch after partial: %v", err) + } +} diff --git a/backend/explore/artifactimport.go b/backend/explore/artifactimport.go new file mode 100644 index 0000000..161b524 --- /dev/null +++ b/backend/explore/artifactimport.go @@ -0,0 +1,389 @@ +package explore + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "strconv" + "time" + + _ "modernc.org/sqlite" // SQLite driver for reading the artifact file. + + "yellowjacket/backend/jobs" +) + +// Import of the prebuilt "core" index artifact. +// +// The catalog half of the index is identical for every user, so deriving +// it on each machine means every install streams ~89GB from +// data.metabrainz.org — a server that caps a client at roughly 2MB/s, so +// better than half a day of downloading to reach a result everyone else +// already has. Instead CI runs that import once (cmd/indexbuild), +// exports a subset (cmd/indexexport), and clients merge the resulting +// artifact in seconds. +// +// The artifact is an ordinary SQLite database holding two tables: +// explore_index with the global catalog columns only, and artifact_meta +// describing what it is. It deliberately carries no FTS table and no +// triggers — rows land in the client's own explore_index, whose AFTER +// INSERT trigger populates the search index as a side effect. +// +// Merging goes through the same ON CONFLICT rules as every other index +// write (upsertIndexConflictSQL), so an artifact can be applied over an +// existing index without clobbering better data: non-empty values win +// over empty, higher listen counts win over lower, and the personal +// columns the artifact does not carry are left untouched. + +const ( + // coreArtifactVersionKey records which artifact version was merged, + // so a client can tell whether it already has one and skip re-import. + coreArtifactVersionKey = "core_artifact_version" + + // supportedArtifactVersion is the artifact schema this build knows how + // to read. The exporter stamps it into artifact_meta; a mismatch is + // refused rather than guessed at, because an artifact written against + // a different explore_index schema would merge wrong columns. + supportedArtifactVersion = "1" +) + +// artifactMergeBatch bounds how many rows are merged per transaction. +// Large enough that per-transaction overhead disappears, small enough +// that a cancelled import doesn't roll back minutes of work. A var so +// tests can shrink it and still cross several batch boundaries. +var artifactMergeBatch = 50_000 + +var ( + // ErrArtifactUnusable means the file is not a core index artifact this + // build can merge. Callers treat it as "fall back to a normal build" + // rather than as a fatal error. + ErrArtifactUnusable = errors.New("core index artifact unusable") + + // ErrArtifactVersion is a version mismatch between the artifact and + // this build. + ErrArtifactVersion = errors.New("core index artifact version mismatch") +) + +// artifactInfo is what the artifact declares about itself. +type artifactInfo struct { + version string + + // builtAt is when the source index finished importing, not when the + // artifact was exported. + builtAt string + + // listensSeries is the incremental listens dump the artifact's + // popularity numbers are baselined on. Stamped into the client's + // index so RefreshListenCounts resumes from the right point instead + // of reapplying deltas already folded in. + listensSeries string + + rows int +} + +// artifactCatalogColumns are the columns an artifact carries. It is the +// global catalog only: the personal columns (in_library, is_similar, +// local_*) describe one person's library and are recomputed locally by +// PopulateLocalCrossReferences. +// +// Kept in sync with cmd/indexexport's catalogColumns by +// TestArtifactColumnsMatchExporter. +const artifactCatalogColumns = `entity_type, mbid, title, artist_name, artist_mbid, + aliases, popularity, listener_count, duration, caa_release_mbid, + release_name, primary_type, secondary_types, release_date, + artist_type, country, disambiguation, sort_name, discog_fetched` + +// inspectArtifact opens the artifact read-only and reports what it +// declares, without touching the live index. Validation happens here so +// a bad download is rejected before anything is attached. +func inspectArtifact(path string) (artifactInfo, error) { + var info artifactInfo + + db, err := sql.Open("sqlite", "file:"+path+"?mode=ro") + if err != nil { + return info, fmt.Errorf("%w: open: %w", ErrArtifactUnusable, err) + } + + defer func() { _ = db.Close() }() + + meta := map[string]string{} + + rows, err := db.Query("SELECT key, value FROM artifact_meta") + if err != nil { + return info, fmt.Errorf("%w: read artifact_meta: %w", ErrArtifactUnusable, err) + } + + defer func() { _ = rows.Close() }() + + for rows.Next() { + var k, v string + if err := rows.Scan(&k, &v); err != nil { + return info, fmt.Errorf("%w: scan artifact_meta: %w", ErrArtifactUnusable, err) + } + + meta[k] = v + } + + if err := rows.Err(); err != nil { + return info, fmt.Errorf("%w: read artifact_meta: %w", ErrArtifactUnusable, err) + } + + info.version = meta["artifact_version"] + info.builtAt = meta["built_at"] + info.listensSeries = meta["listens_applied_series"] + + if info.version != supportedArtifactVersion { + return info, fmt.Errorf("%w: artifact is version %q, this build reads %q", + ErrArtifactVersion, info.version, supportedArtifactVersion) + } + + // A structurally valid but empty artifact would merge cleanly and + // leave Explore just as empty as before, while stamping the index as + // imported. Refuse it. + if err := db.QueryRow( + "SELECT COUNT(*) FROM explore_index", + ).Scan(&info.rows); err != nil { + return info, fmt.Errorf("%w: count rows: %w", ErrArtifactUnusable, err) + } + + if info.rows == 0 { + return info, fmt.Errorf("%w: artifact contains no rows", ErrArtifactUnusable) + } + + return info, nil +} + +// importCoreArtifact merges a validated artifact at path into the live +// index. It is idempotent — the merge is an upsert keyed by MBID, so a +// re-run over an already-imported artifact is a no-op in effect. +// +// The caller keeps ownership of the file; nothing here deletes it. +func (si *SearchIndex) importCoreArtifact(ctx context.Context, path string) error { + info, err := inspectArtifact(path) + if err != nil { + return err + } + + si.logger.Info("core artifact: merging", + "rows", info.rows, + "builtAt", info.builtAt, + "listensSeries", info.listensSeries, + ) + si.logIndexJob(jobs.LevelInfo, fmt.Sprintf( + "Merging prebuilt catalog (%s rows, built %s)", + formatCount(info.rows), info.builtAt, + )) + + // ATTACH cannot run inside a transaction, and it binds to a single + // connection — which is why every statement below goes through the + // writer (SetMaxOpenConns(1)). Reads must not use db.QueryContext: + // that routes to the separate read pool, where "core" does not exist. + if _, err := si.db.ExecContext(`ATTACH DATABASE ? AS core`, path); err != nil { + return fmt.Errorf("%w: attach: %w", ErrArtifactUnusable, err) + } + + defer func() { + if _, err := si.db.ExecContext(`DETACH DATABASE core`); err != nil { + si.logger.Warn("core artifact: detach failed", "error", err) + } + }() + + // Per-row FTS maintenance across a million inserts costs far more + // than the inserts themselves (~31 rows/s against ~4,700), so the + // search index is rebuilt once at the end instead. + ftsSuspended := true + + if err := si.db.SuspendExploreIndexFTS(); err != nil { + si.logger.Warn("core artifact: could not suspend FTS sync", "error", err) + + ftsSuspended = false + } + + merged, mergeErr := si.mergeArtifactRows(ctx, info.rows) + + if ftsSuspended { + start := time.Now() + + if err := si.db.ResumeExploreIndexFTS(); err != nil { + // Leaving search unindexed is worse than a slow import: this + // needs a rebuild to recover, so it is loud. + si.logger.Error("core artifact: FTS rebuild failed — search index is stale", + "error", err, + ) + } else { + si.logger.Info("core artifact: FTS index rebuilt", + "elapsed", time.Since(start).Round(time.Millisecond), + ) + } + } + + if mergeErr != nil { + return mergeErr + } + + si.stampArtifactMeta(info) + si.analyzeIndex() + + si.logger.Info("core artifact: merge complete", "rows", merged) + si.logIndexJob(jobs.LevelInfo, fmt.Sprintf( + "Prebuilt catalog merged (%s rows)", formatCount(merged), + )) + + si.MarkReadyIfPopulated() + si.refreshStatusCounts() + + return nil +} + +// analyzeIndex refreshes the query planner's table statistics. +// +// It runs here rather than at schema creation because an empty database +// has nothing to measure: the numbers that matter only exist once the +// catalog has been merged. Without them the planner mis-estimates the +// partial expression indexes on explore_index and falls back to scanning +// a million rows for queries that should seek. +func (si *SearchIndex) analyzeIndex() { + start := time.Now() + + if _, err := si.db.ExecContext("ANALYZE"); err != nil { + // Only a performance loss, so it must not fail the import. + si.logger.Warn("core artifact: ANALYZE failed", "error", err) + + return + } + + si.logger.Info("core artifact: query planner statistics refreshed", + "elapsed", time.Since(start).Round(time.Millisecond), + ) +} + +// mergeArtifactRows copies the attached artifact into explore_index in +// bounded batches, walking the artifact's MBID primary key so each batch +// is an index range scan and a cancelled import leaves committed work +// behind rather than rolling it all back. +func (si *SearchIndex) mergeArtifactRows(ctx context.Context, total int) (int, error) { + insertSQL := ` + INSERT INTO explore_index (` + artifactCatalogColumns + `) + SELECT ` + artifactCatalogColumns + ` + FROM core.explore_index + WHERE mbid > ?` + upsertIndexConflictSQL + + // The final batch has no upper bound, so the range predicate is + // appended only while one exists. + insertRangeSQL := ` + INSERT INTO explore_index (` + artifactCatalogColumns + `) + SELECT ` + artifactCatalogColumns + ` + FROM core.explore_index + WHERE mbid > ? AND mbid <= ?` + upsertIndexConflictSQL + + var ( + cursor string + merged int + ) + + for { + if err := ctx.Err(); err != nil { + return merged, err + } + + upper, hasUpper, err := si.artifactBatchBound(cursor) + if err != nil { + return merged, err + } + + var res sql.Result + + if hasUpper { + res, err = si.db.ExecContext(insertRangeSQL, cursor, upper) + } else { + res, err = si.db.ExecContext(insertSQL, cursor) + } + + if err != nil { + return merged, fmt.Errorf("%w: merge batch: %w", ErrArtifactUnusable, err) + } + + n, err := res.RowsAffected() + if err != nil { + return merged, fmt.Errorf("%w: merge batch rows: %w", ErrArtifactUnusable, err) + } + + merged += int(n) + + si.setTierDetail( + artifactStageNames[artifactStageMerge], "running", merged, total, + fmt.Sprintf("%s of %s rows", formatCount(merged), formatCount(total)), + ) + + if !hasUpper { + return merged, nil + } + + cursor = upper + } +} + +// artifactBatchBound returns the MBID that ends the next batch, and +// whether one exists — no bound means the remainder is the last batch. +func (si *SearchIndex) artifactBatchBound(cursor string) (string, bool, error) { + var bound string + + err := si.db.QueryRowWriter( + `SELECT mbid FROM core.explore_index + WHERE mbid > ? ORDER BY mbid LIMIT 1 OFFSET ?`, + cursor, artifactMergeBatch-1, + ).Scan(&bound) + + if errors.Is(err, sql.ErrNoRows) { + return "", false, nil + } + + if err != nil { + return "", false, fmt.Errorf("%w: batch bound: %w", ErrArtifactUnusable, err) + } + + return bound, true, nil +} + +// stampArtifactMeta records what the merge established: the catalog half +// is populated, and popularity is baselined on the artifact's listens +// series so the incremental refresh resumes from there. +func (si *SearchIndex) stampArtifactMeta(info artifactInfo) { + si.setMeta(coreArtifactVersionKey, info.version) + + // The catalog is present, so nothing should trigger a full dump + // import on top of the artifact it was meant to replace. + si.setMeta(dumpImportDoneKey, time.Now().UTC().Format(time.RFC3339)) + + // Without a baseline series RefreshListenCounts refuses to run at + // all, so an artifact exported before that key existed leaves the + // index permanently frozen at its shipped popularity. Better to say + // so than to fail silently. + if info.listensSeries == "" { + si.logger.Warn( + "core artifact: no listens series recorded — " + + "popularity refresh will not run until the next full import", + ) + + return + } + + if _, err := strconv.Atoi(info.listensSeries); err != nil { + si.logger.Warn("core artifact: unparseable listens series", + "value", info.listensSeries, + ) + + return + } + + si.setMeta(listensAppliedSeriesKey, info.listensSeries) +} + +// removeArtifactFile deletes a merged artifact. Best-effort: a leftover +// file costs disk, not correctness. +func (si *SearchIndex) removeArtifactFile(path string) { + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + si.logger.Warn("core artifact: cleanup failed", "path", path, "error", err) + } +} diff --git a/backend/explore/artifactimport_test.go b/backend/explore/artifactimport_test.go new file mode 100644 index 0000000..94cdd4f --- /dev/null +++ b/backend/explore/artifactimport_test.go @@ -0,0 +1,444 @@ +package explore + +import ( + "context" + "database/sql" + "os" + "path/filepath" + "strings" + "testing" + + "yellowjacket/backend/database" +) + +// artifactRow is one catalog row written into a test artifact. +type artifactRow struct { + entityType string + mbid string + title string + artistName string + artistMBID string + popularity int +} + +// writeTestArtifact builds an artifact file matching what cmd/indexexport +// produces: catalog columns only, no FTS, no triggers. +func writeTestArtifact( + t *testing.T, meta map[string]string, rows []artifactRow, +) string { + t.Helper() + + path := filepath.Join(t.TempDir(), "core-index.db") + + db, err := sql.Open("sqlite", "file:"+path) + if err != nil { + t.Fatalf("open artifact: %v", err) + } + + defer func() { _ = db.Close() }() + + for _, stmt := range []string{ + `CREATE TABLE explore_index ( + entity_type TEXT NOT NULL, + mbid TEXT NOT NULL, + title TEXT NOT NULL, + artist_name TEXT NOT NULL, + artist_mbid TEXT NOT NULL, + aliases TEXT NOT NULL DEFAULT '', + popularity INTEGER NOT NULL DEFAULT 0, + listener_count INTEGER NOT NULL DEFAULT 0, + duration INTEGER NOT NULL DEFAULT 0, + caa_release_mbid TEXT NOT NULL DEFAULT '', + release_name TEXT NOT NULL DEFAULT '', + primary_type TEXT NOT NULL DEFAULT '', + secondary_types TEXT NOT NULL DEFAULT '', + release_date TEXT NOT NULL DEFAULT '', + artist_type TEXT NOT NULL DEFAULT '', + country TEXT NOT NULL DEFAULT '', + disambiguation TEXT NOT NULL DEFAULT '', + sort_name TEXT NOT NULL DEFAULT '', + discog_fetched INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (mbid) + ) WITHOUT ROWID`, + `CREATE TABLE artifact_meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + )`, + } { + if _, err := db.Exec(stmt); err != nil { + t.Fatalf("create artifact schema: %v", err) + } + } + + for k, v := range meta { + if _, err := db.Exec( + `INSERT INTO artifact_meta (key, value) VALUES (?, ?)`, k, v, + ); err != nil { + t.Fatalf("stamp artifact meta: %v", err) + } + } + + for _, r := range rows { + if _, err := db.Exec(` + INSERT INTO explore_index + (entity_type, mbid, title, artist_name, artist_mbid, popularity) + VALUES (?, ?, ?, ?, ?, ?)`, + r.entityType, r.mbid, r.title, r.artistName, r.artistMBID, r.popularity, + ); err != nil { + t.Fatalf("insert artifact row: %v", err) + } + } + + return path +} + +// validMeta is the artifact_meta a well-formed artifact carries. +func validMeta() map[string]string { + return map[string]string{ + "artifact_version": supportedArtifactVersion, + "built_at": "2026-07-17T03:53:43Z", + "listens_applied_series": "2593", + "source_rows": "2052168", + } +} + +func TestImportCoreArtifactMergesCatalog(t *testing.T) { + db := database.NewTestDB(t) + si := NewSearchIndex(db, nil, nil, testLogger()) + + path := writeTestArtifact(t, validMeta(), []artifactRow{ + {"artist", artA, "Artist A", "Artist A", artA, 5000}, + {"artist", artB, "Artist B", "Artist B", artB, 4000}, + {"release_group", rgA, "Album A", "Artist A", artA, 3000}, + {"recording", recA, "Song A", "Artist A", artA, 2000}, + }) + + if err := si.importCoreArtifact(context.Background(), path); err != nil { + t.Fatalf("importCoreArtifact: %v", err) + } + + var got int + if err := db.QueryRowWriter( + "SELECT COUNT(*) FROM explore_index", + ).Scan(&got); err != nil { + t.Fatalf("count rows: %v", err) + } + + if got != 4 { + t.Errorf("merged %d rows, want 4", got) + } + + // The merge must leave the index searchable: the FTS rebuild that + // closes the bulk-load window is the only thing populating it, since + // the triggers were dropped for the duration. + var hits int + if err := db.QueryRowWriter( + `SELECT COUNT(*) FROM explore_index_fts WHERE explore_index_fts MATCH ?`, + "Song", + ).Scan(&hits); err != nil { + t.Fatalf("query fts: %v", err) + } + + if hits == 0 { + t.Error("FTS index is empty after merge; search would return nothing") + } +} + +func TestImportCoreArtifactStampsMeta(t *testing.T) { + db := database.NewTestDB(t) + si := NewSearchIndex(db, nil, nil, testLogger()) + + path := writeTestArtifact(t, validMeta(), []artifactRow{ + {"artist", artA, "Artist A", "Artist A", artA, 5000}, + }) + + if err := si.importCoreArtifact(context.Background(), path); err != nil { + t.Fatalf("importCoreArtifact: %v", err) + } + + if !si.hasMeta(dumpImportDoneKey) { + t.Error("dump_import_done not stamped; a full dump import would run anyway") + } + + // Without this the incremental refresh refuses to run and the + // shipped popularity never updates again. + if series, ok := si.metaInt(listensAppliedSeriesKey); !ok || series != 2593 { + t.Errorf("listens_applied_series = %d (ok=%v), want 2593", series, ok) + } + + if !si.hasMeta(coreArtifactVersionKey) { + t.Error("core_artifact_version not stamped") + } +} + +// A merge must never downgrade what the index already holds, because a +// user's own library rows and any lazily-fetched detail predate it. +func TestImportCoreArtifactPreservesLocalData(t *testing.T) { + db := database.NewTestDB(t) + si := NewSearchIndex(db, nil, nil, testLogger()) + + si.upsertBatch([]SearchIndexResult{{ + EntityType: "recording", + MBID: recA, + Title: "Song A", + ArtistName: "Artist A", + ArtistMBID: artA, + Popularity: 9999, + Duration: 210000, + InLibrary: true, + DiscogFetched: true, + }}) + + path := writeTestArtifact(t, validMeta(), []artifactRow{ + // Lower popularity and no duration: both must lose. + {"recording", recA, "Song A", "Artist A", artA, 10}, + }) + + if err := si.importCoreArtifact(context.Background(), path); err != nil { + t.Fatalf("importCoreArtifact: %v", err) + } + + var popularity, duration, inLibrary, discogFetched int + + if err := db.QueryRowWriter(` + SELECT popularity, duration, in_library, discog_fetched + FROM explore_index WHERE mbid = ?`, recA, + ).Scan(&popularity, &duration, &inLibrary, &discogFetched); err != nil { + t.Fatalf("read merged row: %v", err) + } + + if popularity != 9999 { + t.Errorf("popularity = %d, want 9999 (higher must win)", popularity) + } + + if duration != 210000 { + t.Errorf("duration = %d, want 210000 (artifact carries none)", duration) + } + + if inLibrary != 1 { + t.Error("in_library was cleared; the artifact must not touch personal columns") + } + + if discogFetched != 1 { + t.Error("discog_fetched was cleared by the merge") + } +} + +// The batch walk is the part most likely to drop or duplicate rows, so +// it is exercised across many batch boundaries rather than one. +func TestImportCoreArtifactBatchWalkCoversAllRows(t *testing.T) { + db := database.NewTestDB(t) + si := NewSearchIndex(db, nil, nil, testLogger()) + + // Shrink the batch rather than growing the fixture: what matters is + // crossing several boundaries, including a short final one. + original := artifactMergeBatch + artifactMergeBatch = 100 + + t.Cleanup(func() { artifactMergeBatch = original }) + + const total = 337 + + rows := make([]artifactRow, 0, total) + for i := range total { + rows = append(rows, artifactRow{ + entityType: "recording", + mbid: syntheticMBID(i), + title: "Song", + artistName: "Artist", + artistMBID: artA, + popularity: i, + }) + } + + path := writeTestArtifact(t, validMeta(), rows) + + if err := si.importCoreArtifact(context.Background(), path); err != nil { + t.Fatalf("importCoreArtifact: %v", err) + } + + var got int + if err := db.QueryRowWriter( + "SELECT COUNT(*) FROM explore_index", + ).Scan(&got); err != nil { + t.Fatalf("count rows: %v", err) + } + + if got != total { + t.Errorf("merged %d rows, want %d", got, total) + } +} + +func TestImportCoreArtifactRejectsBadArtifacts(t *testing.T) { + tests := []struct { + name string + meta map[string]string + rows []artifactRow + want error + }{ + { + name: "version mismatch", + meta: map[string]string{"artifact_version": "99"}, + rows: []artifactRow{{"artist", artA, "A", "A", artA, 1}}, + want: ErrArtifactVersion, + }, + { + name: "no version", + meta: map[string]string{}, + rows: []artifactRow{{"artist", artA, "A", "A", artA, 1}}, + want: ErrArtifactVersion, + }, + { + name: "empty catalog", + meta: validMeta(), + rows: nil, + want: ErrArtifactUnusable, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + db := database.NewTestDB(t) + si := NewSearchIndex(db, nil, nil, testLogger()) + + path := writeTestArtifact(t, tt.meta, tt.rows) + + err := si.importCoreArtifact(context.Background(), path) + if err == nil { + t.Fatal("expected rejection, got nil") + } + + if !strings.Contains(err.Error(), tt.want.Error()) { + t.Errorf("error = %v, want it to wrap %v", err, tt.want) + } + + // A rejected artifact must not leave the index claiming it + // has a catalog, or the real build would never run. + if si.hasMeta(dumpImportDoneKey) { + t.Error("rejected artifact still stamped dump_import_done") + } + }) + } +} + +func TestInspectArtifactRejectsNonArtifactFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "garbage.db") + + if err := os.WriteFile(path, []byte("not a database"), 0o644); err != nil { + t.Fatalf("write file: %v", err) + } + + if _, err := inspectArtifact(path); err == nil { + t.Error("expected rejection of a non-artifact file") + } +} + +// syntheticMBID produces distinct, well-formed MBIDs for bulk fixtures. +func syntheticMBID(i int) string { + const hex = "0123456789abcdef" + + buf := []byte("00000000-0000-0000-0000-000000000000") + + for pos := len(buf) - 1; pos >= 0 && i > 0; pos-- { + if buf[pos] == '-' { + continue + } + + buf[pos] = hex[i%16] + i /= 16 + } + + return string(buf) +} + +// resolveArtistName falls back to the MBID when no name is available. +// That value must never reach the index: the upsert no longer defends +// against it, so the writer is the only thing standing in the way. +func TestAddFromCacheNeverStoresMBIDAsName(t *testing.T) { + db := database.NewTestDB(t) + si := NewSearchIndex(db, nil, nil, testLogger()) + + si.AddFromCache(artA, artA, []MBReleaseGroup{ + {MBID: rgA, Title: "Album A", PrimaryType: "Album"}, + }) + + var artistRows int + if err := db.QueryRowWriter( + `SELECT COUNT(*) FROM explore_index + WHERE entity_type = 'artist' AND title = mbid`, + ).Scan(&artistRows); err != nil { + t.Fatalf("count artist rows: %v", err) + } + + if artistRows != 0 { + t.Errorf("%d artist rows stored the MBID as their title", artistRows) + } + + var artistName string + if err := db.QueryRowWriter( + `SELECT artist_name FROM explore_index WHERE mbid = ?`, rgA, + ).Scan(&artistName); err != nil { + t.Fatalf("read release group: %v", err) + } + + if artistName == artA { + t.Error("release group stored the artist MBID as its artist_name") + } + + // A real name arriving later must still win over the empty one. + si.AddFromCache("Real Artist", artA, []MBReleaseGroup{ + {MBID: rgA, Title: "Album A", PrimaryType: "Album"}, + }) + + if err := db.QueryRowWriter( + `SELECT artist_name FROM explore_index WHERE mbid = ?`, rgA, + ).Scan(&artistName); err != nil { + t.Fatalf("re-read release group: %v", err) + } + + if artistName != "Real Artist" { + t.Errorf("artist_name = %q, want it filled in once known", artistName) + } +} + +// The importer names the artifact's columns independently of the +// exporter that writes them. If the two lists drift, the merge either +// fails outright or silently shifts values into the wrong columns, so +// they are compared directly against cmd/indexexport's source. +func TestArtifactColumnsMatchExporter(t *testing.T) { + src, err := os.ReadFile("../../cmd/indexexport/main.go") + if err != nil { + t.Fatalf("read exporter: %v", err) + } + + const marker = "const catalogColumns = `" + + i := strings.Index(string(src), marker) + if i < 0 { + t.Fatalf("catalogColumns not found in cmd/indexexport/main.go") + } + + rest := string(src)[i+len(marker):] + + j := strings.Index(rest, "`") + if j < 0 { + t.Fatal("unterminated catalogColumns literal") + } + + normalise := func(s string) string { + out := make([]string, 0, 32) + for _, f := range strings.Split(s, ",") { + out = append(out, strings.Join(strings.Fields(f), "")) + } + + return strings.Join(out, ",") + } + + exporter := normalise(rest[:j]) + importer := normalise(artifactCatalogColumns) + + if exporter != importer { + t.Errorf("column lists have drifted:\n exporter: %s\n importer: %s", + exporter, importer) + } +} diff --git a/backend/explore/artistimage.go b/backend/explore/artistimage.go index 2404a6e..5b12704 100644 --- a/backend/explore/artistimage.go +++ b/backend/explore/artistimage.go @@ -36,7 +36,7 @@ const ( artistImageTimeout = 10 * time.Second artistImageCacheTTL = 365 * 24 * time.Hour // positive results: ~permanent artistImageMissCacheTTL = 30 * 24 * time.Hour // negative results: retry monthly - artistImageBaseDir = "artist-images" + artistImageBaseDir = ArtistImageDirName artistImageMaxBytes = 2 * 1024 * 1024 artistImageMaxSize = 500 // max dimension for stored full-res images maxImagesPerArtist = 10 diff --git a/backend/explore/assetdirs.go b/backend/explore/assetdirs.go new file mode 100644 index 0000000..02b601a --- /dev/null +++ b/backend/explore/assetdirs.go @@ -0,0 +1,20 @@ +package explore + +// Asset directory names under the user data directory. +// +// These are exported so the maintenance janitor can sweep them without +// importing the internals of the providers that write them. Each is +// catalogued in backend/datamap as cache data: rebuildable from the +// network, but expensive enough that eviction is by age rather than +// tied to the lifetime of anything else. +const ( + // ArtistImageDirName holds one subdirectory per artist MBID, each + // containing fetched photos, a primary.jpg with its thumbnails, and + // possibly a .miss marker recording that no artwork was found. + ArtistImageDirName = "artist-images" + + // CoverArtCacheDirName holds Cover Art Archive thumbnails fetched + // while browsing Explore, named by release-group MBID. Nothing in + // the database references these files. + CoverArtCacheDirName = "cover-art-cache" +) diff --git a/backend/explore/coverartproxy.go b/backend/explore/coverartproxy.go index 019893a..b7dee0c 100644 --- a/backend/explore/coverartproxy.go +++ b/backend/explore/coverartproxy.go @@ -23,7 +23,7 @@ var ErrCoverArt = errors.New("cover art fetch failed") const ( // thumbnailDir is the subdirectory under the user data dir // where cached cover art thumbnails are stored. - thumbnailDir = "cover-art-cache" + thumbnailDir = CoverArtCacheDirName // thumbnailTimeout is the HTTP timeout for fetching a thumbnail. thumbnailTimeout = 10 * time.Second diff --git a/backend/explore/dumpbuild_stub.go b/backend/explore/dumpbuild_stub.go new file mode 100644 index 0000000..6c977c1 --- /dev/null +++ b/backend/explore/dumpbuild_stub.go @@ -0,0 +1,73 @@ +//go:build !indexbuild + +package explore + +import ( + "context" + "errors" + + "yellowjacket/backend/jobs" +) + +// The app binary does not carry the full dump import. +// +// Building the catalog from source means streaming ~89GB of listens dump +// from a server that caps a client near 2MB/s — better than half a day +// of downloading to derive a catalog that is identical for everyone. +// That work happens once in CI (`go build -tags indexbuild ./cmd/indexbuild`) +// and reaches users as a prebuilt artifact instead. +// +// So on the client the build path is: merge the artifact, then keep +// popularity current with the daily incremental dumps. Artists outside +// the artifact's coverage still resolve lazily, exactly as before. + +// runDumpBuild populates the catalog. In the app build that means the +// prebuilt artifact and nothing else; the tagged build in +// dumpimport.go runs the real import. +func (si *SearchIndex) runDumpBuild(ctx context.Context) { + si.MarkReadyIfPopulated() + + if si.hasMeta(dumpImportDoneKey) { + si.logger.Info("search index: catalog already populated, skipping") + si.refreshStatusCounts() + + return + } + + if si.artifactAlreadyMerged() { + si.refreshStatusCounts() + + return + } + + if err := si.tryCoreArtifact(ctx); err != nil { + if ctx.Err() != nil { + return + } + + si.logArtifactFallback(err) + si.refreshStatusCounts() + } +} + +// logArtifactFallback explains why the catalog is not there. An empty +// Explore with nothing in the log is the worst version of this failure. +func (si *SearchIndex) logArtifactFallback(err error) { + switch { + case errors.Is(err, ErrArtifactUnavailable): + si.logger.Info("search index: no prebuilt catalog available", "error", err) + si.logIndexJob(jobs.LevelWarn, + "No prebuilt catalog available — Explore will cover your own "+ + "library only until one can be fetched.") + + case errors.Is(err, ErrArtifactVersion): + si.logger.Warn("search index: prebuilt catalog is for a different app version", + "error", err) + si.logIndexJob(jobs.LevelWarn, + "The published catalog does not match this app version; skipping it.") + + default: + si.logger.Warn("search index: prebuilt catalog import failed", "error", err) + si.logIndexJob(jobs.LevelWarn, "Prebuilt catalog import failed: "+err.Error()) + } +} diff --git a/backend/explore/dumpcatalog.go b/backend/explore/dumpcatalog.go index 2a2a0c4..a6fb1df 100644 --- a/backend/explore/dumpcatalog.go +++ b/backend/explore/dumpcatalog.go @@ -1,3 +1,5 @@ +//go:build indexbuild + package explore import ( @@ -440,7 +442,7 @@ func (a *artistTopRG) add(rg uuid16, listens uint32) { func (imp *dumpImporter) scanCanonicalDump( ctx context.Context, url string, ks *keptSets, ) (*canonicalScan, error) { - stream := newResumableReader(ctx, imp.httpClient, url, 0) + stream := imp.openDumpStream(ctx, url, 0) defer func() { _ = stream.Close() }() diff --git a/backend/explore/dumpcounts.go b/backend/explore/dumpcounts.go index a8ce318..707f414 100644 --- a/backend/explore/dumpcounts.go +++ b/backend/explore/dumpcounts.go @@ -1,9 +1,10 @@ +//go:build indexbuild + package explore import ( "archive/tar" "bufio" - "bytes" "context" "encoding/binary" "encoding/json" @@ -13,8 +14,8 @@ import ( "os" "strings" "sync" - - "github.com/parquet-go/parquet-go" + "sync/atomic" + "time" ) // Stage 1 of the dump import: stream the ListenBrainz spark listens @@ -27,9 +28,6 @@ import ( const ( // countKindRecording etc. tag entries in the counts map/file. - countKindRecording = byte(1) - countKindRelease = byte(2) - countKindArtist = byte(3) // countsFlushEveryMembers controls checkpoint frequency. Each // flush rewrites counts.bin (~1GB by the end), so this trades @@ -37,105 +35,35 @@ const ( // 19GB of stream progress). countsFlushEveryMembers = 150 - // countsProgressEveryMembers controls progress log frequency. - countsProgressEveryMembers = 50 + // countsUIRefreshInterval is how often the live download line is + // pushed to the UI. A parquet member is ~128MB, so member + // boundaries are minutes apart on a typical connection — sampling + // the stream position instead keeps the stage visibly moving. + countsUIRefreshInterval = 3 * time.Second + + // countsLogInterval and countsJobLogInterval throttle the two log + // surfaces: the app log gets a line every few minutes, the jobs + // panel a coarser one. Checkpoints always log to both. + countsLogInterval = 2 * time.Minute + countsJobLogInterval = 15 * time.Minute + + // countsStallAfter is how long the stream position may stand still + // before progress is reported as stalled rather than as a rate. + countsStallAfter = 45 * time.Second + + // countsRateSmoothing is the EWMA weight given to the newest + // throughput sample, trading responsiveness against jitter. + countsRateSmoothing = 0.25 // parquetParseWorkers is the number of concurrent parquet // decoders. Bounded to limit RAM: each worker holds one // ~128MB member buffer. parquetParseWorkers = 3 - // maxParquetMemberSize guards against unexpected dump format - // changes blowing out RAM. - maxParquetMemberSize = 1 << 30 - // countsFileMagic identifies + versions the counts file format. countsFileMagic = "YJCNTS01" ) -// ErrDumpFormat is returned when dump contents don't match the -// expected format. -var ErrDumpFormat = errors.New("unexpected dump format") - -// mbidKey is a parsed UUID plus an entity-kind tag, used as the counts -// map key. 17 bytes instead of a 36-byte string keeps the ~40M-entry -// map around 2GB. -type mbidKey [17]byte - -func makeMBIDKey(kind byte, mbid string) (mbidKey, bool) { - var k mbidKey - - k[0] = kind - - if !parseUUID(mbid, k[1:]) { - return k, false - } - - return k, true -} - -// parseUUID parses a canonical 36-char UUID string into 16 bytes. -// Returns false for anything malformed. -func parseUUID(s string, out []byte) bool { - if len(s) != 36 || s[8] != '-' || s[13] != '-' || s[18] != '-' || s[23] != '-' { - return false - } - - j := 0 - - for i := 0; i < 36; i++ { - if i == 8 || i == 13 || i == 18 || i == 23 { - continue - } - - hi := hexNibble(s[i]) - i++ - - lo := hexNibble(s[i]) - if hi == 0xFF || lo == 0xFF { - return false - } - - out[j] = hi<<4 | lo - j++ - } - - return true -} - -func hexNibble(c byte) byte { - switch { - case c >= '0' && c <= '9': - return c - '0' - case c >= 'a' && c <= 'f': - return c - 'a' + 10 - case c >= 'A' && c <= 'F': - return c - 'A' + 10 - default: - return 0xFF - } -} - -func formatUUID(b []byte) string { - const hexdigits = "0123456789abcdef" - - out := make([]byte, 36) - j := 0 - - for i := range 16 { - if i == 4 || i == 6 || i == 8 || i == 10 { - out[j] = '-' - j++ - } - - out[j] = hexdigits[b[i]>>4] - out[j+1] = hexdigits[b[i]&0x0F] - j += 2 - } - - return string(out) -} - // countsState is the checkpointed stage-1 state: the counts map plus // the stream position it corresponds to. type countsState struct { @@ -157,14 +85,6 @@ type countsState struct { counts map[mbidKey]uint32 } -// sparkListenRow is the projection of the spark listens parquet schema -// that the aggregator reads. All other columns are skipped. -type sparkListenRow struct { - RecordingMBID string `parquet:"recording_mbid,optional"` - ReleaseMBID string `parquet:"release_mbid,optional"` - ArtistMBIDs []string `parquet:"artist_credit_mbids,optional,list"` -} - type countParseJob struct { idx int endOffset int64 // exact offset of the next member header @@ -180,15 +100,47 @@ type countParseResult struct { // aggregateListenCounts runs stage 1 to completion (or ctx cancel), // checkpointing to the staging counts file as it goes. +// +// Column projection is tried first: it downloads only the three MBID +// columns the aggregator reads, which is well under half the archive. +// It needs a Range-serving origin, so a server that won't range falls +// back to streaming the whole tar. func (imp *dumpImporter) aggregateListenCounts(ctx context.Context, st *countsState) error { if st.counts == nil { st.counts = make(map[mbidKey]uint32, 1<<20) } - stream := newResumableReader(ctx, imp.httpClient, st.SparkURL, st.Offset) + if size, ok := projectionSupported(ctx, imp.httpClient, st.SparkURL); ok { + err := imp.aggregateProjected(ctx, st, size) + if !errors.Is(err, errProjectionUnsupported) { + return err + } + + imp.logger.Warn("dump import: column projection unavailable, streaming whole dump", + "error", err, + ) + } + + return imp.aggregateStreamed(ctx, st) +} + +// aggregateStreamed is the fallback stage-1 path: read the tar end to +// end and parse every parquet member in full. +func (imp *dumpImporter) aggregateStreamed(ctx context.Context, st *countsState) error { + stream := imp.openDumpStream(ctx, st.SparkURL, st.Offset) defer func() { _ = stream.Close() }() + // A live reporter samples the stream position on a timer; without + // it the stage would sit unchanged for minutes at a time between + // parquet members, which reads as "hung" rather than "downloading". + var awaitingWorkers atomic.Bool + + stopReporter := imp.startCountsReporter(ctx, stream, &awaitingWorkers) + defer stopReporter() + + progress := &countsLogger{imp: imp, stream: stream, started: time.Now()} + buffered := bufio.NewReaderSize(stream, 1<<20) tr := tar.NewReader(buffered) @@ -196,7 +148,7 @@ func (imp *dumpImporter) aggregateListenCounts(ctx context.Context, st *countsSt // tar reader has consumed: bytes delivered by HTTP minus bytes // still sitting in the bufio buffer. consumedOffset := func() int64 { - return stream.Offset - int64(buffered.Buffered()) + return stream.Pos() - int64(buffered.Buffered()) } jobs := make(chan countParseJob) @@ -231,64 +183,13 @@ func (imp *dumpImporter) aggregateListenCounts(ctx context.Context, st *countsSt // checkpoint is a contiguous prefix of the stream. It owns // st.counts, st.Offset, and st.MemberIdx until applierDone closes; // on error it keeps draining results so nothing deadlocks. - var applyErr error + applier := newCountsApplier(imp, st, progress) go func() { defer close(applierDone) - pending := make(map[int]countParseResult) - next := st.MemberIdx - lastFlushed := st.MemberIdx - for res := range results { - if applyErr != nil { - continue - } - - pending[res.idx] = res - - for { - r, ok := pending[next] - if !ok { - break - } - - delete(pending, next) - - if r.err != nil { - applyErr = r.err - - break - } - - for k, v := range r.deltas { - st.counts[k] += v - } - - next++ - st.MemberIdx = next - st.Offset = r.endOffset - - if next-lastFlushed >= countsFlushEveryMembers { - if err := imp.writeCountsFile(st); err != nil { - applyErr = err - - break - } - - lastFlushed = next - - imp.logCountsProgress(next, r.endOffset, stream.Size, len(st.counts)) - - if err := imp.checkDiskHeadroom(); err != nil { - applyErr = err - - break - } - } else if next%countsProgressEveryMembers == 0 { - imp.logCountsProgress(next, r.endOffset, stream.Size, len(st.counts)) - } - } + applier.apply(res, nil) } }() @@ -342,9 +243,14 @@ readLoop: // uses PAX extension headers (those start at its header offset). endOffset := consumedOffset() + tarPadding(hdr.Size) + awaitingWorkers.Store(true) + select { case jobs <- countParseJob{idx: memberIdx, endOffset: endOffset, buf: buf}: + awaitingWorkers.Store(false) case <-ctx.Done(): + awaitingWorkers.Store(false) + readErr = ctx.Err() break readLoop @@ -359,7 +265,7 @@ readLoop: <-applierDone if readErr == nil { - readErr = applyErr + readErr = applier.err } if readErr != nil { @@ -378,9 +284,16 @@ readLoop: imp.logger.Info("dump import: listen counts complete", "members", st.MemberIdx, + "gb", fmt.Sprintf("%.1f", float64(st.Offset)/(1<<30)), "entities", len(st.counts), + "elapsed", time.Since(progress.started).Truncate(time.Second).String(), ) + imp.logJob(fmt.Sprintf( + "Listen counts complete — %s of listens read, %s entities ranked", + formatGB(st.Offset), formatCount(len(st.counts)), + )) + return nil } @@ -392,55 +305,6 @@ func tarPadding(size int64) int64 { return (block - size%block) % block } -// parseListenParquet decodes one parquet member and returns the -// per-entity listen-count deltas. -func parseListenParquet(buf []byte) (map[mbidKey]uint32, error) { - reader := parquet.NewGenericReader[sparkListenRow](bytes.NewReader(buf)) - - defer func() { _ = reader.Close() }() - - deltas := make(map[mbidKey]uint32, 1<<18) - rows := make([]sparkListenRow, 4096) - - for { - n, err := reader.Read(rows) - - for _, row := range rows[:n] { - key, ok := makeMBIDKey(countKindRecording, row.RecordingMBID) - if !ok { - // Unmapped listen — no usable recording MBID. - continue - } - - deltas[key]++ - - if relKey, relOK := makeMBIDKey(countKindRelease, row.ReleaseMBID); relOK { - deltas[relKey]++ - } - - for _, artist := range row.ArtistMBIDs { - if artKey, artOK := makeMBIDKey(countKindArtist, artist); artOK { - deltas[artKey]++ - } - } - } - - if errors.Is(err, io.EOF) { - break - } - - if err != nil { - return nil, fmt.Errorf("parquet read: %w", err) - } - - if n == 0 { - break - } - } - - return deltas, nil -} - // --------------------------------------------------------------------------- // counts.bin persistence // --------------------------------------------------------------------------- @@ -567,18 +431,245 @@ func (imp *dumpImporter) readCountsFile() (*countsState, error) { return st, nil } -func (imp *dumpImporter) logCountsProgress(members int, offset, size int64, entities int) { - pct := float64(0) - if size > 0 { - pct = float64(offset) / float64(size) * 100 +// --------------------------------------------------------------------------- +// progress reporting +// --------------------------------------------------------------------------- + +// startCountsReporter runs a goroutine that samples the listens stream +// position every few seconds and publishes it as the stage's UI +// progress. The returned function stops the reporter and waits for it +// to exit, so no stale "running" update can land after the stage is +// marked complete. +func (imp *dumpImporter) startCountsReporter( + ctx context.Context, stream dumpStream, backlog *atomic.Bool, +) func() { + stop := make(chan struct{}) + exited := make(chan struct{}) + + rep := &countsReporter{ + imp: imp, + stream: stream, + backlog: backlog, + lastSample: time.Now(), + lastOffset: stream.Fetched(), + lastMoved: time.Now(), } - imp.logger.Info("dump import: listen counts progress", + go func() { + defer close(exited) + + ticker := time.NewTicker(countsUIRefreshInterval) + defer ticker.Stop() + + for { + select { + case <-stop: + return + case <-ctx.Done(): + return + case now := <-ticker.C: + rep.tick(now) + } + } + }() + + return func() { + close(stop) + <-exited + } +} + +// countsReporter turns stream position samples into a percentage and a +// human-readable throughput line. Only its own goroutine touches it. +type countsReporter struct { + imp *dumpImporter + stream dumpStream + + // backlog is set while the reader is blocked handing a member to + // the parquet workers. The stream stops moving then too, and + // calling that a network stall would be wrong. + backlog *atomic.Bool + + lastSample time.Time + lastOffset int64 + lastMoved time.Time + rate float64 // EWMA bytes/sec +} + +func (rep *countsReporter) tick(now time.Time) { + // Track the downloader, not the consumer: parallel lanes buffer a + // chunk at a time, so delivery stands still for a minute at the + // start of a stream while the network is in fact saturated. Watching + // Pos here would report that as a stall. + offset := rep.stream.Fetched() + size := rep.stream.Total() + + if elapsed := now.Sub(rep.lastSample).Seconds(); elapsed > 0 { + sample := float64(offset-rep.lastOffset) / elapsed + if rep.rate == 0 { + rep.rate = sample + } else { + rep.rate = countsRateSmoothing*sample + (1-countsRateSmoothing)*rep.rate + } + } + + if offset != rep.lastOffset { + rep.lastMoved = now + } + + rep.lastSample = now + rep.lastOffset = offset + + // A stream that has stopped moving is either reconnecting or held + // up by the parsers; either way, reporting a rate that is really + // just an average of nothing would be misleading. + var detail string + + switch { + case now.Sub(rep.lastMoved) <= countsStallAfter: + rep.imp.countsRate.Store(uint64(max(rep.rate, 0))) + + detail = formatStreamProgress(offset, size, rep.rate) + case rep.backlog != nil && rep.backlog.Load(): + rep.imp.countsRate.Store(0) + + detail = formatGB(offset) + " downloaded · parsing, download paused" + default: + rep.imp.countsRate.Store(0) + + detail = formatGB(offset) + " downloaded · stalled, retrying…" + } + + rep.imp.setStageDetail(dumpStageCounts, streamPercent(offset, size), 100, detail) +} + +// countsLogger writes stage-1 progress to the app log and the jobs +// panel on independent time-based schedules. Per-member lines would be +// too sparse to reassure and too noisy to read; checkpoints, which are +// the points a crash would resume from, always log to both. +type countsLogger struct { + imp *dumpImporter + stream dumpStream + + started time.Time + lastLog time.Time + lastJobLog time.Time +} + +// member reports an applied parquet member, logging only if enough time +// has passed since the last line. +func (l *countsLogger) member(members int, offset int64, entities int) { + now := time.Now() + + if now.Sub(l.lastLog) >= countsLogInterval { + l.lastLog = now + + l.logApp("dump import: listen counts progress", members, offset, entities) + } + + if now.Sub(l.lastJobLog) >= countsJobLogInterval { + l.lastJobLog = now + + l.logJob("Listen counts", members, offset, entities) + } +} + +// checkpoint reports a counts.bin flush, which always logs — it is the +// point an interrupted import would resume from. +func (l *countsLogger) checkpoint(members int, offset int64, entities int) { + now := time.Now() + + l.lastLog = now + l.lastJobLog = now + + l.logApp("dump import: listen counts checkpoint", members, offset, entities) + l.logJob("Listen counts checkpointed", members, offset, entities) +} + +func (l *countsLogger) logApp(msg string, members int, offset int64, entities int) { + size := l.stream.Total() + + l.imp.logger.Info(msg, "members", members, "gb", fmt.Sprintf("%.1f", float64(offset)/(1<<30)), - "pct", fmt.Sprintf("%.1f", pct), + "pct", streamPercent(offset, size), + "rate", formatRate(l.imp.streamRate()), + "eta", formatETA(offset, size, l.imp.streamRate()), "entities", entities, ) +} - imp.setStageProgress(dumpStageCounts, int(pct), 100) +func (l *countsLogger) logJob(prefix string, members int, offset int64, entities int) { + l.imp.logJob(fmt.Sprintf("%s: %s · %s members · %s entities", + prefix, + formatStreamProgress(offset, l.stream.Total(), l.imp.streamRate()), + formatCount(members), + formatCount(entities), + )) +} + +// streamRate returns the listens stream throughput most recently +// measured by the reporter, in bytes/sec. +func (imp *dumpImporter) streamRate() float64 { + return float64(imp.countsRate.Load()) +} + +// streamPercent is the whole-percent position in a stream of known +// size; 0 when the size is not yet known. +func streamPercent(offset, size int64) int { + if size <= 0 { + return 0 + } + + return int(float64(offset) / float64(size) * 100) +} + +// formatStreamProgress renders "42.3 / 205.1 GB (20%) · 18 MB/s · +// ~3h20m left", degrading gracefully when the size or rate is unknown. +func formatStreamProgress(offset, size int64, rate float64) string { + parts := make([]string, 0, 3) + + if size > 0 { + parts = append(parts, fmt.Sprintf("%s / %s (%d%%)", + formatGB(offset), formatGB(size), streamPercent(offset, size))) + } else { + parts = append(parts, formatGB(offset)+" downloaded") + } + + if rate > 0 { + parts = append(parts, formatRate(rate)) + } + + if eta := formatETA(offset, size, rate); eta != "" { + parts = append(parts, "~"+eta+" left") + } + + return strings.Join(parts, " · ") +} + +func formatRate(bytesPerSec float64) string { + if bytesPerSec <= 0 { + return "—" + } + + return fmt.Sprintf("%.1f MB/s", bytesPerSec/(1<<20)) +} + +// formatETA estimates remaining time at the current rate. Returns "" +// when the total size or the rate is unknown. +func formatETA(offset, size int64, rate float64) string { + if size <= 0 || rate <= 0 || offset >= size { + return "" + } + + remaining := time.Duration(float64(size-offset)/rate) * time.Second + + switch { + case remaining < time.Minute: + return "<1m" + case remaining < time.Hour: + return fmt.Sprintf("%dm", int(remaining.Minutes())) + default: + return fmt.Sprintf("%dh%02dm", int(remaining.Hours()), int(remaining.Minutes())%60) + } } diff --git a/backend/explore/dumpimport.go b/backend/explore/dumpimport.go index 30190e2..d0acee0 100644 --- a/backend/explore/dumpimport.go +++ b/backend/explore/dumpimport.go @@ -1,25 +1,33 @@ +//go:build indexbuild + package explore import ( "context" + "database/sql" "encoding/json" "errors" "fmt" "log/slog" "net/http" "os" + "path" "path/filepath" "regexp" "strconv" + "sync/atomic" "time" + "yellowjacket/backend/database" + "yellowjacket/backend/jobs" "yellowjacket/backend/system" ) // Dump-based index population. Instead of crawling the ListenBrainz // API artist-by-artist, the index is built from two MetaBrainz dumps: // -// 1. The spark listens dump (~170GB, streamed, never stored) yields +// 1. The spark listens dump (~205GB on the server, of which only the +// three MBID columns are downloaded — see dumpproject.go) yields // listen counts for every recording/release/artist MBID. // 2. The MusicBrainz canonical dump (~2GB, streamed) yields names and // MBIDs, filtered to entities above a popularity floor. @@ -30,14 +38,6 @@ import ( // resumes from checkpoints after interruption. const ( - // dumpImportDoneKey marks a completed import in explore_index_meta. - dumpImportDoneKey = "dump_import_done" - - // listensAppliedSeriesKey stores the dump series number whose listen - // counts are folded into popularity (the high-water-mark for the - // incremental refresh). Set to the full dump's series at import, then - // advanced by each applied incremental. - listensAppliedSeriesKey = "listens_applied_series" // releaseToRGInsertBatch bounds how many rows are written per // transaction when persisting the release→release-group map. @@ -56,9 +56,6 @@ const ( dumpStageAssembled = "assembled" ) -// ErrDiskSpace is returned when free disk falls below the safety floor. -var ErrDiskSpace = errors.New("insufficient free disk space") - // Dump import stages, mapped to status names shown in the UI. const ( dumpStageCounts = iota @@ -115,6 +112,14 @@ type dumpImporter struct { // pendingArtists are kept artists whose names weren't derivable // from the canonical dump; the metadata patch pass resolves them. pendingArtists []string + + // countsRate is the listens stream throughput in bytes/sec, written + // by the stage-1 progress reporter and read by its loggers. + countsRate atomic.Uint64 + + // ftsResumed guards the bulk-load window so the FTS rebuild runs + // exactly once per import. + ftsResumed bool } func newDumpImporter(si *SearchIndex, lb *ListenBrainzClient) (*dumpImporter, error) { @@ -132,10 +137,10 @@ func newDumpImporter(si *SearchIndex, lb *ListenBrainzClient) (*dumpImporter, er si: si, lb: lb, logger: si.logger, - // No client-level timeout: the listens stream runs for hours. - // Discovery requests use per-request context timeouts, and - // resumableReader recovers from stalled connections. - httpClient: &http.Client{}, + // HTTP/1.1-only, no client-level timeout: the listens stream + // runs for hours. Discovery requests use per-request context + // timeouts, and the stream readers recover from stalls. + httpClient: newDumpHTTPClient(), stagingDir: stagingDir, canonicalBaseURL: defaultCanonicalBaseURL, listensBaseURL: defaultListensBaseURL, @@ -192,6 +197,8 @@ func (imp *dumpImporter) run(ctx context.Context) error { } imp.logger.Info("dump import: starting", "listensDump", sparkURL) + imp.logJob("Streaming listens dump " + path.Base(sparkURL) + + " — this stage reads tens of gigabytes and runs for hours") counts = &countsState{SparkURL: sparkURL} } else if !counts.Done { @@ -200,6 +207,10 @@ func (imp *dumpImporter) run(ctx context.Context) error { "members", counts.MemberIdx, "entities", len(counts.counts), ) + imp.logJob(fmt.Sprintf( + "Resuming listen counts from %s (%s members, %s entities so far)", + formatGB(counts.Offset), formatCount(counts.MemberIdx), formatCount(len(counts.counts)), + )) } // Record which dump series this import is baselined on, so the @@ -251,15 +262,15 @@ func (imp *dumpImporter) run(ctx context.Context) error { return err } - // One-time reset when migrating from the legacy API-crawled - // index: its popularity values are on a different scale (the LB - // API includes MLHD+ history) and would permanently outrank - // dump-derived counts via the highest-wins upsert. Re-imports - // (dump→dump) skip this — listen counts only grow. - if !imp.si.hasMeta(dumpImportDoneKey) { - if _, err := imp.si.db.ExecContext("DELETE FROM explore_index"); err == nil { - imp.logger.Info("dump import: cleared legacy index for consistent popularity scale") - } + // Bulk-load window: the wipe below and the millions of upserts that + // follow would otherwise each maintain the FTS5 index row by row, + // which dominates the entire import (~31 rows/s vs ~4,700). The + // index is rebuilt in one pass when the window closes. + if err := imp.si.db.SuspendExploreIndexFTS(); err != nil { + // Not fatal: the import still completes, just slowly. + imp.logger.Warn("dump import: could not suspend FTS sync", "error", err) + } else { + defer imp.resumeFTS() } if err := imp.assembleIndex(ctx, kept, scan); err != nil { @@ -272,6 +283,11 @@ func (imp *dumpImporter) run(ctx context.Context) error { // just built from. imp.persistReleaseToRG(ctx, scan.releaseToRG) + // Close the bulk-load window now rather than at return: the patch + // passes below run at API rate, so per-row FTS upkeep costs nothing + // there and keeps search current while they work. + imp.resumeFTS() + imp.si.setTierStatus(dumpStageNames[dumpStageCatalog], "complete", 0, 0) // Artists that need names from the metadata patch pass. @@ -299,16 +315,38 @@ func (imp *dumpImporter) run(ctx context.Context) error { return imp.finalize() } +// resumeFTS closes the bulk-load window, restoring the FTS sync +// triggers and rebuilding the index. Idempotent, so it can run both at +// its natural point in the pipeline and from a defer covering the +// error and cancellation paths. +func (imp *dumpImporter) resumeFTS() { + if imp.ftsResumed { + return + } + + imp.ftsResumed = true + + start := time.Now() + + if err := imp.si.db.ResumeExploreIndexFTS(); err != nil { + // Leaving search unindexed is worse than a slow import, so this + // is loud: it needs a rebuild to recover. + imp.logger.Error("dump import: FTS rebuild failed — search index is stale", + "error", err, + ) + + return + } + + imp.logger.Info("dump import: FTS index rebuilt", + "elapsed", time.Since(start).Round(time.Millisecond), + ) +} + // finalize records completion and removes all staging data. func (imp *dumpImporter) finalize() error { imp.si.setMeta(dumpImportDoneKey, time.Now().UTC().Format(time.RFC3339)) - // Retire the legacy tier-crawl freshness keys. - _, _ = imp.si.db.ExecContext( - `DELETE FROM explore_index_meta - WHERE key IN ('tier1_built', 'tier2_built', 'tier3_built', 'tier4_built')`, - ) - if err := os.RemoveAll(imp.stagingDir); err != nil { imp.logger.Warn("dump import: staging cleanup failed", "error", err) } @@ -326,25 +364,6 @@ func (imp *dumpImporter) finalize() error { return nil } -// dumpSeriesRe extracts the monotonic series number NNNN from a dump -// URL or directory name (e.g. "listenbrainz-spark-dump-2593-…"). -var dumpSeriesRe = regexp.MustCompile(`listenbrainz-(?:spark-)?dump-(\d+)-`) - -// parseDumpSeries pulls the series number out of a dump URL/name. -func parseDumpSeries(url string) (int, bool) { - m := dumpSeriesRe.FindStringSubmatch(url) - if m == nil { - return 0, false - } - - n, err := strconv.Atoi(m[1]) - if err != nil { - return 0, false - } - - return n, true -} - // recordDumpSeries stores the baseline series number for this import. func (imp *dumpImporter) recordDumpSeries(sparkURL string) { series, ok := parseDumpSeries(sparkURL) @@ -374,7 +393,10 @@ func (imp *dumpImporter) persistReleaseToRG(ctx context.Context, m map[uuid16]rg written := 0 pending := 0 - tx, err := imp.si.db.BeginTx() + // The statement is prepared per transaction rather than passed to + // tx.Exec per row: re-parsing it for each of several million rows + // costs an order of magnitude more than the insert itself. + tx, stmt, err := beginReleaseToRGTx(imp.si.db) if err != nil { imp.logger.Warn("dump import: begin release_to_rg tx failed", "error", err) @@ -383,13 +405,13 @@ func (imp *dumpImporter) persistReleaseToRG(ctx context.Context, m map[uuid16]rg for rel, target := range m { if ctx.Err() != nil { + _ = stmt.Close() _ = tx.Rollback() return } - if _, err := tx.Exec( - "INSERT OR REPLACE INTO release_to_rg (release_mbid, rg_mbid) VALUES (?, ?)", + if _, err := stmt.Exec( formatUUID(rel[:]), formatUUID(target.rg[:]), ); err != nil { imp.logger.Warn("dump import: insert release_to_rg failed", "error", err) @@ -401,6 +423,8 @@ func (imp *dumpImporter) persistReleaseToRG(ctx context.Context, m map[uuid16]rg pending++ if pending >= releaseToRGInsertBatch { + _ = stmt.Close() + if err := tx.Commit(); err != nil { imp.logger.Warn("dump import: commit release_to_rg batch failed", "error", err) @@ -409,7 +433,7 @@ func (imp *dumpImporter) persistReleaseToRG(ctx context.Context, m map[uuid16]rg pending = 0 - tx, err = imp.si.db.BeginTx() + tx, stmt, err = beginReleaseToRGTx(imp.si.db) if err != nil { imp.logger.Warn("dump import: begin release_to_rg tx failed", "error", err) @@ -418,6 +442,8 @@ func (imp *dumpImporter) persistReleaseToRG(ctx context.Context, m map[uuid16]rg } } + _ = stmt.Close() + if err := tx.Commit(); err != nil { imp.logger.Warn("dump import: commit release_to_rg failed", "error", err) @@ -427,6 +453,26 @@ func (imp *dumpImporter) persistReleaseToRG(ctx context.Context, m map[uuid16]rg imp.logger.Info("dump import: persisted release→release-group map", "rows", written) } +// beginReleaseToRGTx opens a batch transaction with the row insert +// already prepared on it. +func beginReleaseToRGTx(db *database.DB) (*sql.Tx, *sql.Stmt, error) { + tx, err := db.BeginTx() + if err != nil { + return nil, nil, fmt.Errorf("release_to_rg begin: %w", err) + } + + stmt, err := tx.Prepare( + "INSERT OR REPLACE INTO release_to_rg (release_mbid, rg_mbid) VALUES (?, ?)", + ) + if err != nil { + _ = tx.Rollback() + + return nil, nil, fmt.Errorf("release_to_rg prepare: %w", err) + } + + return tx, stmt, nil +} + func (imp *dumpImporter) readState() (*dumpImportState, error) { state := &dumpImportState{} @@ -470,28 +516,24 @@ func (imp *dumpImporter) setStageProgress(stage, completed, total int) { imp.si.setTierStatus(dumpStageNames[stage], "running", total, completed) } +// setStageDetail is setStageProgress plus a human-readable line for +// stages where completed/total alone is uninformative. +func (imp *dumpImporter) setStageDetail(stage, completed, total int, detail string) { + imp.si.setTierDetail(dumpStageNames[stage], "running", total, completed, detail) +} + +// logJob appends a line to the index build's job log, which is what the +// user sees in the jobs panel. A build with no registered job (tests, +// headless imports) drops the line. +func (imp *dumpImporter) logJob(message string) { + imp.si.logIndexJob(jobs.LevelInfo, message) +} + // checkDiskHeadroom aborts the import when free disk is critically low. func (imp *dumpImporter) checkDiskHeadroom() error { return checkFreeDisk(imp.stagingDir, imp.abortFreeBytes) } -// checkFreeDisk returns ErrDiskSpace when the volume holding path has -// less than minBytes free. Unknown free space (unsupported platform) -// passes. -func checkFreeDisk(path string, minBytes uint64) error { - free, ok := diskFreeBytes(path) - if !ok { - return nil - } - - if free < minBytes { - return fmt.Errorf("%w: %d MB free, need %d MB", - ErrDiskSpace, free>>20, minBytes>>20) - } - - return nil -} - // --------------------------------------------------------------------------- // SearchIndex integration // --------------------------------------------------------------------------- diff --git a/backend/explore/dumpimport_test.go b/backend/explore/dumpimport_test.go index a0cfe6c..c35a498 100644 --- a/backend/explore/dumpimport_test.go +++ b/backend/explore/dumpimport_test.go @@ -1,43 +1,26 @@ +//go:build indexbuild + package explore import ( - "archive/tar" "bytes" "context" "encoding/csv" "fmt" "io" - "log/slog" "net/http" "net/http/httptest" "os" "strings" + "sync/atomic" "testing" "time" "github.com/klauspost/compress/zstd" - "github.com/parquet-go/parquet-go" "yellowjacket/backend/database" ) -// Fixed MBIDs for fixtures. -const ( - recA = "11111111-1111-1111-1111-111111111111" - recB = "22222222-2222-2222-2222-222222222222" - recC = "33333333-3333-3333-3333-333333333333" - relA = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa" - relB = "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb" - rgA = "cccccccc-cccc-cccc-cccc-cccccccccccc" - rgB = "dddddddd-dddd-dddd-dddd-dddddddddddd" - artA = "eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee" - artB = "ffffffff-ffff-ffff-ffff-ffffffffffff" -) - -func testLogger() *slog.Logger { - return slog.New(slog.DiscardHandler) -} - // --------------------------------------------------------------------------- // Unit tests: parsing helpers // --------------------------------------------------------------------------- @@ -244,67 +227,6 @@ func TestArtistTopRGBoundedAndDeduped(t *testing.T) { // Fixture builders // --------------------------------------------------------------------------- -// sparkFixtureRow mimics the real spark listens schema: the aggregator -// must project just recording/release/artist MBIDs out of it. -type sparkFixtureRow struct { - ListenedAt int64 `parquet:"listened_at"` - UserID int64 `parquet:"user_id"` - ArtistName string `parquet:"artist_name,optional"` - RecordingMBID string `parquet:"recording_mbid,optional"` - ReleaseMBID string `parquet:"release_mbid,optional"` - ArtistMBIDs []string `parquet:"artist_credit_mbids,optional,list"` -} - -func makeParquet(t *testing.T, rows []sparkFixtureRow) []byte { - t.Helper() - - var buf bytes.Buffer - - w := parquet.NewGenericWriter[sparkFixtureRow](&buf) - - if _, err := w.Write(rows); err != nil { - t.Fatalf("parquet write: %v", err) - } - - if err := w.Close(); err != nil { - t.Fatalf("parquet close: %v", err) - } - - return buf.Bytes() -} - -func makeTar(t *testing.T, members map[string][]byte, order []string) []byte { - t.Helper() - - var buf bytes.Buffer - - tw := tar.NewWriter(&buf) - - for _, name := range order { - data := members[name] - hdr := &tar.Header{ - Name: name, - Mode: 0o644, - Size: int64(len(data)), - Typeflag: tar.TypeReg, - } - - if err := tw.WriteHeader(hdr); err != nil { - t.Fatalf("tar header: %v", err) - } - - if _, err := tw.Write(data); err != nil { - t.Fatalf("tar write: %v", err) - } - } - - if err := tw.Close(); err != nil { - t.Fatalf("tar close: %v", err) - } - - return buf.Bytes() -} - func zstdCompress(t *testing.T, data []byte) []byte { t.Helper() @@ -339,23 +261,6 @@ func csvBytes(t *testing.T, rows [][]string) []byte { return buf.Bytes() } -// listensOf builds n identical listen rows for a recording. -func listensOf(n int, recording, release string, artists []string) []sparkFixtureRow { - rows := make([]sparkFixtureRow, n) - for i := range rows { - rows[i] = sparkFixtureRow{ - ListenedAt: 1700000000 + int64(i), - UserID: int64(i), - ArtistName: "Fixture Artist", - RecordingMBID: recording, - ReleaseMBID: release, - ArtistMBIDs: artists, - } - } - - return rows -} - // canonicalDataCSV builds a canonical_musicbrainz_data.csv fixture. func canonicalDataCSV(t *testing.T) []byte { t.Helper() @@ -651,39 +556,37 @@ func TestDumpImportEndToEnd(t *testing.T) { // A legacy API-crawled row with inflated popularity must be // cleared by the first dump import (scale consistency). - legacyMBID := "99999999-9999-9999-9999-999999999999" - - si.upsertBatch([]SearchIndexResult{{ - EntityType: "recording", - MBID: legacyMBID, - Title: "Legacy Row", - ArtistName: "Old Crawl", - ArtistMBID: artA, - Popularity: 123_456_789, - }}) - if err := imp.run(context.Background()); err != nil { t.Fatalf("run: %v", err) } - legacyRows, err := db.QueryContext( - "SELECT COUNT(*) FROM explore_index WHERE mbid = ?", legacyMBID, - ) - if err != nil { - t.Fatalf("legacy query: %v", err) - } + // The import bulk-loads with the FTS sync triggers suspended, so + // the rebuild that closes that window is the only thing keeping + // search usable: assembled rows must be findable afterwards. + ftsCount := func(query string) int { + t.Helper() - if legacyRows.Next() { - var n int - - _ = legacyRows.Scan(&n) - - if n != 0 { - t.Error("legacy API-crawled row survived the first dump import") + rows, err := db.QueryContext( + "SELECT COUNT(*) FROM explore_index_fts WHERE explore_index_fts MATCH ?", query, + ) + if err != nil { + t.Fatalf("fts query %q: %v", query, err) } + + defer func() { _ = rows.Close() }() + + n := 0 + + if rows.Next() { + _ = rows.Scan(&n) + } + + return n } - _ = legacyRows.Close() + if got := ftsCount("Song"); got == 0 { + t.Error("FTS matches no assembled recordings; the rebuild did not run") + } // Index rows landed with dump-derived popularity. assertRow := func(mbid, entityType, title string, popularity int) { @@ -831,6 +734,212 @@ func TestDumpImportResumesAfterCancel(t *testing.T) { } } +// --------------------------------------------------------------------------- +// Progress reporting +// --------------------------------------------------------------------------- + +func TestFormatStreamProgress(t *testing.T) { + const gb = int64(1) << 30 + + tests := []struct { + name string + offset int64 + size int64 + rate float64 + want string + }{ + { + name: "size and rate known", + offset: 40 * gb, + size: 200 * gb, + rate: 20 << 20, + want: "40.0 GB / 200.0 GB (20%) · 20.0 MB/s · ~2h16m left", + }, + { + name: "size unknown before first response", + offset: 2 * gb, + rate: 10 << 20, + want: "2.0 GB downloaded · 10.0 MB/s", + }, + { + name: "rate unknown on the first tick", + offset: 10 * gb, + size: 100 * gb, + want: "10.0 GB / 100.0 GB (10%)", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := formatStreamProgress(tt.offset, tt.size, tt.rate); got != tt.want { + t.Errorf("formatStreamProgress() = %q, want %q", got, tt.want) + } + }) + } +} + +func TestFormatETA(t *testing.T) { + const gb = int64(1) << 30 + + tests := []struct { + name string + offset int64 + size int64 + rate float64 + want string + }{ + {name: "hours", offset: 0, size: 100 * gb, rate: 10 << 20, want: "2h50m"}, + {name: "minutes", offset: 0, size: gb, rate: 10 << 20, want: "1m"}, + {name: "seconds", offset: 0, size: 1 << 20, rate: 10 << 20, want: "<1m"}, + {name: "unknown size", offset: 0, size: -1, rate: 10 << 20, want: ""}, + {name: "stalled", offset: 0, size: 100 * gb, rate: 0, want: ""}, + {name: "past the end", offset: 2 * gb, size: gb, rate: 10 << 20, want: ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := formatETA(tt.offset, tt.size, tt.rate); got != tt.want { + t.Errorf("formatETA() = %q, want %q", got, tt.want) + } + }) + } +} + +func TestFormatCount(t *testing.T) { + tests := []struct { + in int + want string + }{ + {0, "0"}, + {999, "999"}, + {1000, "1,000"}, + {12345, "12,345"}, + {1234567, "1,234,567"}, + } + + for _, tt := range tests { + if got := formatCount(tt.in); got != tt.want { + t.Errorf("formatCount(%d) = %q, want %q", tt.in, got, tt.want) + } + } +} + +// The reporter is what keeps the listens stage from looking frozen, so +// it must publish a detail line for the stage while the stream runs. +func TestCountsReporterPublishesDetail(t *testing.T) { + db := database.NewTestDB(t) + si := NewSearchIndex(db, nil, nil, testLogger()) + imp := &dumpImporter{si: si, logger: testLogger()} + + stream := newResumableReader(context.Background(), nil, "", 0) + stream.offset.Store(50 << 30) + stream.size.Store(200 << 30) + + rep := &countsReporter{ + imp: imp, + stream: stream, + lastSample: time.Now().Add(-time.Second), + lastMoved: time.Now(), + } + + rep.tick(time.Now()) + + tier := findTier(t, si, dumpStageNames[dumpStageCounts]) + + if tier.Completed != 25 { + t.Errorf("tier completed = %d, want 25", tier.Completed) + } + + if !strings.Contains(tier.Detail, "50.0 GB / 200.0 GB (25%)") { + t.Errorf("tier detail = %q, want it to report GB progress", tier.Detail) + } +} + +// A stream that stops moving is reported as stalled rather than as a +// decaying transfer rate. +func TestCountsReporterReportsStall(t *testing.T) { + db := database.NewTestDB(t) + si := NewSearchIndex(db, nil, nil, testLogger()) + imp := &dumpImporter{si: si, logger: testLogger()} + + stream := newResumableReader(context.Background(), nil, "", 0) + stream.offset.Store(50 << 30) + stream.size.Store(200 << 30) + + now := time.Now() + rep := &countsReporter{ + imp: imp, + stream: stream, + lastSample: now.Add(-countsUIRefreshInterval), + lastOffset: 50 << 30, + lastMoved: now.Add(-2 * countsStallAfter), + } + + rep.tick(now) + + tier := findTier(t, si, dumpStageNames[dumpStageCounts]) + + if !strings.Contains(tier.Detail, "stalled") { + t.Errorf("tier detail = %q, want a stall notice", tier.Detail) + } + + if imp.streamRate() != 0 { + t.Errorf("stalled rate = %v, want 0", imp.streamRate()) + } +} + +// A download paused by parser back-pressure is not a network stall and +// must not be reported as one. +func TestCountsReporterDistinguishesBacklog(t *testing.T) { + db := database.NewTestDB(t) + si := NewSearchIndex(db, nil, nil, testLogger()) + imp := &dumpImporter{si: si, logger: testLogger()} + + stream := newResumableReader(context.Background(), nil, "", 0) + stream.offset.Store(50 << 30) + stream.size.Store(200 << 30) + + var backlog atomic.Bool + + backlog.Store(true) + + now := time.Now() + rep := &countsReporter{ + imp: imp, + stream: stream, + backlog: &backlog, + lastSample: now.Add(-countsUIRefreshInterval), + lastOffset: 50 << 30, + lastMoved: now.Add(-2 * countsStallAfter), + } + + rep.tick(now) + + tier := findTier(t, si, dumpStageNames[dumpStageCounts]) + + if strings.Contains(tier.Detail, "stalled") { + t.Errorf("tier detail = %q, want parsing back-pressure, not a stall", tier.Detail) + } + + if !strings.Contains(tier.Detail, "parsing") { + t.Errorf("tier detail = %q, want it to name the parsing pause", tier.Detail) + } +} + +func findTier(t *testing.T, si *SearchIndex, name string) TierStatus { + t.Helper() + + for _, tier := range si.GetIndexStatus().Tiers { + if tier.Name == name { + return tier + } + } + + t.Fatalf("tier %q not found", name) + + return TierStatus{} +} + func TestCheckFreeDisk(t *testing.T) { dir := t.TempDir() diff --git a/backend/explore/dumpparallel.go b/backend/explore/dumpparallel.go new file mode 100644 index 0000000..d73f205 --- /dev/null +++ b/backend/explore/dumpparallel.go @@ -0,0 +1,562 @@ +//go:build indexbuild + +package explore + +import ( + "context" + "crypto/tls" + "fmt" + "io" + "log/slog" + "net/http" + "strconv" + "sync" + "sync/atomic" + "time" +) + +// Parallel dump streaming. MetaBrainz shapes throughput per +// connection, so a single sequential stream of the listens dump tops +// out near 5 MB/s no matter how fast the link is — the 205GB stage-1 +// download alone runs for eleven hours that way. Several concurrent +// Range requests lift that to tens of MB/s. +// +// parallelReader hides the concurrency behind the same sequential +// io.Reader the tar decoders already consume: lanes fetch fixed-size +// chunks ahead of the caller, and chunks are handed over strictly in +// order. Pos() therefore keeps meaning "absolute offset of the next +// byte to be delivered", which is what the stage-1 checkpoint records. +// +// For the listens dump this is now the fallback: dumpproject.go fetches +// only the columns stage 1 reads, which is well under half the bytes. +// This path still serves the canonical dump (a zstd stream that cannot +// be range-projected) and any origin that refuses Range requests. +// +// A caution on the rates quoted below: measured again on 2026-07-28, +// data.metabrainz.org served ~0.65 MB/s aggregate to one client and got +// *slower* past eight connections — the shaping is per-IP, not per +// connection, so raising dumpLanes buys throttling rather than speed. + +const ( + // dumpLanes is the number of concurrent Range requests. One lane + // gets ~5 MB/s because MetaBrainz shapes per connection; four reach + // tens of MB/s. Deliberately conservative: these are a nonprofit's + // servers, they answer sustained heavy use with 503s, and pushing + // past this trades politeness for throughput that backoff eats + // anyway. + dumpLanes = 4 + + // dumpChunkSize is how much a lane fetches per request. Each + // request pays a ramp-up cost, so small chunks squander the gain — + // 32MB chunks measured roughly 40% slower than 128MB ones. + dumpChunkSize = 128 << 20 + + // dumpWindowChunks bounds the chunks in flight or buffered awaiting + // in-order delivery. Kept just above the lane count so lanes never + // idle waiting for the consumer; costs dumpWindowChunks * + // dumpChunkSize of buffer. + dumpWindowChunks = dumpLanes + 2 + + // dumpProbeTimeout bounds the HEAD request that sizes a resource + // before a parallel stream starts. + dumpProbeTimeout = 30 * time.Second + + // dumpMaxIdleConns keeps a pooled connection per lane so chunk + // requests reuse TCP+TLS instead of reconnecting each time. + dumpMaxIdleConns = dumpLanes * 2 + + // dumpRetryAfterCap bounds how long a server-supplied Retry-After is + // honoured, so a bad header can't park a lane indefinitely. + dumpRetryAfterCap = 60 * time.Second +) + +// dumpStream is the streaming surface the dump importers consume, +// implemented by both parallelReader and resumableReader. +type dumpStream interface { + io.ReadCloser + + // Pos is the absolute byte offset of the next byte to be delivered. + // This is what an interrupted import checkpoints and resumes from. + Pos() int64 + + // Fetched is the absolute byte offset the downloader has reached. + // With prefetching lanes this runs ahead of Pos, and it — not Pos — + // is what progress and stall reporting should watch: a reader + // buffering a 128MB chunk is downloading, not stalled. + Fetched() int64 + + // Total is the total resource size, or -1 while unknown. + Total() int64 +} + +// newDumpHTTPClient builds the client used for dump discovery and +// streaming. HTTP/2 is disabled deliberately: it multiplexes every +// lane onto a single TCP connection, which collapses parallel Range +// requests back to one shaped stream (measured at 1-3 MB/s). +func newDumpHTTPClient() *http.Client { + transport := &http.Transport{ + // Explicitly HTTP/1.1: ALPN would otherwise negotiate h2 and + // silently undo the parallelism below. + TLSClientConfig: &tls.Config{ + NextProtos: []string{"http/1.1"}, + MinVersion: tls.VersionTLS12, + }, + ForceAttemptHTTP2: false, + TLSNextProto: map[string]func(string, *tls.Conn) http.RoundTripper{}, + MaxIdleConns: dumpMaxIdleConns, + MaxIdleConnsPerHost: dumpMaxIdleConns, + IdleConnTimeout: 90 * time.Second, + } + + // No client-level timeout: dump streams run for hours. Per-request + // deadlines come from the caller's context, and chunk fetches retry + // on their own. + return &http.Client{Transport: transport} +} + +// probeDumpSize returns the resource size when the server advertises +// one and supports Range requests, else (0, false). +func probeDumpSize(ctx context.Context, client *http.Client, url string) (int64, bool) { + reqCtx, cancel := context.WithTimeout(ctx, dumpProbeTimeout) + defer cancel() + + req, err := http.NewRequestWithContext(reqCtx, http.MethodHead, url, nil) + if err != nil { + return 0, false + } + + req.Header.Set("User-Agent", lbUserAgent) + + resp, err := client.Do(req) + if err != nil { + return 0, false + } + + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK || resp.ContentLength <= 0 { + return 0, false + } + + if resp.Header.Get("Accept-Ranges") != "bytes" { + return 0, false + } + + return resp.ContentLength, true +} + +// chunkResult is one fetched chunk awaiting in-order delivery. bufp is +// the pooled backing array, returned to the pool once drained. +type chunkResult struct { + bufp *[]byte + n int + err error +} + +// parallelReader streams [base, size) of an HTTP resource through +// several concurrent Range requests, delivering bytes in order. +type parallelReader struct { + ctx context.Context + cancel context.CancelFunc + client *http.Client + url string + logger *slog.Logger + + base int64 + size int64 + totalChunks int64 + + // lanes, chunkSize and window are fields rather than constants so + // tests can drive the reader with small chunks. + lanes int + chunkSize int64 + window int64 + + mu sync.Mutex + cond *sync.Cond + ready map[int64]*chunkResult + nextDL int64 // next chunk index to hand to a lane + nextOut int64 // next chunk index to deliver + closed bool + + delivered atomic.Int64 + + // fetched counts bytes pulled down by the lanes, including chunks + // still buffered ahead of the caller. + fetched atomic.Int64 + + // cur is the undelivered tail of the chunk being drained. + cur []byte + curBufp *[]byte + + pool sync.Pool + wg sync.WaitGroup +} + +// newParallelReader starts the lanes and returns a reader positioned at +// offset. Returns nil when the resource can't be range-streamed, so +// callers fall back to a sequential reader. +func newParallelReader( + ctx context.Context, client *http.Client, logger *slog.Logger, url string, offset int64, +) *parallelReader { + return newParallelReaderLogged( + ctx, client, logger, url, offset, dumpLanes, dumpChunkSize, dumpWindowChunks, + ) +} + +// newParallelReaderWith is newParallelReader with the lane geometry +// spelled out, so tests can exercise ordering and resume with chunks +// small enough to be practical. +func newParallelReaderWith( + ctx context.Context, client *http.Client, url string, offset int64, + lanes int, chunkSize, window int64, +) *parallelReader { + return newParallelReaderLogged( + ctx, client, slog.New(slog.DiscardHandler), url, offset, lanes, chunkSize, window, + ) +} + +// newParallelReaderLogged is newParallelReaderWith with a logger, so a +// throttled or retrying stream is diagnosable from the app log. +func newParallelReaderLogged( + ctx context.Context, client *http.Client, logger *slog.Logger, + url string, offset int64, lanes int, chunkSize, window int64, +) *parallelReader { + size, ok := probeDumpSize(ctx, client, url) + if !ok || offset >= size { + return nil + } + + // Below a couple of chunks there's nothing to parallelise. + if size-offset < 2*chunkSize { + return nil + } + + if logger == nil { + logger = slog.New(slog.DiscardHandler) + } + + streamCtx, cancel := context.WithCancel(ctx) + + p := ¶llelReader{ + ctx: streamCtx, + cancel: cancel, + client: client, + logger: logger, + url: url, + base: offset, + size: size, + lanes: lanes, + chunkSize: chunkSize, + window: window, + ready: make(map[int64]*chunkResult, window), + pool: sync.Pool{New: func() any { + b := make([]byte, chunkSize) + + return &b + }}, + } + + remaining := size - offset + p.totalChunks = (remaining + chunkSize - 1) / chunkSize + p.cond = sync.NewCond(&p.mu) + + for range lanes { + p.wg.Add(1) + + go p.lane() + } + + // A cancelled context must wake anyone blocked on the condition. + go func() { + <-streamCtx.Done() + + p.mu.Lock() + p.cond.Broadcast() + p.mu.Unlock() + }() + + return p +} + +// Pos returns the absolute offset of the next byte to be delivered. +func (p *parallelReader) Pos() int64 { + return p.base + p.delivered.Load() +} + +// Fetched returns the absolute offset the lanes have downloaded to. +func (p *parallelReader) Fetched() int64 { + return p.base + p.fetched.Load() +} + +// Total returns the total resource size. +func (p *parallelReader) Total() int64 { + return p.size +} + +// lane fetches chunks until the window is exhausted or the stream ends. +func (p *parallelReader) lane() { + defer p.wg.Done() + + for { + p.mu.Lock() + + for { + if p.closed || p.ctx.Err() != nil || p.nextDL >= p.totalChunks { + p.mu.Unlock() + + return + } + + // Stay inside the delivery window so buffered chunks can't + // outrun the consumer. + if p.nextDL < p.nextOut+p.window { + break + } + + p.cond.Wait() + } + + idx := p.nextDL + p.nextDL++ + + p.mu.Unlock() + + bufp, n, err := p.fetchChunk(idx) + if err == nil { + p.fetched.Add(int64(n)) + } + + p.mu.Lock() + p.ready[idx] = &chunkResult{bufp: bufp, n: n, err: err} + p.cond.Broadcast() + p.mu.Unlock() + } +} + +// fetchChunk retrieves one chunk, retrying transient failures. The +// range is fully specified, so a retry simply re-requests it. +func (p *parallelReader) fetchChunk(idx int64) (*[]byte, int, error) { + lo := p.base + idx*p.chunkSize + + hi := lo + p.chunkSize - 1 + if hi >= p.size { + hi = p.size - 1 + } + + want := int(hi - lo + 1) + + bufp, _ := p.pool.Get().(*[]byte) + + var ( + lastErr error + wait time.Duration + ) + + for attempt := 0; attempt <= maxStreamRetries; attempt++ { + if attempt > 0 { + delay := min(streamRetryBaseDelay<<(attempt-1), streamRetryMaxDelay) + if wait > 0 { + delay = wait + } + + select { + case <-p.ctx.Done(): + p.pool.Put(bufp) + + return nil, 0, p.ctx.Err() + case <-time.After(delay): + } + } + + n, retryAfter, err := p.fetchOnce(lo, hi, (*bufp)[:want]) + if err == nil && n == want { + return bufp, n, nil + } + + if err == nil { + err = io.ErrUnexpectedEOF + } + + lastErr = err + wait = retryAfter + + p.logger.Warn("dump import: chunk fetch failed, retrying", + "chunk", idx, + "attempt", attempt+1, + "retryAfter", retryAfter, + "error", err, + ) + + if p.ctx.Err() != nil { + p.pool.Put(bufp) + + return nil, 0, p.ctx.Err() + } + } + + p.pool.Put(bufp) + + return nil, 0, fmt.Errorf("%w: %s chunk %d after %d retries: %w", + ErrDumpStream, p.url, idx, maxStreamRetries, lastErr) +} + +// fetchOnce performs a single Range request into buf. The second +// return value is the server's requested Retry-After delay, if any. +func (p *parallelReader) fetchOnce(lo, hi int64, buf []byte) (int, time.Duration, error) { + req, err := http.NewRequestWithContext(p.ctx, http.MethodGet, p.url, nil) + if err != nil { + return 0, 0, fmt.Errorf("dump chunk request: %w", err) + } + + req.Header.Set("User-Agent", lbUserAgent) + req.Header.Set("Range", "bytes="+strconv.FormatInt(lo, 10)+"-"+strconv.FormatInt(hi, 10)) + + resp, err := p.client.Do(req) + if err != nil { + return 0, 0, fmt.Errorf("dump chunk fetch: %w", err) + } + + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusPartialContent { + // A loaded dump server answers with 503 (or 429) rather than + // queueing; that is a "come back shortly", not a failure. + return 0, parseRetryAfter(resp.Header.Get("Retry-After")), + fmt.Errorf("%w: HTTP %d from %s", ErrDumpStream, resp.StatusCode, p.url) + } + + n, err := io.ReadFull(resp.Body, buf) + if err != nil { + return n, 0, fmt.Errorf("dump chunk read: %w", err) + } + + return n, 0, nil +} + +// parseRetryAfter reads a Retry-After header given in seconds, clamped +// to dumpRetryAfterCap. Returns 0 when absent or unparseable. +func parseRetryAfter(v string) time.Duration { + if v == "" { + return 0 + } + + secs, err := strconv.Atoi(v) + if err != nil || secs <= 0 { + return 0 + } + + return min(time.Duration(secs)*time.Second, dumpRetryAfterCap) +} + +func (p *parallelReader) Read(b []byte) (int, error) { + for len(p.cur) == 0 { + if err := p.ctx.Err(); err != nil { + return 0, err + } + + res, err := p.nextChunk() + if err != nil { + return 0, err + } + + p.curBufp = res.bufp + p.cur = (*res.bufp)[:res.n] + } + + n := copy(b, p.cur) + p.cur = p.cur[n:] + p.delivered.Add(int64(n)) + + if len(p.cur) == 0 && p.curBufp != nil { + p.pool.Put(p.curBufp) + p.curBufp = nil + } + + return n, nil +} + +// nextChunk blocks until the next in-order chunk is available. +func (p *parallelReader) nextChunk() (*chunkResult, error) { + p.mu.Lock() + defer p.mu.Unlock() + + for { + if p.closed { + return nil, io.ErrClosedPipe + } + + if err := p.ctx.Err(); err != nil { + return nil, err + } + + if p.nextOut >= p.totalChunks { + return nil, io.EOF + } + + res, ok := p.ready[p.nextOut] + if ok { + delete(p.ready, p.nextOut) + + p.nextOut++ + + // A freed window slot may unblock a waiting lane. + p.cond.Broadcast() + + if res.err != nil { + return nil, res.err + } + + return res, nil + } + + p.cond.Wait() + } +} + +// Close stops the lanes and releases buffered chunks. +func (p *parallelReader) Close() error { + p.mu.Lock() + + if p.closed { + p.mu.Unlock() + + return nil + } + + p.closed = true + p.cond.Broadcast() + p.mu.Unlock() + + p.cancel() + p.wg.Wait() + + p.mu.Lock() + clear(p.ready) + p.mu.Unlock() + + return nil +} + +// openDumpStream returns the best available stream for a dump URL, +// preferring parallel Range lanes and falling back to a single +// resumable connection when the server won't serve ranges. +func (imp *dumpImporter) openDumpStream( + ctx context.Context, url string, offset int64, +) dumpStream { + if p := newParallelReader(ctx, imp.httpClient, imp.logger, url, offset); p != nil { + imp.logger.Info("dump import: streaming in parallel", + "lanes", dumpLanes, + "chunkMB", dumpChunkSize>>20, + "url", url, + ) + + return p + } + + imp.logger.Info("dump import: parallel streaming unavailable, using single stream", + "url", url, + ) + + return newResumableReader(ctx, imp.httpClient, url, offset) +} diff --git a/backend/explore/dumpparallel_test.go b/backend/explore/dumpparallel_test.go new file mode 100644 index 0000000..ebe07b6 --- /dev/null +++ b/backend/explore/dumpparallel_test.go @@ -0,0 +1,372 @@ +//go:build indexbuild + +package explore + +import ( + "bytes" + "context" + "io" + "math/rand" + "net/http" + "net/http/httptest" + "strconv" + "sync/atomic" + "testing" + "time" +) + +// serveBlob returns a Range-capable server for a fixed payload, plus a +// counter of the GET requests it served. +func serveBlob(t *testing.T, payload []byte) (*httptest.Server, *atomic.Int64) { + t.Helper() + + var gets atomic.Int64 + + modTime := time.Now() + + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet { + gets.Add(1) + } + + http.ServeContent(w, r, "blob.bin", modTime, bytes.NewReader(payload)) + }, + )) + t.Cleanup(srv.Close) + + return srv, &gets +} + +func randomPayload(n int) []byte { + buf := make([]byte, n) + + rng := rand.New(rand.NewSource(1)) //nolint:gosec // deterministic fixture + _, _ = rng.Read(buf) + + return buf +} + +// The whole point of the reader is that concurrency stays invisible: +// bytes must come out in the same order a single stream would produce. +func TestParallelReaderDeliversBytesInOrder(t *testing.T) { + payload := randomPayload(200_000) + srv, gets := serveBlob(t, payload) + + p := newParallelReaderWith( + context.Background(), srv.Client(), srv.URL, 0, 4, 8_192, 6, + ) + if p == nil { + t.Fatal("newParallelReaderWith returned nil, want a parallel reader") + } + + defer func() { _ = p.Close() }() + + got, err := io.ReadAll(p) + if err != nil { + t.Fatalf("read: %v", err) + } + + if !bytes.Equal(got, payload) { + t.Fatalf("payload mismatch: got %d bytes, want %d", len(got), len(payload)) + } + + // Confirm it really did fan out rather than quietly falling back. + if n := gets.Load(); n < 2 { + t.Errorf("served %d GETs, want one per chunk", n) + } +} + +// Pos is what the stage-1 checkpoint records, so it must track bytes +// handed to the caller — not bytes fetched by the lanes running ahead. +func TestParallelReaderPosTracksDeliveredBytes(t *testing.T) { + payload := randomPayload(100_000) + srv, _ := serveBlob(t, payload) + + p := newParallelReaderWith( + context.Background(), srv.Client(), srv.URL, 0, 4, 4_096, 8, + ) + if p == nil { + t.Fatal("newParallelReaderWith returned nil") + } + + defer func() { _ = p.Close() }() + + if got := p.Total(); got != int64(len(payload)) { + t.Errorf("Total() = %d, want %d", got, len(payload)) + } + + buf := make([]byte, 1_000) + + read, err := io.ReadFull(p, buf) + if err != nil { + t.Fatalf("read: %v", err) + } + + if got := p.Pos(); got != int64(read) { + t.Errorf("Pos() = %d after reading %d bytes, want %d", got, read, read) + } + + // Let the lanes race ahead, then confirm Pos still reflects delivery. + time.Sleep(50 * time.Millisecond) + + if got := p.Pos(); got != int64(read) { + t.Errorf("Pos() = %d after lanes prefetched, want %d", got, read) + } +} + +// Progress reporting watches Fetched rather than Pos, because a reader +// buffering a chunk ahead of the caller is downloading, not stalled. +// Fetched must therefore outrun Pos while lanes prefetch. +func TestParallelReaderFetchedOutrunsPos(t *testing.T) { + payload := randomPayload(200_000) + srv, _ := serveBlob(t, payload) + + p := newParallelReaderWith( + context.Background(), srv.Client(), srv.URL, 0, 4, 8_192, 8, + ) + if p == nil { + t.Fatal("newParallelReaderWith returned nil") + } + + defer func() { _ = p.Close() }() + + // Read a single byte, then let the lanes fill the window. + buf := make([]byte, 1) + if _, err := io.ReadFull(p, buf); err != nil { + t.Fatalf("read: %v", err) + } + + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) && p.Fetched() <= p.Pos() { + time.Sleep(10 * time.Millisecond) + } + + if p.Fetched() <= p.Pos() { + t.Errorf("Fetched() = %d, Pos() = %d; want Fetched ahead while prefetching", + p.Fetched(), p.Pos()) + } +} + +// Resuming an interrupted import constructs a reader at the +// checkpointed offset; it must yield exactly the remaining tail. +func TestParallelReaderResumesFromOffset(t *testing.T) { + payload := randomPayload(120_000) + srv, _ := serveBlob(t, payload) + + const offset = 37_000 + + p := newParallelReaderWith( + context.Background(), srv.Client(), srv.URL, offset, 3, 8_192, 5, + ) + if p == nil { + t.Fatal("newParallelReaderWith returned nil") + } + + defer func() { _ = p.Close() }() + + if got := p.Pos(); got != offset { + t.Errorf("Pos() = %d before reading, want %d", got, offset) + } + + got, err := io.ReadAll(p) + if err != nil { + t.Fatalf("read: %v", err) + } + + if !bytes.Equal(got, payload[offset:]) { + t.Fatalf("resumed payload mismatch: got %d bytes, want %d", + len(got), len(payload)-offset) + } +} + +// A lane that hits a transient failure must retry its range rather than +// tear down the whole multi-hour stream. +func TestParallelReaderRetriesFailedChunk(t *testing.T) { + payload := randomPayload(60_000) + + var attempts atomic.Int64 + + modTime := time.Now() + + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + // Fail the third GET once; every other request succeeds. + if r.Method == http.MethodGet && attempts.Add(1) == 3 { + hj, ok := w.(http.Hijacker) + if ok { + conn, _, err := hj.Hijack() + if err == nil { + _ = conn.Close() + + return + } + } + } + + http.ServeContent(w, r, "blob.bin", modTime, bytes.NewReader(payload)) + }, + )) + t.Cleanup(srv.Close) + + p := newParallelReaderWith( + context.Background(), srv.Client(), srv.URL, 0, 2, 8_192, 4, + ) + if p == nil { + t.Fatal("newParallelReaderWith returned nil") + } + + defer func() { _ = p.Close() }() + + got, err := io.ReadAll(p) + if err != nil { + t.Fatalf("read after transient failure: %v", err) + } + + if !bytes.Equal(got, payload) { + t.Fatalf("payload mismatch after retry: got %d bytes, want %d", + len(got), len(payload)) + } +} + +// A loaded dump server answers with 503 rather than queueing. That is +// "come back shortly", not a failure, so the lane must retry and the +// stream must still complete — this is what stalled a real import. +func TestParallelReaderRecoversFrom503(t *testing.T) { + payload := randomPayload(60_000) + + var gets atomic.Int64 + + modTime := time.Now() + + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + // Refuse the first two range GETs the way a busy + // MetaBrainz mirror does. + if r.Method == http.MethodGet && gets.Add(1) <= 2 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusServiceUnavailable) + + return + } + + http.ServeContent(w, r, "blob.bin", modTime, bytes.NewReader(payload)) + }, + )) + t.Cleanup(srv.Close) + + p := newParallelReaderWith( + context.Background(), srv.Client(), srv.URL, 0, 2, 8_192, 4, + ) + if p == nil { + t.Fatal("newParallelReaderWith returned nil") + } + + defer func() { _ = p.Close() }() + + got, err := io.ReadAll(p) + if err != nil { + t.Fatalf("read after 503s: %v", err) + } + + if !bytes.Equal(got, payload) { + t.Fatalf("payload mismatch after 503s: got %d bytes, want %d", + len(got), len(payload)) + } +} + +// Retry-After is honoured but clamped, so a hostile or buggy header +// can't park a download lane for hours. +func TestParseRetryAfter(t *testing.T) { + tests := []struct { + header string + want time.Duration + }{ + {"", 0}, + {"5", 5 * time.Second}, + {"0", 0}, + {"-3", 0}, + {"not-a-number", 0}, + {"Wed, 21 Oct 2026 07:28:00 GMT", 0}, // HTTP-date form: ignored + {"99999", dumpRetryAfterCap}, + } + + for _, tt := range tests { + if got := parseRetryAfter(tt.header); got != tt.want { + t.Errorf("parseRetryAfter(%q) = %v, want %v", tt.header, got, tt.want) + } + } +} + +// Servers that won't serve ranges must fall back to the sequential +// reader instead of failing the import. +func TestParallelReaderDeclinesWithoutRangeSupport(t *testing.T) { + payload := randomPayload(100_000) + + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + // No Accept-Ranges header: a plain, non-seekable response. + w.Header().Set("Content-Length", strconv.Itoa(len(payload))) + _, _ = w.Write(payload) + }, + )) + t.Cleanup(srv.Close) + + if p := newParallelReaderWith( + context.Background(), srv.Client(), srv.URL, 0, 4, 8_192, 6, + ); p != nil { + _ = p.Close() + + t.Fatal("got a parallel reader for a server without range support, want nil") + } +} + +// Payloads too small to split aren't worth the fan-out. +func TestParallelReaderDeclinesTinyPayload(t *testing.T) { + payload := randomPayload(1_000) + srv, _ := serveBlob(t, payload) + + if p := newParallelReaderWith( + context.Background(), srv.Client(), srv.URL, 0, 4, 8_192, 6, + ); p != nil { + _ = p.Close() + + t.Fatal("got a parallel reader for a sub-chunk payload, want nil") + } +} + +// Cancelling the import must stop the lanes promptly rather than let +// them keep pulling gigabytes in the background. +func TestParallelReaderStopsOnCancel(t *testing.T) { + payload := randomPayload(400_000) + srv, _ := serveBlob(t, payload) + + ctx, cancel := context.WithCancel(context.Background()) + + p := newParallelReaderWith(ctx, srv.Client(), srv.URL, 0, 4, 8_192, 6) + if p == nil { + t.Fatal("newParallelReaderWith returned nil") + } + + buf := make([]byte, 100) + if _, err := io.ReadFull(p, buf); err != nil { + t.Fatalf("initial read: %v", err) + } + + cancel() + + // Close waits for every lane, so returning at all proves they exited. + done := make(chan struct{}) + + go func() { + _ = p.Close() + + close(done) + }() + + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("Close did not return after cancel; lanes are still running") + } +} diff --git a/backend/explore/dumppatch.go b/backend/explore/dumppatch.go index b9b8ec4..5027fbb 100644 --- a/backend/explore/dumppatch.go +++ b/backend/explore/dumppatch.go @@ -1,3 +1,5 @@ +//go:build indexbuild + package explore import ( diff --git a/backend/explore/dumpproject.go b/backend/explore/dumpproject.go new file mode 100644 index 0000000..d878952 --- /dev/null +++ b/backend/explore/dumpproject.go @@ -0,0 +1,679 @@ +//go:build indexbuild + +package explore + +import ( + "archive/tar" + "bytes" + "context" + "encoding/binary" + "errors" + "fmt" + "io" + "net/http" + "slices" + "strconv" + "strings" + "sync/atomic" + "time" + + "github.com/parquet-go/parquet-go" + "github.com/parquet-go/parquet-go/format" +) + +// Column-projected fetching of the spark listens dump. +// +// Streaming the tar end to end pulls all 205GB even though the counts +// aggregator reads three columns. Parquet is columnar and the dump +// server serves Range requests, so the unread columns never have to +// cross the wire: for each member we fetch the footer, look up the byte +// ranges of the wanted column chunks, and download only those. +// +// Measured against listenbrainz-spark-dump-2593 (2026-07-28), the three +// projected columns are 43.4% of the row-group bytes: +// +// recording_msid 26.9% (not read) +// recording_mbid 24.1% ← wanted +// recording_name 15.2% (not read) +// release_mbid 13.5% ← wanted +// release_name 6.0% (not read) +// artist_credit_mbids.list.element 5.8% ← wanted +// artist_name / artist_credit_id / user_id / created / listened_at +// +// The decoder is untouched: the fetched ranges are placed at their real +// offsets in a member-sized buffer, so parseListenParquet still reads +// an ordinary parquet file. It never touches the unfilled bytes, +// because it only projects those three columns. + +const ( + // projectedColumnPaths are the parquet leaf columns sparkListenRow + // projects. Kept in sync with that struct by + // TestProjectedColumnsMatchSchema. + projectedRecordingMBID = "recording_mbid" + projectedReleaseMBID = "release_mbid" + projectedArtistMBIDs = "artist_credit_mbids.list.element" + + // rangeGapCoalesce merges two wanted byte ranges separated by less + // than this much unwanted data. Below a round-trip's worth of + // bytes, downloading the gap is cheaper than a second request. + rangeGapCoalesce = 2 << 20 + + // projectFetchLanes is how many Range requests one member's column + // fetch issues concurrently. Total in-flight requests are this + // times projectMembersInFlight, kept at the dumpLanes budget the + // dump server tolerates. + projectFetchLanes = 2 + + // tarHeaderSize is the size of a tar header block. + tarHeaderSize = 512 + + // walkAheadMembers bounds how far the tar header walk runs ahead of + // the fetchers. The walk is one small request per member and is + // latency-bound, so it needs a long leash to stay off the critical + // path. + walkAheadMembers = 64 +) + +// projectedColumns is the set of leaf column paths to download. +var projectedColumns = []string{ + projectedRecordingMBID, + projectedReleaseMBID, + projectedArtistMBIDs, +} + +// tarMember is one regular file inside the dump tar. +type tarMember struct { + // headerOffset is the absolute offset of the member's tar header. + // This is what the stage-1 checkpoint records: resuming means + // restarting the walk from here. + headerOffset int64 + + // dataOffset is where the member's contents begin, and size how + // many bytes they occupy. + dataOffset int64 + size int64 + + name string + + // typeflag is the tar entry type. Selecting members by name alone + // is not enough: an extension header can carry the name of the + // member it describes, and reading one as data yields PAX records + // where parquet is expected. + typeflag byte +} + +// nextHeaderOffset is the absolute offset of the following tar header. +func (m tarMember) nextHeaderOffset() int64 { + return m.dataOffset + m.size + tarPadding(m.size) +} + +// byteRange is a half-open [lo, hi) span of a resource. +type byteRange struct { + lo, hi int64 +} + +func (r byteRange) len() int64 { return r.hi - r.lo } + +// --------------------------------------------------------------------------- +// Range fetching +// --------------------------------------------------------------------------- + +// rangeFetcher performs retrying HTTP Range reads against one URL. +type rangeFetcher struct { + ctx context.Context + client *http.Client + url string + + // footerProbe is how much of a member's tail to fetch when reading + // its parquet footer; zero means defaultFooterProbe. The real + // dump's footers measure well under that, and an undersized guess + // costs one extra request rather than failing. + footerProbe int64 +} + +// defaultFooterProbe is the tail size used when a fetcher does not +// override it. +const defaultFooterProbe = 64 << 10 + +func (f *rangeFetcher) probeBytes() int64 { + if f.footerProbe > 0 { + return f.footerProbe + } + + return defaultFooterProbe +} + +// fetch reads [lo, hi) into buf (which must be exactly that long), +// retrying transient failures and honouring Retry-After. +func (f *rangeFetcher) fetch(r byteRange, buf []byte) error { + var ( + lastErr error + wait time.Duration + ) + + for attempt := 0; attempt <= maxStreamRetries; attempt++ { + if attempt > 0 { + delay := min(streamRetryBaseDelay<<(attempt-1), streamRetryMaxDelay) + if wait > 0 { + delay = wait + } + + select { + case <-f.ctx.Done(): + return f.ctx.Err() + case <-time.After(delay): + } + } + + retryAfter, err := f.fetchOnce(r, buf) + if err == nil { + return nil + } + + lastErr = err + wait = retryAfter + + if f.ctx.Err() != nil { + return f.ctx.Err() + } + } + + return fmt.Errorf("%w: %s bytes %d-%d after %d retries: %w", + ErrDumpStream, f.url, r.lo, r.hi-1, maxStreamRetries, lastErr) +} + +func (f *rangeFetcher) fetchOnce(r byteRange, buf []byte) (time.Duration, error) { + req, err := http.NewRequestWithContext(f.ctx, http.MethodGet, f.url, nil) + if err != nil { + return 0, fmt.Errorf("dump range request: %w", err) + } + + req.Header.Set("User-Agent", lbUserAgent) + req.Header.Set("Range", "bytes="+strconv.FormatInt(r.lo, 10)+ + "-"+strconv.FormatInt(r.hi-1, 10)) + + resp, err := f.client.Do(req) + if err != nil { + return 0, fmt.Errorf("dump range fetch: %w", err) + } + + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusPartialContent { + // A loaded dump server answers 503/429 rather than queueing. + return parseRetryAfter(resp.Header.Get("Retry-After")), + fmt.Errorf("%w: HTTP %d from %s", ErrDumpStream, resp.StatusCode, f.url) + } + + if _, err := io.ReadFull(resp.Body, buf); err != nil { + return 0, fmt.Errorf("dump range read: %w", err) + } + + return 0, nil +} + +// --------------------------------------------------------------------------- +// Tar member walking +// --------------------------------------------------------------------------- + +// walkTarMembers reads tar headers by Range request, emitting every +// regular member from startOffset onward. Only the 512-byte headers are +// downloaded; member contents are skipped by arithmetic rather than by +// pulling them over the wire, which is the whole point. +// +// PAX extension headers (which the dump uses for long names) are +// themselves regular members and are walked like any other; their +// contents are not needed because the aggregator selects members by +// suffix and the extended name only ever restates the short one. +func walkTarMembers( + ctx context.Context, f *rangeFetcher, startOffset, total int64, out chan<- tarMember, +) error { + defer close(out) + + hdr := make([]byte, tarHeaderSize) + offset := startOffset + + for offset+tarHeaderSize <= total { + if err := ctx.Err(); err != nil { + return err + } + + if err := f.fetch(byteRange{offset, offset + tarHeaderSize}, hdr); err != nil { + return err + } + + m, ok, err := parseTarHeader(hdr, offset) + if err != nil { + return err + } + + if !ok { + // Two zero blocks mark end of archive; one is enough to stop. + return nil + } + + // A GNU long-name entry would leave the following member's name + // truncated in its ustar field, and this walk never reads member + // bodies to recover it. The dump uses PAX, so this is a format + // change rather than something to paper over. + if m.typeflag == tar.TypeGNULongName || m.typeflag == tar.TypeGNULongLink { + return fmt.Errorf( + "%w: GNU long-name entry at %d is not supported", ErrDumpFormat, offset, + ) + } + + select { + case out <- m: + case <-ctx.Done(): + return ctx.Err() + } + + offset = m.nextHeaderOffset() + } + + return nil +} + +// parseTarHeader decodes one 512-byte header block. Returns ok=false +// at the end-of-archive marker. +func parseTarHeader(hdr []byte, offset int64) (tarMember, bool, error) { + if isZeroBlock(hdr) { + return tarMember{}, false, nil + } + + // The fields are decoded by hand rather than with archive/tar. A + // tar.Reader given a lone header block works for plain ustar entries + // but fails on the PAX extension headers the real dump writes before + // every member: Next() reads a PAX record's body to merge its + // attributes, and here that body is not in the block. Those headers + // only restate the name the ustar fields already carry, so decoding + // the fixed fields is both sufficient and immune to that. + if err := verifyTarChecksum(hdr); err != nil { + return tarMember{}, false, fmt.Errorf( + "%w: tar header at %d: %w", ErrDumpFormat, offset, err, + ) + } + + size, err := parseTarSize(hdr[124:136]) + if err != nil { + return tarMember{}, false, fmt.Errorf( + "%w: tar header at %d: %w", ErrDumpFormat, offset, err, + ) + } + + if size < 0 || offset+tarHeaderSize+size < 0 { + return tarMember{}, false, fmt.Errorf( + "%w: tar header at %d declares size %d", ErrDumpFormat, offset, size, + ) + } + + return tarMember{ + headerOffset: offset, + dataOffset: offset + tarHeaderSize, + size: size, + name: tarName(hdr), + typeflag: hdr[156], + }, true, nil +} + +// tarName joins the ustar prefix and name fields. Long names split +// across the two are rejoined; names carried only in a PAX record are +// not needed, because member selection is by suffix and the dump's +// ustar name field always holds the full path. +func tarName(hdr []byte) string { + name := trimTarField(hdr[0:100]) + + if string(hdr[257:262]) != "ustar" { + return name + } + + if prefix := trimTarField(hdr[345:500]); prefix != "" { + return prefix + "/" + name + } + + return name +} + +func trimTarField(b []byte) string { + if i := bytes.IndexByte(b, 0); i >= 0 { + b = b[:i] + } + + return string(b) +} + +// parseTarSize decodes a tar size field, which is octal ASCII in the +// common case and big-endian base-256 (high bit set) for sizes that do +// not fit — GNU's encoding for files above 8GB. +func parseTarSize(field []byte) (int64, error) { + if len(field) > 0 && field[0]&0x80 != 0 { + var n int64 + + // The high bit is a flag, not part of the magnitude. + for i, c := range field { + if i == 0 { + c &= 0x7F + } + + n = n<<8 | int64(c) + } + + return n, nil + } + + trimmed := strings.Trim(string(field), " \x00") + if trimmed == "" { + return 0, nil + } + + n, err := strconv.ParseInt(trimmed, 8, 64) + if err != nil { + return 0, fmt.Errorf("bad size field %q: %w", field, err) + } + + return n, nil +} + +// verifyTarChecksum checks the header's own checksum. The walk seeks to +// computed offsets rather than reading forward, so this is what catches +// a desync before it is mistaken for a member. +func verifyTarChecksum(hdr []byte) error { + stored, err := parseTarSize(hdr[148:156]) + if err != nil { + return fmt.Errorf("bad checksum field: %w", err) + } + + var signed, unsigned int64 + + for i, c := range hdr { + // The checksum field itself is treated as spaces. + if i >= 148 && i < 156 { + c = ' ' + } + + unsigned += int64(c) + signed += int64(int8(c)) + } + + if stored != unsigned && stored != signed { + return fmt.Errorf( + "%w: checksum %d does not match %d", ErrDumpFormat, stored, unsigned, + ) + } + + return nil +} + +func isZeroBlock(b []byte) bool { + for _, c := range b { + if c != 0 { + return false + } + } + + return true +} + +// --------------------------------------------------------------------------- +// Column projection +// --------------------------------------------------------------------------- + +// projectedMemberRanges returns the byte ranges of a member that must be +// downloaded to decode projectedColumns: the parquet header magic, the +// wanted column chunks, and the footer. Offsets are member-relative. +func projectedMemberRanges(meta *format.FileMetaData, size int64, footerLen int64) []byteRange { + ranges := []byteRange{ + // Leading "PAR1" magic — parquet readers verify it. + {0, 4}, + } + + for i := range meta.RowGroups { + for j := range meta.RowGroups[i].Columns { + col := &meta.RowGroups[i].Columns[j] + + path := strings.Join(col.MetaData.PathInSchema, ".") + if !slices.Contains(projectedColumns, path) { + continue + } + + lo := col.MetaData.DataPageOffset + if col.MetaData.DictionaryPageOffset != 0 { + lo = col.MetaData.DictionaryPageOffset + } + + hi := lo + col.MetaData.TotalCompressedSize + if lo < 0 || hi > size || hi <= lo { + continue + } + + ranges = append(ranges, byteRange{lo, hi}) + } + } + + // The footer was already fetched to get here, but including it keeps + // the buffer self-describing for the decoder. + ranges = append(ranges, byteRange{size - footerLen, size}) + + return coalesceRanges(ranges) +} + +// coalesceRanges sorts and merges overlapping or near-adjacent ranges so +// each becomes one HTTP request. +func coalesceRanges(ranges []byteRange) []byteRange { + if len(ranges) == 0 { + return nil + } + + slices.SortFunc(ranges, func(a, b byteRange) int { + return int(a.lo - b.lo) + }) + + merged := ranges[:1] + + for _, r := range ranges[1:] { + last := &merged[len(merged)-1] + if r.lo <= last.hi+rangeGapCoalesce { + last.hi = max(last.hi, r.hi) + + continue + } + + merged = append(merged, r) + } + + return merged +} + +// --------------------------------------------------------------------------- +// Member fetching +// --------------------------------------------------------------------------- + +// fetchProjectedMember downloads only the projected columns of one +// parquet member and returns a member-sized buffer with those bytes at +// their real offsets. The returned count is how many bytes actually +// crossed the wire. +func fetchProjectedMember( + ctx context.Context, f *rangeFetcher, m tarMember, buf []byte, +) (int64, error) { + if int64(len(buf)) != m.size { + return 0, fmt.Errorf("%w: buffer %d for member of %d bytes", + ErrDumpFormat, len(buf), m.size) + } + + // Unfilled gaps must not carry data from a previous member: a stale + // page header there could be read as valid parquet. + clear(buf) + + footerLen := min(f.probeBytes(), m.size) + + fetchTail := func(n int64) error { + return f.fetch( + byteRange{m.dataOffset + m.size - n, m.dataOffset + m.size}, + buf[m.size-n:], + ) + } + + if err := fetchTail(footerLen); err != nil { + return 0, err + } + + fetched := footerLen + + // A member smaller than the probe arrived whole; there is nothing + // left to project out of it. + if footerLen == m.size { + return fetched, nil + } + + // A footer larger than the probe leaves the metadata truncated; the + // trailing length field says how much is really needed. + if need := declaredFooterLen(buf, m.size); need > footerLen { + footerLen = min(need, m.size) + + if err := fetchTail(footerLen); err != nil { + return 0, err + } + + fetched += footerLen + } + + meta, err := readFooterMetadata(buf, m.size, footerLen) + if err != nil { + return 0, err + } + + ranges := projectedMemberRanges(meta, m.size, footerLen) + + got, err := fetchRangesConcurrent(ctx, f, m.dataOffset, ranges, buf) + if err != nil { + return 0, err + } + + return fetched + got, nil +} + +// declaredFooterLen reads the footer length a parquet file advertises in +// its last eight bytes, plus those eight bytes. Returns 0 when the tail +// in buf is too short to hold the field. +func declaredFooterLen(buf []byte, size int64) int64 { + const trailer = 8 + + if size < trailer { + return 0 + } + + return int64(binary.LittleEndian.Uint32(buf[size-trailer:size-4])) + trailer +} + +// readFooterMetadata parses the parquet footer sitting in the tail of +// buf. parquet.OpenFile is given a reader over the tail alone, offset +// so that footer-relative seeks land correctly. +func readFooterMetadata(buf []byte, size, footerLen int64) (*format.FileMetaData, error) { + // A parquet file ends with the 4-byte footer length followed by + // "PAR1"; the metadata precedes it. + if footerLen < 8 { + return nil, fmt.Errorf("%w: member too small for a parquet footer", ErrDumpFormat) + } + + // Only the tail of buf holds real bytes at this point, which is all + // footer parsing needs. SkipMagicBytes suppresses the leading-magic + // check: those four bytes are fetched with the column ranges and are + // verified by the decoder when the assembled buffer is parsed. + file, err := parquet.OpenFile(bytes.NewReader(buf), size, + parquet.SkipMagicBytes(true), + parquet.SkipPageIndex(true), + parquet.SkipBloomFilters(true), + ) + if err != nil { + return nil, fmt.Errorf("%w: parquet footer: %w", ErrDumpFormat, err) + } + + return file.Metadata(), nil +} + +// fetchRangesConcurrent downloads ranges (member-relative) into buf, +// using a few lanes so a member's chunks aren't fetched one round trip +// at a time. +func fetchRangesConcurrent( + ctx context.Context, f *rangeFetcher, base int64, ranges []byteRange, buf []byte, +) (int64, error) { + type job struct{ r byteRange } + + jobs := make(chan job) + errs := make(chan error, projectFetchLanes) + + var fetched atomic.Int64 + + fetchCtx, cancel := context.WithCancel(ctx) + defer cancel() + + laneFetcher := &rangeFetcher{ctx: fetchCtx, client: f.client, url: f.url} + + for range projectFetchLanes { + go func() { + var firstErr error + + for j := range jobs { + if firstErr != nil { + continue + } + + dst := buf[j.r.lo:j.r.hi] + abs := byteRange{base + j.r.lo, base + j.r.hi} + + if err := laneFetcher.fetch(abs, dst); err != nil { + firstErr = err + + continue + } + + fetched.Add(j.r.len()) + } + + errs <- firstErr + }() + } + + for _, r := range ranges { + select { + case jobs <- job{r}: + case <-fetchCtx.Done(): + close(jobs) + + for range projectFetchLanes { + <-errs + } + + return 0, fetchCtx.Err() + } + } + + close(jobs) + + var firstErr error + + for range projectFetchLanes { + if err := <-errs; err != nil && firstErr == nil { + firstErr = err + } + } + + if firstErr != nil { + return 0, firstErr + } + + return fetched.Load(), nil +} + +// projectionSupported reports whether the dump server will serve the +// Range requests column projection depends on. +func projectionSupported(ctx context.Context, client *http.Client, url string) (int64, bool) { + size, ok := probeDumpSize(ctx, client, url) + if !ok || size <= 0 { + return 0, false + } + + return size, true +} + +var errProjectionUnsupported = errors.New("column projection unavailable") diff --git a/backend/explore/dumpproject_test.go b/backend/explore/dumpproject_test.go new file mode 100644 index 0000000..2a7f328 --- /dev/null +++ b/backend/explore/dumpproject_test.go @@ -0,0 +1,716 @@ +//go:build indexbuild + +package explore + +import ( + "archive/tar" + "bytes" + "context" + "fmt" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" + + "yellowjacket/backend/database" +) + +// serveRangeBlob serves payload with Range support, counting the bytes +// actually delivered so a test can assert how much crossed the wire. +func serveRangeBlob(t *testing.T, payload []byte) (*httptest.Server, *atomic.Int64) { + t.Helper() + + var served atomic.Int64 + + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + rec := &countingResponseWriter{ResponseWriter: w, n: &served} + http.ServeContent(rec, r, "dump.tar", time.Unix(0, 0), bytes.NewReader(payload)) + }, + )) + + t.Cleanup(srv.Close) + + return srv, &served +} + +type countingResponseWriter struct { + http.ResponseWriter + + n *atomic.Int64 +} + +func (w *countingResponseWriter) Write(b []byte) (int, error) { + n, err := w.ResponseWriter.Write(b) + w.n.Add(int64(n)) + + return n, err +} + +// bigSparkTar builds a tar whose single parquet member has enough rows +// that the unprojected columns dominate its size. +func bigSparkTar(t *testing.T) []byte { + t.Helper() + + rows := make([]sparkFixtureRow, 0, 20_000) + + for i := range 20_000 { + rows = append(rows, sparkFixtureRow{ + ListenedAt: int64(i), + UserID: int64(i % 977), + // A high-cardinality column that is not projected: it is what + // projection must avoid downloading. + ArtistName: strings.Repeat("padding-", 12) + string(rune('a'+i%26)) + itoa(i), + RecordingMBID: recA, + ReleaseMBID: relA, + ArtistMBIDs: []string{artA}, + }) + } + + name := "listenbrainz-spark-dump-1-20260101-000003-full/1.parquet" + + return makeTar(t, map[string][]byte{name: makeParquet(t, rows)}, []string{name}) +} + +func itoa(i int) string { + if i == 0 { + return "0" + } + + var b []byte + + for i > 0 { + b = append([]byte{byte('0' + i%10)}, b...) + i /= 10 + } + + return string(b) +} + +func TestWalkTarMembersFindsMembersWithoutReadingThem(t *testing.T) { + t.Parallel() + + payload := fixtureSparkTar(t) + srv, served := serveRangeBlob(t, payload) + + f := &rangeFetcher{ctx: t.Context(), client: srv.Client(), url: srv.URL} + out := make(chan tarMember, 8) + + if err := walkTarMembers(t.Context(), f, 0, int64(len(payload)), out); err != nil { + t.Fatalf("walk: %v", err) + } + + var members []tarMember + for m := range out { + members = append(members, m) + } + + if len(members) != 2 { + t.Fatalf("got %d members, want 2", len(members)) + } + + for i, m := range members { + if !strings.HasSuffix(m.name, ".parquet") { + t.Errorf("member %d: name %q", i, m.name) + } + + if m.size <= 0 { + t.Errorf("member %d: size %d", i, m.size) + } + + // The member's declared bytes must match what the tar really holds. + want := payload[m.dataOffset : m.dataOffset+m.size] + if !bytes.HasPrefix(want, []byte("PAR1")) { + t.Errorf("member %d: data offset %d does not start a parquet file", i, m.dataOffset) + } + } + + // The walk must read headers only — not the multi-KB member bodies. + if n := served.Load(); n > int64(4*tarHeaderSize) { + t.Errorf("walk downloaded %d bytes, want only tar headers", n) + } +} + +func TestFetchProjectedMemberMatchesFullParse(t *testing.T) { + t.Parallel() + + payload := bigSparkTar(t) + srv, served := serveRangeBlob(t, payload) + + f := &rangeFetcher{ctx: t.Context(), client: srv.Client(), url: srv.URL} + out := make(chan tarMember, 4) + + if err := walkTarMembers(t.Context(), f, 0, int64(len(payload)), out); err != nil { + t.Fatalf("walk: %v", err) + } + + var member tarMember + + for m := range out { + if strings.HasSuffix(m.name, ".parquet") { + member = m + } + } + + if member.size == 0 { + t.Fatal("no parquet member found") + } + + // Ground truth: parse the member as the sequential path would. + full := payload[member.dataOffset : member.dataOffset+member.size] + + wantDeltas, err := parseListenParquet(full) + if err != nil { + t.Fatalf("full parse: %v", err) + } + + served.Store(0) + + buf := make([]byte, member.size) + + fetched, err := fetchProjectedMember(t.Context(), f, member, buf) + if err != nil { + t.Fatalf("projected fetch: %v", err) + } + + gotDeltas, err := parseListenParquet(buf) + if err != nil { + t.Fatalf("projected parse: %v", err) + } + + if len(gotDeltas) != len(wantDeltas) { + t.Fatalf("projected parse produced %d entities, want %d", len(gotDeltas), len(wantDeltas)) + } + + for k, want := range wantDeltas { + if got := gotDeltas[k]; got != want { + t.Errorf("entity %x: count %d, want %d", k, got, want) + } + } + + // The point of the exercise: materially fewer bytes than the member. + if fetched >= member.size { + t.Errorf("projected fetch pulled %d bytes of a %d byte member", fetched, member.size) + } + + t.Logf("projected fetch: %d of %d bytes (%.1f%%)", + fetched, member.size, 100*float64(fetched)/float64(member.size)) +} + +func TestProjectedMemberRangesSkipsUnwantedColumns(t *testing.T) { + t.Parallel() + + payload := bigSparkTar(t) + srv, _ := serveRangeBlob(t, payload) + + f := &rangeFetcher{ctx: t.Context(), client: srv.Client(), url: srv.URL} + out := make(chan tarMember, 4) + + if err := walkTarMembers(t.Context(), f, 0, int64(len(payload)), out); err != nil { + t.Fatalf("walk: %v", err) + } + + var member tarMember + + for m := range out { + if strings.HasSuffix(m.name, ".parquet") { + member = m + } + } + + buf := make([]byte, member.size) + footerLen := min(f.probeBytes(), member.size) + tailStart := member.dataOffset + member.size - footerLen + copy(buf[member.size-footerLen:], payload[tailStart:member.dataOffset+member.size]) + + meta, err := readFooterMetadata(buf, member.size, footerLen) + if err != nil { + t.Fatalf("footer: %v", err) + } + + ranges := projectedMemberRanges(meta, member.size, footerLen) + if len(ranges) == 0 { + t.Fatal("no ranges computed") + } + + var total int64 + + for _, r := range ranges { + if r.lo < 0 || r.hi > member.size || r.hi <= r.lo { + t.Fatalf("range %v out of bounds for member size %d", r, member.size) + } + + total += r.len() + } + + if total >= member.size { + t.Errorf("projected ranges cover %d of %d bytes", total, member.size) + } +} + +func TestCoalesceRangesMergesNeighbours(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + in []byteRange + want []byteRange + }{ + { + name: "adjacent merge", + in: []byteRange{{0, 100}, {100, 200}}, + want: []byteRange{{0, 200}}, + }, + { + name: "small gap merges", + in: []byteRange{{0, 100}, {100 + rangeGapCoalesce - 1, 500}}, + want: []byteRange{{0, 500}}, + }, + { + name: "large gap stays split", + in: []byteRange{{0, 100}, {100 + rangeGapCoalesce + 1, 500}}, + want: []byteRange{{0, 100}, {100 + rangeGapCoalesce + 1, 500}}, + }, + { + name: "unsorted input", + in: []byteRange{{10 * rangeGapCoalesce, 10*rangeGapCoalesce + 100}, {0, 100}}, + want: []byteRange{{0, 100}, {10 * rangeGapCoalesce, 10*rangeGapCoalesce + 100}}, + }, + { + name: "overlap", + in: []byteRange{{0, 300}, {100, 200}}, + want: []byteRange{{0, 300}}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := coalesceRanges(tt.in) + if len(got) != len(tt.want) { + t.Fatalf("got %v, want %v", got, tt.want) + } + + for i := range got { + if got[i] != tt.want[i] { + t.Fatalf("got %v, want %v", got, tt.want) + } + } + }) + } +} + +func TestFetchProjectedMemberRejectsWrongBuffer(t *testing.T) { + t.Parallel() + + f := &rangeFetcher{ + ctx: context.Background(), + client: http.DefaultClient, + url: "http://example.invalid", + } + + if _, err := fetchProjectedMember( + t.Context(), f, tarMember{size: 100}, make([]byte, 50), + ); err == nil { + t.Fatal("expected an error for a mis-sized buffer") + } +} + +// serveDumpNoRanges serves the spark dump without advertising Range +// support, which is what forces the streamed fallback. +func serveDumpNoRanges(t *testing.T, sparkTar []byte) *httptest.Server { + t.Helper() + + const ( + listensDir = "listenbrainz-dump-1-20260101-000003-full" + sparkFile = "listenbrainz-spark-dump-1-20260101-000003-full.tar" + ) + + mux := http.NewServeMux() + + mux.HandleFunc("/listens/", func(w http.ResponseWriter, r *http.Request) { + switch strings.TrimPrefix(r.URL.Path, "/listens/") { + case "": + _, _ = fmt.Fprintf(w, `%s/`, listensDir, listensDir) + case listensDir + "/": + _, _ = fmt.Fprintf(w, `%s`, sparkFile, sparkFile) + case listensDir + "/" + sparkFile: + // No Accept-Ranges, and Range headers are ignored. + w.Header().Set("Content-Length", strconv.Itoa(len(sparkTar))) + _, _ = w.Write(sparkTar) + default: + http.NotFound(w, r) + } + }) + + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + + return srv +} + +// The projected and streamed paths must agree exactly: projection is an +// optimisation, not a different answer. +func TestProjectedAndStreamedCountsAgree(t *testing.T) { + t.Parallel() + + sparkTar := bigSparkTar(t) + + counts := func(srv *httptest.Server) *countsState { + t.Helper() + + db := database.NewTestDB(t) + si := NewSearchIndex(db, nil, nil, testLogger()) + imp := testImporter(t, si, srv) + + url := srv.URL + "/listens/listenbrainz-dump-1-20260101-000003-full/" + + "listenbrainz-spark-dump-1-20260101-000003-full.tar" + + st := &countsState{SparkURL: url} + if err := imp.aggregateListenCounts(t.Context(), st); err != nil { + t.Fatalf("aggregate: %v", err) + } + + if !st.Done { + t.Fatal("aggregation did not complete") + } + + return st + } + + ranged := serveDumps(t, sparkTar, nil) + plain := serveDumpNoRanges(t, sparkTar) + + projected := counts(ranged) + streamed := counts(plain) + + if len(projected.counts) == 0 { + t.Fatal("projected run produced no counts") + } + + if len(projected.counts) != len(streamed.counts) { + t.Fatalf("projected %d entities, streamed %d", + len(projected.counts), len(streamed.counts)) + } + + for k, want := range streamed.counts { + if got := projected.counts[k]; got != want { + t.Errorf("entity %x: projected %d, streamed %d", k, got, want) + } + } + + if projected.Offset != streamed.Offset { + t.Errorf("checkpoint offset: projected %d, streamed %d", + projected.Offset, streamed.Offset) + } + + if projected.MemberIdx != streamed.MemberIdx { + t.Errorf("member index: projected %d, streamed %d", + projected.MemberIdx, streamed.MemberIdx) + } +} + +func TestDeclaredFooterLenReadsTrailer(t *testing.T) { + t.Parallel() + + payload := bigSparkTar(t) + srv, _ := serveRangeBlob(t, payload) + + f := &rangeFetcher{ctx: t.Context(), client: srv.Client(), url: srv.URL} + out := make(chan tarMember, 4) + + if err := walkTarMembers(t.Context(), f, 0, int64(len(payload)), out); err != nil { + t.Fatalf("walk: %v", err) + } + + var m tarMember + + for got := range out { + if strings.HasSuffix(got.name, ".parquet") { + m = got + } + } + + member := payload[m.dataOffset : m.dataOffset+m.size] + + got := declaredFooterLen(member, m.size) + if got <= 8 || got > m.size { + t.Fatalf("declared footer length %d for a %d byte member", got, m.size) + } + + // A tail of exactly that length must be enough to parse the footer. + buf := make([]byte, m.size) + copy(buf[m.size-got:], member[m.size-got:]) + + if _, err := readFooterMetadata(buf, m.size, got); err != nil { + t.Fatalf("footer parse with declared length: %v", err) + } +} + +// A footer bigger than the initial probe must trigger a second, larger +// tail fetch rather than failing. +func TestFetchProjectedMemberRefetchesOversizedFooter(t *testing.T) { + t.Parallel() + + payload := bigSparkTar(t) + srv, _ := serveRangeBlob(t, payload) + + f := &rangeFetcher{ctx: t.Context(), client: srv.Client(), url: srv.URL} + out := make(chan tarMember, 4) + + if err := walkTarMembers(t.Context(), f, 0, int64(len(payload)), out); err != nil { + t.Fatalf("walk: %v", err) + } + + var m tarMember + + for got := range out { + if strings.HasSuffix(got.name, ".parquet") { + m = got + } + } + + member := payload[m.dataOffset : m.dataOffset+m.size] + + want, err := parseListenParquet(member) + if err != nil { + t.Fatalf("full parse: %v", err) + } + + // Force the probe to land short of the real footer. + footer := declaredFooterLen(member, m.size) + + short := &rangeFetcher{ + ctx: t.Context(), + client: srv.Client(), + url: srv.URL, + footerProbe: footer / 2, + } + + buf := make([]byte, m.size) + + if _, err := fetchProjectedMember(t.Context(), short, m, buf); err != nil { + t.Fatalf("projected fetch with short probe: %v", err) + } + + got, err := parseListenParquet(buf) + if err != nil { + t.Fatalf("projected parse: %v", err) + } + + if len(got) != len(want) { + t.Fatalf("got %d entities, want %d", len(got), len(want)) + } +} + +// makePaxTar writes members preceded by PAX extension headers, which is +// how the real ListenBrainz dump is written. A lone header block from +// such an archive cannot be decoded with archive/tar, so this is the +// layout the walker must handle directly. +func makePaxTar(t *testing.T, members map[string][]byte, order []string) []byte { + t.Helper() + + var buf bytes.Buffer + + tw := tar.NewWriter(&buf) + + for _, name := range order { + data := members[name] + hdr := &tar.Header{ + Name: name, + Mode: 0o644, + Size: int64(len(data)), + Typeflag: tar.TypeReg, + // Sub-second precision cannot be expressed in ustar, so the + // writer emits a PAX extension header ahead of the member. + ModTime: time.Unix(1700000000, 123456789), + Format: tar.FormatPAX, + } + + if err := tw.WriteHeader(hdr); err != nil { + t.Fatalf("tar header: %v", err) + } + + if _, err := tw.Write(data); err != nil { + t.Fatalf("tar write: %v", err) + } + } + + if err := tw.Close(); err != nil { + t.Fatalf("tar close: %v", err) + } + + return buf.Bytes() +} + +func TestWalkTarMembersHandlesPaxHeaders(t *testing.T) { + t.Parallel() + + rows := listensOf(4, recA, relA, []string{artA}) + name := "listenbrainz-spark-dump-1-20260101-000003-full/1.parquet" + member := makeParquet(t, rows) + + payload := makePaxTar(t, map[string][]byte{name: member}, []string{name}) + + // Guard the premise: the fixture really does contain a PAX header. + if !bytes.Contains(payload[:4096], []byte("PaxHeader")) { + t.Fatal("fixture has no PAX extension header") + } + + srv, _ := serveRangeBlob(t, payload) + + f := &rangeFetcher{ctx: t.Context(), client: srv.Client(), url: srv.URL} + out := make(chan tarMember, 16) + + if err := walkTarMembers(t.Context(), f, 0, int64(len(payload)), out); err != nil { + t.Fatalf("walk: %v", err) + } + + var found tarMember + + for m := range out { + if isProjectableMember(m) { + found = m + } + } + + if found.size != int64(len(member)) { + t.Fatalf("member size %d, want %d", found.size, len(member)) + } + + if got := payload[found.dataOffset : found.dataOffset+4]; string(got) != "PAR1" { + t.Fatalf("data offset %d does not start a parquet file (%q)", found.dataOffset, got) + } +} + +// End to end through the aggregator, against a PAX archive. +func TestAggregateProjectedWithPaxHeaders(t *testing.T) { + t.Parallel() + + prefix := "listenbrainz-spark-dump-1-20260101-000003-full/" + m1 := prefix + "1.parquet" + m2 := prefix + "2.parquet" + + payload := makePaxTar(t, + map[string][]byte{ + m1: makeParquet(t, listensOf(12, recA, relA, []string{artA})), + m2: makeParquet(t, listensOf(11, recB, relA, []string{artA})), + }, + []string{m1, m2}, + ) + + srv := serveDumps(t, payload, nil) + + db := database.NewTestDB(t) + si := NewSearchIndex(db, nil, nil, testLogger()) + imp := testImporter(t, si, srv) + + url := srv.URL + "/listens/listenbrainz-dump-1-20260101-000003-full/" + + "listenbrainz-spark-dump-1-20260101-000003-full.tar" + + st := &countsState{SparkURL: url} + if err := imp.aggregateListenCounts(t.Context(), st); err != nil { + t.Fatalf("aggregate: %v", err) + } + + if !st.Done { + t.Fatal("aggregation did not complete") + } + + assert := func(kind byte, mbid string, want uint32) { + t.Helper() + + key, ok := makeMBIDKey(kind, mbid) + if !ok { + t.Fatalf("bad fixture mbid %s", mbid) + } + + if got := st.counts[key]; got != want { + t.Errorf("count(kind=%d, %s) = %d, want %d", kind, mbid, got, want) + } + } + + assert(countKindRecording, recA, 12) + assert(countKindRecording, recB, 11) + assert(countKindRelease, relA, 23) + assert(countKindArtist, artA, 23) +} + +func TestParseTarHeaderRejectsCorruptBlock(t *testing.T) { + t.Parallel() + + payload := fixtureSparkTar(t) + + valid := make([]byte, tarHeaderSize) + copy(valid, payload[:tarHeaderSize]) + + if _, ok, err := parseTarHeader(valid, 0); err != nil || !ok { + t.Fatalf("valid header rejected: ok=%v err=%v", ok, err) + } + + // A desynced walk lands mid-member; the checksum must catch it. + corrupt := make([]byte, tarHeaderSize) + copy(corrupt, valid) + corrupt[10] ^= 0xFF + + if _, _, err := parseTarHeader(corrupt, 0); err == nil { + t.Fatal("corrupt header accepted") + } +} + +// A member smaller than the footer probe arrives in the probe request; +// it must not then be fetched a second time. +func TestFetchProjectedMemberSkipsRefetchForTinyMembers(t *testing.T) { + t.Parallel() + + rows := listensOf(2, recA, relA, []string{artA}) + name := "listenbrainz-spark-dump-1-20260101-000003-full/1.parquet" + member := makeParquet(t, rows) + + if int64(len(member)) >= defaultFooterProbe { + t.Skipf("fixture member is %d bytes, not smaller than the probe", len(member)) + } + + payload := makeTar(t, map[string][]byte{name: member}, []string{name}) + srv, served := serveRangeBlob(t, payload) + + f := &rangeFetcher{ctx: t.Context(), client: srv.Client(), url: srv.URL} + out := make(chan tarMember, 8) + + if err := walkTarMembers(t.Context(), f, 0, int64(len(payload)), out); err != nil { + t.Fatalf("walk: %v", err) + } + + var m tarMember + + for got := range out { + if isProjectableMember(got) { + m = got + } + } + + served.Store(0) + + buf := make([]byte, m.size) + + fetched, err := fetchProjectedMember(t.Context(), f, m, buf) + if err != nil { + t.Fatalf("projected fetch: %v", err) + } + + if fetched != m.size { + t.Errorf("fetched %d bytes for a %d byte member", fetched, m.size) + } + + if n := served.Load(); n > m.size { + t.Errorf("server delivered %d bytes for a %d byte member", n, m.size) + } + + if _, err := parseListenParquet(buf); err != nil { + t.Fatalf("parse: %v", err) + } +} diff --git a/backend/explore/dumpprojectrun.go b/backend/explore/dumpprojectrun.go new file mode 100644 index 0000000..3c9cb68 --- /dev/null +++ b/backend/explore/dumpprojectrun.go @@ -0,0 +1,339 @@ +//go:build indexbuild + +package explore + +import ( + "archive/tar" + "context" + "errors" + "fmt" + "strings" + "sync" + "sync/atomic" + "time" +) + +// Stage 1 driven by column projection. The tar is never streamed: a +// walker reads member headers by Range request, and workers download and +// parse each parquet member's projected columns independently. Results +// are applied in member order, so the checkpoint stays a contiguous +// prefix of the archive exactly as it is on the streamed path. + +const ( + // projectMemberWorkers is how many members are fetched and parsed + // concurrently. Each holds one member-sized buffer, and each issues + // projectFetchLanes concurrent Range requests, so the product is the + // in-flight request count the dump server sees. + projectMemberWorkers = 3 + + // minParquetMemberSize is the smallest member that can hold a + // parquet footer ("PAR1" + length + "PAR1"). Anything shorter is + // not a parquet file whatever its name says. + minParquetMemberSize = 12 +) + +// indexedMember is a parquet member with its position in the aggregation +// order, which is what the applier reassembles results by. +type indexedMember struct { + idx int + m tarMember +} + +// aggregateProjected runs stage 1 by downloading only the projected +// columns of each parquet member. Returns errProjectionUnsupported +// (wrapped) when the dump's layout defeats projection, so the caller can +// fall back before any counts are applied. +func (imp *dumpImporter) aggregateProjected( + ctx context.Context, st *countsState, total int64, +) error { + imp.logger.Info("dump import: streaming listen counts with column projection", + "columns", strings.Join(projectedColumns, ","), + "workers", projectMemberWorkers, + "lanesPerWorker", projectFetchLanes, + "resumeOffset", st.Offset, + ) + + runCtx, cancel := context.WithCancel(ctx) + defer cancel() + + fetcher := &rangeFetcher{ctx: runCtx, client: imp.httpClient, url: st.SparkURL} + + prog := &projectedProgress{total: total} + prog.position.Store(st.Offset) + + stopReporter := imp.startCountsReporter(runCtx, prog, nil) + defer stopReporter() + + progress := &countsLogger{imp: imp, stream: prog, started: time.Now()} + + // The walker runs ahead of the workers: each member costs it one + // small request, so given a leash it never becomes the bottleneck. + rawMembers := make(chan tarMember, walkAheadMembers) + walkErr := make(chan error, 1) + + go func() { + walkErr <- walkTarMembers(runCtx, fetcher, st.Offset, total, rawMembers) + }() + + // Number the parquet members the aggregation actually consumes, + // continuing from the checkpoint so indices stay stable on resume. + work := make(chan indexedMember) + + go func() { + defer close(work) + + idx := st.MemberIdx + + for m := range rawMembers { + if !isProjectableMember(m) { + continue + } + + select { + case work <- indexedMember{idx: idx, m: m}: + idx++ + case <-runCtx.Done(): + return + } + } + }() + + results := make(chan countParseResult, projectMemberWorkers) + + var workerWG sync.WaitGroup + + for range projectMemberWorkers { + workerWG.Add(1) + + go func() { + defer workerWG.Done() + + buf := []byte(nil) + + for job := range work { + if cap(buf) < int(job.m.size) { + buf = make([]byte, job.m.size) + } + + buf = buf[:job.m.size] + + fetched, err := fetchProjectedMember(runCtx, fetcher, job.m, buf) + if err == nil { + prog.addFetched(fetched) + } + + var deltas map[mbidKey]uint32 + + if err == nil { + deltas, err = parseListenParquet(buf) + } + + results <- countParseResult{ + idx: job.idx, + endOffset: job.m.nextHeaderOffset(), + deltas: deltas, + err: err, + } + } + }() + } + + applier := newCountsApplier(imp, st, progress) + applierDone := make(chan struct{}) + + go func() { + defer close(applierDone) + + for res := range results { + applier.apply(res, prog) + } + }() + + workerWG.Wait() + close(results) + <-applierDone + + // Drain the walker so its error (if any) is observed and its + // goroutine cannot outlive this call. + cancel() + + for range rawMembers { //nolint:revive // draining + } + + err := applier.err + if err == nil { + err = walkFailure(ctx, <-walkErr) + } else { + <-walkErr + } + + if err != nil { + // Best-effort checkpoint so even a cancelled run resumes where + // it left off. + _ = imp.writeCountsFile(st) + + return err + } + + st.Done = true + + if err := imp.writeCountsFile(st); err != nil { + return err + } + + imp.logger.Info("dump import: listen counts complete", + "members", st.MemberIdx, + "gb", fmt.Sprintf("%.1f", float64(st.Offset)/(1<<30)), + "downloadedGB", fmt.Sprintf("%.1f", float64(prog.Downloaded())/(1<<30)), + "entities", len(st.counts), + "elapsed", time.Since(progress.started).Truncate(time.Second).String(), + ) + + imp.logJob(fmt.Sprintf( + "Listen counts complete — %s of listens read (%s downloaded), %s entities ranked", + formatGB(st.Offset), formatGB(prog.Downloaded()), formatCount(len(st.counts)), + )) + + return nil +} + +// walkFailure reports a walker error worth surfacing. A walk cancelled +// because the workers finished first is not a failure. +func walkFailure(ctx context.Context, err error) error { + if err == nil || (errors.Is(err, context.Canceled) && ctx.Err() == nil) { + return nil + } + + return err +} + +// isProjectableMember reports whether a tar member is a parquet file the +// aggregator should consume. This must match the streamed path's member +// selection exactly, or the two paths would produce different counts. +func isProjectableMember(m tarMember) bool { + if m.typeflag != tar.TypeReg && m.typeflag != 0 { + return false + } + + return strings.HasSuffix(m.name, ".parquet") && m.size >= minParquetMemberSize +} + +// --------------------------------------------------------------------------- +// Applier +// --------------------------------------------------------------------------- + +// countsApplier merges per-member deltas into the counts map in member +// order, checkpointing every countsFlushEveryMembers members. It owns +// st.counts, st.Offset and st.MemberIdx for the duration of a stage. +type countsApplier struct { + imp *dumpImporter + st *countsState + progress *countsLogger + + pending map[int]countParseResult + next int + lastFlushed int + + err error +} + +func newCountsApplier( + imp *dumpImporter, st *countsState, progress *countsLogger, +) *countsApplier { + return &countsApplier{ + imp: imp, + st: st, + progress: progress, + pending: make(map[int]countParseResult), + next: st.MemberIdx, + lastFlushed: st.MemberIdx, + } +} + +// apply buffers a result and folds in every member that is now +// contiguous with the checkpoint. pos, when non-nil, is advanced to the +// archive offset the checkpoint has reached. +func (a *countsApplier) apply(res countParseResult, pos *projectedProgress) { + if a.err != nil { + return + } + + a.pending[res.idx] = res + + for { + r, ok := a.pending[a.next] + if !ok { + return + } + + delete(a.pending, a.next) + + if r.err != nil { + a.err = r.err + + return + } + + for k, v := range r.deltas { + a.st.counts[k] += v + } + + a.next++ + a.st.MemberIdx = a.next + a.st.Offset = r.endOffset + + if pos != nil { + pos.position.Store(r.endOffset) + } + + if a.next-a.lastFlushed >= countsFlushEveryMembers { + if err := a.imp.writeCountsFile(a.st); err != nil { + a.err = err + + return + } + + a.lastFlushed = a.next + + a.progress.checkpoint(a.next, r.endOffset, len(a.st.counts)) + + if err := a.imp.checkDiskHeadroom(); err != nil { + a.err = err + + return + } + } else { + a.progress.member(a.next, r.endOffset, len(a.st.counts)) + } + } +} + +// --------------------------------------------------------------------------- +// Progress +// --------------------------------------------------------------------------- + +// projectedProgress presents the projected import to the stage-1 +// reporter through the same interface a sequential stream uses. +// +// Position — not bytes downloaded — is what is reported as stream +// progress: with projection those diverge (under half the archive is +// downloaded), and it is position that gives a percentage and an ETA the +// user can act on. Bytes actually downloaded are tracked separately and +// logged at the end. +type projectedProgress struct { + total int64 + + position atomic.Int64 + downloaded atomic.Int64 +} + +func (p *projectedProgress) Read([]byte) (int, error) { return 0, errProjectionUnsupported } +func (p *projectedProgress) Close() error { return nil } +func (p *projectedProgress) Pos() int64 { return p.position.Load() } +func (p *projectedProgress) Fetched() int64 { return p.position.Load() } +func (p *projectedProgress) Total() int64 { return p.total } + +func (p *projectedProgress) addFetched(n int64) { p.downloaded.Add(n) } + +// Downloaded is how many bytes actually crossed the wire. +func (p *projectedProgress) Downloaded() int64 { return p.downloaded.Load() } diff --git a/backend/explore/dumpshared.go b/backend/explore/dumpshared.go new file mode 100644 index 0000000..150e629 --- /dev/null +++ b/backend/explore/dumpshared.go @@ -0,0 +1,257 @@ +package explore + +import ( + "bytes" + "errors" + "fmt" + "io" + "regexp" + "strconv" + "strings" + + "github.com/parquet-go/parquet-go" +) + +// Plumbing shared between the client and the CI-only index builder. +// +// The full dump import (dumpimport.go and friends) is behind the +// `indexbuild` build tag so it is not linked into the app: a user's +// machine never streams the ~89GB listens dump, it merges the prebuilt +// artifact instead. What stays here is what the client genuinely still +// needs — the daily incremental refresh (dumpincremental.go) and the +// artifact download (artifactfetch.go) — plus the small helpers both +// sides share. + +// mbidKey is a parsed UUID plus an entity-kind tag, used as the counts +// map key. 17 bytes instead of a 36-byte string keeps the ~40M-entry +// map around 2GB. +type mbidKey [17]byte + +func makeMBIDKey(kind byte, mbid string) (mbidKey, bool) { + var k mbidKey + + k[0] = kind + + if !parseUUID(mbid, k[1:]) { + return k, false + } + + return k, true +} + +func formatUUID(b []byte) string { + const hexdigits = "0123456789abcdef" + + out := make([]byte, 36) + j := 0 + + for i := range 16 { + if i == 4 || i == 6 || i == 8 || i == 10 { + out[j] = '-' + j++ + } + + out[j] = hexdigits[b[i]>>4] + out[j+1] = hexdigits[b[i]&0x0F] + j += 2 + } + + return string(out) +} + +func formatGB(n int64) string { + return fmt.Sprintf("%.1f GB", float64(n)/(1<<30)) +} + +// formatCount renders a number with thousands separators, matching how +// the frontend shows counts. +func formatCount(n int) string { + s := strconv.Itoa(n) + if len(s) <= 3 { + return s + } + + var b strings.Builder + + lead := len(s) % 3 + if lead > 0 { + b.WriteString(s[:lead]) + } + + for i := lead; i < len(s); i += 3 { + if b.Len() > 0 { + b.WriteByte(',') + } + + b.WriteString(s[i : i+3]) + } + + return b.String() +} + +// parseListenParquet decodes one parquet member and returns the +// per-entity listen-count deltas. +func parseListenParquet(buf []byte) (map[mbidKey]uint32, error) { + reader := parquet.NewGenericReader[sparkListenRow](bytes.NewReader(buf)) + + defer func() { _ = reader.Close() }() + + deltas := make(map[mbidKey]uint32, 1<<18) + rows := make([]sparkListenRow, 4096) + + for { + n, err := reader.Read(rows) + + for _, row := range rows[:n] { + key, ok := makeMBIDKey(countKindRecording, row.RecordingMBID) + if !ok { + // Unmapped listen — no usable recording MBID. + continue + } + + deltas[key]++ + + if relKey, relOK := makeMBIDKey(countKindRelease, row.ReleaseMBID); relOK { + deltas[relKey]++ + } + + for _, artist := range row.ArtistMBIDs { + if artKey, artOK := makeMBIDKey(countKindArtist, artist); artOK { + deltas[artKey]++ + } + } + } + + if errors.Is(err, io.EOF) { + break + } + + if err != nil { + return nil, fmt.Errorf("parquet read: %w", err) + } + + if n == 0 { + break + } + } + + return deltas, nil +} + +// sparkListenRow is the projection of the spark listens parquet schema +// that the aggregator reads. All other columns are skipped. +type sparkListenRow struct { + RecordingMBID string `parquet:"recording_mbid,optional"` + ReleaseMBID string `parquet:"release_mbid,optional"` + ArtistMBIDs []string `parquet:"artist_credit_mbids,optional,list"` +} + +// checkFreeDisk returns ErrDiskSpace when the volume holding path has +// less than minBytes free. Unknown free space (unsupported platform) +// passes. +func checkFreeDisk(path string, minBytes uint64) error { + free, ok := diskFreeBytes(path) + if !ok { + return nil + } + + if free < minBytes { + return fmt.Errorf("%w: %d MB free, need %d MB", + ErrDiskSpace, free>>20, minBytes>>20) + } + + return nil +} + +// dumpSeriesRe extracts the monotonic series number NNNN from a dump +// URL or directory name (e.g. "listenbrainz-spark-dump-2593-…"). +var dumpSeriesRe = regexp.MustCompile(`listenbrainz-(?:spark-)?dump-(\d+)-`) + +// parseDumpSeries pulls the series number out of a dump URL/name. +func parseDumpSeries(url string) (int, bool) { + m := dumpSeriesRe.FindStringSubmatch(url) + if m == nil { + return 0, false + } + + n, err := strconv.Atoi(m[1]) + if err != nil { + return 0, false + } + + return n, true +} + +// Meta keys describing the catalog's provenance. They are read by the +// client (the incremental refresh and the artifact import) and written +// by whichever path populated the index. +const ( + // dumpImportDoneKey marks a populated catalog in explore_index_meta. + dumpImportDoneKey = "dump_import_done" + + // listensAppliedSeriesKey stores the listens dump series the + // popularity numbers are folded up to — the high-water-mark the + // incremental refresh resumes from. + listensAppliedSeriesKey = "listens_applied_series" +) + +// Entity kinds, used as the first byte of an mbidKey so one map can hold +// counts for all three entity types. +const ( + countKindRecording = byte(1) + countKindRelease = byte(2) + countKindArtist = byte(3) +) + +// maxParquetMemberSize caps how large a single parquet member may be +// before it is treated as a malformed dump rather than buffered whole. +const maxParquetMemberSize = 1 << 30 + +// ErrDiskSpace is returned when free disk falls below the safety floor. +var ErrDiskSpace = errors.New("insufficient free disk space") + +// parseUUID parses a canonical 36-char UUID string into 16 bytes. +// Returns false for anything malformed. +func parseUUID(s string, out []byte) bool { + if len(s) != 36 || s[8] != '-' || s[13] != '-' || s[18] != '-' || s[23] != '-' { + return false + } + + j := 0 + + for i := 0; i < 36; i++ { + if i == 8 || i == 13 || i == 18 || i == 23 { + continue + } + + hi := hexNibble(s[i]) + i++ + + lo := hexNibble(s[i]) + if hi == 0xFF || lo == 0xFF { + return false + } + + out[j] = hi<<4 | lo + j++ + } + + return true +} + +func hexNibble(c byte) byte { + switch { + case c >= '0' && c <= '9': + return c - '0' + case c >= 'a' && c <= 'f': + return c - 'a' + 10 + case c >= 'A' && c <= 'F': + return c - 'A' + 10 + default: + return 0xFF + } +} + +// ErrDumpFormat is returned when dump contents don't match the +// expected format. +var ErrDumpFormat = errors.New("unexpected dump format") diff --git a/backend/explore/dumpstream.go b/backend/explore/dumpstream.go index 62ca61d..88357f7 100644 --- a/backend/explore/dumpstream.go +++ b/backend/explore/dumpstream.go @@ -7,8 +7,8 @@ import ( "io" "net/http" "regexp" - "sort" "strconv" + "sync/atomic" "time" ) @@ -25,9 +25,14 @@ const ( maxStreamRetries = 8 // streamRetryBaseDelay is the initial reconnect backoff; it - // doubles per consecutive failure. + // doubles per consecutive failure, up to streamRetryMaxDelay. streamRetryBaseDelay = 2 * time.Second + // streamRetryMaxDelay caps the backoff. Uncapped doubling reaches + // four minutes by the last attempt, which is a long time to leave a + // download lane idle over a transient 503 from a busy dump server. + streamRetryMaxDelay = 20 * time.Second + // dumpDiscoveryTimeout bounds the small directory-listing // requests (not the multi-hour stream requests). dumpDiscoveryTimeout = 30 * time.Second @@ -42,103 +47,6 @@ var ErrDumpStream = errors.New("dump stream failed") var hrefRe = regexp.MustCompile(`href="([^"?/][^"?]*)"`) -// listHrefs fetches an Apache-style index page and returns the href -// values (directory entries end with a trailing slash). -func listHrefs(ctx context.Context, client *http.Client, url string) ([]string, error) { - reqCtx, cancel := context.WithTimeout(ctx, dumpDiscoveryTimeout) - defer cancel() - - req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, url, nil) - if err != nil { - return nil, fmt.Errorf("dump listing request: %w", err) - } - - req.Header.Set("User-Agent", lbUserAgent) - - resp, err := client.Do(req) - if err != nil { - return nil, fmt.Errorf("dump listing fetch: %w", err) - } - - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf( - "%w: listing %s returned HTTP %d", ErrDumpDiscovery, url, resp.StatusCode, - ) - } - - body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20)) - if err != nil { - return nil, fmt.Errorf("dump listing read: %w", err) - } - - var hrefs []string - - for _, m := range hrefRe.FindAllStringSubmatch(string(body), -1) { - hrefs = append(hrefs, m[1]) - } - - return hrefs, nil -} - -// discoverDumpFile walks a dump base directory, finds subdirectories -// matching dirRe (newest first, lexicographically — MetaBrainz dump -// directory names embed sortable timestamps), and returns the full URL -// of the first file inside matching fileRe. Directories that don't -// contain a matching file (e.g. partial uploads) are skipped. -func discoverDumpFile( - ctx context.Context, - client *http.Client, - baseURL string, - dirRe, fileRe *regexp.Regexp, -) (string, error) { - hrefs, err := listHrefs(ctx, client, baseURL) - if err != nil { - return "", err - } - - var dirs []string - - for _, h := range hrefs { - trimmed := trimTrailingSlash(h) - if dirRe.MatchString(trimmed) { - dirs = append(dirs, trimmed) - } - } - - if len(dirs) == 0 { - return "", fmt.Errorf("%w: no dump directories under %s", ErrDumpDiscovery, baseURL) - } - - sort.Sort(sort.Reverse(sort.StringSlice(dirs))) - - for _, dir := range dirs { - dirURL := baseURL + dir + "/" - - files, err := listHrefs(ctx, client, dirURL) - if err != nil { - continue - } - - for _, f := range files { - if fileRe.MatchString(f) { - return dirURL + f, nil - } - } - } - - return "", fmt.Errorf("%w: no matching dump file under %s", ErrDumpDiscovery, baseURL) -} - -func trimTrailingSlash(s string) string { - if len(s) > 0 && s[len(s)-1] == '/' { - return s[:len(s)-1] - } - - return s -} - // resumableReader is an io.Reader over an HTTP resource that survives // connection failures by reconnecting with a Range request at the // current offset. Offset is the absolute position of the next byte to @@ -149,27 +57,31 @@ type resumableReader struct { client *http.Client url string - // Offset is the absolute byte position of the next read. - Offset int64 - - // Size is the total resource size, learned from the first - // response. -1 until known. - Size int64 + // offset is the absolute byte position of the next read, and size + // the total resource size (-1 until the first response reveals it). + // Both are atomic so a progress reporter on another goroutine can + // sample them while the stream is being read. + offset atomic.Int64 + size atomic.Int64 body io.ReadCloser retries int } -func newResumableReader( - ctx context.Context, client *http.Client, url string, offset int64, -) *resumableReader { - return &resumableReader{ - ctx: ctx, - client: client, - url: url, - Offset: offset, - Size: -1, - } +// Pos returns the absolute byte position of the next read. +func (r *resumableReader) Pos() int64 { + return r.offset.Load() +} + +// Fetched matches Pos: a single sequential connection reads no further +// ahead than it delivers. +func (r *resumableReader) Fetched() int64 { + return r.offset.Load() +} + +// Total returns the total resource size, or -1 while unknown. +func (r *resumableReader) Total() int64 { + return r.size.Load() } func (r *resumableReader) Read(p []byte) (int, error) { @@ -185,7 +97,7 @@ func (r *resumableReader) Read(p []byte) (int, error) { } n, err := r.body.Read(p) - r.Offset += int64(n) + offset := r.offset.Add(int64(n)) if n > 0 { r.retries = 0 @@ -197,7 +109,7 @@ func (r *resumableReader) Read(p []byte) (int, error) { case errors.Is(err, io.EOF): // A server that closes early looks like EOF; only // trust it when we've seen the advertised size. - if r.Size >= 0 && r.Offset < r.Size { + if size := r.size.Load(); size >= 0 && offset < size { r.closeBody() if retryErr := r.backoff(err); retryErr != nil { @@ -236,7 +148,7 @@ func (r *resumableReader) backoff(cause error) error { ) } - delay := streamRetryBaseDelay << (r.retries - 1) + delay := min(streamRetryBaseDelay<<(r.retries-1), streamRetryMaxDelay) select { case <-r.ctx.Done(): @@ -254,8 +166,9 @@ func (r *resumableReader) connect() error { req.Header.Set("User-Agent", lbUserAgent) - if r.Offset > 0 { - req.Header.Set("Range", "bytes="+strconv.FormatInt(r.Offset, 10)+"-") + offset := r.offset.Load() + if offset > 0 { + req.Header.Set("Range", "bytes="+strconv.FormatInt(offset, 10)+"-") } resp, err := r.client.Do(req) @@ -265,22 +178,22 @@ func (r *resumableReader) connect() error { switch resp.StatusCode { case http.StatusPartialContent: - if r.Size < 0 { - r.Size = parseContentRangeTotal(resp.Header.Get("Content-Range")) + if r.size.Load() < 0 { + r.size.Store(parseContentRangeTotal(resp.Header.Get("Content-Range"))) } r.body = resp.Body return nil case http.StatusOK: - if r.Size < 0 && resp.ContentLength > 0 { - r.Size = resp.ContentLength + if r.size.Load() < 0 && resp.ContentLength > 0 { + r.size.Store(resp.ContentLength) } // Server ignored the Range header: discard the prefix so // the caller still reads from the requested offset. - if r.Offset > 0 { - if _, err := io.CopyN(io.Discard, resp.Body, r.Offset); err != nil { + if offset > 0 { + if _, err := io.CopyN(io.Discard, resp.Body, offset); err != nil { _ = resp.Body.Close() return r.backoff(err) @@ -327,3 +240,66 @@ func parseContentRangeTotal(v string) int64 { return -1 } + +func newResumableReader( + ctx context.Context, client *http.Client, url string, offset int64, +) *resumableReader { + r := &resumableReader{ + ctx: ctx, + client: client, + url: url, + } + + r.offset.Store(offset) + r.size.Store(-1) + + return r +} + +// listHrefs fetches an Apache-style index page and returns the href +// values (directory entries end with a trailing slash). +func listHrefs(ctx context.Context, client *http.Client, url string) ([]string, error) { + reqCtx, cancel := context.WithTimeout(ctx, dumpDiscoveryTimeout) + defer cancel() + + req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("dump listing request: %w", err) + } + + req.Header.Set("User-Agent", lbUserAgent) + + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("dump listing fetch: %w", err) + } + + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf( + "%w: listing %s returned HTTP %d", ErrDumpDiscovery, url, resp.StatusCode, + ) + } + + body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20)) + if err != nil { + return nil, fmt.Errorf("dump listing read: %w", err) + } + + var hrefs []string + + for _, m := range hrefRe.FindAllStringSubmatch(string(body), -1) { + hrefs = append(hrefs, m[1]) + } + + return hrefs, nil +} + +func trimTrailingSlash(s string) string { + if len(s) > 0 && s[len(s)-1] == '/' { + return s[:len(s)-1] + } + + return s +} diff --git a/backend/explore/explore.go b/backend/explore/explore.go index 4107515..37ff06c 100644 --- a/backend/explore/explore.go +++ b/backend/explore/explore.go @@ -149,6 +149,12 @@ func (e *Service) StopIndexBuild() { e.index.StopBuild() } +// CoreCatalogImported reports whether a prebuilt catalog artifact has +// been merged into this index. +func (e *Service) CoreCatalogImported() bool { + return e.index.artifactAlreadyMerged() +} + // SetJobRegistry wires the background job registry into the search // index so its build reports progress and controls to the frontend. func (e *Service) SetJobRegistry(reg *jobs.Registry) { @@ -565,8 +571,7 @@ func (e *Service) resolveArtistName(artistMBID string, rgs []MBReleaseGroup) str // Check the index for a previously-indexed artist row. if indexed := e.index.LookupArtistByMBID( artistMBID, - ); indexed != nil && indexed.Title != "" && - indexed.Title != artistMBID { + ); indexed != nil && indexed.Title != "" { return indexed.Title } @@ -1048,8 +1053,7 @@ func (e *Service) GetArtistImageURL(artistMBID string) string { // GetArtistImageCached returns a base64 data URL for the artist's // photo ONLY if it's already on disk — no MB/Wikidata resolution -// or Wikimedia fetch. Safe to call from library-only mode. -// Returns "" if not cached. +// or Wikimedia fetch. Returns "" if not cached. func (e *Service) GetArtistImageCached(artistMBID string) string { return e.artistImg.GetCachedImage(artistMBID) } diff --git a/backend/explore/indexjob.go b/backend/explore/indexjob.go index e67f8ae..5fb36c3 100644 --- a/backend/explore/indexjob.go +++ b/backend/explore/indexjob.go @@ -153,6 +153,12 @@ func (si *SearchIndex) applyStagesToJob(h *jobs.Handle, status IndexStatus) { phase = t.Name current = int64(t.Completed) total = int64(t.Total) + + // The listens stream's raw completed/total is a bare + // percentage; the detail line is what makes it legible. + if t.Detail != "" { + phase = t.Name + " — " + t.Detail + } } } diff --git a/backend/explore/indexpatch.go b/backend/explore/indexpatch.go new file mode 100644 index 0000000..a977a14 --- /dev/null +++ b/backend/explore/indexpatch.go @@ -0,0 +1,213 @@ +//go:build indexbuild + +package explore + +import ( + "context" + "fmt" + "net/http" + "regexp" + "sort" + "sync" + + "yellowjacket/backend/jobs" +) + +// Helpers used only while building the catalog from the MetaBrainz +// dumps: the similar-artist patch pass and the per-stage error +// reporting that goes with it. They live behind the `indexbuild` tag +// with the importer they serve, so the app binary carries neither. + +const ( + // indexSimilarPerArtist is how many similar artists to store per + // library artist in similar_artist_map. + indexSimilarPerArtist = 20 + + // similarArtistsBatchSize is the number of seed MBIDs processed in + // one logging "batch". The labs multi-seed POST form is broken, so + // one GET per seed is issued (concurrency bounded by indexerRate); + // batching here just keeps progress log output bounded. + similarArtistsBatchSize = 50 +) + +// setTierError marks a build stage as errored. +func (si *SearchIndex) setTierError(name, errMsg string) { + si.mu.Lock() + + for i := range si.buildStatus.Tiers { + if si.buildStatus.Tiers[i].Name == name { + si.buildStatus.Tiers[i].State = "error" + si.buildStatus.Tiers[i].Error = errMsg + si.mu.Unlock() + + si.logIndexJob(jobs.LevelError, name+": "+errMsg) + si.emitStatus() + + return + } + } + + si.mu.Unlock() +} + +// fetchSimilarArtistsBatch queries the labs similar-artists endpoint +// for multiple seed MBIDs. Despite the name, this actually fans +// out one request per seed: the labs API's multi-seed mode is +// broken (results for different seeds get mis-labeled, and some +// seeds return zero), so batching with multiple artist_mbids is +// not viable. Concurrency is bounded by indexerRate to respect +// the labs rate limit; each call goes through the provided LB +// client's rate limiter and cache. +func (si *SearchIndex) fetchSimilarArtistsBatch( + ctx context.Context, lb *ListenBrainzClient, seedMBIDs []string, +) map[string][]lbSimilarArtistWire { + if len(seedMBIDs) == 0 { + return nil + } + + var ( + mu sync.Mutex + grouped = make(map[string][]lbSimilarArtistWire, len(seedMBIDs)) + wg sync.WaitGroup + ) + + sem := make(chan struct{}, indexerRate) + + for _, seedMBID := range seedMBIDs { + if ctx.Err() != nil { + break + } + + sem <- struct{}{} + + wg.Add(1) + + go func(seed string) { + defer func() { + <-sem + wg.Done() + }() + + // Use the LB client's per-seed GET form — goes through + // the shared rate limiter and cache. The multi-seed + // POST form is not viable (see function comment). + similar, err := lb.SimilarArtists(ctx, seed) + if err != nil || len(similar) == 0 { + return + } + + // Convert to the internal wire type used by the caller + // and trim to indexSimilarPerArtist. + if len(similar) > indexSimilarPerArtist { + similar = similar[:indexSimilarPerArtist] + } + + results := make([]lbSimilarArtistWire, len(similar)) + for i, s := range similar { + results[i] = lbSimilarArtistWire{ + ArtistMBID: s.ArtistMBID, + Name: s.Name, + Score: int(s.Score), + ReferenceMBID: seed, + } + } + + mu.Lock() + grouped[seed] = results + mu.Unlock() + }(seedMBID) + } + + wg.Wait() + + return grouped +} + +// chunkStrings splits a slice into chunks of at most size n. +func chunkStrings(s []string, n int) [][]string { + var chunks [][]string + + for i := 0; i < len(s); i += n { + end := i + n + if end > len(s) { + end = len(s) + } + + chunks = append(chunks, s[i:end]) + } + + return chunks +} + +// getLibraryArtistMBIDs returns MBIDs for all library artists that have one. +// Used when Tier 3 was skipped but Tier 4 needs the library MBID list. +func (si *SearchIndex) getLibraryArtistMBIDs() []string { + rows, err := si.db.QueryContext( + "SELECT DISTINCT mbid FROM artists WHERE mbid IS NOT NULL AND mbid != ''", + ) + if err != nil { + return nil + } + + defer func() { _ = rows.Close() }() + + var mbids []string + + for rows.Next() { + var mbid string + if err := rows.Scan(&mbid); err == nil { + mbids = append(mbids, mbid) + } + } + + return mbids +} + +// discoverDumpFile walks a dump base directory, finds subdirectories +// matching dirRe (newest first, lexicographically — MetaBrainz dump +// directory names embed sortable timestamps), and returns the full URL +// of the first file inside matching fileRe. Directories that don't +// contain a matching file (e.g. partial uploads) are skipped. +func discoverDumpFile( + ctx context.Context, + client *http.Client, + baseURL string, + dirRe, fileRe *regexp.Regexp, +) (string, error) { + hrefs, err := listHrefs(ctx, client, baseURL) + if err != nil { + return "", err + } + + var dirs []string + + for _, h := range hrefs { + trimmed := trimTrailingSlash(h) + if dirRe.MatchString(trimmed) { + dirs = append(dirs, trimmed) + } + } + + if len(dirs) == 0 { + return "", fmt.Errorf("%w: no dump directories under %s", ErrDumpDiscovery, baseURL) + } + + sort.Sort(sort.Reverse(sort.StringSlice(dirs))) + + for _, dir := range dirs { + dirURL := baseURL + dir + "/" + + files, err := listHrefs(ctx, client, dirURL) + if err != nil { + continue + } + + for _, f := range files { + if fileRe.MatchString(f) { + return dirURL + f, nil + } + } + } + + return "", fmt.Errorf("%w: no matching dump file under %s", ErrDumpDiscovery, baseURL) +} diff --git a/backend/explore/searchindex.go b/backend/explore/searchindex.go index f2c7fa4..8d9d4a0 100644 --- a/backend/explore/searchindex.go +++ b/backend/explore/searchindex.go @@ -94,23 +94,12 @@ const ( // across launches instead of running for the better part of an hour. discogBackfillMaxPerRun = 2000 - // indexSimilarPerArtist is how many similar artists to store - // per library artist in similar_artist_map. - indexSimilarPerArtist = 20 - // labsBaseURL is the base URL for the ListenBrainz labs API. labsBaseURL = "https://labs.api.listenbrainz.org" // labsSimilarAlgorithm is the algorithm parameter for the // similar-artists endpoint. labsSimilarAlgorithm = "session_based_days_7500_session_300_contribution_5_threshold_10_limit_100_filter_True_skip_30" - - // similarArtistsBatchSize is the number of seed MBIDs processed - // in one logging "batch" during Tier 4. The labs multi-seed - // POST form is broken, so we actually issue one GET per seed - // (concurrency bounded by indexerRate); batching here just - // keeps progress log output bounded. - similarArtistsBatchSize = 50 ) // SearchIndexResult is a single hit from the local popularity index. @@ -160,16 +149,8 @@ type SearchIndexResult struct { LocalArtistID int64 `json:"localArtistId,omitempty"` LocalReleaseGroupID int64 `json:"localReleaseGroupId,omitempty"` LocalRecordingID int64 `json:"localRecordingId,omitempty"` - - // Schema version for staleness detection. - SchemaVersion int `json:"-"` } -// currentSchemaVersion is bumped when we add new fields that should -// trigger re-indexing of existing rows. The build logic checks each -// artist's rows against this version and re-fetches if stale. -const currentSchemaVersion = 1 - // lbSitewideArtist is the response shape from the LB sitewide // top-artists endpoint. type lbSitewideArtist struct { @@ -239,6 +220,11 @@ type TierStatus struct { Total int `json:"total"` Completed int `json:"completed"` Error string `json:"error,omitempty"` + + // Detail is a human-readable progress line for stages whose raw + // completed/total numbers say little on their own — the listens + // stream reports "42.3 / 205.1 GB · 18 MB/s · ~3h20m left" here. + Detail string `json:"detail,omitempty"` } // IndexStatus is the full index build status, exposed to the frontend. @@ -456,7 +442,7 @@ func (si *SearchIndex) BackfillLibraryDiscographies(ctx context.Context) { // itself. Used to seed the discography fetch's artist entry. func (si *SearchIndex) artistDisplayName(mbid string) string { for _, q := range []string{ - "SELECT title FROM explore_index WHERE entity_type = 'artist' AND mbid = ? AND title != '' AND title != mbid LIMIT 1", + "SELECT title FROM explore_index WHERE entity_type = 'artist' AND mbid = ? AND title != '' LIMIT 1", "SELECT name FROM artists WHERE mbid = ? AND name != '' LIMIT 1", } { rows, err := si.db.QueryContext(q, mbid) @@ -651,6 +637,11 @@ func (si *SearchIndex) refreshStatusCounts() { // setTierStatus updates the build status for a named tier. func (si *SearchIndex) setTierStatus(name, state string, total, completed int) { + si.setTierDetail(name, state, total, completed, "") +} + +// setTierDetail is setTierStatus plus a human-readable progress line. +func (si *SearchIndex) setTierDetail(name, state string, total, completed int, detail string) { si.mu.Lock() for i := range si.buildStatus.Tiers { @@ -659,6 +650,7 @@ func (si *SearchIndex) setTierStatus(name, state string, total, completed int) { si.buildStatus.Tiers[i].State = state si.buildStatus.Tiers[i].Total = total si.buildStatus.Tiers[i].Completed = completed + si.buildStatus.Tiers[i].Detail = detail si.mu.Unlock() if transitioned { @@ -676,32 +668,13 @@ func (si *SearchIndex) setTierStatus(name, state string, total, completed int) { State: state, Total: total, Completed: completed, + Detail: detail, }) si.mu.Unlock() si.emitStatus() } -// setTierError marks a build stage as errored. -func (si *SearchIndex) setTierError(name, errMsg string) { - si.mu.Lock() - - for i := range si.buildStatus.Tiers { - if si.buildStatus.Tiers[i].Name == name { - si.buildStatus.Tiers[i].State = "error" - si.buildStatus.Tiers[i].Error = errMsg - si.mu.Unlock() - - si.logIndexJob(jobs.LevelError, name+": "+errMsg) - si.emitStatus() - - return - } - } - - si.mu.Unlock() -} - // emitStatus pushes the current index status to the frontend via Wails event. func (si *SearchIndex) emitStatus() { if si.runtimeCtx == nil { @@ -1086,25 +1059,36 @@ func (si *SearchIndex) TopReleaseGroupsByArtist(artistMBID string, limit int) [] return results } -// AddFromCache inserts entries from a cached discography browse -// into the search index (Tier 5: organic growth). Called when a -// user views an artist page and the discography is fetched. +// AddFromCache inserts entries from a cached discography browse into +// the search index. Called when a user views an artist page and the +// discography is fetched — organic growth beyond the shipped catalog. func (si *SearchIndex) AddFromCache(artistName, artistMBID string, rgs []MBReleaseGroup) { if len(rgs) == 0 { return } + // resolveArtistName falls back to the MBID when it cannot find a + // name, which is fine for a one-off render but must never be + // persisted: an MBID stored as a title is unsearchable and shows up + // as a UUID in the UI. Writing nothing lets the upsert's + // "non-empty wins" rule keep whatever real name arrives later. + if artistName == artistMBID { + artistName = "" + } + entries := make([]SearchIndexResult, 0, len(rgs)+1) - // Add the artist itself. - entries = append(entries, SearchIndexResult{ - EntityType: "artist", - MBID: artistMBID, - Title: artistName, - ArtistName: artistName, - ArtistMBID: artistMBID, - Popularity: 0, // Unknown from this path. - }) + // Add the artist itself, unless there is no name to add. + if artistName != "" { + entries = append(entries, SearchIndexResult{ + EntityType: "artist", + MBID: artistMBID, + Title: artistName, + ArtistName: artistName, + ArtistMBID: artistMBID, + Popularity: 0, // Unknown from this path. + }) + } for _, rg := range rgs { entries = append(entries, SearchIndexResult{ @@ -1862,79 +1846,6 @@ type lbSimilarArtistWire struct { ReferenceMBID string `json:"reference_mbid"` // which seed artist this result belongs to } -// fetchSimilarArtistsBatch queries the labs similar-artists endpoint -// for multiple seed MBIDs. Despite the name, this actually fans -// out one request per seed: the labs API's multi-seed mode is -// broken (results for different seeds get mis-labeled, and some -// seeds return zero), so batching with multiple artist_mbids is -// not viable. Concurrency is bounded by indexerRate to respect -// the labs rate limit; each call goes through the provided LB -// client's rate limiter and cache. -func (si *SearchIndex) fetchSimilarArtistsBatch( - ctx context.Context, lb *ListenBrainzClient, seedMBIDs []string, -) map[string][]lbSimilarArtistWire { - if len(seedMBIDs) == 0 { - return nil - } - - var ( - mu sync.Mutex - grouped = make(map[string][]lbSimilarArtistWire, len(seedMBIDs)) - wg sync.WaitGroup - ) - - sem := make(chan struct{}, indexerRate) - - for _, seedMBID := range seedMBIDs { - if ctx.Err() != nil { - break - } - - sem <- struct{}{} - - wg.Add(1) - - go func(seed string) { - defer func() { - <-sem - wg.Done() - }() - - // Use the LB client's per-seed GET form — goes through - // the shared rate limiter and cache. The multi-seed - // POST form is not viable (see function comment). - similar, err := lb.SimilarArtists(ctx, seed) - if err != nil || len(similar) == 0 { - return - } - - // Convert to the internal wire type used by the caller - // and trim to indexSimilarPerArtist. - if len(similar) > indexSimilarPerArtist { - similar = similar[:indexSimilarPerArtist] - } - - results := make([]lbSimilarArtistWire, len(similar)) - for i, s := range similar { - results[i] = lbSimilarArtistWire{ - ArtistMBID: s.ArtistMBID, - Name: s.Name, - Score: int(s.Score), - ReferenceMBID: seed, - } - } - - mu.Lock() - grouped[seed] = results - mu.Unlock() - }(seedMBID) - } - - wg.Wait() - - return grouped -} - // --------------------------------------------------------------------------- // Shared: index artist discographies // --------------------------------------------------------------------------- @@ -2155,22 +2066,6 @@ func (si *SearchIndex) fetchTopRecordings( return results } -// chunkStrings splits a slice into chunks of at most size n. -func chunkStrings(s []string, n int) [][]string { - var chunks [][]string - - for i := 0; i < len(s); i += n { - end := i + n - if end > len(s) { - end = len(s) - } - - chunks = append(chunks, s[i:end]) - } - - return chunks -} - // --------------------------------------------------------------------------- // Database writes // --------------------------------------------------------------------------- @@ -2186,6 +2081,69 @@ func chunkStrings(s []string, n int) [][]string { // empty values, and numeric fields use "highest wins" for popularity/ // listener_count/duration so older richer data survives refreshes. +// upsertIndexSQL is the single index write statement. It is kept as +// a const so assembly can prepare it once per transaction instead of +// re-parsing this large upsert for every row. +const upsertIndexSQL = ` + INSERT INTO explore_index ( + entity_type, mbid, title, artist_name, artist_mbid, aliases, + popularity, listener_count, + duration, caa_release_mbid, release_name, + primary_type, secondary_types, release_date, + artist_type, country, disambiguation, sort_name, + in_library, is_similar, + local_artist_id, local_release_group_id, local_recording_id, + discog_fetched + ) VALUES ( + ?, ?, ?, ?, ?, ?, + ?, ?, + ?, ?, ?, + ?, ?, ?, + ?, ?, ?, ?, + ?, ?, + NULLIF(?, 0), NULLIF(?, 0), NULLIF(?, 0), + ? + )` + upsertIndexConflictSQL + +// upsertIndexConflictSQL is the merge half of every index write, split +// out so the bulk artifact import (which inserts by SELECT rather than +// by parameter list) resolves conflicts identically instead of carrying +// a second, drifting copy of these rules. +const upsertIndexConflictSQL = ` + ON CONFLICT(mbid) DO UPDATE SET + -- Title and artist info: never clobber a good value with an + -- empty one. Writers are responsible for not offering an MBID + -- as a name; AddFromCache is the path that used to. + title = CASE WHEN excluded.title != '' THEN excluded.title ELSE title END, + artist_name = CASE WHEN excluded.artist_name != '' THEN excluded.artist_name ELSE artist_name END, + artist_mbid = CASE WHEN excluded.artist_mbid != '' THEN excluded.artist_mbid ELSE artist_mbid END, + aliases = CASE WHEN excluded.aliases != '' THEN excluded.aliases ELSE aliases END, + + -- Highest wins for popularity + listener_count (refreshes can go up). + popularity = CASE WHEN excluded.popularity > popularity THEN excluded.popularity ELSE popularity END, + listener_count = CASE WHEN excluded.listener_count > listener_count THEN excluded.listener_count ELSE listener_count END, + + -- Non-empty wins for all other optional fields (never clobber with empty). + duration = CASE WHEN excluded.duration > 0 THEN excluded.duration ELSE duration END, + caa_release_mbid = CASE WHEN excluded.caa_release_mbid != '' THEN excluded.caa_release_mbid ELSE caa_release_mbid END, + release_name = CASE WHEN excluded.release_name != '' THEN excluded.release_name ELSE release_name END, + primary_type = CASE WHEN excluded.primary_type != '' THEN excluded.primary_type ELSE primary_type END, + secondary_types = CASE WHEN excluded.secondary_types != '' THEN excluded.secondary_types ELSE secondary_types END, + release_date = CASE WHEN excluded.release_date != '' THEN excluded.release_date ELSE release_date END, + artist_type = CASE WHEN excluded.artist_type != '' THEN excluded.artist_type ELSE artist_type END, + country = CASE WHEN excluded.country != '' THEN excluded.country ELSE country END, + disambiguation = CASE WHEN excluded.disambiguation != '' THEN excluded.disambiguation ELSE disambiguation END, + sort_name = CASE WHEN excluded.sort_name != '' THEN excluded.sort_name ELSE sort_name END, + + -- Flags and cross-references: non-null wins. + in_library = MAX(in_library, excluded.in_library), + is_similar = MAX(is_similar, excluded.is_similar), + discog_fetched = MAX(discog_fetched, excluded.discog_fetched), + local_artist_id = COALESCE(excluded.local_artist_id, local_artist_id), + local_release_group_id = COALESCE(excluded.local_release_group_id, local_release_group_id), + local_recording_id = COALESCE(excluded.local_recording_id, local_recording_id) +` + // upsertBatch writes a batch of SearchIndexResult entries to the index // inside a single transaction. This is the ONE function that all // writes go through. All fields are handled — callers don't need to @@ -2202,6 +2160,17 @@ func (si *SearchIndex) upsertBatch(entries []SearchIndexResult) { return } + stmt, err := tx.Prepare(upsertIndexSQL) + if err != nil { + si.logger.Warn("search index: prepare upsert error", "error", err) + + _ = tx.Rollback() + + return + } + + defer func() { _ = stmt.Close() }() + for _, e := range entries { if e.MBID == "" { continue // skip entries without MBIDs — can't be looked up @@ -2222,69 +2191,7 @@ func (si *SearchIndex) upsertBatch(entries []SearchIndexResult) { discogFetched = 1 } - if _, err := tx.Exec(` - INSERT INTO explore_index ( - entity_type, mbid, title, artist_name, artist_mbid, aliases, - popularity, listener_count, - duration, caa_release_mbid, release_name, - primary_type, secondary_types, release_date, - artist_type, country, disambiguation, sort_name, - in_library, is_similar, - local_artist_id, local_release_group_id, local_recording_id, - discog_fetched, - schema_version - ) VALUES ( - ?, ?, ?, ?, ?, ?, - ?, ?, - ?, ?, ?, - ?, ?, ?, - ?, ?, ?, ?, - ?, ?, - NULLIF(?, 0), NULLIF(?, 0), NULLIF(?, 0), - ?, - ? - ) - ON CONFLICT(mbid) DO UPDATE SET - -- Title and artist info: don't clobber a good value with - -- an empty string or with the MBID itself (which can sneak - -- in via fallback paths in AddFromCache). - title = CASE - WHEN excluded.title != '' AND excluded.title != excluded.mbid THEN excluded.title - ELSE title - END, - artist_name = CASE - WHEN excluded.artist_name != '' AND excluded.artist_name != excluded.artist_mbid THEN excluded.artist_name - ELSE artist_name - END, - artist_mbid = CASE WHEN excluded.artist_mbid != '' THEN excluded.artist_mbid ELSE artist_mbid END, - aliases = CASE WHEN excluded.aliases != '' THEN excluded.aliases ELSE aliases END, - - -- Highest wins for popularity + listener_count (refreshes can go up). - popularity = CASE WHEN excluded.popularity > popularity THEN excluded.popularity ELSE popularity END, - listener_count = CASE WHEN excluded.listener_count > listener_count THEN excluded.listener_count ELSE listener_count END, - - -- Non-empty wins for all other optional fields (never clobber with empty). - duration = CASE WHEN excluded.duration > 0 THEN excluded.duration ELSE duration END, - caa_release_mbid = CASE WHEN excluded.caa_release_mbid != '' THEN excluded.caa_release_mbid ELSE caa_release_mbid END, - release_name = CASE WHEN excluded.release_name != '' THEN excluded.release_name ELSE release_name END, - primary_type = CASE WHEN excluded.primary_type != '' THEN excluded.primary_type ELSE primary_type END, - secondary_types = CASE WHEN excluded.secondary_types != '' THEN excluded.secondary_types ELSE secondary_types END, - release_date = CASE WHEN excluded.release_date != '' THEN excluded.release_date ELSE release_date END, - artist_type = CASE WHEN excluded.artist_type != '' THEN excluded.artist_type ELSE artist_type END, - country = CASE WHEN excluded.country != '' THEN excluded.country ELSE country END, - disambiguation = CASE WHEN excluded.disambiguation != '' THEN excluded.disambiguation ELSE disambiguation END, - sort_name = CASE WHEN excluded.sort_name != '' THEN excluded.sort_name ELSE sort_name END, - - -- Flags and cross-references: non-null wins. - in_library = MAX(in_library, excluded.in_library), - is_similar = MAX(is_similar, excluded.is_similar), - discog_fetched = MAX(discog_fetched, excluded.discog_fetched), - local_artist_id = COALESCE(excluded.local_artist_id, local_artist_id), - local_release_group_id = COALESCE(excluded.local_release_group_id, local_release_group_id), - local_recording_id = COALESCE(excluded.local_recording_id, local_recording_id), - - schema_version = MAX(schema_version, excluded.schema_version) - `, + if _, err := stmt.Exec( e.EntityType, e.MBID, e.Title, e.ArtistName, e.ArtistMBID, e.Aliases, e.Popularity, e.ListenerCount, e.Duration, e.CAAReleaseMBID, e.ReleaseName, @@ -2293,7 +2200,6 @@ func (si *SearchIndex) upsertBatch(entries []SearchIndexResult) { inLib, isSim, e.LocalArtistID, e.LocalReleaseGroupID, e.LocalRecordingID, discogFetched, - currentSchemaVersion, ); err != nil { si.logger.Warn("search index: upsert error", "mbid", e.MBID, @@ -2311,30 +2217,6 @@ func (si *SearchIndex) upsertBatch(entries []SearchIndexResult) { // Helpers // --------------------------------------------------------------------------- -// getLibraryArtistMBIDs returns MBIDs for all library artists that have one. -// Used when Tier 3 was skipped but Tier 4 needs the library MBID list. -func (si *SearchIndex) getLibraryArtistMBIDs() []string { - rows, err := si.db.QueryContext( - "SELECT DISTINCT mbid FROM artists WHERE mbid IS NOT NULL AND mbid != ''", - ) - if err != nil { - return nil - } - - defer func() { _ = rows.Close() }() - - var mbids []string - - for rows.Next() { - var mbid string - if err := rows.Scan(&mbid); err == nil { - mbids = append(mbids, mbid) - } - } - - return mbids -} - // hasMeta reports whether a key exists in explore_index_meta. func (si *SearchIndex) hasMeta(key string) bool { rows, err := si.db.QueryContext( diff --git a/backend/explore/testfixtures_test.go b/backend/explore/testfixtures_test.go new file mode 100644 index 0000000..2fccc8e --- /dev/null +++ b/backend/explore/testfixtures_test.go @@ -0,0 +1,108 @@ +package explore + +import ( + "archive/tar" + "bytes" + "log/slog" + "testing" + + "github.com/parquet-go/parquet-go" +) + +// Fixtures shared by the client tests and the tagged index-builder +// tests. They live in an untagged file so both builds compile. + +// Fixed MBIDs for fixtures. +const ( + recA = "11111111-1111-1111-1111-111111111111" + recB = "22222222-2222-2222-2222-222222222222" + recC = "33333333-3333-3333-3333-333333333333" + relA = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa" + relB = "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb" + rgA = "cccccccc-cccc-cccc-cccc-cccccccccccc" + rgB = "dddddddd-dddd-dddd-dddd-dddddddddddd" + artA = "eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee" + artB = "ffffffff-ffff-ffff-ffff-ffffffffffff" +) + +func testLogger() *slog.Logger { + return slog.New(slog.DiscardHandler) +} + +func makeTar(t *testing.T, members map[string][]byte, order []string) []byte { + t.Helper() + + var buf bytes.Buffer + + tw := tar.NewWriter(&buf) + + for _, name := range order { + data := members[name] + hdr := &tar.Header{ + Name: name, + Mode: 0o644, + Size: int64(len(data)), + Typeflag: tar.TypeReg, + } + + if err := tw.WriteHeader(hdr); err != nil { + t.Fatalf("tar header: %v", err) + } + + if _, err := tw.Write(data); err != nil { + t.Fatalf("tar write: %v", err) + } + } + + if err := tw.Close(); err != nil { + t.Fatalf("tar close: %v", err) + } + + return buf.Bytes() +} + +func makeParquet(t *testing.T, rows []sparkFixtureRow) []byte { + t.Helper() + + var buf bytes.Buffer + + w := parquet.NewGenericWriter[sparkFixtureRow](&buf) + + if _, err := w.Write(rows); err != nil { + t.Fatalf("parquet write: %v", err) + } + + if err := w.Close(); err != nil { + t.Fatalf("parquet close: %v", err) + } + + return buf.Bytes() +} + +// sparkFixtureRow mimics the real spark listens schema: the aggregator +// must project just recording/release/artist MBIDs out of it. +type sparkFixtureRow struct { + ListenedAt int64 `parquet:"listened_at"` + UserID int64 `parquet:"user_id"` + ArtistName string `parquet:"artist_name,optional"` + RecordingMBID string `parquet:"recording_mbid,optional"` + ReleaseMBID string `parquet:"release_mbid,optional"` + ArtistMBIDs []string `parquet:"artist_credit_mbids,optional,list"` +} + +// listensOf builds n identical listen rows for a recording. +func listensOf(n int, recording, release string, artists []string) []sparkFixtureRow { + rows := make([]sparkFixtureRow, n) + for i := range rows { + rows[i] = sparkFixtureRow{ + ListenedAt: 1700000000 + int64(i), + UserID: int64(i), + ArtistName: "Fixture Artist", + RecordingMBID: recording, + ReleaseMBID: release, + ArtistMBIDs: artists, + } + } + + return rows +} diff --git a/backend/jobs/jobs.go b/backend/jobs/jobs.go index 88d58f6..e274aaa 100644 --- a/backend/jobs/jobs.go +++ b/backend/jobs/jobs.go @@ -26,6 +26,7 @@ type Kind string const ( KindLibraryScan Kind = "library-scan" KindIndexBuild Kind = "index-build" + KindDownload Kind = "download" ) // State is the lifecycle position of a job. diff --git a/backend/library/coverart.go b/backend/library/coverart.go index 0bb4aea..cc8e6e4 100644 --- a/backend/library/coverart.go +++ b/backend/library/coverart.go @@ -48,6 +48,32 @@ var thumbnailTiers = []thumbnailTier{ // multi-tier system. Kept for migration purposes only. const legacyThumbSuffix = "_thumb" +// CoverArtFileSet returns every file on disk belonging to one cover art +// entry: the original plus each generated size variant, plus the legacy +// _thumb file for databases that predate the multi-tier thumbnails. +// +// Only the original is recorded in cover_art.file_path — the variants +// are derived filenames — so any code deleting cover art has to expand +// the set or the thumbnails are orphaned. +func CoverArtFileSet(originalPath string) []string { + dir := filepath.Dir(originalPath) + base := filepath.Base(originalPath) + + paths := make([]string, 0, len(thumbnailTiers)+2) //nolint:mnd + + paths = append(paths, originalPath) + + for _, tier := range thumbnailTiers { + paths = append(paths, filepath.Join( + dir, coverart.SizedFilename(base, tier.Suffix), + )) + } + + return append(paths, filepath.Join( + dir, coverart.SizedFilename(base, legacyThumbSuffix), + )) +} + // isSizedVariant reports whether a filename contains any known size suffix // (current tiers or legacy). func isSizedVariant(name string) bool { diff --git a/backend/library/crud.go b/backend/library/crud.go index 794d010..0540ebb 100644 --- a/backend/library/crud.go +++ b/backend/library/crud.go @@ -9,8 +9,6 @@ import ( "strings" "time" - "github.com/wailsapp/wails/v2/pkg/runtime" - "yellowjacket/backend/database" "yellowjacket/backend/database/sql/sqlcgen" "yellowjacket/backend/events" @@ -104,7 +102,7 @@ func (l *Library) AddLibrary(path string) (*sqlcgen.Library, error) { "libraryID", lib.ID, "claimed", claimed) } - runtime.EventsEmit(l.ctx, events.LibraryAdded, lib) + l.emit(events.LibraryAdded, lib) go func() { if scanErr := l.ScanLibrary(lib.ID); scanErr != nil { @@ -149,7 +147,7 @@ func (l *Library) RenameLibrary(id int64, newName string) error { return fmt.Errorf("could not rename library: %w", err) } - runtime.EventsEmit(l.ctx, events.LibraryRenamed, map[string]any{ + l.emit(events.LibraryRenamed, map[string]any{ "id": id, "name": newName, }) @@ -427,7 +425,18 @@ func (l *Library) RemoveLibrary(id int64) (*RemovalSummary, error) { return nil, fmt.Errorf("could not delete orphaned cover_art: %w", err) } - // 17. Delete library row. + // 17. Delete the library's tagging queue. tagging_items holds a + // FOREIGN KEY to libraries with no ON DELETE clause, so leaving these + // rows behind makes the DELETE below fail the whole transaction and + // the library becomes unremovable. tagging_candidates is tied to + // tagging_items by ON DELETE CASCADE and goes with it. + // SAFETY: Hand-crafted DELETE — sqlc has no query for this. Parameterized. + if _, err := tx.ExecContext(l.ctx, + `DELETE FROM tagging_items WHERE library_id = ?`, id); err != nil { + return nil, fmt.Errorf("could not delete tagging items: %w", err) + } + + // 18. Delete library row. // SAFETY: Hand-crafted DELETE matching sqlc DeleteLibrary but within // the same transaction. Parameterized. if _, err := tx.ExecContext(l.ctx, @@ -435,36 +444,41 @@ func (l *Library) RemoveLibrary(id int64) (*RemovalSummary, error) { return nil, fmt.Errorf("could not delete library: %w", err) } - // 18. Commit transaction. + // 19. Commit transaction. if err := tx.Commit(); err != nil { return nil, fmt.Errorf("could not commit removal transaction: %w", err) } committed = true - // 19. FTS5 rebuild skipped — contentless FTS5 (content='') cannot + // 20. FTS5 rebuild skipped — contentless FTS5 (content='') cannot // delete individual rows, but stale entries are harmless: search // queries JOIN against track_metadata which filters out deleted // rows. The index is rebuilt on the next full rescan. Skipping // avoids a costly full re-index of all remaining tracks (~10s for // 25K tracks). - // 20. Post-commit: Delete orphaned cover art files. + // 21. Post-commit: Delete orphaned cover art files and their sized + // variants. Only the original is stored in cover_art.file_path; the + // _sm/_md/_lg thumbnails are derived filenames beside it, so they + // have to be removed by name or they accumulate forever. for _, coverPath := range orphanedCoverArtPaths { - if err := os.Remove(coverPath); err != nil && !os.IsNotExist(err) { - l.logger.Warn("could not remove orphaned cover art file", - "path", coverPath, - "err", err, - ) + for _, path := range CoverArtFileSet(coverPath) { + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + l.logger.Warn("could not remove orphaned cover art file", + "path", path, + "err", err, + ) + } } } - // 21. Post-commit: Compact queue. + // 22. Post-commit: Compact queue. if l.removalHooks.CompactQueue != nil { l.removalHooks.CompactQueue() } - // 22. Post-commit: invalidate library-sync markers so the gated + // 23. Post-commit: invalidate library-sync markers so the gated // index/lyric re-sync runs on the next launch. if l.removalHooks.PostRemove != nil { l.removalHooks.PostRemove() @@ -480,7 +494,7 @@ func (l *Library) RemoveLibrary(id int64) (*RemovalSummary, error) { } // 22. Emit events. - runtime.EventsEmit(l.ctx, events.LibraryRemoved, map[string]any{ + l.emit(events.LibraryRemoved, map[string]any{ "id": id, "summary": summary, }) diff --git a/backend/library/leak_test.go b/backend/library/leak_test.go new file mode 100644 index 0000000..7cc6aa8 --- /dev/null +++ b/backend/library/leak_test.go @@ -0,0 +1,129 @@ +package library + +import ( + "testing" + + "yellowjacket/backend/datamap" +) + +// staleTolerated lists tables that deliberately keep rows after the data +// they describe is gone. Each needs a reason: the point of this list is +// that tolerating a leak becomes a decision somebody wrote down, not an +// oversight nobody noticed. +var staleTolerated = map[string]string{ + "file_types": "static lookup rows seeded from code, not user data", + "search_index": "contentless FTS5 cannot delete individual rows; " + + "stale entries are filtered by joining track_metadata and are " + + "cleared by a full rescan", + "lyrics_index": "contentless FTS5, same constraint as search_index", +} + +// Removing the only library must leave no owned or derived rows behind. +// +// The table list comes from the datamap catalog rather than being +// hardcoded, so a newly added table is covered by this test the moment it +// is catalogued — which is the mechanism that would have caught +// tagging_items blocking removal, and the cover art variants leaking. +func TestRemoveLibraryLeavesNoOwnedOrDerivedRows(t *testing.T) { + t.Parallel() + + lib, _ := setupTestLibrary(t) + + library := seedRemovableLibrary(t, lib, "/nonexistent/cover.jpg") + + if _, err := lib.RemoveLibrary(library.ID); err != nil { + t.Fatalf("RemoveLibrary: %v", err) + } + + for _, entry := range datamap.Tables() { + if entry.Kind != datamap.Owned && entry.Kind != datamap.Derived { + continue + } + + // FTS5 virtual tables do not answer COUNT(*) meaningfully. + if entry.FTS { + continue + } + + if reason, exempt := staleTolerated[entry.Name]; exempt { + t.Logf("skipping %s: %s", entry.Name, reason) + + continue + } + + if n := countRows(t, lib, entry.Name); n != 0 { + t.Errorf( + "%s (%s) has %d rows after the only library was removed. "+ + "Either delete them in RemoveLibrary, or add an entry "+ + "to staleTolerated explaining why they stay.", + entry.Name, entry.Kind, n, + ) + } + } +} + +// Authored data must survive removal of the library it was created +// against — losing it is unrecoverable, so it must never be a casualty +// of cleaning up owned data. +func TestRemoveLibraryPreservesAuthoredData(t *testing.T) { + t.Parallel() + + lib, _ := setupTestLibrary(t) + + library := seedRemovableLibrary(t, lib, "/nonexistent/cover.jpg") + + if _, err := lib.db.ExecContext( + `INSERT INTO playlists (name) VALUES ('Keep Me')`, + ); err != nil { + t.Fatalf("seed playlist: %v", err) + } + + if _, err := lib.RemoveLibrary(library.ID); err != nil { + t.Fatalf("RemoveLibrary: %v", err) + } + + if n := countRows(t, lib, "playlists"); n != 1 { + t.Errorf("playlists = %d rows after removal, want 1 preserved", n) + } +} + +// Every table the catalog marks as needing an explicit sweep must +// actually reach zero, or be listed as tolerated. This is a narrower +// restatement of the leak test aimed at the Lifetime axis rather than +// the Kind axis, so a table declared "swept" that nothing sweeps is +// caught. +func TestSweptTablesAreActuallySwept(t *testing.T) { + t.Parallel() + + lib, _ := setupTestLibrary(t) + + library := seedRemovableLibrary(t, lib, "/nonexistent/cover.jpg") + + if _, err := lib.RemoveLibrary(library.ID); err != nil { + t.Fatalf("RemoveLibrary: %v", err) + } + + for _, entry := range datamap.Tables() { + if entry.Lifetime != datamap.Swept || entry.FTS { + continue + } + + // Cache tables are swept by the janitor on their own schedule, + // not by library removal. + if entry.Kind == datamap.Cache { + continue + } + + if _, exempt := staleTolerated[entry.Name]; exempt { + continue + } + + if n := countRows(t, lib, entry.Name); n != 0 { + t.Errorf( + "%s declares Lifetime swept but still has %d rows after "+ + "removal — nothing is sweeping it", + entry.Name, n, + ) + } + } +} diff --git a/backend/library/library.go b/backend/library/library.go index fe560b8..a0de79f 100644 --- a/backend/library/library.go +++ b/backend/library/library.go @@ -193,6 +193,31 @@ func (l *Library) SetContext(ctx context.Context) { l.registerEventHandlers() } +// emit publishes a Wails event, tolerating a context that carries no +// Wails runtime. +// +// runtime.EventsEmit calls log.Fatalf when the context is nil or lacks +// the runtime's "events" value, which terminates the process rather +// than returning an error. Background workers that outlive a context +// and tests that construct a Library directly both hit that path, so +// every emit in this package routes through here. +func (l *Library) emit(event string, data ...any) { + l.mu.Lock() + ctx := l.ctx + l.mu.Unlock() + + if ctx == nil || ctx.Value("events") == nil { + l.logger.Debug( + "skipping event emit, no Wails runtime in context", + "event", event, + ) + + return + } + + runtime.EventsEmit(ctx, event, data...) +} + // registerEventHandlers sets up Wails runtime event listeners. // The legacy LibraryConfigChanged handler was removed — in the // multi-library model, libraries are managed through the CRUD @@ -309,11 +334,11 @@ func (l *Library) scanInternal( // legacy LibraryScanProgress event and the shared job registry. // Routing everything through here keeps the two from drifting. emitProgress := func(p ScanProgress) { - runtime.EventsEmit(l.ctx, events.LibraryScanProgress, p) + l.emit(events.LibraryScanProgress, p) reportScanProgress(jobHandle, p) } - runtime.EventsEmit(l.ctx, events.LibraryScanStarted, map[string]any{ + l.emit(events.LibraryScanStarted, map[string]any{ "libraryId": libraryID, "libraryName": libraryName, }) @@ -369,6 +394,12 @@ func (l *Library) scanInternal( var errMu sync.Mutex + // statBackfill collects staleness baselines for skipped files whose + // rows predate migration 47. Appended to only by the walk goroutine + // and read after workChan closes, which orders the writes before the + // flush. + var statBackfill []sqlcgen.UpdateAudioFileStatParams + // --- Phase 2: directory walk --- walkStart := time.Now() @@ -406,14 +437,38 @@ func (l *Library) scanInternal( return nil } + // Stat the entry for the staleness comparison below. + // This happens before the file is read, so a file + // modified mid-scan records the pre-read mtime and is + // picked up again next scan — the safe direction. + var ( + diskModTime int64 + diskSize int64 + ) + + if info, infoErr := d.Info(); infoErr == nil { + diskModTime = info.ModTime().Unix() + diskSize = info.Size() + } else { + l.logger.Debug( + "could not stat file, treating as unchanged", + "path", absoluteFilePath, "err", infoErr, + ) + } + // Check if file already exists in database. if existing, exists := existingPaths.LoadAndDelete(absoluteFilePath); exists { audioFile := existing.(sqlcgen.AudioFile) - if audioFile.RecordingID == 0 { + contentChanged := fileContentChanged( + audioFile, diskModTime, diskSize, + ) + + if audioFile.RecordingID == 0 || contentChanged { l.logger.Debug( "file needs metadata update", "path", absoluteFilePath, + "contentChanged", contentChanged, ) select { @@ -423,6 +478,8 @@ func (l *Library) scanInternal( existingFileID: audioFile.ID, needsUpdate: true, existingLength: audioFile.LengthMilliseconds, + contentChanged: contentChanged, + modTime: diskModTime, }: case <-scanCtx.Done(): return scanCtx.Err() @@ -438,6 +495,21 @@ func (l *Library) scanInternal( ) skipped.Add(1) + // Record the baseline for a row that lacks one so the + // next scan can detect edits. Collected here and + // flushed in one transaction after the walk rather + // than issuing an UPDATE per file. + if audioFile.ModifiedAt == 0 && diskModTime != 0 { + statBackfill = append( + statBackfill, + sqlcgen.UpdateAudioFileStatParams{ + ModifiedAt: diskModTime, + FileSize: diskSize, + ID: audioFile.ID, + }, + ) + } + return nil } @@ -450,6 +522,7 @@ func (l *Library) scanInternal( case workChan <- scanWork{ absolutePath: absoluteFilePath, fileType: fileType, + modTime: diskModTime, }: case <-scanCtx.Done(): return scanCtx.Err() @@ -658,6 +731,11 @@ func (l *Library) scanInternal( metrics.ThumbnailWallClock = time.Since(thumbStart) + // Establish staleness baselines for unchanged files that lacked one. + // Safe to run even on a cancelled scan: every entry was individually + // confirmed against the file on disk during the walk. + l.flushStatBackfill(statBackfill) + // Skip orphan cleanup if the scan was cancelled — existingPaths // still contains unvisited files that would be incorrectly deleted. cancelled := scanCtx.Err() != nil @@ -776,13 +854,9 @@ func (l *Library) scanInternal( finishScanJob(jobHandle, metrics, cancelled) if cancelled { - runtime.EventsEmit( - l.ctx, events.LibraryScanCancelled, metrics, - ) + l.emit(events.LibraryScanCancelled, metrics) } else { - runtime.EventsEmit( - l.ctx, events.LibraryScanComplete, metrics, - ) + l.emit(events.LibraryScanComplete, metrics) } return metrics @@ -792,6 +866,84 @@ func (l *Library) scanInternal( // emitted to the frontend. const progressInterval = 300 * time.Millisecond +// fileContentChanged reports whether a file on disk differs from what +// was imported, by comparing mtime and size against the recorded +// baseline. This is what catches another application retagging a file +// in place — without it the scan skips every path already in the +// database and the edit stays invisible until a full rescan. +// +// Two cases are deliberately treated as unchanged: +// +// - A recorded mtime of 0 means no baseline exists (the row predates +// migration 47). There is nothing to compare against, so reporting +// a change would re-import the entire library on first upgrade. +// - A disk mtime of 0 means the stat failed. Skipping is preferable +// to re-importing a file on no evidence. +// +// A writer that preserves mtime and lands on an identical file size +// defeats this check. That needs content hashing to catch, which costs +// a full read of every file — deliberately out of scope. +func fileContentChanged( + audioFile sqlcgen.AudioFile, + diskModTime, diskSize int64, +) bool { + if audioFile.ModifiedAt == 0 || diskModTime == 0 { + return false + } + + return diskModTime != audioFile.ModifiedAt || + diskSize != audioFile.FileSize +} + +// flushStatBackfill writes mtime/size baselines for files the scan +// skipped but that had no baseline recorded. Failures are logged and +// not fatal — a missing baseline only means the file is re-checked on +// the next scan. +func (l *Library) flushStatBackfill( + entries []sqlcgen.UpdateAudioFileStatParams, +) { + if len(entries) == 0 { + return + } + + tx, err := l.db.BeginTx() + if err != nil { + l.logger.Warn( + "could not begin stat backfill transaction", + "count", len(entries), "err", err, + ) + + return + } + + defer func() { _ = tx.Rollback() }() // no-op after commit + + txq := l.db.Queries.WithTx(tx) + + for _, e := range entries { + if updErr := txq.UpdateAudioFileStat(l.ctx, e); updErr != nil { + l.logger.Warn( + "could not backfill file stat", + "audioFileID", e.ID, "err", updErr, + ) + } + } + + if err := tx.Commit(); err != nil { + l.logger.Warn( + "could not commit stat backfill", + "count", len(entries), "err", err, + ) + + return + } + + l.logger.Info( + "recorded staleness baselines for existing files", + "count", len(entries), + ) +} + // countAudioFiles performs a fast walk of the library directory, // counting only files with supported audio extensions. No per-file // I/O is performed — this reads only directory entries. @@ -817,6 +969,46 @@ func countAudioFiles(basePath string) int64 { return count } +// surveyAudioFiles walks the library directory and returns both the +// number of supported audio files and the newest mtime among them +// (Unix seconds). The soft scan compares both against the database: +// the count catches added and removed files, the mtime catches files +// another application edited in place. +// +// Unlike countAudioFiles this stats every entry, so it is the more +// expensive of the two walks. Only the startup soft scan uses it — +// the in-scan progress total does not need mtimes. +func surveyAudioFiles(basePath string) (count, maxModTime int64) { + _ = fs.WalkDir( + os.DirFS(basePath), ".", + func(_ string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return nil + } + + ext := filepath.Ext(d.Name()) + if _, ok := metadata.GetSupportedFileType(ext); !ok { + return nil + } + + count++ + + info, infoErr := d.Info() + if infoErr != nil { + return nil + } + + if mt := info.ModTime().Unix(); mt > maxModTime { + maxModTime = mt + } + + return nil + }, + ) + + return count, maxModTime +} + // hddWorkerCount is the maximum number of concurrent extraction // workers when the library resides on a spinning disk. const hddWorkerCount = 2 @@ -851,6 +1043,14 @@ type scanWork struct { existingFileID int64 // non-zero if this is an update needsUpdate bool existingLength int64 // existing length if updating + // contentChanged marks a file whose bytes differ from what was + // imported (mtime/size mismatch), as opposed to one merely missing + // its metadata link. The audio itself may have changed, so cached + // values like duration cannot be reused. + contentChanged bool + // modTime is the file's mtime (Unix seconds) observed during the + // walk, stored as the new staleness baseline. + modTime int64 } // importResult holds metadata extracted by workers, ready for DB insertion. @@ -863,6 +1063,7 @@ type importResult struct { existingFileID int64 // non-zero if this is an update needsUpdate bool libraryID int64 // library this file belongs to + modTime int64 // mtime baseline to persist (Unix seconds) } // extractAudioMetadata reads and extracts metadata from an audio file. @@ -877,10 +1078,15 @@ func (l *Library) extractAudioMetadata( fileType: work.fileType, existingFileID: work.existingFileID, needsUpdate: work.needsUpdate, + modTime: work.modTime, } // Skip duration decode if we already have it from a previous import. - skipDuration := work.needsUpdate && work.existingLength > 0 + // A file whose bytes changed is decoded again — a re-encode or a + // replaced file can have a different duration than the one on record. + skipDuration := work.needsUpdate && + work.existingLength > 0 && + !work.contentChanged tags, lengthMillis, audioProps, timing, err := metadata.ExtractAllMetadata( work.absolutePath, skipDuration, @@ -902,6 +1108,19 @@ func (l *Library) extractAudioMetadata( ) } + // A degraded tag read is reported but never fatal — the track is + // imported either way, falling back to the filename if the tag + // yielded nothing. + if tags.TagReadWarning != nil { + l.logger.Warn( + "degraded tag read", + "path", work.absolutePath, + "err", tags.TagReadWarning, + ) + + metrics.addWarning(work.absolutePath, "tags", tags.TagReadWarning) + } + result.tags = tags result.audioProps = audioProps @@ -1055,6 +1274,7 @@ func (l *Library) saveAudioFile( LibraryID: result.libraryID, GroupKey: groupKey, TagStatus: tagStatus, + ModifiedAt: result.modTime, }) if err != nil { return fmt.Errorf( @@ -1144,13 +1364,15 @@ func (l *Library) updateAudioFileMetadata( if err := q.UpdateAudioFileRecording( l.ctx, sqlcgen.UpdateAudioFileRecordingParams{ - RecordingID: recordingID, - SampleRate: int64(props.SampleRate), - BitDepth: int64(props.BitDepth), - Channels: int64(props.Channels), - Bitrate: int64(props.Bitrate), - FileSize: props.FileSize, - ID: result.existingFileID, + RecordingID: recordingID, + SampleRate: int64(props.SampleRate), + BitDepth: int64(props.BitDepth), + Channels: int64(props.Channels), + Bitrate: int64(props.Bitrate), + FileSize: props.FileSize, + LengthMilliseconds: result.lengthMillis, + ModifiedAt: result.modTime, + ID: result.existingFileID, }); err != nil { return fmt.Errorf( "could not update audio file recording: %w", err, diff --git a/backend/library/removal_test.go b/backend/library/removal_test.go new file mode 100644 index 0000000..498a56c --- /dev/null +++ b/backend/library/removal_test.go @@ -0,0 +1,217 @@ +package library + +import ( + "os" + "path/filepath" + "testing" + + "yellowjacket/backend/coverart" + "yellowjacket/backend/database/sql/sqlcgen" +) + +// seedRemovableLibrary builds a library with one track, its recording +// chain, a cover art row, and a tagging queue entry — the shape a real +// scan leaves behind. +func seedRemovableLibrary( + t *testing.T, + lib *Library, + coverPath string, +) sqlcgen.Library { + t.Helper() + + ctx := lib.ctx + q := lib.db.Queries + + library, err := q.CreateLibrary(ctx, sqlcgen.CreateLibraryParams{ + Name: "Test Library", + Path: "/music", + }) + if err != nil { + t.Fatalf("create library: %v", err) + } + + ac, err := q.UpsertArtistCredit(ctx, "Test Artist") + if err != nil { + t.Fatalf("upsert artist credit: %v", err) + } + + rec, err := q.CreateRecordingFull(ctx, sqlcgen.CreateRecordingFullParams{ + Name: "Test Song", + ArtistCreditID: ac.ID, + }) + if err != nil { + t.Fatalf("create recording: %v", err) + } + + if _, err := lib.db.ExecContext( + `INSERT INTO cover_art (is_embedded, file_path, mime_type) + VALUES (0, ?, 'image/jpeg')`, coverPath, + ); err != nil { + t.Fatalf("insert cover art: %v", err) + } + + if _, err := q.CreateAudioFile(ctx, sqlcgen.CreateAudioFileParams{ + FilePath: "/music/song.mp3", + LengthMilliseconds: 180000, + RecordingID: rec.ID, + LibraryID: library.ID, + Basename: "song.mp3", + }); err != nil { + t.Fatalf("create audio file: %v", err) + } + + // Every scanned library gets tagging_items rows, one per album + // folder. These FK-reference libraries. + if _, err := lib.db.ExecContext( + `INSERT INTO tagging_items (group_key, library_id, album_name) + VALUES ('grp1', ?, 'Test Album')`, library.ID, + ); err != nil { + t.Fatalf("insert tagging_items: %v", err) + } + + if _, err := lib.db.ExecContext( + `INSERT INTO tagging_candidates (group_key, candidates) + VALUES ('grp1', '[]')`, + ); err != nil { + t.Fatalf("insert tagging_candidates: %v", err) + } + + return library +} + +func countRows( + t *testing.T, + lib *Library, + table string, + args ...any, +) int64 { + t.Helper() + + query := "SELECT COUNT(*) FROM " + table + + rows, err := lib.db.QueryContext(query, args...) + if err != nil { + t.Fatalf("count %s: %v", table, err) + } + + defer func() { _ = rows.Close() }() + + var n int64 + + if rows.Next() { + if err := rows.Scan(&n); err != nil { + t.Fatalf("scan count %s: %v", table, err) + } + } + + return n +} + +// A library with tagging_items must still be removable. tagging_items +// FK-references libraries with no ON DELETE clause, so leaving those +// rows behind fails the DELETE and rolls back the entire removal. +func TestRemoveLibrary_WithTaggingItems(t *testing.T) { + t.Parallel() + + lib, _ := setupTestLibrary(t) + + library := seedRemovableLibrary(t, lib, "/nonexistent/cover.jpg") + + summary, err := lib.RemoveLibrary(library.ID) + if err != nil { + t.Fatalf("RemoveLibrary: %v", err) + } + + if summary.TracksDeleted != 1 { + t.Errorf("TracksDeleted = %d, want 1", summary.TracksDeleted) + } + + // The test DB keeps a sentinel library at id=0 so audio_files rows + // using the default library_id satisfy their FK, so scope this one + // to the library actually removed. + if n := countRows( + t, lib, "libraries WHERE id = ?", library.ID, + ); n != 0 { + t.Errorf("library row still present after removal") + } + + for _, table := range []string{ + "audio_files", + "recordings", + "artist_credit", + "artists", + "tagging_items", + "tagging_candidates", + "cover_art", + } { + if n := countRows(t, lib, table); n != 0 { + t.Errorf("%s has %d rows after removal, want 0", table, n) + } + } +} + +// Removing a library must delete the cover art original *and* its +// derived size variants, which are not recorded in the database. +func TestRemoveLibrary_DeletesCoverArtVariants(t *testing.T) { + t.Parallel() + + lib, _ := setupTestLibrary(t) + + dir := t.TempDir() + original := filepath.Join(dir, "abc123.jpg") + + paths := []string{original} + for _, tier := range thumbnailTiers { + paths = append(paths, filepath.Join( + dir, coverart.SizedFilename("abc123.jpg", tier.Suffix), + )) + } + + paths = append(paths, filepath.Join( + dir, coverart.SizedFilename("abc123.jpg", legacyThumbSuffix), + )) + + for _, p := range paths { + if err := os.WriteFile(p, []byte("img"), 0o600); err != nil { + t.Fatalf("write %s: %v", p, err) + } + } + + library := seedRemovableLibrary(t, lib, original) + + if _, err := lib.RemoveLibrary(library.ID); err != nil { + t.Fatalf("RemoveLibrary: %v", err) + } + + for _, p := range paths { + if _, err := os.Stat(p); !os.IsNotExist(err) { + t.Errorf("cover art file still present: %s", filepath.Base(p)) + } + } +} + +// CoverArtFileSet must cover the original, every generated tier, and +// the legacy _thumb name. +func TestCoverArtFileSet(t *testing.T) { + t.Parallel() + + got := CoverArtFileSet("/covers/abc123.jpg") + + want := []string{ + "/covers/abc123.jpg", + "/covers/abc123_sm.jpg", + "/covers/abc123_md.jpg", + "/covers/abc123_lg.jpg", + "/covers/abc123_thumb.jpg", + } + + if len(got) != len(want) { + t.Fatalf("got %d paths, want %d: %v", len(got), len(want), got) + } + + for i, w := range want { + if got[i] != w { + t.Errorf("path %d = %q, want %q", i, got[i], w) + } + } +} diff --git a/backend/library/scan_control.go b/backend/library/scan_control.go index b36f698..4a616ab 100644 --- a/backend/library/scan_control.go +++ b/backend/library/scan_control.go @@ -3,8 +3,6 @@ package library import ( "context" - "github.com/wailsapp/wails/v2/pkg/runtime" - "yellowjacket/backend/events" "yellowjacket/backend/jobs" ) @@ -41,7 +39,7 @@ func (l *Library) PauseScan() { reg := l.jobs l.mu.Unlock() - runtime.EventsEmit(l.ctx, events.LibraryScanPaused) + l.emit(events.LibraryScanPaused) // Confirm the pause on the job — the registry moved it to "pausing" // when the request came in. Writing the durable pause record is a @@ -76,7 +74,7 @@ func (l *Library) ResumeScan() { reg := l.jobs l.mu.Unlock() - runtime.EventsEmit(l.ctx, events.LibraryScanResumed) + l.emit(events.LibraryScanResumed) // Clears the durable pause record as a side effect of leaving // StatePaused. diff --git a/backend/library/scan_queue.go b/backend/library/scan_queue.go index f08a004..27ccdef 100644 --- a/backend/library/scan_queue.go +++ b/backend/library/scan_queue.go @@ -3,8 +3,6 @@ package library import ( "fmt" - "github.com/wailsapp/wails/v2/pkg/runtime" - "yellowjacket/backend/events" ) @@ -65,7 +63,7 @@ func (l *Library) ScanLibrary(id int64) error { queueLength := len(l.scanQueue) l.mu.Unlock() - runtime.EventsEmit(l.ctx, events.LibraryScanQueued, map[string]any{ + l.emit(events.LibraryScanQueued, map[string]any{ "libraryId": lib.ID, "libraryName": lib.Name, "queueLength": queueLength, @@ -167,9 +165,31 @@ func (l *Library) SoftScanAllLibraries() error { continue } - diskCount := countAudioFiles(lib.Path) + dbModTime, modErr := l.db.Queries.GetLibraryMaxModifiedAt( + l.ctx, lib.ID, + ) + if modErr != nil { + l.logger.Warn( + "soft scan: could not read newest mtime, queueing full scan", + "libraryID", lib.ID, + "libraryName", lib.Name, + "err", modErr, + ) - if diskCount == dbCount { + _ = l.ScanLibrary(lib.ID) + + continue + } + + diskCount, diskModTime := surveyAudioFiles(lib.Path) + + // A newer file on disk than anything on record means something + // was edited in place since the last scan. An older newest-mtime + // is not evidence of a change: deleting the newest file lowers it + // while the count check already covers that case. + staleTags := diskModTime > dbModTime + + if diskCount == dbCount && !staleTags { l.logger.Info( "soft scan: library unchanged, skipping", "libraryID", lib.ID, @@ -181,11 +201,14 @@ func (l *Library) SoftScanAllLibraries() error { } l.logger.Info( - "soft scan: file count mismatch, queueing scan", + "soft scan: library changed, queueing scan", "libraryID", lib.ID, "libraryName", lib.Name, "diskFiles", diskCount, "dbTracks", dbCount, + "diskModTime", diskModTime, + "dbModTime", dbModTime, + "reason", softScanReason(diskCount != dbCount, staleTags), ) if err := l.ScanLibrary(lib.ID); err != nil { @@ -201,6 +224,18 @@ func (l *Library) SoftScanAllLibraries() error { return nil } +// softScanReason labels why the soft scan queued a library, for the log. +func softScanReason(countChanged, staleTags bool) string { + switch { + case countChanged && staleTags: + return "file count mismatch and modified files" + case countChanged: + return "file count mismatch" + default: + return "modified files" + } +} + // CancelCurrentScan cancels only the currently scanning library. // The next queued library (if any) starts automatically when the // current scan's goroutine completes. @@ -281,7 +316,7 @@ func (l *Library) drainQueue() { hooks := l.scanHooks l.mu.Unlock() - runtime.EventsEmit(l.ctx, events.LibraryScanQueueDrained) + l.emit(events.LibraryScanQueueDrained) if hooks.OnAllScansComplete != nil { hooks.OnAllScansComplete() diff --git a/backend/library/staleness_test.go b/backend/library/staleness_test.go new file mode 100644 index 0000000..621dd77 --- /dev/null +++ b/backend/library/staleness_test.go @@ -0,0 +1,287 @@ +package library + +import ( + "os" + "path/filepath" + "testing" + "time" + + "yellowjacket/backend/database/sql/sqlcgen" +) + +// --------------------------------------------------------------------------- +// fileContentChanged — the staleness predicate +// --------------------------------------------------------------------------- + +func TestFileContentChanged(t *testing.T) { + t.Parallel() + + const ( + baseMod int64 = 1700000000 + baseSize int64 = 5_000_000 + ) + + tests := []struct { + name string + recordedMod int64 + recordedSz int64 + diskMod int64 + diskSz int64 + want bool + }{ + { + name: "unchanged file", + recordedMod: baseMod, + recordedSz: baseSize, + diskMod: baseMod, + diskSz: baseSize, + want: false, + }, + { + name: "retagged in place, mtime bumped and size grew", + recordedMod: baseMod, + recordedSz: baseSize, + diskMod: baseMod + 60, + diskSz: baseSize + 2048, + want: true, + }, + { + name: "mtime bumped, size absorbed by tag padding", + recordedMod: baseMod, + recordedSz: baseSize, + diskMod: baseMod + 60, + diskSz: baseSize, + want: true, + }, + { + name: "size changed but mtime preserved by the writer", + recordedMod: baseMod, + recordedSz: baseSize, + diskMod: baseMod, + diskSz: baseSize + 2048, + want: true, + }, + { + name: "no recorded baseline is never stale", + recordedMod: 0, + recordedSz: baseSize, + diskMod: baseMod, + diskSz: baseSize + 4096, + want: false, + }, + { + name: "failed stat is never stale", + recordedMod: baseMod, + recordedSz: baseSize, + diskMod: 0, + diskSz: 0, + want: false, + }, + { + name: "file replaced with an older copy", + recordedMod: baseMod, + recordedSz: baseSize, + diskMod: baseMod - 3600, + diskSz: baseSize, + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + af := sqlcgen.AudioFile{ + ModifiedAt: tt.recordedMod, + FileSize: tt.recordedSz, + } + + got := fileContentChanged(af, tt.diskMod, tt.diskSz) + if got != tt.want { + t.Errorf( + "fileContentChanged() = %v, want %v", + got, tt.want, + ) + } + }) + } +} + +// --------------------------------------------------------------------------- +// surveyAudioFiles — soft scan change signal +// --------------------------------------------------------------------------- + +func TestSurveyAudioFiles(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + + // Two audio files plus one unsupported file that must be ignored. + writeFile(t, filepath.Join(dir, "a.mp3"), 1024) + writeFile(t, filepath.Join(dir, "nested", "b.flac"), 2048) + writeFile(t, filepath.Join(dir, "cover.jpg"), 512) + + older := time.Now().Add(-48 * time.Hour) + newer := time.Now().Add(-1 * time.Hour) + + setModTime(t, filepath.Join(dir, "a.mp3"), older) + setModTime(t, filepath.Join(dir, "nested", "b.flac"), newer) + // The ignored file is the newest on disk — it must not influence + // the result, or every artwork change would trigger a rescan. + setModTime(t, filepath.Join(dir, "cover.jpg"), time.Now()) + + count, maxMod := surveyAudioFiles(dir) + + if count != 2 { + t.Errorf("count = %d, want 2", count) + } + + if maxMod != newer.Unix() { + t.Errorf("maxModTime = %d, want %d", maxMod, newer.Unix()) + } + + // Retagging the older file in place makes it the newest, which is + // what the soft scan compares against the database. + touched := time.Now() + setModTime(t, filepath.Join(dir, "a.mp3"), touched) + + _, afterMod := surveyAudioFiles(dir) + + if afterMod != touched.Unix() { + t.Errorf( + "maxModTime after touch = %d, want %d", + afterMod, touched.Unix(), + ) + } +} + +func TestSurveyAudioFiles_EmptyDir(t *testing.T) { + t.Parallel() + + count, maxMod := surveyAudioFiles(t.TempDir()) + + if count != 0 || maxMod != 0 { + t.Errorf( + "surveyAudioFiles(empty) = (%d, %d), want (0, 0)", + count, maxMod, + ) + } +} + +// --------------------------------------------------------------------------- +// flushStatBackfill — baseline backfill for pre-migration rows +// --------------------------------------------------------------------------- + +func TestFlushStatBackfill(t *testing.T) { + t.Parallel() + + lib, db := setupTestLibrary(t) + ctx := lib.ctx + q := db.Queries + + ac, err := q.UpsertArtistCredit(ctx, "Test Artist") + if err != nil { + t.Fatalf("upsert artist credit: %v", err) + } + + rec, err := q.CreateRecordingFull(ctx, sqlcgen.CreateRecordingFullParams{ + Name: "Test Song", + ArtistCreditID: ac.ID, + }) + if err != nil { + t.Fatalf("create recording: %v", err) + } + + // Seed two rows with no baseline, as migration 47 leaves them. + first, err := q.CreateAudioFile(ctx, sqlcgen.CreateAudioFileParams{ + FilePath: "/music/first.mp3", + LengthMilliseconds: 180000, + RecordingID: rec.ID, + Basename: "first.mp3", + }) + if err != nil { + t.Fatalf("create first audio file: %v", err) + } + + second, err := q.CreateAudioFile(ctx, sqlcgen.CreateAudioFileParams{ + FilePath: "/music/second.mp3", + LengthMilliseconds: 200000, + RecordingID: rec.ID, + Basename: "second.mp3", + }) + if err != nil { + t.Fatalf("create second audio file: %v", err) + } + + if first.ModifiedAt != 0 { + t.Fatalf("seeded ModifiedAt = %d, want 0", first.ModifiedAt) + } + + lib.flushStatBackfill([]sqlcgen.UpdateAudioFileStatParams{ + {ModifiedAt: 1700000000, FileSize: 4096, ID: first.ID}, + {ModifiedAt: 1700000500, FileSize: 8192, ID: second.ID}, + }) + + got, err := q.GetAudioFile(ctx, first.ID) + if err != nil { + t.Fatalf("get first audio file: %v", err) + } + + if got.ModifiedAt != 1700000000 || got.FileSize != 4096 { + t.Errorf( + "first row = (mtime %d, size %d), want (1700000000, 4096)", + got.ModifiedAt, got.FileSize, + ) + } + + // A backfilled row now has a baseline, so the same file on disk is + // no longer treated as stale. + if fileContentChanged(got, 1700000000, 4096) { + t.Error("backfilled row reported stale against identical stat") + } + + // The backfill must not disturb unrelated columns. + if got.LengthMilliseconds != 180000 { + t.Errorf( + "LengthMilliseconds = %d, want 180000 (backfill overwrote it)", + got.LengthMilliseconds, + ) + } + + if got.FilePath != "/music/first.mp3" { + t.Errorf("FilePath = %q, want /music/first.mp3", got.FilePath) + } +} + +func TestFlushStatBackfill_Empty(t *testing.T) { + t.Parallel() + + lib, _ := setupTestLibrary(t) + + // Must be a no-op rather than opening an empty transaction. + lib.flushStatBackfill(nil) +} + +// --------------------------------------------------------------------------- +// helpers +// --------------------------------------------------------------------------- + +func writeFile(t *testing.T, path string, size int) { + t.Helper() + + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", filepath.Dir(path), err) + } + + if err := os.WriteFile(path, make([]byte, size), 0o600); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} + +func setModTime(t *testing.T, path string, mt time.Time) { + t.Helper() + + if err := os.Chtimes(path, mt, mt); err != nil { + t.Fatalf("chtimes %s: %v", path, err) + } +} diff --git a/backend/maintenance/maintenance.go b/backend/maintenance/maintenance.go new file mode 100644 index 0000000..c18286e --- /dev/null +++ b/backend/maintenance/maintenance.go @@ -0,0 +1,214 @@ +// Package maintenance runs the janitorial work that keeps persisted +// data from accumulating without bound. +// +// It exists because cleanup used to have no owner. Functions that +// deleted expired rows were written and then never called; files written +// by one package had no counterpart anywhere that removed them. A +// registry makes the set of janitorial jobs a single visible list, so a +// new cache that forgets to register is obvious in review rather than +// discovered years later as unbounded growth. +// +// Policies come from the classification in backend/datamap: +// +// - Derived data is swept against a live set computed from the data it +// was derived from. Anything not in the live set is garbage. +// - Cache data is evicted by age, because it has no owner to be +// compared against and is merely expensive — not impossible — to +// re-fetch. +// +// Sweeps are idempotent and safe to interrupt: each deletes only what it +// has positively identified as unreferenced, so a partial run simply +// leaves work for the next one. +package maintenance + +import ( + "context" + "log/slog" + "sync" + "time" +) + +// Result reports what a single job reclaimed. +type Result struct { + // RowsDeleted counts database rows removed. + RowsDeleted int64 + // FilesDeleted counts files removed from disk. + FilesDeleted int64 + // BytesFreed is the size of those files. + BytesFreed int64 +} + +// empty reports whether the job found nothing to do, so quiet runs can +// be logged at a lower level. +func (r Result) empty() bool { + return r.RowsDeleted == 0 && r.FilesDeleted == 0 +} + +// Job is one unit of janitorial work. +type Job struct { + // Name identifies the job in logs and in the run record. + Name string + // MinInterval is the minimum time between runs. A job is skipped if + // it ran more recently than this, so hooking the runner to a + // frequently-firing trigger stays cheap. + MinInterval time.Duration + // Run performs the work. It must be idempotent and must respect + // context cancellation. + Run func(ctx context.Context) (Result, error) +} + +// Runner holds the registered jobs and enforces their intervals. +type Runner struct { + mu sync.Mutex + jobs []Job + lastRun map[string]time.Time + logger *slog.Logger +} + +// NewRunner returns an empty runner. +func NewRunner(logger *slog.Logger) *Runner { + return &Runner{ + lastRun: make(map[string]time.Time), + logger: logger, + } +} + +// Register adds a job. Registering a name twice replaces the earlier +// job, so wiring code can be re-run without accumulating duplicates. +func (r *Runner) Register(job Job) { + r.mu.Lock() + defer r.mu.Unlock() + + for i, existing := range r.jobs { + if existing.Name == job.Name { + r.jobs[i] = job + + return + } + } + + r.jobs = append(r.jobs, job) +} + +// JobNames returns the registered job names, for tests and diagnostics. +func (r *Runner) JobNames() []string { + r.mu.Lock() + defer r.mu.Unlock() + + names := make([]string, 0, len(r.jobs)) + for _, j := range r.jobs { + names = append(names, j.Name) + } + + return names +} + +// due reports whether a job's interval has elapsed. +func (r *Runner) due(job Job, now time.Time) bool { + r.mu.Lock() + defer r.mu.Unlock() + + last, ran := r.lastRun[job.Name] + if !ran { + return true + } + + return now.Sub(last) >= job.MinInterval +} + +func (r *Runner) markRun(name string, at time.Time) { + r.mu.Lock() + defer r.mu.Unlock() + + r.lastRun[name] = at +} + +// snapshot copies the job list so a run does not hold the lock while +// executing jobs. +func (r *Runner) snapshot() []Job { + r.mu.Lock() + defer r.mu.Unlock() + + out := make([]Job, len(r.jobs)) + copy(out, r.jobs) + + return out +} + +// RunDue runs every job whose interval has elapsed. A job that fails is +// logged and does not prevent the others from running; janitorial work +// is best-effort by nature and the next run will retry. +func (r *Runner) RunDue(ctx context.Context) Result { + var total Result + + for _, job := range r.snapshot() { + if ctx.Err() != nil { + r.logger.Info("maintenance cancelled", "after", job.Name) + + break + } + + now := time.Now() + if !r.due(job, now) { + continue + } + + start := time.Now() + + result, err := job.Run(ctx) + + r.markRun(job.Name, now) + + if err != nil { + r.logger.Warn("maintenance job failed", + "job", job.Name, "err", err, + "duration", time.Since(start), + ) + + continue + } + + total.RowsDeleted += result.RowsDeleted + total.FilesDeleted += result.FilesDeleted + total.BytesFreed += result.BytesFreed + + if result.empty() { + r.logger.Debug("maintenance job found nothing", + "job", job.Name, "duration", time.Since(start), + ) + + continue + } + + r.logger.Info("maintenance job reclaimed", + "job", job.Name, + "rows", result.RowsDeleted, + "files", result.FilesDeleted, + "bytes", result.BytesFreed, + "duration", time.Since(start), + ) + } + + return total +} + +// Start runs the due jobs immediately and then on every tick until the +// context is cancelled. It returns straight away; the loop runs in its +// own goroutine. +func (r *Runner) Start(ctx context.Context, tick time.Duration) { + go func() { + r.RunDue(ctx) + + ticker := time.NewTicker(tick) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + r.RunDue(ctx) + } + } + }() +} diff --git a/backend/maintenance/maintenance_test.go b/backend/maintenance/maintenance_test.go new file mode 100644 index 0000000..246c247 --- /dev/null +++ b/backend/maintenance/maintenance_test.go @@ -0,0 +1,434 @@ +package maintenance + +import ( + "context" + "errors" + "log/slog" + "os" + "path/filepath" + "slices" + "testing" + "time" + + "yellowjacket/backend/database" +) + +// errTestJobFailed stands in for a job returning an error. +var errTestJobFailed = errors.New("job failed") + +func testRunner() *Runner { + return NewRunner(slog.Default()) +} + +func TestRunnerRunsRegisteredJobs(t *testing.T) { + t.Parallel() + + r := testRunner() + + var ran int + + r.Register(Job{ + Name: "counter", + Run: func(_ context.Context) (Result, error) { + ran++ + + return Result{RowsDeleted: 3}, nil + }, + }) + + total := r.RunDue(context.Background()) + + if ran != 1 { + t.Errorf("job ran %d times, want 1", ran) + } + + if total.RowsDeleted != 3 { + t.Errorf("RowsDeleted = %d, want 3", total.RowsDeleted) + } +} + +// A job must not run again before its interval has elapsed, so hooking +// the runner to a frequent trigger stays cheap. +func TestRunnerRespectsMinInterval(t *testing.T) { + t.Parallel() + + r := testRunner() + + var ran int + + r.Register(Job{ + Name: "throttled", + MinInterval: time.Hour, + Run: func(_ context.Context) (Result, error) { + ran++ + + return Result{}, nil + }, + }) + + r.RunDue(context.Background()) + r.RunDue(context.Background()) + r.RunDue(context.Background()) + + if ran != 1 { + t.Errorf("job ran %d times despite 1h interval, want 1", ran) + } +} + +// One failing job must not prevent the others from running — janitorial +// work is best-effort and the next run retries. +func TestRunnerContinuesAfterFailure(t *testing.T) { + t.Parallel() + + r := testRunner() + + var secondRan bool + + r.Register(Job{ + Name: "failing", + Run: func(_ context.Context) (Result, error) { + return Result{}, errTestJobFailed + }, + }) + r.Register(Job{ + Name: "healthy", + Run: func(_ context.Context) (Result, error) { + secondRan = true + + return Result{}, nil + }, + }) + + r.RunDue(context.Background()) + + if !secondRan { + t.Error("second job did not run after the first failed") + } +} + +// Registering the same name twice replaces the job rather than +// accumulating duplicates, so wiring code is safe to re-run. +func TestRunnerRegisterReplaces(t *testing.T) { + t.Parallel() + + r := testRunner() + + noop := func(_ context.Context) (Result, error) { return Result{}, nil } + + r.Register(Job{Name: "dup", Run: noop}) + r.Register(Job{Name: "dup", Run: noop}) + + if names := r.JobNames(); len(names) != 1 { + t.Errorf("JobNames() = %v, want one entry", names) + } +} + +func TestRunnerStopsOnCancelledContext(t *testing.T) { + t.Parallel() + + r := testRunner() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + var ran bool + + r.Register(Job{ + Name: "should-not-run", + Run: func(_ context.Context) (Result, error) { + ran = true + + return Result{}, nil + }, + }) + + r.RunDue(ctx) + + if ran { + t.Error("job ran despite cancelled context") + } +} + +// --------------------------------------------------------------------------- +// Sweeps +// --------------------------------------------------------------------------- + +func TestExpiredHTTPCacheJob(t *testing.T) { + t.Parallel() + + db := database.NewTestDB(t) + + if _, err := db.ExecContext( + `INSERT INTO http_cache (url_key, response, expires_at) + VALUES ('stale', '{}', datetime('now', '-1 day')), + ('fresh', '{}', datetime('now', '+1 day'))`, + ); err != nil { + t.Fatalf("seed http_cache: %v", err) + } + + result, err := ExpiredHTTPCacheJob(db).Run(context.Background()) + if err != nil { + t.Fatalf("run job: %v", err) + } + + if result.RowsDeleted != 1 { + t.Errorf("RowsDeleted = %d, want 1", result.RowsDeleted) + } + + var remaining string + + rows, err := db.QueryContext("SELECT url_key FROM http_cache") + if err != nil { + t.Fatalf("query http_cache: %v", err) + } + + defer func() { _ = rows.Close() }() + + if rows.Next() { + _ = rows.Scan(&remaining) + } + + if remaining != "fresh" { + t.Errorf("remaining row = %q, want the unexpired one", remaining) + } +} + +// The covers sweep must delete files no cover_art row references while +// keeping the referenced original and every derived variant. +func TestOrphanedCoverFilesJob(t *testing.T) { + t.Parallel() + + db := database.NewTestDB(t) + dir := t.TempDir() + + // Stand-in for library.CoverArtFileSet. + expand := func(original string) []string { + base := filepath.Base(original) + base = base[:len(base)-len(filepath.Ext(base))] + + return []string{ + original, + filepath.Join(filepath.Dir(original), base+"_sm.jpg"), + filepath.Join(filepath.Dir(original), base+"_md.jpg"), + } + } + + keep := []string{"live.jpg", "live_sm.jpg", "live_md.jpg"} + drop := []string{"orphan.jpg", "orphan_sm.jpg", "stray_md.jpg"} + + for _, name := range slices.Concat(keep, drop) { + if err := os.WriteFile( + filepath.Join(dir, name), []byte("img"), 0o600, + ); err != nil { + t.Fatalf("write %s: %v", name, err) + } + } + + if _, err := db.ExecContext( + `INSERT INTO cover_art (is_embedded, file_path, mime_type) + VALUES (0, ?, 'image/jpeg')`, + filepath.Join(dir, "live.jpg"), + ); err != nil { + t.Fatalf("seed cover_art: %v", err) + } + + result, err := OrphanedCoverFilesJob(db, dir, expand). + Run(context.Background()) + if err != nil { + t.Fatalf("run job: %v", err) + } + + if result.FilesDeleted != int64(len(drop)) { + t.Errorf("FilesDeleted = %d, want %d", result.FilesDeleted, len(drop)) + } + + for _, name := range keep { + if _, err := os.Stat(filepath.Join(dir, name)); err != nil { + t.Errorf("referenced file %s was deleted", name) + } + } + + for _, name := range drop { + if _, err := os.Stat(filepath.Join(dir, name)); !os.IsNotExist(err) { + t.Errorf("orphan %s survived the sweep", name) + } + } +} + +// An empty live set means the query saw nothing, not that every cover is +// garbage. The sweep must refuse to empty the directory in that case. +func TestOrphanedCoverFilesJob_EmptyLiveSetIsNoOp(t *testing.T) { + t.Parallel() + + db := database.NewTestDB(t) + dir := t.TempDir() + + path := filepath.Join(dir, "something.jpg") + if err := os.WriteFile(path, []byte("img"), 0o600); err != nil { + t.Fatalf("write file: %v", err) + } + + expand := func(p string) []string { return []string{p} } + + result, err := OrphanedCoverFilesJob(db, dir, expand). + Run(context.Background()) + if err != nil { + t.Fatalf("run job: %v", err) + } + + if result.FilesDeleted != 0 { + t.Errorf("FilesDeleted = %d, want 0", result.FilesDeleted) + } + + if _, err := os.Stat(path); err != nil { + t.Error("sweep emptied the directory on an empty live set") + } +} + +// Artwork for an artist in the library is kept regardless of age; +// artwork for a browsed artist ages out. +func TestOrphanedArtistImagesJob(t *testing.T) { + t.Parallel() + + db := database.NewTestDB(t) + dir := t.TempDir() + + const ( + ownedMBID = "11111111-1111-1111-1111-111111111111" + browsedMBID = "22222222-2222-2222-2222-222222222222" + recentMBID = "33333333-3333-3333-3333-333333333333" + ) + + for _, mbid := range []string{ownedMBID, browsedMBID, recentMBID} { + artistDir := filepath.Join(dir, mbid) + if err := os.MkdirAll(artistDir, 0o755); err != nil { + t.Fatalf("mkdir %s: %v", mbid, err) + } + + if err := os.WriteFile( + filepath.Join(artistDir, "primary.jpg"), []byte("img"), 0o600, + ); err != nil { + t.Fatalf("write image: %v", err) + } + } + + // The owned artist is in the library. + if _, err := db.ExecContext( + `INSERT INTO artists (name, mbid) VALUES ('Owned', ?)`, ownedMBID, + ); err != nil { + t.Fatalf("seed artists: %v", err) + } + + old := time.Now().Add(-200 * 24 * time.Hour) + + for _, tc := range []struct { + mbid string + created time.Time + }{ + {ownedMBID, old}, + {browsedMBID, old}, + {recentMBID, time.Now()}, + } { + if _, err := db.ExecContext( + `INSERT INTO artist_images + (artist_mbid, source, source_url, file_path, created_at) + VALUES (?, 'test', 'http://x', ?, ?)`, + tc.mbid, + filepath.Join(dir, tc.mbid, "primary.jpg"), + tc.created, + ); err != nil { + t.Fatalf("seed artist_images for %s: %v", tc.mbid, err) + } + } + + if _, err := OrphanedArtistImagesJob(db, dir). + Run(context.Background()); err != nil { + t.Fatalf("run job: %v", err) + } + + if _, err := os.Stat(filepath.Join(dir, ownedMBID)); err != nil { + t.Error("artwork for a library artist was evicted") + } + + if _, err := os.Stat(filepath.Join(dir, recentMBID)); err != nil { + t.Error("recently fetched artwork was evicted") + } + + if _, err := os.Stat(filepath.Join(dir, browsedMBID)); !os.IsNotExist(err) { + t.Error("stale browsed artwork survived the sweep") + } + + // The rows must go with the files. + rows, err := db.QueryContext( + "SELECT COUNT(*) FROM artist_images WHERE artist_mbid = ?", + browsedMBID, + ) + if err != nil { + t.Fatalf("count rows: %v", err) + } + + defer func() { _ = rows.Close() }() + + var n int + + if rows.Next() { + _ = rows.Scan(&n) + } + + if n != 0 { + t.Errorf("artist_images rows for evicted artist = %d, want 0", n) + } +} + +func TestExpiredProxyCacheJob(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + + stale := filepath.Join(dir, "stale.jpg") + fresh := filepath.Join(dir, "fresh.jpg") + + for _, p := range []string{stale, fresh} { + if err := os.WriteFile(p, []byte("img"), 0o600); err != nil { + t.Fatalf("write %s: %v", p, err) + } + } + + old := time.Now().Add(-60 * 24 * time.Hour) + if err := os.Chtimes(stale, old, old); err != nil { + t.Fatalf("chtimes: %v", err) + } + + result, err := ExpiredProxyCacheJob(dir).Run(context.Background()) + if err != nil { + t.Fatalf("run job: %v", err) + } + + if result.FilesDeleted != 1 { + t.Errorf("FilesDeleted = %d, want 1", result.FilesDeleted) + } + + if _, err := os.Stat(fresh); err != nil { + t.Error("recently written thumbnail was evicted") + } + + if _, err := os.Stat(stale); !os.IsNotExist(err) { + t.Error("stale thumbnail survived the sweep") + } +} + +// A missing directory is normal on a fresh install and must not error. +func TestSweepMissingDirectory(t *testing.T) { + t.Parallel() + + result, err := ExpiredProxyCacheJob( + filepath.Join(t.TempDir(), "does-not-exist"), + ).Run(context.Background()) + if err != nil { + t.Fatalf("missing directory returned an error: %v", err) + } + + if result.FilesDeleted != 0 { + t.Errorf("FilesDeleted = %d, want 0", result.FilesDeleted) + } +} diff --git a/backend/maintenance/sweeps.go b/backend/maintenance/sweeps.go new file mode 100644 index 0000000..7a0aca6 --- /dev/null +++ b/backend/maintenance/sweeps.go @@ -0,0 +1,356 @@ +package maintenance + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + "yellowjacket/backend/database" +) + +// Retention windows for cache data. Cached artwork for artists the user +// actually owns is kept indefinitely; art fetched while browsing Explore +// is transient and ages out. +const ( + // browsedArtRetention is how long artwork for a non-library artist + // survives after it was fetched. + browsedArtRetention = 90 * 24 * time.Hour + + // proxyCacheRetention is how long an Explore cover-art thumbnail + // survives after it was last written. + proxyCacheRetention = 30 * 24 * time.Hour +) + +// Default intervals. These are minimums, not schedules — the runner +// skips a job that ran more recently. +const ( + frequentInterval = 6 * time.Hour + dailyInterval = 24 * time.Hour +) + +// ExpiredHTTPCacheJob deletes HTTP cache rows past their TTL. +// +// Reads already filter on expires_at, so expired rows are inert — but +// nothing was deleting them, so the table grew without bound for the +// life of the install. +func ExpiredHTTPCacheJob(db *database.DB) Job { + return Job{ + Name: "http-cache-evict", + MinInterval: frequentInterval, + Run: func(_ context.Context) (Result, error) { + res, err := db.ExecContext( + "DELETE FROM http_cache WHERE expires_at < datetime('now')", + ) + if err != nil { + return Result{}, fmt.Errorf( + "delete expired http_cache rows: %w", err, + ) + } + + rows, _ := res.RowsAffected() + + return Result{RowsDeleted: rows}, nil + }, + } +} + +// OrphanedCoverFilesJob removes files from the covers directory that no +// cover_art row references. +// +// Cover art is derived data, so the live set is authoritative: every +// file that is not the original named by a cover_art row, or one of that +// original's derived size variants, is garbage. This reclaims art left +// behind by earlier versions that deleted only the original and left its +// thumbnails. +func OrphanedCoverFilesJob( + db *database.DB, + coversDir string, + expandVariants func(originalPath string) []string, +) Job { + return Job{ + Name: "covers-sweep", + MinInterval: dailyInterval, + Run: func(ctx context.Context) (Result, error) { + live, err := liveCoverFiles(db, coversDir, expandVariants) + if err != nil { + return Result{}, err + } + + // A covers directory with no live entries almost certainly + // means the query failed to see the real table rather than + // that every cover is garbage. Refuse to empty the + // directory on that basis. + if len(live) == 0 { + return Result{}, nil + } + + return sweepDir(ctx, coversDir, func(name string) bool { + return !live[name] + }) + }, + } +} + +// liveCoverFiles returns the basenames of every file the covers +// directory is supposed to contain. +func liveCoverFiles( + db *database.DB, + coversDir string, + expandVariants func(string) []string, +) (map[string]bool, error) { + rows, err := db.QueryContext("SELECT file_path FROM cover_art") + if err != nil { + return nil, fmt.Errorf("read cover_art paths: %w", err) + } + + defer func() { _ = rows.Close() }() + + live := make(map[string]bool) + + for rows.Next() { + var path string + + if err := rows.Scan(&path); err != nil { + return nil, fmt.Errorf("scan cover_art path: %w", err) + } + + // Rows may store an absolute path from a previous install + // location, so compare by basename within the covers directory. + original := filepath.Join(coversDir, filepath.Base(path)) + + for _, variant := range expandVariants(original) { + live[filepath.Base(variant)] = true + } + } + + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate cover_art paths: %w", err) + } + + return live, nil +} + +// OrphanedArtistImagesJob evicts cached artist artwork. +// +// Artist images are cache data with no owner to compare against — they +// are fetched for any artist the user browses in Explore, most of whom +// are not in the library. The policy is therefore twofold: artwork for +// an artist the user owns is kept indefinitely, and everything else ages +// out. Rows whose file has vanished are dropped so the table matches +// what is actually on disk. +func OrphanedArtistImagesJob(db *database.DB, artistImagesDir string) Job { + return Job{ + Name: "artist-images-sweep", + MinInterval: dailyInterval, + Run: func(ctx context.Context) (Result, error) { + var result Result + + cutoff := time.Now().Add(-browsedArtRetention) + + // Collect the directories to remove before deleting rows, so + // a failure partway leaves rows pointing at real files + // rather than the reverse. + stale, err := staleArtistMBIDs(db, cutoff) + if err != nil { + return Result{}, err + } + + for _, mbid := range stale { + if ctx.Err() != nil { + return result, nil + } + + dir := filepath.Join(artistImagesDir, mbid) + + freed, files := dirSize(dir) + + if err := os.RemoveAll(dir); err != nil && !os.IsNotExist(err) { + continue + } + + result.FilesDeleted += files + result.BytesFreed += freed + } + + if len(stale) > 0 { + rows, delErr := deleteArtistImageRows(db, stale) + if delErr != nil { + return result, delErr + } + + result.RowsDeleted += rows + } + + return result, nil + }, + } +} + +// staleArtistMBIDs returns artist MBIDs whose cached artwork may be +// evicted: fetched before the cutoff and not an artist in the library. +func staleArtistMBIDs( + db *database.DB, + cutoff time.Time, +) ([]string, error) { + rows, err := db.QueryContext( + `SELECT DISTINCT artist_mbid FROM artist_images + WHERE created_at < ? + AND artist_mbid NOT IN ( + SELECT mbid FROM artists + WHERE mbid IS NOT NULL AND mbid != '' + )`, + cutoff, + ) + if err != nil { + return nil, fmt.Errorf("query stale artist images: %w", err) + } + + defer func() { _ = rows.Close() }() + + var mbids []string + + for rows.Next() { + var mbid string + + if err := rows.Scan(&mbid); err != nil { + return nil, fmt.Errorf("scan artist mbid: %w", err) + } + + if mbid != "" { + mbids = append(mbids, mbid) + } + } + + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate stale artist images: %w", err) + } + + return mbids, nil +} + +// deleteArtistImageRows removes the rows for the given artist MBIDs. +func deleteArtistImageRows( + db *database.DB, + mbids []string, +) (int64, error) { + var total int64 + + for _, mbid := range mbids { + res, err := db.ExecContext( + "DELETE FROM artist_images WHERE artist_mbid = ?", mbid, + ) + if err != nil { + return total, fmt.Errorf( + "delete artist_images rows for %s: %w", mbid, err, + ) + } + + n, _ := res.RowsAffected() + total += n + } + + return total, nil +} + +// ExpiredProxyCacheJob evicts Explore cover-art thumbnails that have not +// been rewritten within the retention window. +// +// This cache has no database table at all — it is keyed by release-group +// MBID on the filesystem — so age is the only signal available. +func ExpiredProxyCacheJob(proxyCacheDir string) Job { + return Job{ + Name: "cover-art-proxy-sweep", + MinInterval: dailyInterval, + Run: func(ctx context.Context) (Result, error) { + cutoff := time.Now().Add(-proxyCacheRetention) + + return sweepDirFunc(ctx, proxyCacheDir, + func(_ string, info os.FileInfo) bool { + return info.ModTime().Before(cutoff) + }, + ) + }, + } +} + +// sweepDir removes every file in dir for which shouldDelete reports true. +func sweepDir( + ctx context.Context, + dir string, + shouldDelete func(name string) bool, +) (Result, error) { + return sweepDirFunc(ctx, dir, func(name string, _ os.FileInfo) bool { + return shouldDelete(name) + }) +} + +// sweepDirFunc removes files from a flat directory based on a predicate +// over the name and stat info. Subdirectories are left alone; sweeps +// that own directory trees handle them explicitly. +func sweepDirFunc( + ctx context.Context, + dir string, + shouldDelete func(name string, info os.FileInfo) bool, +) (Result, error) { + entries, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return Result{}, nil + } + + return Result{}, fmt.Errorf("read %s: %w", dir, err) + } + + var result Result + + for _, entry := range entries { + if ctx.Err() != nil { + return result, nil + } + + if entry.IsDir() { + continue + } + + info, infoErr := entry.Info() + if infoErr != nil { + continue + } + + if !shouldDelete(entry.Name(), info) { + continue + } + + if err := os.Remove(filepath.Join(dir, entry.Name())); err != nil { + continue + } + + result.FilesDeleted++ + result.BytesFreed += info.Size() + } + + return result, nil +} + +// dirSize totals the files in a directory tree. +func dirSize(dir string) (bytes, files int64) { + _ = filepath.WalkDir(dir, func(_ string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() { + return nil //nolint:nilerr // best-effort accounting + } + + info, infoErr := d.Info() + if infoErr != nil { + return nil + } + + bytes += info.Size() + files++ + + return nil + }) + + return bytes, files +} diff --git a/backend/metadata/tags.go b/backend/metadata/tags.go index 034e636..c734848 100644 --- a/backend/metadata/tags.go +++ b/backend/metadata/tags.go @@ -44,6 +44,13 @@ type TrackMetadata struct { // Format info TagFormat string // "ID3v2.3", "VORBIS", etc. FileFormat string // "MP3", "FLAC", etc. + + // TagReadWarning is set when the tag could not be read cleanly: + // ErrTagsRecovered if the lenient parser salvaged the fields above, + // ErrTagsUnreadable if they are empty because nothing could read the + // tag. Nil on a clean read. Either way the file is still usable — + // callers should surface the warning, not discard the track. + TagReadWarning error } // PictureData holds embedded artwork. @@ -74,7 +81,11 @@ func ExtractTagsFromReader(r io.ReadSeeker) (*TrackMetadata, error) { return &TrackMetadata{}, nil } - return nil, fmt.Errorf("could not read tags: %w", err) + // A single malformed frame must not cost us the whole file: retry + // with a more forgiving parser and, failing that, hand back empty + // metadata carrying a warning. The audio is still playable and + // the caller can fall back to the filename. + return recoverTags(r, err), nil } trackNum, totalTracks := m.Track() diff --git a/backend/metadata/tags_lenient.go b/backend/metadata/tags_lenient.go new file mode 100644 index 0000000..eaf793d --- /dev/null +++ b/backend/metadata/tags_lenient.go @@ -0,0 +1,253 @@ +package metadata + +import ( + "errors" + "fmt" + "io" + "strconv" + "strings" + + "github.com/bogem/id3v2/v2" +) + +// Sentinel errors describing a degraded tag read. Both travel on +// TrackMetadata.TagReadWarning rather than being returned, so that one +// malformed frame never costs the caller the entire file. +var ( + // ErrTagsRecovered means the strict parser rejected the tag but the + // lenient ID3v2 fallback read it. The metadata is populated. + ErrTagsRecovered = errors.New("tags recovered by lenient parser") + + // ErrTagsUnreadable means no parser could read the tag. The + // metadata is empty and callers should fall back to the filename. + ErrTagsUnreadable = errors.New("tags could not be parsed") +) + +// recoverTags retries a failed tag read with bogem/id3v2, which +// tolerates frames that dhowden/tag rejects outright — a stray NUL +// inside a UTF-16 TXXX frame, for example. It always returns usable +// metadata: when nothing can be salvaged the metadata is empty and only +// TagReadWarning is set. +func recoverTags(r io.ReadSeeker, cause error) *TrackMetadata { + if _, err := r.Seek(0, io.SeekStart); err != nil { + return &TrackMetadata{ + TagReadWarning: fmt.Errorf("%w: %w", ErrTagsUnreadable, cause), + } + } + + meta, err := extractID3v2Lenient(r) + if err != nil { + return &TrackMetadata{ + TagReadWarning: fmt.Errorf("%w: %w", ErrTagsUnreadable, cause), + } + } + + meta.TagReadWarning = fmt.Errorf("%w: %w", ErrTagsRecovered, cause) + + return meta +} + +// extractID3v2Lenient reads an ID3v2 tag with bogem/id3v2. Only MP3 +// (and other ID3v2-carrying containers) can be recovered this way; +// for anything else the tag has no frames and the read fails. +func extractID3v2Lenient(r io.Reader) (*TrackMetadata, error) { + // The tag is not Closed here: it wraps a reader the caller owns. + t, err := id3v2.ParseReader(r, id3v2.Options{Parse: true}) + if err != nil { + return nil, fmt.Errorf("lenient id3v2 parse: %w", err) + } + + if !t.HasFrames() { + return nil, ErrTagsUnreadable + } + + meta := &TrackMetadata{ + Title: t.Title(), + Artist: t.Artist(), + Album: t.Album(), + AlbumArtist: id3Text(t, "Band/Orchestra/Accompaniment"), + Composer: id3Text(t, "Composer"), + Genre: t.Genre(), + Year: parseLeadingInt(t.Year()), + Lyrics: id3Lyrics(t), + Comment: id3Comment(t), + TagFormat: fmt.Sprintf("ID3v2.%d", t.Version()), + FileFormat: strings.ToUpper(strings.TrimPrefix(string(MP3), ".")), + } + + meta.TrackNumber, meta.TotalTracks = parsePosition( + id3Text(t, "Track number/Position in set"), + ) + meta.DiscNumber, meta.TotalDiscs = parsePosition( + id3Text(t, "Part of a set"), + ) + + extractMBIDsID3v2(t, meta) + + meta.Picture = id3Picture(t) + + return meta, nil +} + +// id3Text returns the text of the frame registered under the given +// common description, empty if the frame is absent. +func id3Text(t *id3v2.Tag, description string) string { + return strings.TrimRight( + t.GetTextFrame(t.CommonID(description)).Text, "\x00 \t\n\r", + ) +} + +// id3Lyrics returns the first non-empty USLT frame. +func id3Lyrics(t *id3v2.Tag) string { + for _, f := range t.GetFrames(t.CommonID("Unsynchronised lyrics/text transcription")) { + if uslf, ok := f.(id3v2.UnsynchronisedLyricsFrame); ok && uslf.Lyrics != "" { + return uslf.Lyrics + } + } + + return "" +} + +// id3Comment returns the first non-empty COMM frame, skipping the +// machine-written iTunes frames that carry no user comment. +func id3Comment(t *id3v2.Tag) string { + for _, f := range t.GetFrames(t.CommonID("Comments")) { + cf, ok := f.(id3v2.CommentFrame) + if !ok || cf.Text == "" { + continue + } + + if strings.HasPrefix(cf.Description, "iTun") { + continue + } + + return cf.Text + } + + return "" +} + +// id3Picture returns the front cover if one is attached, otherwise the +// first attached picture of any type. +func id3Picture(t *id3v2.Tag) *PictureData { + var first *PictureData + + for _, f := range t.GetFrames(t.CommonID("Attached picture")) { + pf, ok := f.(id3v2.PictureFrame) + if !ok || len(pf.Picture) == 0 { + continue + } + + pic := &PictureData{ + Data: pf.Picture, + MIMEType: pf.MimeType, + Ext: imageExtFromMIME(pf.MimeType), + } + + if pf.PictureType == id3v2.PTFrontCover { + return pic + } + + if first == nil { + first = pic + } + } + + return first +} + +// extractMBIDsID3v2 populates the MBID fields of meta from TXXX and +// UFID frames, mirroring extractMBIDs for the lenient parser. +func extractMBIDsID3v2(t *id3v2.Tag, meta *TrackMetadata) { + normalized := make(map[string]string) + + for _, f := range t.GetFrames(t.CommonID("User defined text information frame")) { + if udtf, ok := f.(id3v2.UserDefinedTextFrame); ok && udtf.Description != "" { + normalized[strings.ToLower(udtf.Description)] = strings.TrimRight( + udtf.Value, "\x00 \t\n\r", + ) + } + } + + for _, f := range t.GetFrames(t.CommonID("Unique file identifier")) { + if ufid, ok := f.(id3v2.UFIDFrame); ok && + ufid.OwnerIdentifier == "http://musicbrainz.org" { + meta.RecordingMBID = strings.TrimRight( + string(ufid.Identifier), "\x00 \t\n\r", + ) + } + } + + targets := map[string]*string{ + "ArtistMBID": &meta.ArtistMBID, + "AlbumArtistMBID": &meta.AlbumArtistMBID, + "ReleaseGroupMBID": &meta.ReleaseGroupMBID, + "ReleaseMBID": &meta.ReleaseMBID, + "RecordingMBID": &meta.RecordingMBID, + } + + for field, keys := range mbidTagKeys { + for _, key := range keys { + if val, ok := normalized[key]; ok && val != "" { + *targets[field] = val + + break + } + } + } +} + +// parsePosition splits an ID3 "n/total" position string such as the +// TRCK or TPOS payload. Missing parts come back as zero. +func parsePosition(raw string) (int, int) { + raw = strings.TrimSpace(raw) + if raw == "" { + return 0, 0 + } + + number, total, found := strings.Cut(raw, "/") + if !found { + return parseLeadingInt(number), 0 + } + + return parseLeadingInt(number), parseLeadingInt(total) +} + +// parseLeadingInt reads the leading run of digits from s, returning +// zero when there is none. Tolerates values like "2021-06-11" (a +// TDRC date) and "3 " (a padded track number). +func parseLeadingInt(s string) int { + s = strings.TrimSpace(s) + + end := 0 + for end < len(s) && s[end] >= '0' && s[end] <= '9' { + end++ + } + + if end == 0 { + return 0 + } + + n, err := strconv.Atoi(s[:end]) + if err != nil { + return 0 + } + + return n +} + +// imageExtFromMIME returns a file extension for common image MIME types. +func imageExtFromMIME(mimeType string) string { + switch strings.ToLower(strings.TrimSpace(mimeType)) { + case "image/png": + return "png" + case "image/gif": + return "gif" + case "image/webp": + return "webp" + case "image/bmp": + return "bmp" + default: + return "jpg" + } +} diff --git a/backend/metadata/tags_lenient_test.go b/backend/metadata/tags_lenient_test.go new file mode 100644 index 0000000..3641428 --- /dev/null +++ b/backend/metadata/tags_lenient_test.go @@ -0,0 +1,214 @@ +package metadata + +import ( + "bytes" + "encoding/binary" + "errors" + "strings" + "testing" +) + +// errTestParseFailure stands in for the strict parser's error when +// exercising the fallback directly. +var errTestParseFailure = errors.New("original parse failure") + +// synchsafe encodes n as a 4-byte ID3v2.4 synchsafe integer. +func synchsafe(n int) []byte { + return []byte{ + byte(n>>21) & 0x7f, + byte(n>>14) & 0x7f, + byte(n>>7) & 0x7f, + byte(n) & 0x7f, + } +} + +// id3Frame builds a single ID3v2.4 frame from a raw body. +func id3Frame(id string, body []byte) []byte { + var buf bytes.Buffer + + buf.WriteString(id) + buf.Write(synchsafe(len(body))) + buf.Write([]byte{0, 0}) // Flags. + buf.Write(body) + + return buf.Bytes() +} + +// utf8TextBody builds a UTF-8 text frame body (encoding byte $03). +func utf8TextBody(s string) []byte { + return append([]byte{3}, []byte(s)...) +} + +// malformedUTF16TXXX reproduces the real-world frame that motivated the +// lenient fallback: a UTF-16BE TXXX frame carrying two stray NUL bytes, +// one after the description terminator and one at the very end. Both +// the description and the value end up an odd number of bytes, which +// dhowden/tag rejects outright. +func malformedUTF16TXXX(description, value string) []byte { + var buf bytes.Buffer + + buf.WriteByte(1) // Encoding: UTF-16 with BOM. + + writeUTF16BE := func(s string) { + buf.Write([]byte{0xfe, 0xff}) // Big-endian BOM. + + for _, r := range s { + _ = binary.Write(&buf, binary.BigEndian, uint16(r)) + } + } + + writeUTF16BE(description) + buf.Write([]byte{0, 0}) // Terminator. + buf.WriteByte(0) // Stray NUL. + writeUTF16BE(value) + buf.WriteByte(0) // Stray NUL. + + return buf.Bytes() +} + +// buildID3v24 assembles a tag from frames and appends a stub of MPEG +// audio so the result looks like a real file to a parser. +func buildID3v24(frames ...[]byte) []byte { + body := bytes.Join(frames, nil) + + var buf bytes.Buffer + + buf.WriteString("ID3") + buf.Write([]byte{4, 0}) // Version 2.4.0. + buf.WriteByte(0) // Flags. + buf.Write(synchsafe(len(body))) + buf.Write(body) + buf.Write([]byte{0xff, 0xfb, 0x90, 0x00}) // MPEG frame header stub. + + return buf.Bytes() +} + +func TestExtractTagsFromReaderRecoversMalformedFrame(t *testing.T) { + data := buildID3v24( + id3Frame("TIT2", utf8TextBody("Evolution's a Lie")), + id3Frame("TPE1", utf8TextBody("Ariel Pink")), + id3Frame("TALB", utf8TextBody("Sit n' Spin")), + id3Frame("TRCK", utf8TextBody("1/17")), + id3Frame("TXXX", malformedUTF16TXXX("LABEL", "Mexican Summer")), + ) + + meta, err := ExtractTagsFromReader(bytes.NewReader(data)) + if err != nil { + t.Fatalf("ExtractTagsFromReader returned a hard error: %v", err) + } + + if !errors.Is(meta.TagReadWarning, ErrTagsRecovered) { + t.Errorf( + "TagReadWarning = %v, want it to wrap ErrTagsRecovered", + meta.TagReadWarning, + ) + } + + if meta.Title != "Evolution's a Lie" { + t.Errorf("Title = %q, want %q", meta.Title, "Evolution's a Lie") + } + + if meta.Artist != "Ariel Pink" { + t.Errorf("Artist = %q, want %q", meta.Artist, "Ariel Pink") + } + + if meta.Album != "Sit n' Spin" { + t.Errorf("Album = %q, want %q", meta.Album, "Sit n' Spin") + } + + if meta.TrackNumber != 1 || meta.TotalTracks != 17 { + t.Errorf( + "track = %d/%d, want 1/17", + meta.TrackNumber, meta.TotalTracks, + ) + } +} + +// TestExtractTagsFromReaderCleanTagHasNoWarning guards against the +// fallback firing on tags the strict parser handles. +func TestExtractTagsFromReaderCleanTagHasNoWarning(t *testing.T) { + data := buildID3v24( + id3Frame("TIT2", utf8TextBody("Clean Title")), + id3Frame("TXXX", utf8TextBody("LABEL\x00Mexican Summer")), + ) + + meta, err := ExtractTagsFromReader(bytes.NewReader(data)) + if err != nil { + t.Fatalf("ExtractTagsFromReader: %v", err) + } + + if meta.TagReadWarning != nil { + t.Errorf("TagReadWarning = %v, want nil", meta.TagReadWarning) + } + + if meta.Title != "Clean Title" { + t.Errorf("Title = %q, want %q", meta.Title, "Clean Title") + } +} + +// TestRecoverTagsUnsalvageable covers the case where the fallback finds +// no ID3v2 frames either: metadata comes back empty but usable, with a +// warning that tells the caller to fall back to the filename. +func TestRecoverTagsUnsalvageable(t *testing.T) { + r := strings.NewReader("not an audio file at all") + + meta := recoverTags(r, errTestParseFailure) + + if !errors.Is(meta.TagReadWarning, ErrTagsUnreadable) { + t.Errorf( + "TagReadWarning = %v, want it to wrap ErrTagsUnreadable", + meta.TagReadWarning, + ) + } + + if !errors.Is(meta.TagReadWarning, errTestParseFailure) { + t.Errorf("TagReadWarning = %v, want it to wrap the cause", meta.TagReadWarning) + } + + if meta.Title != "" || meta.Artist != "" { + t.Errorf("expected empty metadata, got %+v", meta) + } +} + +func TestParsePosition(t *testing.T) { + tests := []struct { + raw string + want, wantOf int + }{ + {"", 0, 0}, + {"3", 3, 0}, + {"3/17", 3, 17}, + {" 3 / 17 ", 3, 17}, + {"03/17", 3, 17}, + {"A/B", 0, 0}, + {"1/", 1, 0}, + } + + for _, tt := range tests { + t.Run(tt.raw, func(t *testing.T) { + got, gotOf := parsePosition(tt.raw) + if got != tt.want || gotOf != tt.wantOf { + t.Errorf( + "parsePosition(%q) = %d/%d, want %d/%d", + tt.raw, got, gotOf, tt.want, tt.wantOf, + ) + } + }) + } +} + +func TestParseLeadingInt(t *testing.T) { + tests := map[string]int{ + "": 0, + "2021": 2021, + "2021-06-11": 2021, + "1995\t": 1995, + "none": 0, + } + + for raw, want := range tests { + if got := parseLeadingInt(raw); got != want { + t.Errorf("parseLeadingInt(%q) = %d, want %d", raw, got, want) + } + } +} diff --git a/backend/tagwriter/dbsync.go b/backend/tagwriter/dbsync.go index b240d75..9030c9f 100644 --- a/backend/tagwriter/dbsync.go +++ b/backend/tagwriter/dbsync.go @@ -417,6 +417,26 @@ func syncDatabase( } } + // ------------------------------------------------------------------ + // 7d. Re-baseline the staleness fields. Writing tags rewrites the + // file, changing its mtime and possibly its size. Recording the + // new values here keeps the scan from mistaking YellowJacket's + // own edit for an external one and re-importing the track. + // ------------------------------------------------------------------ + if info, statErr := os.Stat(params.filePath); statErr != nil { + logger.Warn("could not stat file after tag write", + "path", params.filePath, "err", statErr) + } else if updErr := txq.UpdateAudioFileStat(ctx, + sqlcgen.UpdateAudioFileStatParams{ + ModifiedAt: info.ModTime().Unix(), + FileSize: info.Size(), + ID: params.audioFileID, + }, + ); updErr != nil { + logger.Warn("could not update file stat after tag write", + "path", params.filePath, "err", updErr) + } + // ------------------------------------------------------------------ // 8. Commit. // ------------------------------------------------------------------ diff --git a/backend/tagwriter/pipeline.go b/backend/tagwriter/pipeline.go index 6ccec26..b22f6f6 100644 --- a/backend/tagwriter/pipeline.go +++ b/backend/tagwriter/pipeline.go @@ -194,6 +194,51 @@ func (tw *TagWriter) WriteTrackTags(trackID int64, changes TagChanges) error { return nil } +// WriteUntrackedFileTags writes tags to a file that is not in the +// library, skipping every step of the full pipeline that assumes it is: +// no audio_file lookup, no database sync, no event. +// +// This exists for the download import path, which tags files while they +// are still in the staging directory. Tagging before the move is what +// makes the import atomic from the library's point of view — the +// scanner only ever sees a finished, correctly tagged file, instead of +// ingesting a mislabelled one and being corrected afterwards. +// +// Callers are responsible for ensuring the file is not in a library +// path; using this on a tracked file would leave the database stale. +func (tw *TagWriter) WriteUntrackedFileTags( + filePath string, + changes TagChanges, +) error { + if len(changes) == 0 { + return errNoChanges + } + + format, err := DetectFormat(filePath) + if err != nil { + return fmt.Errorf("detect format: %w", err) + } + + switch format { + case FormatMP3: + err = writeMp3Tags(tw.logger, filePath, changes) + case FormatFLAC: + err = writeFlacTags(tw.logger, filePath, changes) + case FormatWAV: + err = writeWavTags(tw.logger, filePath, changes) + case FormatOGG: + err = writeOggTags(tw.logger, filePath, changes) + default: + err = fmt.Errorf("%w: %s", errUnsupportedFormat, format) + } + + if err != nil { + return fmt.Errorf("write file tags: %w", err) + } + + return nil +} + // WriteTrackTagsByPath resolves a file path to its audio_file.id and // delegates to WriteTrackTags. This is the frontend-facing entry // point since the frontend identifies tracks by FilePath. diff --git a/cmd/indexbuild/decide_test.go b/cmd/indexbuild/decide_test.go index 088fa9f..6ec211f 100644 --- a/cmd/indexbuild/decide_test.go +++ b/cmd/indexbuild/decide_test.go @@ -1,3 +1,5 @@ +//go:build indexbuild + package main import ( diff --git a/cmd/indexbuild/main.go b/cmd/indexbuild/main.go index 69f2b59..bc8ea9c 100644 --- a/cmd/indexbuild/main.go +++ b/cmd/indexbuild/main.go @@ -1,3 +1,5 @@ +//go:build indexbuild + // Command indexbuild maintains the explore search index outside the // desktop app, so the catalog can be built once centrally instead of by // every install. @@ -9,7 +11,7 @@ // import older than 3mo → rebuild (re-import from the newest dump) // otherwise → refresh (fold in new incremental listens) // -// A full build streams ~205GB from the ListenBrainz spark dump — far +// A full build streams ~89GB from the ListenBrainz spark dump — far // more than one CI job should attempt — so builds are budgeted and // resumable: the importer checkpoints its absolute stream offset, and // each run continues where the last stopped. A refresh is cheap @@ -87,7 +89,7 @@ func main() { "stop and checkpoint a build after this long (0 = no limit)") modeFlag = flag.String("mode", string(modeAuto), "auto | build | refresh | rebuild") - rebuildAfter = flag.Duration("rebuild-after", 90*24*time.Hour, + rebuildAfter = flag.Duration("rebuild-after", 180*24*time.Hour, "re-import from a fresh dump once the last import is older than this") refreshAfter = flag.Duration("refresh-after", 7*24*time.Hour, "minimum gap between incremental refreshes (0 = always)") diff --git a/frontend/index.css b/frontend/index.css index 39e0f2d..4ca74a9 100644 --- a/frontend/index.css +++ b/frontend/index.css @@ -40,68 +40,6 @@ p { flex: 0 1 320px; } -.mode-toggle { - display: flex; - align-items: center; - gap: 6px; - cursor: pointer; - flex-shrink: 0; -} - -.mode-toggle-track { - position: relative; - width: 36px; - height: 20px; - border-radius: 10px; - background: var(--yj-bg-overlay, rgba(255, 255, 255, 0.15)); - transition: background 0.2s ease; -} - -.mode-toggle:hover .mode-toggle-track { - background: rgba(255, 255, 255, 0.22); -} - -.mode-toggle-thumb { - position: absolute; - top: 2px; - left: 2px; - width: 16px; - height: 16px; - border-radius: 50%; - background: var(--yj-text-primary, #fff); - transition: left 0.2s ease, background 0.2s ease; -} - -.mode-toggle.active .mode-toggle-track { - background: var(--yj-accent, #ffd43b); -} - -.mode-toggle.active .mode-toggle-thumb { - left: 18px; - background: #000; -} - -.mode-icon { - font-size: 14px; - transition: color 0.2s ease, opacity 0.2s ease; -} - -.mode-icon-globe { - color: var(--yj-text-primary, #fff); -} - -.mode-icon-local { - color: var(--yj-text-secondary, #888); -} - -.mode-toggle.active .mode-icon-globe { - color: var(--yj-text-secondary, #888); -} - -.mode-toggle.active .mode-icon-local { - color: var(--yj-accent, #ffd43b); -} - ul { list-style-type: none; } diff --git a/frontend/index.html b/frontend/index.html index c76149d..2878466 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -15,13 +15,6 @@

YellowJacket

Music how it was meant to bee.

-
- -
-
-
- -
diff --git a/frontend/index.ts b/frontend/index.ts index 203b9c1..985157e 100644 --- a/frontend/index.ts +++ b/frontend/index.ts @@ -23,6 +23,7 @@ import '@components/autotag-view/autotag-view.ts'; import '@components/first-run-wizard/first-run-wizard.ts'; import '@components/jobs/job-indicator.ts'; import '@components/jobs/jobs-view.ts'; +import '@components/wanted-view/wanted-view.ts'; import '@awesome.me/webawesome/dist/styles/themes/default.css'; import '@awesome.me/webawesome/dist/components/icon/icon.js'; import { setBasePath } from '@awesome.me/webawesome/dist/webawesome.js'; @@ -36,7 +37,6 @@ import '@store/theme-store'; // Importing the keyboard shortcut service triggers initialization: // registers the document keydown listener for global shortcuts. import './src/services/keyboard-shortcut-service'; -import { exploreSettings } from '@store/explore-settings'; import { hasTrackPayload, getDragPayload, @@ -64,6 +64,7 @@ const VIEW_TAGS: Record = { playlists: 'playlist-view', explore: 'explore-view', autotag: 'autotag-view', + wanted: 'wanted-view', jobs: 'jobs-view', settings: 'config-page', }; @@ -310,24 +311,3 @@ if (queueButton && queuePanel) { // or timing assumptions needed. void Player.EmitCurrentState(); void Queue.EmitCurrentState(); - -// --------------------------------------------------------------------------- -// Library Only toggle -// --------------------------------------------------------------------------- -const libraryOnlyToggle = document.getElementById('library-only-toggle'); - -if (libraryOnlyToggle) { - // Sync initial state. - if (exploreSettings.libraryOnly) { - libraryOnlyToggle.classList.add('active'); - } - - libraryOnlyToggle.addEventListener('click', () => { - exploreSettings.toggle(); - libraryOnlyToggle.classList.toggle('active', exploreSettings.libraryOnly); - }); - - exploreSettings.subscribe(() => { - libraryOnlyToggle.classList.toggle('active', exploreSettings.libraryOnly); - }); -} diff --git a/frontend/src/components/config-page/config-page.ts b/frontend/src/components/config-page/config-page.ts index 3d91ee8..f5a4ddd 100644 --- a/frontend/src/components/config-page/config-page.ts +++ b/frontend/src/components/config-page/config-page.ts @@ -32,6 +32,7 @@ import { import './config-field'; import './config-section'; +import './download-clients'; import './shortcut-capture'; import { shortcutsStore } from '../../store/shortcuts-store'; import { ShortcutsController } from '../../store/controllers/shortcuts-controller'; @@ -959,6 +960,12 @@ export class ConfigPage extends LitElement { font-variant-numeric: tabular-nums; } + .tier-detail { + color: var(--yj-text-tertiary, #888); + font-size: var(--yj-text-xs, 11px); + font-variant-numeric: tabular-nums; + } + .tier-error { color: var(--yj-accent-error, #f44); font-size: var(--yj-text-xs, 11px); @@ -1457,6 +1464,7 @@ export class ConfigPage extends LitElement { ${this.renderFavoritesSection()} ${this.renderTrackListSection()} ${this.renderShortcutsSection()} + ${this.renderLibrarySection()} `; } @@ -1499,6 +1507,9 @@ export class ConfigPage extends LitElement { ${t.state === 'running' && t.total > 0 ? html`${t.completed}/${t.total}` : nothing} + ${t.state === 'running' && t.detail + ? html`${t.detail}` + : nothing} ${t.state === 'error' ? html`${t.error}` : nothing} diff --git a/frontend/src/components/config-page/download-clients.ts b/frontend/src/components/config-page/download-clients.ts new file mode 100644 index 0000000..c5f10ac --- /dev/null +++ b/frontend/src/components/config-page/download-clients.ts @@ -0,0 +1,514 @@ +import { LitElement, html, css, nothing } from 'lit'; +import { customElement, state } from 'lit/decorators.js'; +import '@awesome.me/webawesome/dist/components/button/button.js'; +import '@awesome.me/webawesome/dist/components/input/input.js'; +import '@awesome.me/webawesome/dist/components/select/select.js'; +import '@awesome.me/webawesome/dist/components/option/option.js'; +import '@awesome.me/webawesome/dist/components/switch/switch.js'; +import '@awesome.me/webawesome/dist/components/spinner/spinner.js'; +import '@awesome.me/webawesome/dist/components/callout/callout.js'; +import { designTokens } from '../../styles/tokens.css'; +import type { + DownloadDescriptor, + DownloadProvider, +} from '@store/download-store'; +import { downloadStore } from '@store/download-store'; +import './config-section'; + +/** + * Download client configuration. + * + * The forms are rendered from the descriptors the backend publishes, not + * from anything hard-coded here, so adding a provider on the backend + * gives it a settings UI with no frontend change. That is also why + * secret fields render as password inputs purely on the descriptor's + * say-so — the frontend never needs to know which services have keys. + */ +@customElement('download-clients') +export class DownloadClients extends LitElement { + @state() + private providers: DownloadProvider[] = []; + + @state() + private descriptors: DownloadDescriptor[] = []; + + /** Provider being edited, or 'new' while adding one. */ + @state() + private editing: number | 'new' | null = null; + + /** Kind selected in the add form. */ + @state() + private newKind = ''; + + /** Working copy of the form's field values. */ + @state() + private draft: Record = {}; + + @state() + private draftName = ''; + + /** Per-provider connection test results, keyed by provider ID. */ + @state() + private testResults: Record = {}; + + @state() + private testing: number | null = null; + + @state() + private errorMessage = ''; + + private unsubscribe: (() => void) | null = null; + + override connectedCallback(): void { + super.connectedCallback(); + + this.unsubscribe = downloadStore.subscribe(() => this.syncFromStore()); + + void downloadStore.init().then(() => this.syncFromStore()); + } + + override disconnectedCallback(): void { + super.disconnectedCallback(); + + this.unsubscribe?.(); + this.unsubscribe = null; + } + + private syncFromStore(): void { + this.providers = downloadStore.providers; + this.descriptors = downloadStore.descriptors; + } + + static override styles = [ + designTokens, + css` + :host { + display: block; + } + + .clients { + display: flex; + flex-direction: column; + gap: 0.6em; + } + + .client { + display: grid; + grid-template-columns: 1fr auto; + gap: 0.75em; + align-items: center; + padding: 0.7em 0.85em; + border: 1px solid var(--wa-color-surface-border, #333); + border-radius: 8px; + } + + .client-name { + font-weight: 600; + } + + .client-meta { + font-size: 0.82em; + opacity: 0.7; + margin-top: 0.15em; + } + + .client-actions { + display: flex; + gap: 0.4em; + align-items: center; + } + + .test-result { + font-size: 0.8em; + margin-top: 0.35em; + } + + .test-result.ok { + color: var(--wa-color-success-fill-loud, #4c9f70); + } + + .test-result.fail { + color: var(--wa-color-danger-fill-loud, #c65f5f); + } + + .form { + display: flex; + flex-direction: column; + gap: 0.7em; + padding: 0.9em; + border: 1px solid var(--wa-color-surface-border, #333); + border-radius: 8px; + margin-top: 0.6em; + } + + .form-actions { + display: flex; + gap: 0.5em; + justify-content: flex-end; + margin-top: 0.3em; + } + + .requires { + font-size: 0.82em; + opacity: 0.75; + } + + .empty { + opacity: 0.7; + font-size: 0.9em; + padding: 0.5em 0; + } + + .add-row { + margin-top: 0.8em; + } + `, + ]; + + override render() { + return html` + + ${this.errorMessage + ? html`${this.errorMessage}` + : nothing} + +
+ ${this.providers.length === 0 && this.editing !== 'new' + ? html`
No download clients connected.
` + : nothing} + ${this.providers.map((provider) => this.renderProvider(provider))} +
+ + ${this.editing === 'new' + ? this.renderAddForm() + : html` +
+ + Add download client + +
+ `} + + `; + } + + private renderProvider(provider: DownloadProvider) { + const descriptor = this.descriptorFor(provider.kind); + const test = this.testResults[provider.id]; + + if (this.editing === provider.id) { + return this.renderEditForm(provider); + } + + return html` +
+
+
${provider.name}
+
+ ${descriptor?.name ?? provider.kind} · + ${provider.enabled ? 'Enabled' : 'Disabled'} · + priority ${provider.priority} +
+ ${test + ? html`
+ ${test.message} +
` + : nothing} +
+
+ this.testProvider(provider)} + > + ${this.testing === provider.id + ? html`` + : 'Test'} + + this.startEdit(provider)} + > + Edit + + this.deleteProvider(provider)} + > + Remove + +
+
+ `; + } + + private renderAddForm() { + const descriptor = this.descriptorFor(this.newKind); + + return html` +
+ + ${this.descriptors.map( + (d) => html`${d.name}`, + )} + + + ${descriptor + ? html` +
+ ${descriptor.summary} + ${descriptor.requiresExternal + ? html`
Requires a running + ${descriptor.requiresExternal} instance.` + : nothing} +
+ + { + this.draftName = (e.target as HTMLInputElement).value; + }} + > + + ${this.renderFields(descriptor)} + ` + : nothing} + +
+ + Cancel + + + Add + +
+
+ `; + } + + private renderEditForm(provider: DownloadProvider) { + const descriptor = this.descriptorFor(provider.kind); + + return html` +
+ { + this.draftName = (e.target as HTMLInputElement).value; + }} + > + + ${descriptor ? this.renderFields(descriptor) : nothing} + + { + this.draft['__priority'] = (e.target as HTMLInputElement).value; + }} + > + + { + this.draft['__enabled'] = (e.target as HTMLInputElement) + .checked + ? '1' + : ''; + }} + > + Enabled + + +
+ + Cancel + + this.saveEdit(provider)} + > + Save + +
+
+ `; + } + + /** Renders one input per descriptor field. */ + private renderFields(descriptor: DownloadDescriptor) { + return (descriptor.fields ?? []).map( + (field) => html` + { + this.draft = { + ...this.draft, + [field.key]: (e.target as HTMLInputElement).value, + }; + }} + > + ${field.help ? html`${field.help}` : nothing} + + `, + ); + } + + private descriptorFor(kind: string): DownloadDescriptor | undefined { + return this.descriptors.find((d) => d.kind === kind); + } + + private startAdd = () => { + this.editing = 'new'; + this.errorMessage = ''; + this.newKind = this.descriptors[0]?.kind ?? ''; + this.draftName = this.descriptorFor(this.newKind)?.name ?? ''; + this.draft = this.defaultsFor(this.newKind); + }; + + private startEdit(provider: DownloadProvider) { + this.editing = provider.id; + this.errorMessage = ''; + this.draftName = provider.name; + // Secrets are never sent back to the frontend, so their fields + // start blank; a blank secret on save means "leave it alone" + // rather than "clear it". + this.draft = { ...(provider.settings ?? {}) }; + } + + private cancelEdit = () => { + this.editing = null; + this.draft = {}; + this.errorMessage = ''; + }; + + private onKindChange = (event: Event) => { + this.newKind = (event.target as HTMLInputElement).value; + this.draftName = this.descriptorFor(this.newKind)?.name ?? ''; + this.draft = this.defaultsFor(this.newKind); + }; + + private defaultsFor(kind: string): Record { + const descriptor = this.descriptorFor(kind); + const out: Record = {}; + + for (const field of descriptor?.fields ?? []) { + if (field.default) out[field.key] = field.default; + } + + return out; + } + + private saveNew = async () => { + this.errorMessage = ''; + + try { + await downloadStore.addProvider( + this.newKind, + this.draftName || this.newKind, + this.cleanDraft(), + ); + + this.cancelEdit(); + } catch (err) { + this.errorMessage = String(err); + } + }; + + private async saveEdit(provider: DownloadProvider) { + this.errorMessage = ''; + + const priority = this.draft['__priority'] + ? Number(this.draft['__priority']) + : provider.priority; + + const enabled = + '__enabled' in this.draft + ? this.draft['__enabled'] === '1' + : provider.enabled; + + try { + await downloadStore.updateProvider( + provider.id, + this.draftName || provider.name, + enabled, + priority, + this.cleanDraft(), + ); + + this.cancelEdit(); + } catch (err) { + this.errorMessage = String(err); + } + } + + /** Strips the form's internal bookkeeping keys before saving. */ + private cleanDraft(): Record { + const out: Record = {}; + + for (const [key, value] of Object.entries(this.draft)) { + if (!key.startsWith('__')) out[key] = value; + } + + return out; + } + + private async deleteProvider(provider: DownloadProvider) { + this.errorMessage = ''; + + try { + await downloadStore.deleteProvider(provider.id); + } catch (err) { + this.errorMessage = String(err); + } + } + + private async testProvider(provider: DownloadProvider) { + this.testing = provider.id; + + try { + await downloadStore.testProvider(provider.id); + + this.testResults = { + ...this.testResults, + [provider.id]: { ok: true, message: 'Connected.' }, + }; + } catch (err) { + this.testResults = { + ...this.testResults, + [provider.id]: { ok: false, message: String(err) }, + }; + } finally { + this.testing = null; + } + } +} + +declare global { + interface HTMLElementTagNameMap { + 'download-clients': DownloadClients; + } +} diff --git a/frontend/src/components/download-picker/candidate-row.ts b/frontend/src/components/download-picker/candidate-row.ts new file mode 100644 index 0000000..037d74f --- /dev/null +++ b/frontend/src/components/download-picker/candidate-row.ts @@ -0,0 +1,254 @@ +import { LitElement, html, css, nothing } from 'lit'; +import { customElement, property } from 'lit/decorators.js'; +import '@awesome.me/webawesome/dist/components/icon/icon.js'; +import '@awesome.me/webawesome/dist/components/button/button.js'; +import { designTokens } from '../../styles/tokens.css'; +import type { DownloadCandidate } from '@store/download-store'; +import { candidateSummary, scorePercent } from '@store/download-store'; + +/** + * One candidate in the download picker. + * + * The row shows match and quality as two separate meters rather than + * one blended score, because they fail differently: a flawless copy of + * the wrong album is useless, a mediocre copy of the right one is + * merely disappointing, and only the user knows which they will accept. + * Collapsing them into a single number would make the ranking + * impossible to argue with. + */ +@customElement('candidate-row') +export class CandidateRow extends LitElement { + @property({ type: Object }) + candidate!: DownloadCandidate; + + /** Marks the row the ranking put first. */ + @property({ type: Boolean, attribute: 'is-best' }) + isBest = false; + + @property({ type: Boolean }) + busy = false; + + static override styles = [ + designTokens, + css` + :host { + display: block; + } + + .row { + display: grid; + grid-template-columns: 1fr auto; + gap: 1em; + align-items: center; + padding: 0.75em 0.9em; + border: 1px solid var(--wa-color-surface-border, #333); + border-radius: 8px; + background: var(--wa-color-surface-raised, #1c1c1c); + } + + .row.best { + border-color: var(--wa-color-brand-fill-loud, #d9a441); + } + + .title { + font-weight: 600; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + } + + .summary { + font-size: 0.85em; + opacity: 0.75; + margin-top: 0.15em; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + } + + .badges { + display: flex; + gap: 0.4em; + margin-top: 0.4em; + flex-wrap: wrap; + } + + .badge { + font-size: 0.72em; + padding: 0.1em 0.45em; + border-radius: 4px; + background: rgba(255, 255, 255, 0.08); + white-space: nowrap; + } + + .badge.best { + background: var(--wa-color-brand-fill-loud, #d9a441); + color: #111; + font-weight: 600; + } + + .badge.warn { + background: rgba(217, 119, 65, 0.25); + } + + .meters { + display: grid; + grid-template-columns: auto 1fr auto; + gap: 0.3em 0.5em; + align-items: center; + margin-top: 0.5em; + font-size: 0.75em; + max-width: 340px; + } + + .meter-label { + opacity: 0.7; + } + + .track { + height: 5px; + border-radius: 3px; + background: rgba(255, 255, 255, 0.1); + overflow: hidden; + } + + .fill { + height: 100%; + border-radius: 3px; + transition: width 150ms ease; + } + + .fill.match { + background: var(--wa-color-success-fill-loud, #4c9f70); + } + + .fill.match.low { + background: var(--wa-color-warning-fill-loud, #d97741); + } + + .fill.quality { + background: var(--wa-color-brand-fill-loud, #6a8cc7); + } + + .value { + font-variant-numeric: tabular-nums; + opacity: 0.85; + } + `, + ]; + + /** Match below this reads as "probably not what you asked for". */ + private static readonly LOW_MATCH = 0.7; + + override render() { + const candidate = this.candidate; + if (!candidate) return nothing; + + const match = candidate.match?.overall ?? 0; + const quality = candidate.quality?.overall ?? 0; + + return html` +
+
+
+ ${candidate.title} +
+
${candidateSummary(candidate)}
+ ${this.renderBadges()} +
+ Match +
+
+
+ ${scorePercent(match)} + + Quality +
+
+
+ ${scorePercent(quality)} +
+
+ + + Download + +
+ `; + } + + private renderBadges() { + const candidate = this.candidate; + const badges = []; + + if (this.isBest) { + badges.push(html`Best match`); + } + + // An unanchored match is a guess: there was no MusicBrainz ID to + // check the result against, so the score cannot mean much and + // saying so is more honest than showing a confident number. + if (candidate.match && !candidate.match.anchored) { + badges.push( + html` + Unverified + `, + ); + } + + if (candidate.quality?.mixed) { + badges.push( + html` + Mixed formats + `, + ); + } + + const completeness = candidate.match?.completeness ?? 1; + + if (completeness < 1 && completeness > 0) { + badges.push( + html` + ${scorePercent(completeness)} of tracks + `, + ); + } + + if (candidate.protocol && candidate.protocol !== 'direct') { + badges.push(html`${candidate.protocol}`); + } + + return badges.length > 0 + ? html`
${badges}
` + : nothing; + } + + private onPick() { + this.dispatchEvent( + new CustomEvent('candidate-pick', { + detail: { candidateId: this.candidate.id }, + bubbles: true, + composed: true, + }), + ); + } +} + +declare global { + interface HTMLElementTagNameMap { + 'candidate-row': CandidateRow; + } +} diff --git a/frontend/src/components/download-picker/download-picker.ts b/frontend/src/components/download-picker/download-picker.ts new file mode 100644 index 0000000..8b7c1b6 --- /dev/null +++ b/frontend/src/components/download-picker/download-picker.ts @@ -0,0 +1,278 @@ +import { LitElement, html, css, nothing } from 'lit'; +import { customElement, property, state } from 'lit/decorators.js'; +import '@awesome.me/webawesome/dist/components/dialog/dialog.js'; +import '@awesome.me/webawesome/dist/components/button/button.js'; +import '@awesome.me/webawesome/dist/components/spinner/spinner.js'; +import '@awesome.me/webawesome/dist/components/callout/callout.js'; +import { designTokens } from '../../styles/tokens.css'; +import type { DownloadCandidate } from '@store/download-store'; +import { downloadStore } from '@store/download-store'; +import type { download } from '@go/models'; +import './candidate-row'; + +/** + * The "find this album" dialog: searches every enabled download client, + * ranks what comes back, and asks the user to choose. + * + * When the pipeline finds a clear winner it starts on its own and this + * dialog reports that rather than asking a question with one obvious + * answer. When it does not — two equally good candidates, or a free-text + * request with nothing to verify against — the choice is the user's, + * because guessing wrong puts the wrong files in their library. + */ +@customElement('download-picker') +export class DownloadPicker extends LitElement { + @property({ type: Boolean, reflect: true }) + open = false; + + /** Library the imported files belong to. */ + @property({ type: Number, attribute: 'library-id' }) + libraryId = 0; + + @property({ type: String }) + artist = ''; + + @property({ type: String }) + album = ''; + + /** MusicBrainz release-group ID, when the caller has one. */ + @property({ type: String, attribute: 'release-group-mbid' }) + releaseGroupMbid = ''; + + @property({ type: String, attribute: 'release-mbid' }) + releaseMbid = ''; + + /** + * Expected tracklist. Supplying it is what makes the result + * trustworthy: without it there is nothing to check a candidate + * against, and the pipeline will never auto-pick. + */ + @property({ type: Array }) + expected: download.ExpectedTrack[] = []; + + @state() + private searching = false; + + @state() + private candidates: DownloadCandidate[] = []; + + @state() + private requestId = ''; + + @state() + private autoPicked = false; + + @state() + private picking = false; + + @state() + private errorMessage = ''; + + static override styles = [ + designTokens, + css` + :host { + display: contents; + } + + .heading { + display: flex; + flex-direction: column; + gap: 0.15em; + margin-bottom: 1em; + } + + .album { + font-size: 1.05em; + font-weight: 600; + } + + .artist { + opacity: 0.75; + font-size: 0.9em; + } + + .status { + display: flex; + align-items: center; + gap: 0.6em; + padding: 1.5em 0; + justify-content: center; + opacity: 0.85; + } + + .list { + display: flex; + flex-direction: column; + gap: 0.6em; + max-height: 55vh; + overflow-y: auto; + } + + .footnote { + margin-top: 1em; + font-size: 0.8em; + opacity: 0.65; + } + `, + ]; + + override updated(changed: Map) { + if (changed.has('open') && this.open) { + void this.search(); + } + } + + /** Runs the search that populates the dialog. */ + private async search(): Promise { + this.searching = true; + this.errorMessage = ''; + this.candidates = []; + this.autoPicked = false; + + try { + const result = await downloadStore.start({ + libraryId: this.libraryId, + releaseMbid: this.releaseMbid, + releaseGroupMbid: this.releaseGroupMbid, + artist: this.artist, + album: this.album, + query: '', + expected: this.expected ?? [], + } as download.SearchRequest); + + this.requestId = result.requestId; + this.candidates = result.candidates ?? []; + this.autoPicked = result.autoPicked; + } catch (err) { + this.errorMessage = String(err); + } finally { + this.searching = false; + } + } + + private async onPick(event: CustomEvent<{ candidateId: string }>) { + if (this.picking) return; + + this.picking = true; + this.errorMessage = ''; + + try { + await downloadStore.pick(this.requestId, event.detail.candidateId); + this.close(); + } catch (err) { + this.errorMessage = String(err); + } finally { + this.picking = false; + } + } + + private close() { + this.open = false; + + this.dispatchEvent( + new CustomEvent('picker-close', { bubbles: true, composed: true }), + ); + } + + override render() { + return html` + this.close()} + > +
+ ${this.album || 'Unknown album'} + ${this.artist} +
+ + ${this.renderBody()} + + this.close()}> + Close + + + `; + } + + private renderBody() { + if (this.errorMessage) { + return html` + ${this.errorMessage} + `; + } + + if (this.searching) { + return html` +
+ + Searching your download clients… +
+ `; + } + + if (this.autoPicked) { + return html` + + Found a clear match and started downloading it. Progress is + in the background jobs panel. + + `; + } + + if (this.candidates.length === 0) { + return html` + + Nothing found. Try a different spelling, or connect more + download clients in Settings. + + `; + } + + return html` +
+ ${this.candidates.map( + (candidate, index) => html` + + `, + )} +
+ ${this.renderFootnote()} + `; + } + + private renderFootnote() { + const best = this.candidates[0]; + if (!best?.match) return nothing; + + // Say plainly why nothing was auto-picked, so the dialog does + // not look like it is asking a question it could have answered. + if (!best.match.anchored) { + return html` +
+ This search had no MusicBrainz match to verify against, so + these results could not be checked automatically. +
+ `; + } + + return html` +
+ Downloads are checked and tagged before they are added to your + library. +
+ `; + } +} + +declare global { + interface HTMLElementTagNameMap { + 'download-picker': DownloadPicker; + } +} diff --git a/frontend/src/components/explore-album-details/explore-album-details.ts b/frontend/src/components/explore-album-details/explore-album-details.ts index b6cede1..b0e9015 100644 --- a/frontend/src/components/explore-album-details/explore-album-details.ts +++ b/frontend/src/components/explore-album-details/explore-album-details.ts @@ -8,18 +8,20 @@ import { } from '@go/explore/Service'; import { GetAlbumTracks } from '@go/library/Library'; import { library } from '@go/models'; -import type { explore } from '@go/models'; +import type { download, explore } from '@go/models'; type MBReleaseGroup = explore.MBReleaseGroup; type MBRelease = explore.MBRelease; type MBTrack = explore.MBTrack; import { exploreCache } from '../../store/explore-cache'; -import { exploreSettings } from '../../store/explore-settings'; import { libraryStore } from '../../store/library-store'; import { artistLink, exploreLinkStyles } from '../../utils/explore-link'; import { EventsOn } from '@runtime/runtime'; import { Events } from '../../events'; import '@awesome.me/webawesome/dist/components/icon/icon.js'; import '../library-status-indicator/library-status-indicator.js'; +import '@awesome.me/webawesome/dist/components/button/button.js'; +import '../download-picker/download-picker'; +import { downloadStore } from '../../store/download-store'; /* ── Utility functions (duplicated per Knowledge Pattern #9 — no cross-component imports) ── */ @@ -112,6 +114,18 @@ export class ExploreAlbumDetails extends LitElement { @state() private selectedVersionKey: string = ''; @state() private coverArtURL = ''; + /** Open state of the "find this album" dialog. */ + @state() private pickerOpen = false; + + /** True once a download client is configured and enabled. */ + @state() private canDownload = false; + + /** True when this album is already on the wanted list. */ + @state() private isWanted = false; + + /** Unsubscribe handle for the download store. */ + private downloadUnsub: (() => void) | null = null; + /* ── Styles ── */ static override styles = [ @@ -454,7 +468,6 @@ export class ExploreAlbumDetails extends LitElement { /* ── Lifecycle ── */ - private unsubSettings?: () => void; private unsubReleasesReady?: () => void; /** Release-group MBIDs whose AlbumReleasesReady event we've handled, * so a background BrowseReleases fetch re-hydrates versions once. */ @@ -469,12 +482,16 @@ export class ExploreAlbumDetails extends LitElement { void this.loadAllData(); } - this.unsubSettings = exploreSettings.subscribe(() => { - // Re-run data loading — library-only mode may show/hide - // API-sourced content or hydrate from local tracks. - if (this.releaseGroupMBID || this.localAlbumId) { - void this.loadAllData(); - } + // The download button only appears once a client is connected, + // so this tracks the provider list rather than assuming. + this.downloadUnsub = downloadStore.subscribe(() => { + this.canDownload = downloadStore.available; + this.syncWanted(); + }); + + void downloadStore.init().then(() => { + this.canDownload = downloadStore.available; + this.syncWanted(); }); // A background BrowseReleases fetch (cold album, versions + @@ -496,7 +513,8 @@ export class ExploreAlbumDetails extends LitElement { override disconnectedCallback() { super.disconnectedCallback(); - this.unsubSettings?.(); + this.downloadUnsub?.(); + this.downloadUnsub = null; this.unsubReleasesReady?.(); if (this.releasesFallbackTimer) clearTimeout(this.releasesFallbackTimer); } @@ -631,14 +649,6 @@ export class ExploreAlbumDetails extends LitElement { // tracklist; the return value isn't currently consumed. await this.hydrateFromLibrary(mbid); - // Library-only mode: local data is all we show. - if (exploreSettings.libraryOnly) { - this.loadingInfo = false; - this.loadingReleases = false; - console.log(`[explore-album] loaded (library-only): "${this.albumName}"`); - return; - } - // Phase 2: fire API calls independently so each section // renders as its data arrives. Allow the versions section one // background-fetch re-fetch and arm a fallback so it can't spin @@ -1336,7 +1346,7 @@ export class ExploreAlbumDetails extends LitElement { * - localAlbumId set → owned * - releaseGroup.inLibrary set → owned (backend cross-ref) * - cachedAlbums has MBID match → owned - * - any selected version has → owned (covers library-only mode + * - any selected version has → owned (covers local-only albums * a track marked inLibrary where releaseGroup may be null) * - else → not owned * @@ -1478,11 +1488,137 @@ export class ExploreAlbumDetails extends LitElement { > ${this.renderAlbumMeta()} + ${this.renderDownloadAction()} + ${this.renderPicker()} `; } + /** + * Offers to acquire the album, but only when the user has actually + * connected a download client and does not already own it. Showing + * the button otherwise would advertise a feature that cannot work. + */ + private renderDownloadAction() { + if (this.albumLibraryStatus() === 'in-library') return nothing; + + return html` +
+ ${this.canDownload + ? html` + { + this.pickerOpen = true; + }} + > + + Find this album + + ` + : nothing} + ${this.renderWantAction()} +
+ `; + } + + /** + * Adds the album to the wanted list, which is the answer to "look + * for it, but not right now". + * + * Unlike the download button this shows whether or not a client is + * connected: wanting something is a durable statement about the + * library, and it stays true — and stays queued — until a client + * exists to act on it. + */ + private renderWantAction() { + if (!this.releaseGroupMBID) return nothing; + + const want = downloadStore.wantFor(this.releaseGroupMBID); + + return html` + void this.toggleWanted(want?.id)} + > + + ${this.isWanted ? 'Wanted' : 'Want this'} + + `; + } + + /** Reflects the store's view of whether this album is wanted. */ + private syncWanted(): void { + this.isWanted = this.releaseGroupMBID + ? downloadStore.isWanted(this.releaseGroupMBID) + : false; + } + + private async toggleWanted(wantId: number | undefined): Promise { + if (!this.releaseGroupMBID) return; + + try { + if (wantId) { + await downloadStore.removeWant(wantId); + } else { + await downloadStore.addWant({ + mbid: this.releaseGroupMBID, + entity: 'release-group', + libraryId: libraryStore.getSelectedLibraryId() ?? 0, + artist: this.releaseGroup?.artistCredit ?? '', + title: this.albumName, + scope: 'future', + secondary: false, + } as download.WantRequest); + } + } catch (err) { + console.error('Could not update the wanted list:', err); + } + + this.syncWanted(); + } + + private renderPicker() { + if (!this.pickerOpen) return nothing; + + const tracks = this.currentTracks(); + + return html` + ({ + position: t.position || index + 1, + discNumber: t.discNumber ?? 0, + title: t.title, + artist: '', + lengthMillis: t.length ?? 0, + }))} + @picker-close=${() => { + this.pickerOpen = false; + }} + > + `; + } + + /** Tracks of the version currently selected in the dropdown. */ + private currentTracks(): MBTrack[] { + const entry = this.versionEntries.find( + (e) => e.key === this.selectedVersionKey, + ); + + return entry?.tracks ?? []; + } + private renderAlbumMeta() { if (this.loadingInfo) { return html` @@ -1683,19 +1816,6 @@ export class ExploreAlbumDetails extends LitElement { } const current = this.currentVersion(); if (!current) { - // In library-only mode with no local tracks, show a gentle message. - if (exploreSettings.libraryOnly) { - return html` -
-

Tracklist

-
- This album is not in your library. -
-
- `; - } return html`

Tracklist

diff --git a/frontend/src/components/explore-artist-details/explore-artist-details.ts b/frontend/src/components/explore-artist-details/explore-artist-details.ts index 63649a4..70fbdf5 100644 --- a/frontend/src/components/explore-artist-details/explore-artist-details.ts +++ b/frontend/src/components/explore-artist-details/explore-artist-details.ts @@ -9,7 +9,6 @@ import { SimilarArtists, GetArtistImageURL, GetArtistImageCachedPath, - GetLibrarySimilarArtists, GetThumbnail, GetThumbnails, GetTrackThumbnail, @@ -24,8 +23,9 @@ type LBTopRecording = explore.LBTopRecording; type LBTopReleaseGroup = explore.LBTopReleaseGroup; type LBSimilarArtist = explore.LBSimilarArtist; import { exploreCache } from '../../store/explore-cache'; -import { exploreSettings } from '../../store/explore-settings'; import { libraryStore } from '../../store/library-store'; +import { downloadStore } from '../../store/download-store'; +import '@awesome.me/webawesome/dist/components/button/button.js'; import { trackLink, exploreLinkStyles } from '../../utils/explore-link'; import { GetAlbumsByArtist } from '@go/library/Library'; import { EventsOn } from '@runtime/runtime'; @@ -194,6 +194,10 @@ export class ExploreArtistDetails extends LitElement { object-fit: cover; } + .artist-follow { + margin-top: 10px; + } + .artist-info { display: flex; flex-direction: column; @@ -830,7 +834,6 @@ export class ExploreArtistDetails extends LitElement { /* ── Lifecycle ── */ - private unsubSettings?: () => void; private unsubDiscogReady?: () => void; private unsubSimilarReady?: () => void; /** MBIDs whose ArtistSimilarReady event we've already handled, so a @@ -843,19 +846,20 @@ export class ExploreArtistDetails extends LitElement { * background discography fetch never signals readiness. */ private discogFallbackTimer?: number; + /** Unsubscribe handle for the wanted list. */ + private unsubWanted: (() => void) | null = null; + override connectedCallback() { super.connectedCallback(); if (this.artistMBID || this.localArtistId) { void this.loadAllData(); } - // Re-render when library-only mode toggles. - this.unsubSettings = exploreSettings.subscribe(() => { - this.requestUpdate(); - if (this.artistMBID || this.localArtistId) { - void this.loadAllData(); - } - }); + // Keep the follow button in step with the wanted list, which a + // background reconcile pass can change without this page doing + // anything. + this.unsubWanted = downloadStore.subscribe(() => this.requestUpdate()); + void downloadStore.init().then(() => this.requestUpdate()); // A background discography fetch (top tracks / top releases for an // artist that wasn't indexed yet) finished — re-fetch those two @@ -893,7 +897,8 @@ export class ExploreArtistDetails extends LitElement { override disconnectedCallback() { super.disconnectedCallback(); - this.unsubSettings?.(); + this.unsubWanted?.(); + this.unsubWanted = null; this.unsubDiscogReady?.(); this.unsubSimilarReady?.(); if (this.discogFallbackTimer) clearTimeout(this.discogFallbackTimer); @@ -1027,90 +1032,6 @@ export class ExploreArtistDetails extends LitElement { // Phase 0: hydrate from caches (instant, no Go calls). this.hydrateFromCache(mbid); - if (exploreSettings.libraryOnly) { - // Library-only mode: no external API calls. - // Discography comes from library store (already hydrated). - // Similar artists from pre-computed DB table. - this.loadingArtist = false; - this.loadingTracks = false; - this.loadingTopReleases = false; - this.loadingReleases = false; - this.loadingSimilar = false; - - // If the library store hasn't eagerly fetched yet, - // await it and re-hydrate. Covers the race between - // navigation and the deferred eagerFetch on DOMContentLoaded. - if (!libraryStore.cachedArtists || !libraryStore.cachedAlbums) { - try { - const pending: Promise[] = []; - if (!libraryStore.cachedArtists) { - pending.push(libraryStore.getArtists()); - } - if (!libraryStore.cachedAlbums) { - pending.push(libraryStore.getAlbums()); - } - await Promise.all(pending); - this.hydrateFromCache(mbid); - } catch { - // Ignore — we'll fall through to the Wails-cached path. - } - } - - // Artist image: if hydrateFromCache didn't find one (e.g. - // the library store's cached artist row has an empty - // ImageMedium because the on-disk file post-dates the - // store's last fetch), fall back to a disk-only Wails - // call. This just asks the backend whether - // /artist-images/.../primary_md.jpg exists — no network, - // no base64 transfer. - if (!this.artistImageURL && mbid) { - void GetArtistImageCachedPath(mbid) - .then((url) => { - if (url) { - this.artistImageURL = url; - } else { - // Final fallback: album art by artist name. - this.fallbackArtistImageFromAlbumArt(); - } - }) - .catch(() => { - this.fallbackArtistImageFromAlbumArt(); - }); - } - - // Fetch library-only similar artists (single Go call, no external API). - try { - const similar = await GetLibrarySimilarArtists(mbid); - // Dedupe by MBID as a safety net — the backend query - // should already return unique rows but multiple library - // artists can share an MBID (ensemble credits), so this - // guards against any future query regression. - const seen = new Set(); - const deduped: LBSimilarArtist[] = []; - for (const s of similar ?? []) { - const key = s.artistMbid || s.name; - if (!key || seen.has(key)) continue; - seen.add(key); - deduped.push(s); - } - this.similarArtists = deduped; - - // Resolve similar artist images from library cache — - // no Go calls, no network. These artists are all in - // the library (that's the filter GetLibrarySimilarArtists - // applies), so the library store has their image paths. - this.seedSimilarArtistImagesFromLibrary(); - } catch { - this.similarArtists = []; - } - - console.log( - `[explore-artist] loaded (library-only): "${this.artistName}"`, - ); - - return; - } - // Fresh load for this artist: allow the top sections one // background-fetch re-fetch, and arm a fallback so they can't spin // forever if ArtistDiscographyReady never arrives. @@ -1604,7 +1525,7 @@ export class ExploreArtistDetails extends LitElement { /** * Populate similarImageURLs for the current similarArtists list * using only library-store data and disk-cached artist images. - * Makes ZERO network calls — safe for library-only mode. + * Makes ZERO network calls. * * Resolution priority per artist: * 1. libraryStore.cachedArtists[mbid].ImageMedium (in-memory) @@ -1694,10 +1615,10 @@ export class ExploreArtistDetails extends LitElement { } private async fetchSimilarArtistImages() { - // Phase 1: instant seed from library store + disk cache. - // This mirrors the library-only path so any card whose image - // is already on disk appears immediately, without waiting - // for a network-enabled GetArtistImageURL round-trip. + // Phase 1: instant seed from library store + disk cache, so any + // card whose image is already on disk appears immediately, + // without waiting for a network-enabled GetArtistImageURL + // round-trip. await this.seedSimilarArtistImagesFromLibrary(); // Phase 2: network fetch for any similars still without @@ -1961,9 +1882,10 @@ export class ExploreArtistDetails extends LitElement { ? html`
${this.artist.name}
` : nothing} ${this.renderArtistMeta()} - ${this.artist?.popularity && this.artist.popularity > 0 && !exploreSettings.libraryOnly + ${this.artist?.popularity && this.artist.popularity > 0 ? html`${formatListenCount(this.artist.popularity)} plays on ListenBrainz` : nothing} + ${this.renderFollowAction()}
@@ -1973,6 +1895,61 @@ export class ExploreArtistDetails extends LitElement { `; } + /** + * Subscribes to an artist: their new releases go on the wanted list + * as they come out. + * + * The default is new releases only. Following an artist should not + * silently queue forty albums — someone who wants the back + * catalogue can widen it from the wanted list, and will not be + * surprised by having done so. + */ + private renderFollowAction() { + if (!this.artistMBID) return nothing; + + const want = downloadStore.wantFor(this.artistMBID); + + return html` +
+ void this.toggleFollow(want?.id)} + > + + ${want ? 'Following' : 'Follow for new releases'} + +
+ `; + } + + private async toggleFollow(wantId: number | undefined): Promise { + if (!this.artistMBID) return; + + try { + if (wantId) { + await downloadStore.removeWant(wantId); + } else { + await downloadStore.addWant({ + mbid: this.artistMBID, + entity: 'artist', + libraryId: libraryStore.getSelectedLibraryId() ?? 0, + artist: this.displayName, + title: this.displayName, + scope: 'future', + secondary: false, + } as never); + } + } catch (err) { + console.error('Could not update the wanted list:', err); + } + + this.requestUpdate(); + } + private renderArtistMeta() { if (this.loadingArtist) { return html` 0; const hasReleases = !this.loadingTopReleases && this.topReleaseGroups.length > 0; const tracksLoading = this.loadingTracks; @@ -2362,10 +2336,8 @@ export class ExploreArtistDetails extends LitElement { return nothing; } - // Library-only mode: show all library-matching similar artists - // (up to the 20 stored per seed). Online mode: cap at 10 to - // avoid a very long list. - const maxSimilar = exploreSettings.libraryOnly ? 20 : 10; + // Cap the similar-artists list at 10 to avoid a very long list. + const maxSimilar = 10; const artists = this.similarArtists.slice(0, maxSimilar); const showToggle = artists.length > this.discoRowSize; const collapsed = !this.similarExpanded && showToggle; diff --git a/frontend/src/components/explore-view/explore-view.ts b/frontend/src/components/explore-view/explore-view.ts index 3ff2ac0..8b2f7ba 100644 --- a/frontend/src/components/explore-view/explore-view.ts +++ b/frontend/src/components/explore-view/explore-view.ts @@ -869,10 +869,10 @@ export class ExploreView extends LitElement { * Wails call. Called after search results are set. */ /** - * Seed the thumbnail cache from local library data only. Safe - * to call in library-only mode — does no API calls. Reads from - * cachedAlbums (by MBID) and from any `_coverArt` underscore - * field that searchLibraryCache stamped on the release group. + * Seed the thumbnail cache from local library data only — does + * no API calls. Reads from cachedAlbums (by MBID) and from any + * `_coverArt` underscore field that searchLibraryCache stamped + * on the release group. */ private seedThumbnailsFromLibrary() { if (!this.results?.releaseGroups?.length) return; @@ -906,8 +906,8 @@ export class ExploreView extends LitElement { } /** - * Seed the artist image cache from local library data only. - * Safe to call in library-only mode. Reads from cachedArtists + * Seed the artist image cache from local library data only — + * does no API calls. Reads from cachedArtists * by MBID and from any `_imageMedium`/`_imageSmall` underscore * field that searchLibraryCache stamped on the artist. Falls * back to library album art when an artist has no portrait. diff --git a/frontend/src/components/first-run-wizard/first-run-wizard.ts b/frontend/src/components/first-run-wizard/first-run-wizard.ts index 3c0b8a3..b7a3842 100644 --- a/frontend/src/components/first-run-wizard/first-run-wizard.ts +++ b/frontend/src/components/first-run-wizard/first-run-wizard.ts @@ -3,21 +3,20 @@ import { customElement, state, query } from 'lit/decorators.js'; import '@awesome.me/webawesome/dist/components/dialog/dialog.js'; import '@awesome.me/webawesome/dist/components/icon/icon.js'; import { - GetLibraryDirectory, - SetLibraryDirectory, -} from '@go/config/Config'; + AddLibrary, + GetAllLibrariesWithTrackCounts, +} from '@go/library/Library'; import { DirectoryPicker } from '@go/frontendutil/FrontendUtil'; /** * First-run setup wizard. * - * On startup it checks whether a library directory has already been - * configured. If one exists the wizard stays hidden and the app - * proceeds as normal. If none is set (fresh install), it presents a - * non-dismissable modal prompting the user to pick their music folder, - * saves it to the config, and dismisses itself. Saving the directory - * emits LibraryConfigChanged on the backend, which kicks off the - * initial scan automatically. + * On startup it checks whether any library has already been registered. + * If one exists the wizard stays hidden and the app proceeds as normal. + * If there are none (fresh install), it presents a non-dismissable modal + * prompting the user to pick their music folder, registers it through the + * library CRUD API, and dismisses itself. AddLibrary emits LibraryAdded + * and kicks off the initial scan automatically. */ @customElement('first-run-wizard') export class FirstRunWizard extends LitElement { @@ -40,13 +39,13 @@ export class FirstRunWizard extends LitElement { super.connectedCallback(); try { - const existing = await GetLibraryDirectory(); + const existing = await GetAllLibrariesWithTrackCounts(); - // A configured directory means setup is already complete. - if (existing) return; + // An existing library means setup is already complete. + if (existing && existing.length > 0) return; } catch (err) { console.error( - 'First-run wizard: failed to read library directory:', + 'First-run wizard: failed to read libraries:', err, ); @@ -249,7 +248,7 @@ export class FirstRunWizard extends LitElement { this.errorMessage = ''; try { - await SetLibraryDirectory(this.selectedDirectory); + await AddLibrary(this.selectedDirectory); this.finished = true; @@ -257,8 +256,8 @@ export class FirstRunWizard extends LitElement { this.active = false; } catch (err) { - this.errorMessage = `Could not save the folder: ${err}`; - console.error('First-run wizard: save failed:', err); + this.errorMessage = `Could not add the folder: ${err}`; + console.error('First-run wizard: add library failed:', err); } finally { this.saving = false; } diff --git a/frontend/src/components/sidebar/app-sidebar.ts b/frontend/src/components/sidebar/app-sidebar.ts index 1a6f4b3..3ecfe70 100644 --- a/frontend/src/components/sidebar/app-sidebar.ts +++ b/frontend/src/components/sidebar/app-sidebar.ts @@ -5,7 +5,7 @@ import { designTokens } from '../../styles/tokens.css'; import type { DragActiveDetail } from '@utils/drag-controller'; -type View = 'home' | 'playlists' | 'artists' | 'genres' | 'albums' | 'tracks' | 'explore' | 'autotag' | 'jobs' | 'settings'; +type View = 'home' | 'playlists' | 'artists' | 'genres' | 'albums' | 'tracks' | 'explore' | 'wanted' | 'autotag' | 'jobs' | 'settings'; interface NavItem { id: View; @@ -149,6 +149,7 @@ export class AppSidebar extends LitElement { { id: 'albums', label: 'Albums', icon: 'compact-disc' }, { id: 'tracks', label: 'Tracks', icon: 'music' }, { id: 'explore', label: 'Explore', icon: 'globe' }, + { id: 'wanted', label: 'Wanted', icon: 'bookmark' }, { id: 'autotag', label: 'Autotag', icon: 'tag' }, { id: 'jobs', label: 'Jobs', icon: 'list-check' }, { id: 'settings', label: 'Settings', icon: 'gear' }, diff --git a/frontend/src/components/wanted-view/wanted-view.ts b/frontend/src/components/wanted-view/wanted-view.ts new file mode 100644 index 0000000..289d2ee --- /dev/null +++ b/frontend/src/components/wanted-view/wanted-view.ts @@ -0,0 +1,377 @@ +import { LitElement, html, css, nothing } from 'lit'; +import { customElement, state } from 'lit/decorators.js'; +import '@awesome.me/webawesome/dist/components/icon/icon.js'; +import '@awesome.me/webawesome/dist/components/button/button.js'; +import { designTokens } from '../../styles/tokens.css'; +import { downloadStore } from '@store/download-store'; +import type { Want, WantSummary } from '@store/download-store'; +import { libraryStore } from '@store/library-store'; + +/** + * The wanted list: music the user has said they want but does not have. + * + * The list is the durable thing here, not the downloads it produces. + * Something unfindable today stays on the list and is retried on a + * backoff, so this view is mostly about making the waiting legible — + * what is being looked for, when it was last tried, and why it has not + * turned up. A row is not a failure just because it is still here. + */ +@customElement('wanted-view') +export class WantedView extends LitElement { + @state() private wants: Want[] = []; + + @state() private checking = false; + + @state() private lastSummary: WantSummary | null = null; + + private unsubscribe: (() => void) | null = null; + + static override styles = [ + designTokens, + css` + :host { + display: block; + height: 100%; + overflow-y: auto; + padding: 20px; + box-sizing: border-box; + } + + header { + display: flex; + align-items: center; + gap: 12px; + margin-bottom: 4px; + } + + h1 { + margin: 0; + font-size: 22px; + font-weight: 700; + color: var(--yj-text-primary, #fff); + flex: 1; + } + + .subtitle { + margin: 0 0 20px; + font-size: 13px; + color: var(--yj-text-secondary, #b3b3b3); + } + + h2 { + margin: 24px 0 8px; + font-size: 13px; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.06em; + color: var(--yj-text-secondary, #b3b3b3); + } + + .row { + display: flex; + align-items: center; + gap: 12px; + padding: 10px 12px; + border-radius: 6px; + background: var(--yj-bg-surface, #181818); + } + + .row + .row { + margin-top: 6px; + } + + .row-main { + flex: 1; + min-width: 0; + } + + .title { + font-size: 14px; + color: var(--yj-text-primary, #fff); + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } + + .detail { + font-size: 12px; + color: var(--yj-text-tertiary, #888); + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } + + .badge { + font-size: 11px; + padding: 2px 8px; + border-radius: 10px; + background: var(--yj-bg-overlay, rgba(255, 255, 255, 0.08)); + color: var(--yj-text-secondary, #b3b3b3); + flex-shrink: 0; + } + + .empty { + padding: 40px 20px; + text-align: center; + color: var(--yj-text-tertiary, #888); + font-size: 14px; + } + + .actions { + display: flex; + gap: 8px; + flex-shrink: 0; + } + + .summary { + font-size: 12px; + color: var(--yj-text-secondary, #b3b3b3); + margin: 8px 0 0; + } + `, + ]; + + override connectedCallback(): void { + super.connectedCallback(); + + this.unsubscribe = downloadStore.subscribe(() => { + this.wants = downloadStore.wants; + }); + + void downloadStore.init().then(() => { + this.wants = downloadStore.wants; + }); + } + + override disconnectedCallback(): void { + super.disconnectedCallback(); + + this.unsubscribe?.(); + this.unsubscribe = null; + } + + override render() { + const subscriptions = this.wants.filter((w) => w.entity === 'artist'); + const wanted = this.wants.filter( + (w) => w.entity !== 'artist' && w.state === 'wanted', + ); + const paused = this.wants.filter((w) => w.state === 'paused'); + const satisfied = this.wants.filter((w) => w.state === 'satisfied'); + + return html` +
+

Wanted

+ void this.checkNow()} + > + + ${this.checking ? 'Checking…' : 'Check now'} + + ${satisfied.length > 0 + ? html` + + void downloadStore.clearSatisfiedWants()} + > + Clear found + + ` + : nothing} +
+ +

+ Music you want but do not have. Anything that cannot be found + stays here and is looked for again later. +

+ + ${this.renderSummary()} + ${this.wants.length === 0 ? this.renderEmpty() : nothing} + ${this.renderSection( + 'Following', + subscriptions, + (w) => this.renderSubscription(w), + )} + ${this.renderSection('Looking for', wanted, (w) => this.renderWant(w))} + ${this.renderSection('Paused', paused, (w) => this.renderWant(w))} + ${this.renderSection('Found', satisfied, (w) => this.renderWant(w))} + `; + } + + private renderEmpty() { + return html` +
+ Nothing wanted yet. Use “Want this” on an album or artist to + add it here. +
+ `; + } + + private renderSummary() { + if (!this.lastSummary) return nothing; + + const s = this.lastSummary; + + const parts = [ + s.expanded > 0 ? `${s.expanded} new album${s.expanded === 1 ? '' : 's'} found` : '', + s.satisfied > 0 ? `${s.satisfied} already owned` : '', + s.started > 0 ? `${s.started} downloading` : '', + s.attempted > 0 ? `${s.attempted} searched for` : '', + ].filter(Boolean); + + return html` +

+ ${parts.length > 0 ? parts.join(' · ') : 'Nothing new this time.'} +

+ `; + } + + private renderSection( + title: string, + items: Want[], + renderer: (want: Want) => unknown, + ) { + if (items.length === 0) return nothing; + + return html` +

${title}

+ ${items.map((want) => renderer(want))} + `; + } + + /** + * An artist row is a subscription, not a queued download, so it + * shows what it covers rather than a retry count — the albums it + * produced appear in their own section. + */ + private renderSubscription(want: Want) { + return html` +
+ +
+
${want.artist || want.title || want.mbid}
+
+ ${want.scope === 'all' + ? 'Whole discography, plus new releases' + : 'New releases only'} +
+
+ Following +
+ void this.toggleScope(want)} + > + ${want.scope === 'all' ? 'New only' : 'Everything'} + + ${this.renderRemove(want)} +
+
+ `; + } + + private renderWant(want: Want) { + return html` +
+ +
+
+ ${want.artist ? `${want.artist} — ` : ''}${want.title || + want.mbid} +
+
${wantDetail(want)}
+
+
+ ${want.state === 'satisfied' + ? nothing + : html` + + void downloadStore.pauseWant( + want.id, + want.state !== 'paused', + )} + > + ${want.state === 'paused' ? 'Resume' : 'Pause'} + + `} + ${this.renderRemove(want)} +
+
+ `; + } + + private renderRemove(want: Want) { + return html` + void downloadStore.removeWant(want.id)} + > + + + `; + } + + /** Widens or narrows what an artist subscription covers. */ + private async toggleScope(want: Want): Promise { + try { + await downloadStore.addWant({ + mbid: want.mbid, + entity: 'artist', + libraryId: want.libraryId || (libraryStore.getSelectedLibraryId() ?? 0), + artist: want.artist, + title: want.title, + scope: want.scope === 'all' ? 'future' : 'all', + secondary: want.secondary, + } as never); + } catch (err) { + console.error('Could not change what this subscription covers:', err); + } + } + + private async checkNow(): Promise { + this.checking = true; + + try { + this.lastSummary = await downloadStore.reconcileWanted(); + } catch (err) { + console.error('Could not check the wanted list:', err); + } finally { + this.checking = false; + } + } +} + +/** + * The second line of a want row: what is happening, in the user's terms. + * + * A want that has been tried and not found is reported as still being + * looked for rather than as an error, because that is what it is — the + * retry is already scheduled and there is nothing for the user to do. + */ +function wantDetail(want: Want): string { + if (want.state === 'satisfied') return 'In your library'; + if (want.state === 'paused') return 'Paused'; + + if (want.attempts === 0) return 'Not looked for yet'; + + const reason = want.lastError ? ` — ${want.lastError}` : ''; + + return `Looked for ${want.attempts} time${want.attempts === 1 ? '' : 's'}${reason}`; +} + +declare global { + interface HTMLElementTagNameMap { + 'wanted-view': WantedView; + } +} diff --git a/frontend/src/events.ts b/frontend/src/events.ts index 2430506..7e3f97e 100644 --- a/frontend/src/events.ts +++ b/frontend/src/events.ts @@ -69,6 +69,9 @@ export const Events = { ArtistDiscographyReady: "ArtistDiscographyReady", ArtistSimilarReady: "ArtistSimilarReady", AlbumReleasesReady: "AlbumReleasesReady", + DownloadProvidersChanged: "DownloadProvidersChanged", + DownloadsChanged: "DownloadsChanged", + WantedListChanged: "WantedListChanged", } as const; export type EventName = (typeof Events)[keyof typeof Events]; diff --git a/frontend/src/store/download-store.ts b/frontend/src/store/download-store.ts new file mode 100644 index 0000000..611c7bc --- /dev/null +++ b/frontend/src/store/download-store.ts @@ -0,0 +1,474 @@ +import { EventsOn } from '@runtime/runtime'; +import { + AddProvider, + AddWant, + Cancel, + Candidates, + ClearFinished, + ClearSatisfiedWants, + DeleteProvider, + ImportExternalWants, + ListProviders, + ListRequests, + ListWants, + PauseWant, + Pick, + ProviderKinds, + ReconcileWanted, + RemoveWant, + Start, + TestProvider, + UpdateProvider, +} from '@go/download/Service'; +import type { download } from '@go/models'; +import { Events } from '../events'; + +export type DownloadCandidate = download.Candidate; +export type DownloadProvider = download.Config; +export type DownloadDescriptor = download.Descriptor; +export type DownloadRequest = download.RequestView; +export type ProviderField = download.Field; +export type Want = download.Want; +export type WantSummary = download.Summary; + +/** + * What a want's MBID names. Mirrors backend/download.Entity — the + * wanted list makes no other type distinction, because an MBID plus + * what it names is the whole of a want. + */ +export type WantEntity = 'artist' | 'release-group' | 'release' | 'recording'; + +/** + * Where a want sits. There is deliberately no "failed": an attempt can + * fail, a want cannot — something unfindable today is still wanted. + */ +export type WantState = 'wanted' | 'satisfied' | 'paused'; + +/** + * How much of an artist's output a subscription covers. 'future' is the + * default so subscribing does not silently queue a back catalogue. + */ +export type WantScope = 'future' | 'all'; + +/** Lifecycle states a request can be in. Mirrors backend/download.State. */ +export type DownloadState = + | 'searching' + | 'found' + | 'queued' + | 'grabbing' + | 'verifying' + | 'tagging' + | 'importing' + | 'complete' + | 'cancelled' + | 'failed'; + +type Subscriber = () => void; + +const TERMINAL_STATES: ReadonlySet = new Set([ + 'complete', + 'cancelled', + 'failed', +]); + +export function isRequestTerminal(request: DownloadRequest): boolean { + return TERMINAL_STATES.has(request.state); +} + +/** + * Human-readable label for a request state. Kept here rather than in the + * components so the downloads list and the picker never disagree about + * what a state is called. + */ +export function stateLabel(state: string): string { + switch (state) { + case 'searching': + return 'Searching'; + case 'found': + return 'Waiting for you to choose'; + case 'queued': + return 'Queued'; + case 'grabbing': + return 'Downloading'; + case 'verifying': + return 'Verifying'; + case 'tagging': + return 'Tagging'; + case 'importing': + return 'Importing'; + case 'complete': + return 'Complete'; + case 'cancelled': + return 'Cancelled'; + case 'failed': + return 'Failed'; + default: + return state; + } +} + +/** + * Formats a 0..1 score as a percentage for display. + */ +export function scorePercent(score: number): string { + return `${Math.round(score * 100)}%`; +} + +/** + * Describes why a candidate ranks where it does, in the user's terms. + * + * Match and quality are reported separately on purpose: a perfect match + * at low bitrate and a great-sounding copy of the wrong album are + * different problems, and only the user knows which they will accept. + */ +export function candidateSummary(candidate: DownloadCandidate): string { + const audio = (candidate.files ?? []).filter((f) => f.isAudio); + const formats = new Set(audio.map((f) => f.format).filter(Boolean)); + + const parts: string[] = []; + + const [onlyFormat] = [...formats]; + + if (formats.size === 1 && onlyFormat) { + parts.push(onlyFormat.toUpperCase()); + } else if (formats.size > 1) { + parts.push('Mixed formats'); + } + + if (audio.length > 0) { + parts.push(`${audio.length} track${audio.length === 1 ? '' : 's'}`); + } + + if (candidate.totalSize > 0) { + parts.push(formatBytes(candidate.totalSize)); + } + + if (candidate.origin) { + parts.push(candidate.origin); + } + + return parts.join(' · '); +} + +export function formatBytes(bytes: number): string { + if (!bytes || bytes <= 0) return ''; + + const units = ['B', 'KB', 'MB', 'GB', 'TB']; + let value = bytes; + let unit = 0; + + while (value >= 1024 && unit < units.length - 1) { + value /= 1024; + unit += 1; + } + + return `${value < 10 && unit > 0 ? value.toFixed(1) : Math.round(value)} ${units[unit]}`; +} + +/** + * Reactive singleton for the download subsystem. + * + * Per-transfer progress deliberately does not flow through here — that + * lives in the jobs registry, which already coalesces high-frequency + * updates into one event. This store handles the coarse changes: which + * providers exist, which requests exist, and what the user is being + * asked to choose between. + */ +class DownloadStore { + private providersValue: DownloadProvider[] = []; + + private descriptorsValue: DownloadDescriptor[] = []; + + private requestsValue: DownloadRequest[] = []; + + private wantsValue: Want[] = []; + + private subscribers = new Set(); + + private notifyScheduled = false; + + private initialized = false; + + constructor() { + EventsOn(Events.DownloadProvidersChanged, () => { + void this.refreshProviders(); + }); + + EventsOn(Events.DownloadsChanged, () => { + void this.refreshRequests(); + }); + + // The wanted list changes on its own — a background reconcile + // pass expands an artist, retires something the library gained, + // or starts a download nobody asked for just now. So it is + // event-driven rather than fetched once on mount. + EventsOn(Events.WantedListChanged, () => { + void this.refreshWants(); + }); + } + + /** + * Loads providers and requests once. Safe to call from every + * component's connectedCallback — subsequent calls are no-ops. + */ + async init(): Promise { + if (this.initialized) return; + + this.initialized = true; + + await Promise.all([ + this.refreshDescriptors(), + this.refreshProviders(), + this.refreshRequests(), + this.refreshWants(), + ]); + } + + get providers(): DownloadProvider[] { + return this.providersValue; + } + + /** Providers the user has switched on. */ + get enabledProviders(): DownloadProvider[] { + return this.providersValue.filter((p) => p.enabled); + } + + /** Provider types available to add. */ + get descriptors(): DownloadDescriptor[] { + return this.descriptorsValue; + } + + get requests(): DownloadRequest[] { + return this.requestsValue; + } + + get activeRequests(): DownloadRequest[] { + return this.requestsValue.filter((r) => !isRequestTerminal(r)); + } + + /** + * True when at least one provider is configured and enabled. The UI + * uses this to decide whether to offer downloading at all, rather + * than letting the user start a search that cannot succeed. + */ + get available(): boolean { + return this.enabledProviders.length > 0; + } + + subscribe(callback: Subscriber): () => void { + this.subscribers.add(callback); + + return () => this.subscribers.delete(callback); + } + + /** + * Coalesces notifications into one microtask so a burst of refreshes + * causes a single render pass. + */ + private notify(): void { + if (this.notifyScheduled) return; + + this.notifyScheduled = true; + + queueMicrotask(() => { + this.notifyScheduled = false; + this.subscribers.forEach((callback) => callback()); + }); + } + + async refreshDescriptors(): Promise { + try { + this.descriptorsValue = (await ProviderKinds()) ?? []; + this.notify(); + } catch (err) { + console.error('Failed to load download client types:', err); + } + } + + async refreshProviders(): Promise { + try { + this.providersValue = (await ListProviders()) ?? []; + this.notify(); + } catch (err) { + console.error('Failed to load download clients:', err); + } + } + + async refreshRequests(): Promise { + try { + this.requestsValue = (await ListRequests(50)) ?? []; + this.notify(); + } catch (err) { + console.error('Failed to load downloads:', err); + } + } + + // ----------------------------------------------------------------- + // Provider configuration + // ----------------------------------------------------------------- + + async addProvider( + kind: string, + name: string, + settings: Record, + ): Promise { + const id = await AddProvider(kind, name, settings); + + await this.refreshProviders(); + + return id; + } + + async updateProvider( + id: number, + name: string, + enabled: boolean, + priority: number, + settings: Record, + ): Promise { + await UpdateProvider(id, name, enabled, priority, settings); + await this.refreshProviders(); + } + + async deleteProvider(id: number): Promise { + await DeleteProvider(id); + await this.refreshProviders(); + } + + /** + * Tests a provider's connection. Resolves on success and rejects + * with the backend's message, which is what the settings page + * shows — these errors are the user's main debugging tool for a + * misconfigured client. + */ + async testProvider(id: number): Promise { + await TestProvider(id); + } + + // ----------------------------------------------------------------- + // Requests + // ----------------------------------------------------------------- + + /** + * Starts a download. Returns the ranked candidates plus whether the + * pipeline already picked one, so the caller knows whether to open + * the picker or just show progress. + */ + async start(request: download.SearchRequest): Promise { + const result = await Start(request); + + await this.refreshRequests(); + + return result; + } + + async pick(requestId: string, candidateId: string): Promise { + await Pick(requestId, candidateId); + await this.refreshRequests(); + } + + async cancel(requestId: string): Promise { + await Cancel(requestId); + await this.refreshRequests(); + } + + async candidates(requestId: string): Promise { + return (await Candidates(requestId)) ?? []; + } + + async clearFinished(): Promise { + await ClearFinished(); + await this.refreshRequests(); + } + + // ----------------------------------------------------------------- + // Wanted list + // ----------------------------------------------------------------- + + get wants(): Want[] { + return this.wantsValue; + } + + /** Wants still being looked for. */ + get activeWants(): Want[] { + return this.wantsValue.filter((w) => w.state === 'wanted'); + } + + /** Artist subscriptions, which expand rather than download. */ + get subscriptions(): Want[] { + return this.wantsValue.filter((w) => w.entity === 'artist'); + } + + async refreshWants(): Promise { + try { + this.wantsValue = (await ListWants()) ?? []; + this.notify(); + } catch (err) { + console.error('Failed to load the wanted list:', err); + } + } + + /** True when this MBID is already on the list. */ + isWanted(mbid: string): boolean { + const needle = mbid.trim().toLowerCase(); + + return this.wantsValue.some((w) => w.mbid === needle); + } + + /** The want for an MBID, if it is on the list. */ + wantFor(mbid: string): Want | undefined { + const needle = mbid.trim().toLowerCase(); + + return this.wantsValue.find((w) => w.mbid === needle); + } + + async addWant(want: download.WantRequest): Promise { + const id = await AddWant(want); + + await this.refreshWants(); + + return id; + } + + async removeWant(id: number): Promise { + await RemoveWant(id); + await this.refreshWants(); + } + + async pauseWant(id: number, paused: boolean): Promise { + await PauseWant(id, paused); + await this.refreshWants(); + } + + async clearSatisfiedWants(): Promise { + await ClearSatisfiedWants(); + await this.refreshWants(); + } + + /** + * Runs a reconcile pass now, for the "check now" button. Resolves + * with what the pass did so the UI can say something concrete + * rather than just stopping its spinner. + */ + async reconcileWanted(): Promise { + const summary = await ReconcileWanted(); + + await Promise.all([this.refreshWants(), this.refreshRequests()]); + + return summary; + } + + /** Adopts a provider's own list, e.g. Lidarr's monitored artists. */ + async importExternalWants( + providerId: number, + libraryId: number, + ): Promise { + const count = await ImportExternalWants(providerId, libraryId); + + await this.refreshWants(); + + return count; + } +} + +export const downloadStore = new DownloadStore(); diff --git a/frontend/src/store/explore-settings.ts b/frontend/src/store/explore-settings.ts deleted file mode 100644 index 84ad8b3..0000000 --- a/frontend/src/store/explore-settings.ts +++ /dev/null @@ -1,41 +0,0 @@ -/** - * ExploreSettingsStore — global settings for the explore feature. - * Persists to localStorage so the toggle state survives restarts. - */ - -type Listener = () => void; - -class ExploreSettingsStore { - private _libraryOnly: boolean; - private listeners = new Set(); - - constructor() { - this._libraryOnly = localStorage.getItem('explore:libraryOnly') === 'true'; - } - - get libraryOnly(): boolean { - return this._libraryOnly; - } - - setLibraryOnly(value: boolean) { - if (this._libraryOnly === value) return; - this._libraryOnly = value; - localStorage.setItem('explore:libraryOnly', String(value)); - this.notify(); - } - - toggle() { - this.setLibraryOnly(!this._libraryOnly); - } - - subscribe(fn: Listener): () => void { - this.listeners.add(fn); - return () => this.listeners.delete(fn); - } - - private notify() { - for (const fn of this.listeners) fn(); - } -} - -export const exploreSettings = new ExploreSettingsStore(); diff --git a/frontend/wailsjs/go/download/Service.d.ts b/frontend/wailsjs/go/download/Service.d.ts new file mode 100755 index 0000000..9979801 --- /dev/null +++ b/frontend/wailsjs/go/download/Service.d.ts @@ -0,0 +1,48 @@ +// Cynhyrchwyd y ffeil hon yn awtomatig. PEIDIWCH Â MODIWL +// This file is automatically generated. DO NOT EDIT +import {download} from '../models'; +import {context} from '../models'; + +export function AddProvider(arg1:string,arg2:string,arg3:Record):Promise; + +export function AddWant(arg1:download.WantRequest):Promise; + +export function Cancel(arg1:string):Promise; + +export function Candidates(arg1:string):Promise>; + +export function ClearFinished():Promise; + +export function ClearSatisfiedWants():Promise; + +export function DeleteProvider(arg1:number):Promise; + +export function ImportExternalWants(arg1:number,arg2:number):Promise; + +export function IsWanted(arg1:string,arg2:number):Promise; + +export function ListProviders():Promise>; + +export function ListRequests(arg1:number):Promise>; + +export function ListWants():Promise>; + +export function PauseWant(arg1:number,arg2:boolean):Promise; + +export function Pick(arg1:string,arg2:string):Promise; + +export function ProviderKinds():Promise>; + +export function ReconcileWanted():Promise; + +export function RemoveWant(arg1:number):Promise; + +export function SetContext(arg1:context.Context):Promise; + +export function SetReconciler(arg1:download.Reconciler):Promise; + +export function Start(arg1:download.SearchRequest):Promise; + +export function TestProvider(arg1:number):Promise; + +export function UpdateProvider(arg1:number,arg2:string,arg3:boolean,arg4:number,arg5:Record):Promise; diff --git a/frontend/wailsjs/go/download/Service.js b/frontend/wailsjs/go/download/Service.js new file mode 100755 index 0000000..b11ea1d --- /dev/null +++ b/frontend/wailsjs/go/download/Service.js @@ -0,0 +1,91 @@ +// @ts-check +// Cynhyrchwyd y ffeil hon yn awtomatig. PEIDIWCH Â MODIWL +// This file is automatically generated. DO NOT EDIT + +export function AddProvider(arg1, arg2, arg3) { + return window['go']['download']['Service']['AddProvider'](arg1, arg2, arg3); +} + +export function AddWant(arg1) { + return window['go']['download']['Service']['AddWant'](arg1); +} + +export function Cancel(arg1) { + return window['go']['download']['Service']['Cancel'](arg1); +} + +export function Candidates(arg1) { + return window['go']['download']['Service']['Candidates'](arg1); +} + +export function ClearFinished() { + return window['go']['download']['Service']['ClearFinished'](); +} + +export function ClearSatisfiedWants() { + return window['go']['download']['Service']['ClearSatisfiedWants'](); +} + +export function DeleteProvider(arg1) { + return window['go']['download']['Service']['DeleteProvider'](arg1); +} + +export function ImportExternalWants(arg1, arg2) { + return window['go']['download']['Service']['ImportExternalWants'](arg1, arg2); +} + +export function IsWanted(arg1, arg2) { + return window['go']['download']['Service']['IsWanted'](arg1, arg2); +} + +export function ListProviders() { + return window['go']['download']['Service']['ListProviders'](); +} + +export function ListRequests(arg1) { + return window['go']['download']['Service']['ListRequests'](arg1); +} + +export function ListWants() { + return window['go']['download']['Service']['ListWants'](); +} + +export function PauseWant(arg1, arg2) { + return window['go']['download']['Service']['PauseWant'](arg1, arg2); +} + +export function Pick(arg1, arg2) { + return window['go']['download']['Service']['Pick'](arg1, arg2); +} + +export function ProviderKinds() { + return window['go']['download']['Service']['ProviderKinds'](); +} + +export function ReconcileWanted() { + return window['go']['download']['Service']['ReconcileWanted'](); +} + +export function RemoveWant(arg1) { + return window['go']['download']['Service']['RemoveWant'](arg1); +} + +export function SetContext(arg1) { + return window['go']['download']['Service']['SetContext'](arg1); +} + +export function SetReconciler(arg1) { + return window['go']['download']['Service']['SetReconciler'](arg1); +} + +export function Start(arg1) { + return window['go']['download']['Service']['Start'](arg1); +} + +export function TestProvider(arg1) { + return window['go']['download']['Service']['TestProvider'](arg1); +} + +export function UpdateProvider(arg1, arg2, arg3, arg4, arg5) { + return window['go']['download']['Service']['UpdateProvider'](arg1, arg2, arg3, arg4, arg5); +} diff --git a/frontend/wailsjs/go/explore/Service.d.ts b/frontend/wailsjs/go/explore/Service.d.ts index ad37294..7a7aa3d 100755 --- a/frontend/wailsjs/go/explore/Service.d.ts +++ b/frontend/wailsjs/go/explore/Service.d.ts @@ -1,6 +1,7 @@ // Cynhyrchwyd y ffeil hon yn awtomatig. PEIDIWCH Â MODIWL // This file is automatically generated. DO NOT EDIT import {explore} from '../models'; +import {time} from '../models'; import {context} from '../models'; import {jobs} from '../models'; @@ -18,6 +19,8 @@ export function CAALimiter():Promise; export function CheckLibraryMBIDs(arg1:Array):Promise>; +export function CoreCatalogImported():Promise; + export function CoverArtGroupURL(arg1:string):Promise; export function CoverArtURL(arg1:string):Promise; @@ -52,6 +55,12 @@ export function GetTrackThumbnail(arg1:string,arg2:string,arg3:string,arg4:strin export function GetTrackThumbnails(arg1:Array):Promise>; +export function IndexBaselineSeries():Promise; + +export function IndexImportComplete():Promise; + +export function IndexLastImported():Promise; + export function InvalidateIndexDiscographies():Promise; export function InvalidateLibrarySync():Promise; @@ -70,12 +79,16 @@ export function PopulateLocalCrossReferencesIfNeeded():Promise; export function PrefetchReleases(arg1:Array):Promise; +export function PrepareIndexRebuild():Promise; + export function RebuildLyricsIndex():Promise; export function RebuildLyricsIndexIfNeeded():Promise; export function RecordSearchClick(arg1:string,arg2:string,arg3:string):Promise; +export function RefreshIndexNow(arg1:time.Duration):Promise; + export function RefreshListenCounts():Promise; export function ResolveReleaseGroupMBIDs(arg1:Array):Promise>; diff --git a/frontend/wailsjs/go/explore/Service.js b/frontend/wailsjs/go/explore/Service.js index 01eec9d..f52d433 100755 --- a/frontend/wailsjs/go/explore/Service.js +++ b/frontend/wailsjs/go/explore/Service.js @@ -30,6 +30,10 @@ export function CheckLibraryMBIDs(arg1) { return window['go']['explore']['Service']['CheckLibraryMBIDs'](arg1); } +export function CoreCatalogImported() { + return window['go']['explore']['Service']['CoreCatalogImported'](); +} + export function CoverArtGroupURL(arg1) { return window['go']['explore']['Service']['CoverArtGroupURL'](arg1); } @@ -98,6 +102,18 @@ export function GetTrackThumbnails(arg1) { return window['go']['explore']['Service']['GetTrackThumbnails'](arg1); } +export function IndexBaselineSeries() { + return window['go']['explore']['Service']['IndexBaselineSeries'](); +} + +export function IndexImportComplete() { + return window['go']['explore']['Service']['IndexImportComplete'](); +} + +export function IndexLastImported() { + return window['go']['explore']['Service']['IndexLastImported'](); +} + export function InvalidateIndexDiscographies() { return window['go']['explore']['Service']['InvalidateIndexDiscographies'](); } @@ -134,6 +150,10 @@ export function PrefetchReleases(arg1) { return window['go']['explore']['Service']['PrefetchReleases'](arg1); } +export function PrepareIndexRebuild() { + return window['go']['explore']['Service']['PrepareIndexRebuild'](); +} + export function RebuildLyricsIndex() { return window['go']['explore']['Service']['RebuildLyricsIndex'](); } @@ -146,6 +166,10 @@ export function RecordSearchClick(arg1, arg2, arg3) { return window['go']['explore']['Service']['RecordSearchClick'](arg1, arg2, arg3); } +export function RefreshIndexNow(arg1) { + return window['go']['explore']['Service']['RefreshIndexNow'](arg1); +} + export function RefreshListenCounts() { return window['go']['explore']['Service']['RefreshListenCounts'](); } diff --git a/frontend/wailsjs/go/models.ts b/frontend/wailsjs/go/models.ts index 9e9c1e5..cea8cd0 100755 --- a/frontend/wailsjs/go/models.ts +++ b/frontend/wailsjs/go/models.ts @@ -287,6 +287,571 @@ export namespace autotagservice { } +export namespace download { + + export class QualityScore { + overall: number; + formatRank: number; + bitrate: number; + health: number; + priority: number; + mixed: boolean; + + static createFrom(source: any = {}) { + return new QualityScore(source); + } + + constructor(source: any = {}) { + if ('string' === typeof source) source = JSON.parse(source); + this.overall = source["overall"]; + this.formatRank = source["formatRank"]; + this.bitrate = source["bitrate"]; + this.health = source["health"]; + this.priority = source["priority"]; + this.mixed = source["mixed"]; + } + } + export class MatchScore { + overall: number; + titleFit: number; + artistFit: number; + albumFit: number; + completeness: number; + anchored: boolean; + + static createFrom(source: any = {}) { + return new MatchScore(source); + } + + constructor(source: any = {}) { + if ('string' === typeof source) source = JSON.parse(source); + this.overall = source["overall"]; + this.titleFit = source["titleFit"]; + this.artistFit = source["artistFit"]; + this.albumFit = source["albumFit"]; + this.completeness = source["completeness"]; + this.anchored = source["anchored"]; + } + } + export class CandidateFile { + path: string; + size: number; + format: string; + bitrate?: number; + isAudio: boolean; + matchedTo?: number; + + static createFrom(source: any = {}) { + return new CandidateFile(source); + } + + constructor(source: any = {}) { + if ('string' === typeof source) source = JSON.parse(source); + this.path = source["path"]; + this.size = source["size"]; + this.format = source["format"]; + this.bitrate = source["bitrate"]; + this.isAudio = source["isAudio"]; + this.matchedTo = source["matchedTo"]; + } + } + export class Candidate { + id: string; + providerId: number; + kind: string; + protocol: string; + title: string; + artist?: string; + origin?: string; + files: CandidateFile[]; + totalSize: number; + health: number; + match: MatchScore; + quality: QualityScore; + score: number; + + static createFrom(source: any = {}) { + return new Candidate(source); + } + + constructor(source: any = {}) { + if ('string' === typeof source) source = JSON.parse(source); + this.id = source["id"]; + this.providerId = source["providerId"]; + this.kind = source["kind"]; + this.protocol = source["protocol"]; + this.title = source["title"]; + this.artist = source["artist"]; + this.origin = source["origin"]; + this.files = this.convertValues(source["files"], CandidateFile); + this.totalSize = source["totalSize"]; + this.health = source["health"]; + this.match = this.convertValues(source["match"], MatchScore); + this.quality = this.convertValues(source["quality"], QualityScore); + this.score = source["score"]; + } + + convertValues(a: any, classs: any, asMap: boolean = false): any { + if (!a) { + return a; + } + if (a.slice && a.map) { + return (a as any[]).map(elem => this.convertValues(elem, classs)); + } else if ("object" === typeof a) { + if (asMap) { + for (const key of Object.keys(a)) { + a[key] = new classs(a[key]); + } + return a; + } + return new classs(a); + } + return a; + } + } + + export class Caps { + canSearch: boolean; + canTransport: boolean; + canDelegate: boolean; + canList: boolean; + canResume: boolean; + canCancel: boolean; + reportsSize: boolean; + transports: string[]; + + static createFrom(source: any = {}) { + return new Caps(source); + } + + constructor(source: any = {}) { + if ('string' === typeof source) source = JSON.parse(source); + this.canSearch = source["canSearch"]; + this.canTransport = source["canTransport"]; + this.canDelegate = source["canDelegate"]; + this.canList = source["canList"]; + this.canResume = source["canResume"]; + this.canCancel = source["canCancel"]; + this.reportsSize = source["reportsSize"]; + this.transports = source["transports"]; + } + } + export class Config { + id: number; + kind: string; + name: string; + enabled: boolean; + priority: number; + settings: Record; + + static createFrom(source: any = {}) { + return new Config(source); + } + + constructor(source: any = {}) { + if ('string' === typeof source) source = JSON.parse(source); + this.id = source["id"]; + this.kind = source["kind"]; + this.name = source["name"]; + this.enabled = source["enabled"]; + this.priority = source["priority"]; + this.settings = source["settings"]; + } + } + export class Field { + key: string; + label: string; + placeholder?: string; + help?: string; + secret: boolean; + required: boolean; + default?: string; + + static createFrom(source: any = {}) { + return new Field(source); + } + + constructor(source: any = {}) { + if ('string' === typeof source) source = JSON.parse(source); + this.key = source["key"]; + this.label = source["label"]; + this.placeholder = source["placeholder"]; + this.help = source["help"]; + this.secret = source["secret"]; + this.required = source["required"]; + this.default = source["default"]; + } + } + export class Descriptor { + kind: string; + name: string; + summary: string; + caps: Caps; + fields: Field[]; + requiresExternal?: string; + + static createFrom(source: any = {}) { + return new Descriptor(source); + } + + constructor(source: any = {}) { + if ('string' === typeof source) source = JSON.parse(source); + this.kind = source["kind"]; + this.name = source["name"]; + this.summary = source["summary"]; + this.caps = this.convertValues(source["caps"], Caps); + this.fields = this.convertValues(source["fields"], Field); + this.requiresExternal = source["requiresExternal"]; + } + + convertValues(a: any, classs: any, asMap: boolean = false): any { + if (!a) { + return a; + } + if (a.slice && a.map) { + return (a as any[]).map(elem => this.convertValues(elem, classs)); + } else if ("object" === typeof a) { + if (asMap) { + for (const key of Object.keys(a)) { + a[key] = new classs(a[key]); + } + return a; + } + return new classs(a); + } + return a; + } + } + export class ExpectedTrack { + position: number; + discNumber: number; + title: string; + artist: string; + lengthMillis: number; + + static createFrom(source: any = {}) { + return new ExpectedTrack(source); + } + + constructor(source: any = {}) { + if ('string' === typeof source) source = JSON.parse(source); + this.position = source["position"]; + this.discNumber = source["discNumber"]; + this.title = source["title"]; + this.artist = source["artist"]; + this.lengthMillis = source["lengthMillis"]; + } + } + + export class Item { + id: string; + requestId: string; + providerId: number; + transportId?: number; + externalId?: string; + candidate: Candidate; + state: string; + bytesDone: number; + bytesTotal: number; + error?: string; + createdAt: time.Time; + updatedAt: time.Time; + + static createFrom(source: any = {}) { + return new Item(source); + } + + constructor(source: any = {}) { + if ('string' === typeof source) source = JSON.parse(source); + this.id = source["id"]; + this.requestId = source["requestId"]; + this.providerId = source["providerId"]; + this.transportId = source["transportId"]; + this.externalId = source["externalId"]; + this.candidate = this.convertValues(source["candidate"], Candidate); + this.state = source["state"]; + this.bytesDone = source["bytesDone"]; + this.bytesTotal = source["bytesTotal"]; + this.error = source["error"]; + this.createdAt = this.convertValues(source["createdAt"], time.Time); + this.updatedAt = this.convertValues(source["updatedAt"], time.Time); + } + + convertValues(a: any, classs: any, asMap: boolean = false): any { + if (!a) { + return a; + } + if (a.slice && a.map) { + return (a as any[]).map(elem => this.convertValues(elem, classs)); + } else if ("object" === typeof a) { + if (asMap) { + for (const key of Object.keys(a)) { + a[key] = new classs(a[key]); + } + return a; + } + return new classs(a); + } + return a; + } + } + + + export class Reconciler { + + + static createFrom(source: any = {}) { + return new Reconciler(source); + } + + constructor(source: any = {}) { + if ('string' === typeof source) source = JSON.parse(source); + + } + } + export class RequestView { + id: string; + releaseMbid?: string; + releaseGroupMbid?: string; + recordingMbid?: string; + wantId?: number; + source?: string; + artist: string; + album: string; + query?: string; + expected?: ExpectedTrack[]; + libraryId: number; + createdAt: time.Time; + state: string; + error?: string; + items: Item[]; + + static createFrom(source: any = {}) { + return new RequestView(source); + } + + constructor(source: any = {}) { + if ('string' === typeof source) source = JSON.parse(source); + this.id = source["id"]; + this.releaseMbid = source["releaseMbid"]; + this.releaseGroupMbid = source["releaseGroupMbid"]; + this.recordingMbid = source["recordingMbid"]; + this.wantId = source["wantId"]; + this.source = source["source"]; + this.artist = source["artist"]; + this.album = source["album"]; + this.query = source["query"]; + this.expected = this.convertValues(source["expected"], ExpectedTrack); + this.libraryId = source["libraryId"]; + this.createdAt = this.convertValues(source["createdAt"], time.Time); + this.state = source["state"]; + this.error = source["error"]; + this.items = this.convertValues(source["items"], Item); + } + + convertValues(a: any, classs: any, asMap: boolean = false): any { + if (!a) { + return a; + } + if (a.slice && a.map) { + return (a as any[]).map(elem => this.convertValues(elem, classs)); + } else if ("object" === typeof a) { + if (asMap) { + for (const key of Object.keys(a)) { + a[key] = new classs(a[key]); + } + return a; + } + return new classs(a); + } + return a; + } + } + export class SearchRequest { + libraryId: number; + releaseMbid: string; + releaseGroupMbid: string; + artist: string; + album: string; + query: string; + expected: ExpectedTrack[]; + + static createFrom(source: any = {}) { + return new SearchRequest(source); + } + + constructor(source: any = {}) { + if ('string' === typeof source) source = JSON.parse(source); + this.libraryId = source["libraryId"]; + this.releaseMbid = source["releaseMbid"]; + this.releaseGroupMbid = source["releaseGroupMbid"]; + this.artist = source["artist"]; + this.album = source["album"]; + this.query = source["query"]; + this.expected = this.convertValues(source["expected"], ExpectedTrack); + } + + convertValues(a: any, classs: any, asMap: boolean = false): any { + if (!a) { + return a; + } + if (a.slice && a.map) { + return (a as any[]).map(elem => this.convertValues(elem, classs)); + } else if ("object" === typeof a) { + if (asMap) { + for (const key of Object.keys(a)) { + a[key] = new classs(a[key]); + } + return a; + } + return new classs(a); + } + return a; + } + } + export class StartResult { + requestId: string; + candidates: Candidate[]; + autoPicked: boolean; + + static createFrom(source: any = {}) { + return new StartResult(source); + } + + constructor(source: any = {}) { + if ('string' === typeof source) source = JSON.parse(source); + this.requestId = source["requestId"]; + this.candidates = this.convertValues(source["candidates"], Candidate); + this.autoPicked = source["autoPicked"]; + } + + convertValues(a: any, classs: any, asMap: boolean = false): any { + if (!a) { + return a; + } + if (a.slice && a.map) { + return (a as any[]).map(elem => this.convertValues(elem, classs)); + } else if ("object" === typeof a) { + if (asMap) { + for (const key of Object.keys(a)) { + a[key] = new classs(a[key]); + } + return a; + } + return new classs(a); + } + return a; + } + } + export class Summary { + expanded: number; + satisfied: number; + attempted: number; + started: number; + synced: number; + + static createFrom(source: any = {}) { + return new Summary(source); + } + + constructor(source: any = {}) { + if ('string' === typeof source) source = JSON.parse(source); + this.expanded = source["expanded"]; + this.satisfied = source["satisfied"]; + this.attempted = source["attempted"]; + this.started = source["started"]; + this.synced = source["synced"]; + } + } + export class Want { + id: number; + mbid: string; + entity: string; + libraryId: number; + artist: string; + title: string; + scope: string; + secondary: boolean; + state: string; + parentId?: number; + attempts: number; + lastError?: string; + lastTriedAt?: time.Time; + nextTryAt?: time.Time; + externalIds?: Record; + createdAt: time.Time; + updatedAt: time.Time; + + static createFrom(source: any = {}) { + return new Want(source); + } + + constructor(source: any = {}) { + if ('string' === typeof source) source = JSON.parse(source); + this.id = source["id"]; + this.mbid = source["mbid"]; + this.entity = source["entity"]; + this.libraryId = source["libraryId"]; + this.artist = source["artist"]; + this.title = source["title"]; + this.scope = source["scope"]; + this.secondary = source["secondary"]; + this.state = source["state"]; + this.parentId = source["parentId"]; + this.attempts = source["attempts"]; + this.lastError = source["lastError"]; + this.lastTriedAt = this.convertValues(source["lastTriedAt"], time.Time); + this.nextTryAt = this.convertValues(source["nextTryAt"], time.Time); + this.externalIds = source["externalIds"]; + this.createdAt = this.convertValues(source["createdAt"], time.Time); + this.updatedAt = this.convertValues(source["updatedAt"], time.Time); + } + + convertValues(a: any, classs: any, asMap: boolean = false): any { + if (!a) { + return a; + } + if (a.slice && a.map) { + return (a as any[]).map(elem => this.convertValues(elem, classs)); + } else if ("object" === typeof a) { + if (asMap) { + for (const key of Object.keys(a)) { + a[key] = new classs(a[key]); + } + return a; + } + return new classs(a); + } + return a; + } + } + export class WantRequest { + mbid: string; + entity: string; + libraryId: number; + artist: string; + title: string; + scope: string; + secondary: boolean; + + static createFrom(source: any = {}) { + return new WantRequest(source); + } + + constructor(source: any = {}) { + if ('string' === typeof source) source = JSON.parse(source); + this.mbid = source["mbid"]; + this.entity = source["entity"]; + this.libraryId = source["libraryId"]; + this.artist = source["artist"]; + this.title = source["title"]; + this.scope = source["scope"]; + this.secondary = source["secondary"]; + } + } + +} + export namespace explore { export class TierStatus { @@ -295,6 +860,7 @@ export namespace explore { total: number; completed: number; error?: string; + detail?: string; static createFrom(source: any = {}) { return new TierStatus(source); @@ -307,6 +873,7 @@ export namespace explore { this.total = source["total"]; this.completed = source["completed"]; this.error = source["error"]; + this.detail = source["detail"]; } } export class IndexStatus { @@ -1635,8 +2202,7 @@ export namespace sqlcgen { ID: number; Name: string; Path: string; - // Go type: time - CreatedAt: any; + CreatedAt: time.Time; AutotagWarningAcked: number; static createFrom(source: any = {}) { @@ -1648,7 +2214,7 @@ export namespace sqlcgen { this.ID = source["ID"]; this.Name = source["Name"]; this.Path = source["Path"]; - this.CreatedAt = this.convertValues(source["CreatedAt"], null); + this.CreatedAt = this.convertValues(source["CreatedAt"], time.Time); this.AutotagWarningAcked = source["AutotagWarningAcked"]; } @@ -1730,6 +2296,23 @@ export namespace tagwriter { } +export namespace time { + + export class Time { + + + static createFrom(source: any = {}) { + return new Time(source); + } + + constructor(source: any = {}) { + if ('string' === typeof source) source = JSON.parse(source); + + } + } + +} + export namespace tracklist { export class Column { diff --git a/frontend/wailsjs/go/tagwriter/TagWriter.d.ts b/frontend/wailsjs/go/tagwriter/TagWriter.d.ts index 4b8b643..9657f52 100755 --- a/frontend/wailsjs/go/tagwriter/TagWriter.d.ts +++ b/frontend/wailsjs/go/tagwriter/TagWriter.d.ts @@ -12,3 +12,5 @@ export function SetContext(arg1:context.Context):Promise; export function WriteTrackTags(arg1:number,arg2:tagwriter.TagChanges):Promise; export function WriteTrackTagsByPath(arg1:string,arg2:tagwriter.TagChanges):Promise; + +export function WriteUntrackedFileTags(arg1:string,arg2:tagwriter.TagChanges):Promise; diff --git a/frontend/wailsjs/go/tagwriter/TagWriter.js b/frontend/wailsjs/go/tagwriter/TagWriter.js index f3f7226..bb5fa8b 100755 --- a/frontend/wailsjs/go/tagwriter/TagWriter.js +++ b/frontend/wailsjs/go/tagwriter/TagWriter.js @@ -21,3 +21,7 @@ export function WriteTrackTags(arg1, arg2) { export function WriteTrackTagsByPath(arg1, arg2) { return window['go']['tagwriter']['TagWriter']['WriteTrackTagsByPath'](arg1, arg2); } + +export function WriteUntrackedFileTags(arg1, arg2) { + return window['go']['tagwriter']['TagWriter']['WriteUntrackedFileTags'](arg1, arg2); +}