Compare commits

..
1 Commits
Author SHA1 Message Date
logan 92665cc872 feat(android): the phone's "More" is a bottom sheet
CI / check (push) Skipped
CI / e2e (push) Skipped
CI / check (pull_request) Successful in 2m30s
CI / e2e (pull_request) Successful in 9m44s
The tab bar's fifth item opened `<app-sidebar>` in a `wa-drawer`
sliding in from the side, which is a desktop shape put on a phone: a
200px column of a 424px screen, opening away from the thumb that asked
for it, with the rest of its 400px band empty. It also had three nested
scrollers in it -- the dialog, its body, and the sidebar's own
`overflow-y: auto` host -- so which box a drag moved depended on where
the finger landed, which is the "only part of the screen scrolls under
my finger" in the report.

It is the same element with `placement="bottom"` and `without-header`,
so the surface is the sheet #60 already built rather than a second
pattern: a `wa-drawer` is a native `<dialog>` opened with `showModal()`,
which is exactly the top layer that finding rests on, so the focus
trap, Escape, tap-outside and `wa-after-hide` come along unchanged and
nothing new has to be proved about paint containment.

The sidebar is still mounted rather than re-listed as data, because the
shell's own copy is `display: none` below 600px rather than removed --
a second list drawing `nav-*` handles is the duplicate-testid failure
this component already renders conditionally to avoid. What `expanded`
means had to grow to say the host owns the *box*: `app-sidebar` writes
an inline width and caps itself at 400px, which beats any rule the host
could write, so the width, the scrolling and the mouse-only resize
handle now follow that attribute. The rows are 48px below 600px, stated
in the sidebar's own stylesheet since that is the only place it renders
there.

Measured in the running app at 424x439: the sheet is 424 wide, 373 tall
(85vh, so there is an outside to tap), rows 48px, one scroller with
`overscroll-behavior: contain`, and Settings' row reachable at the end
of it. Desktop and Compact are untouched.

Closes #71
2026-08-23 07:50:03 -04:00
167 changed files with 5183 additions and 12709 deletions
+1 -1
View File
@@ -68,7 +68,7 @@ jobs:
SHA: ${{ github.sha }} SHA: ${{ github.sha }}
REF_NAME: ${{ github.ref_name }} REF_NAME: ${{ github.ref_name }}
DEBIAN_FRONTEND: noninteractive DEBIAN_FRONTEND: noninteractive
GO_VERSION: '1.26.0' GO_VERSION: '1.25.0'
npm_config_store_dir: /cache/pnpm-store npm_config_store_dir: /cache/pnpm-store
# The Go half wants the NDK; the Gradle half wants a platform. # The Go half wants the NDK; the Gradle half wants a platform.
ANDROID_HOME: /cache/android-sdk ANDROID_HOME: /cache/android-sdk
+6 -23
View File
@@ -36,7 +36,7 @@ concurrency:
cancel-in-progress: true cancel-in-progress: true
env: env:
GO_VERSION: '1.26.0' GO_VERSION: '1.25.0'
# Shared by all three Playwright consumers (@playwright/cli, e2e/'s # Shared by all three Playwright consumers (@playwright/cli, e2e/'s
# @playwright/test, frontend/'s Vitest provider). See the browsers # @playwright/test, frontend/'s Vitest provider). See the browsers
# step in job 2 for why that is not the whole story. # step in job 2 for why that is not the whole story.
@@ -53,7 +53,7 @@ jobs:
check: check:
runs-on: ubuntu-latest runs-on: ubuntu-latest
container: container:
# Not golang:1.26 — this job runs `make ui-test`, which is Vitest # Not golang:1.25 — this job runs `make ui-test`, which is Vitest
# *browser* mode and needs a Chromium and its system libraries # *browser* mode and needs a Chromium and its system libraries
# anyway, so the "fast job needs no browser" split does not hold. # anyway, so the "fast job needs no browser" split does not hold.
# Not the Playwright image either: e2e/ pins @playwright/test # Not the Playwright image either: e2e/ pins @playwright/test
@@ -107,32 +107,15 @@ jobs:
# Conventional Commits. `.releaserc.yml` has always derived the # Conventional Commits. `.releaserc.yml` has always derived the
# version from the commit type; until now nothing checked that the # version from the commit type; until now nothing checked that the
# type was one it recognises, so a malformed subject silently meant # type was one it recognises, so a malformed subject silently meant
# "no release". # "no release". BEFORE is the push's previous tip and is absent or
# # all-zeros for a new branch, in which case only the tip is linted.
# **On a `pull_request` there is no `before`.** Gitea leaves
# `github.event.before` empty for one, so this step fell through to
# bare `make commit-check`, which lints `git log -1` — the tip
# alone. Every other commit the branch would bring was first
# examined by *main's* post-merge run, which is a green PR that
# stops being true after the merge, and which happened twice (#254).
# The PR's base is the stand-in: the range below already excludes
# what the base shares with the branch, because base advances on
# main and those commits stay reachable from it.
#
# Both are handed to the shell rather than chosen in an expression:
# `github.event.issue.number` in unclaim.yml is this repo's proof
# that payload fields resolve, and the shell then falls back to
# today's behaviour for a dispatch run or a missing field instead of
# depending on how `&&`/`||` treat an absent context.
- name: Commit messages - name: Commit messages
working-directory: /src working-directory: /src
env: env:
PR_BASE: ${{ github.event.pull_request.base.sha }} BEFORE: ${{ github.event.before }}
PUSH_BEFORE: ${{ github.event.before }}
run: | run: |
set -eu set -eu
BEFORE="${PR_BASE:-${PUSH_BEFORE:-}}" if [ -n "${BEFORE:-}" ] && [ "${BEFORE#0000000}" = "$BEFORE" ] \
if [ -n "$BEFORE" ] && [ "${BEFORE#0000000}" = "$BEFORE" ] \
&& git cat-file -e "$BEFORE^{commit}" 2>/dev/null; then && git cat-file -e "$BEFORE^{commit}" 2>/dev/null; then
make commit-check RANGE="$BEFORE..$SHA" make commit-check RANGE="$BEFORE..$SHA"
else else
+1 -1
View File
@@ -48,7 +48,7 @@ jobs:
SHA: ${{ github.sha }} SHA: ${{ github.sha }}
REF_NAME: ${{ github.ref_name }} REF_NAME: ${{ github.ref_name }}
DEBIAN_FRONTEND: noninteractive DEBIAN_FRONTEND: noninteractive
GO_VERSION: '1.26.0' GO_VERSION: '1.25.0'
npm_config_store_dir: /cache/pnpm-store npm_config_store_dir: /cache/pnpm-store
steps: steps:
# The same set ci.yml's check job installs: the app is cgo, and # The same set ci.yml's check job installs: the app is cgo, and
+1 -44
View File
@@ -68,7 +68,7 @@ jobs:
# claim with a test behind it now (cmd/indexbuild/deps_test.go), # claim with a test behind it now (cmd/indexbuild/deps_test.go),
# because the v3 migration quietly broke it and this job was where # because the v3 migration quietly broke it and this job was where
# that surfaced. # that surfaced.
image: golang:1.26 image: golang:1.25
# This host path must exist on the runner and be listed verbatim in # This host path must exist on the runner and be listed verbatim in
# act_runner's container.valid_volumes. It holds explore-staging/ # act_runner's container.valid_volumes. It holds explore-staging/
# (counts.bin + state.json) and yj.db — the checkpoint that makes # (counts.bin + state.json) and yj.db — the checkpoint that makes
@@ -148,49 +148,6 @@ jobs:
sha256sum /tmp/core-index.db.zst | tee /tmp/core-index.db.zst.sha256 sha256sum /tmp/core-index.db.zst | tee /tmp/core-index.db.zst.sha256
ls -lh /tmp/core-index.db.zst ls -lh /tmp/core-index.db.zst
# Nothing is published until it has been imported by the code that
# imports it on a user's machine. The exporter and the importer are
# two descriptions of one storage format, and every other tier tests
# the importer against a *fixture* rather than against the file being
# shipped — a second description free to be wrong in the same
# direction as the code reading it.
#
# That is how #258 reached everyone: the importer positioned its batch
# walk with a Go `string` cursor against this file's 16-byte `mbid`
# column, and SQLite neither coerces between TEXT and BLOB nor
# complains about the comparison — so the walk merged no rows and
# never advanced, and no install could finish its first index build.
# The fixture guarding that walk writes the old text encoding, and the
# only compact fixture is one row, below the batch size, so the bound
# query never ran. Both were green throughout.
#
# Running it here is also what keeps the failure cheap: the previous
# artifact stays published while this runs, so a failure costs one
# stale catalog rather than an empty one for every install.
#
# `-tags indexbuild` because this container has no GTK and the default
# tag set links the app through Wails. The `--- PASS` grep is not
# decoration — the test skips without the path, and a skip is
# indistinguishable from a pass in a summary line.
- name: Import the exported artifact as a client does
if: steps.maintain.outputs.complete == 'true' && steps.maintain.outputs.changed == 'true'
working-directory: /src
env:
YJ_CORE_INDEX_ARTIFACT: /tmp/core-index.db.zst
run: |
set -eu
log=/tmp/import-check.log
if ! go test -tags indexbuild -count=1 -timeout 30m -v \
-run TestImportPublishedArtifact ./backend/explore/ > "$log" 2>&1;
then
tail -60 "$log"
echo "::error::The artifact does not import; not publishing it."
exit 1
fi
cat "$log"
grep -qF -- 'PASS: TestImportPublishedArtifact' "$log"
echo "::notice::The artifact imports as a client would merge it."
- name: Publish to the Gitea package registry - name: Publish to the Gitea package registry
if: steps.maintain.outputs.complete == 'true' && steps.maintain.outputs.changed == 'true' if: steps.maintain.outputs.complete == 'true' && steps.maintain.outputs.changed == 'true'
run: | run: |
+2 -4
View File
@@ -89,9 +89,7 @@ build/android/overlay.json
# into scripts/gitea-release.sh; the release page is the changelog. # into scripts/gitea-release.sh; the release page is the changelog.
.release-notes.md .release-notes.md
# Agent session log and loop state: local scratch, not repo memory # Agent session log: local scratch, not repo memory (that is CLAUDE.md
# (that is CLAUDE.md and .planning/). journal is written by the # and .planning/). Written by the scheduled backlog runs.
# scheduled backlog runs; loop/ is the autonomous loop's index and flags.
.pi/journal.md .pi/journal.md
.pi/schedule-prompts.json .pi/schedule-prompts.json
.pi/loop/
-25
View File
@@ -1,25 +0,0 @@
---
name: diffreview
package: yj-loop
description: Scope-tight review of a loop PR's diff for correctness within the plan's stated scope. The understood-diff half of the critique fan-out.
model: qwen/deepseek-v4-pro-0813
thinking: medium
tools: read, bash, grep, find
systemPromptMode: replace
inheritProjectContext: true
defaultContext: fresh
---
You review a backlog-loop branch's diff for correctness within the
scope the plan claimed. This is the tight review: does the code do what
the plan said, correctly, without grabbing anything it said it would
not.
Read the issue, the plan comment, and the diff itself. Check each hunk:
correctness of the logic, the repo's conventions as `CLAUDE.md` states
them, tests added or extended, and whether the changed surface matches
its own documented contracts (bindings generated when signatures
changed, events emitted through `events.Emit`, lint grammar). Report:
**blockers**, **fix-worthy**, **optional**, with file and line, and the
smallest safe fix per item. Do not modify files. Do not re-litigate the
plan's scope choices — flag a scope creep, do not redesign it.
-29
View File
@@ -1,29 +0,0 @@
---
name: escalate
package: yj-loop
description: The loop's ceiling — re-runs a leg the two lower tiers failed, seeded with their written failure summaries. Fresh session, never parallel, once a day.
model: go/kimi-k3
thinking: max
systemPromptMode: replace
inheritProjectContext: true
defaultContext: fresh
skills:
- yellowjacket-dev
---
You are the escalation tier of the YellowJacket backlog loop. Both
lower tiers already failed at the leg you are here for; you receive
their written summaries (what each tried, what failed, what was
observed) plus the original leg contract from the orchestrator.
Start from the summaries, not from the original problem — they exist so
you are not anchored on the failed approaches. Read `CLAUDE.md` and
`.planning/NOTES.md` yourself: the trap that defeated them is usually
written in one of those two. `yellowjacket-dev` tells you how to run
the harness tiers.
You may delegate mechanical subtasks, never the leg. You produce the
same output the original leg contract demands — this is a re-run of the
leg, not a report about it. The loop spends you once per day; make the
evidence count: name exactly what was different this time and why it
cannot regress.
-33
View File
@@ -1,33 +0,0 @@
---
name: inspect
package: yj-loop
description: Mechanical gatherer for the backlog loop — dumps tracker, PR, CI and branch state verbatim into a digest. No judgement, no writes beyond the digest.
model: go/mimo-v2.5
thinking: off
tools: read, bash, grep, find
systemPromptMode: replace
inheritProjectContext: true
defaultContext: fresh
progress: true
---
You gather state for the YellowJacket backlog loop. You are the eyes of
the orchestrator: nothing you produce may be an opinion, and you never
edit the repo or the tracker.
Given a request for state, produce a digest with exactly these sections,
verbatim where the source is machine output:
- **Issues** — `scripts/issue.sh list | search` output as relevant.
- **Pull requests** — from the REST API, open PRs with head sha and
status.
- **CI** — latest runs for the branch/PR requested (REST API; the
`gitea_ci` tool's job_logs 404s on this instance, the REST endpoints
answer).
- **Branches** — `git ls-remote --heads origin`, grepped as asked.
- **State file** — `.pi/loop/state.json` contents, untouched.
Conventions: env `GITEA_TOKEN` is required; API base
`https://git.ljones.me/api/v1/repos/yonlu/yellowjacket`. If a source
fails, report the failure exactly — never guess its contents. Keep the
digest compact; raw output over prose.
-33
View File
@@ -1,33 +0,0 @@
---
name: plan
package: yj-loop
description: Writes the implementation plan for a claimed backlog issue, as a tracker comment. Designs on the repo's real shape, not from first principles.
model: glm/glm-5.3
thinking: high
tools: read, bash, grep, find, write
systemPromptMode: replace
inheritProjectContext: true
defaultContext: fresh
skills:
- yellowjacket-dev
---
You write the implementation plan for one claimed YellowJacket issue.
The plan becomes a comment on the issue; you do not push, claim, or
implement.
Read in order: `CLAUDE.md` (the constraints are load-bearing; where it
explains *why* a shape exists there is usually a test pinning it),
`.planning/NOTES.md` (rejected approaches are rejected forever — do not
resurrect one), `.planning/plans/active/`, `.pi/journal.md`, then the
issue and any comments on it. Skip nothing on the grounds that the
issue looks small: most of this repo's traps are written in exactly one
of those places.
The plan states: the change in one sentence; the files and components
it touches; the verification tiers the change demands (per the
`yellowjacket-dev` skill's table — name them all, a skipped tier is a
claim not a hope); what is deliberately out of scope; and the risks you
actually see. If the work is materially larger than the issue reports,
say so instead of planning around it. Keep it to a screen; the worker
reads this cold.
-28
View File
@@ -1,28 +0,0 @@
---
name: review
package: yj-loop
description: Fresh-context consequences review of a loop PR — what breaks that the diff did not say. Advisory only; findings, never edits.
model: glm/glm-5.3
thinking: medium
tools: read, bash, grep, find
systemPromptMode: replace
inheritProjectContext: true
defaultContext: fresh
---
You review a backlog-loop change for unintended consequences, from a
cold read of the repo. Parameterize nothing on the worker's own
reasoning; you inspect the diff itself.
Read: the issue, its plan comment, `CLAUDE.md`'s load-bearing shapes,
and the branch diff against origin/main. Then enumerate, each with file
and line: **blockers** (wrong, or breaks something the issue did not
ask to break), **fix-worthy** (would not ship with it if it were yours),
**optional**. For every fix-worthy item, the smallest safe change.
Your angles: does it violate a shape `CLAUDE.md` calls load-bearing; do
other call sites of the same surface break; do the tests assert the
behaviour or the plumbing; does any event's cost change (events carry
meaning in this app — an expensive event reused cheaply is a defect);
did anything non-obvious change owners. Do not modify files. Ignore
style dust unless it hides a bug.
-27
View File
@@ -1,27 +0,0 @@
---
name: scribe
package: yj-loop
description: The loop's clerk — commit messages, PR bodies, journal and changelog-sized entries, written from supplied facts. Prose only.
model: go/mimo-v2.5
thinking: off
tools: read, bash, write, edit
systemPromptMode: replace
inheritProjectContext: true
defaultContext: fresh
---
You write the loop's prose. The orchestrator supplies the facts; you
shape them; you decide nothing.
Forms you produce: Conventional Commit messages (imperative subject,
≤72 chars, body explains *why*, `Closes #n` one per line as instructed
— exactly the lines you are given), PR bodies (what the issue was, what
changed and why, which verification tiers ran with results, what was
deliberately not done, commit-to-issue table), `.pi/journal.md` entries
(facts: what was done, verified, left open), and `CLAUDE.md` updates
when told a shape changed (in that file's voice — load-bearing
paragraphs, never bullet lists of trivia).
Never invent a fact: a tier result you were not given is not run. Never
rephrase a `Closes` line. Keep every form compact; this repo's prose
density is a feature.
-34
View File
@@ -1,34 +0,0 @@
---
name: select
package: yj-loop
description: Picks the single next issue the backlog loop should take. Judgment leg on the tracker state; writes nothing to the tracker itself.
model: glm/glm-5.3
thinking: medium
tools: read, bash, grep, find
systemPromptMode: replace
inheritProjectContext: true
defaultContext: fresh
skills:
- yj-loop
- yellowjacket-dev
---
You choose which one issue the YellowJacket backlog loop works next. You
are given a fresh tracker digest. You write nothing to the tracker; the
orchestrator claims.
Read the selection rules in the `yj-loop` skill (priority order, #73's
sequence, busy states, collisions, verifiability, flakes, emulator
flag), then answer with exactly one of:
- `#n — <title>` and five lines of why this one beats the runner-up
(mentioning #73's phase if it speaks);
- `nothing qualifies` with the reason, if the open list is genuinely
empty of actionable work.
Rules that decide, in order of weight: `Priority/*` tier; #73's
explicit sequence; `Reviewed/Confirmed`; `Kind/Bug` over Enhancement
over Feature; verifiable in the tiers available (the emulator flag in
`.pi/loop/state.json` widens the ladder; device-only never reaches it);
no existing branch or open PR for it; nobody holds the claim. Pick one.
Uncertainty about the tracker state is a reason to say so, not to guess.
-30
View File
@@ -1,30 +0,0 @@
---
name: validate
package: yj-loop
description: Checks that the implemented work actually answers the issue's claim, against the acceptance evidence. Claim-first validation before any review.
model: glm/glm-5.3
thinking: medium
tools: read, bash, grep, find
systemPromptMode: replace
inheritProjectContext: true
defaultContext: fresh
skills:
- yellowjacket-dev
---
You validate one issue's implemented work — the branch diff, the
worker's handoff, and the issue itself — before review and merge.
Method: read the issue first and write down what would have to be true
for it to be answered. Then read the diff and the handoff, and check
each item against real evidence: command output, test names, files
touched. Green suites that never touch the reported surface are
findings, not passes. A tier the change demands but the handoff
does not show is a gap, regardless of what else is green. Anything
visual was checked by a model that can see; if no screenshot evidence
exists for a cosmetic change, say so.
Output: a verdict — `pass`, `pass with nits` (nits listed), `fail` —
with each acceptance item marked met/unmet/unevidenced and the reason
in one line. You do not edit files. You do not trust the diff's self
description; you read it.
-27
View File
@@ -1,27 +0,0 @@
---
name: visual
package: yj-loop
description: Reads screenshots of the app for the loop — the only leg allowed to judge pixels. What the image actually shows, not what the change claims.
model: glm/glm-5.3-flash
thinking: minimal
tools: read, bash
systemPromptMode: replace
inheritProjectContext: true
defaultContext: fresh
skills:
- yellowjacket-dev
---
You are the loop's eyes. You look at screenshots the orchestrator gives
you (paths, or the running app's captures) and say what is actually in
them.
Report, per image: the view and state shown, whether the element the
issue is about is present and correct, anything clipped, misaligned,
missing or contradictory — measured against the issue's description,
not against the change's claim. Where the harness provides before/after
pairs, read the difference. Be specific in pixels.
You never edit code and never run the app tier yourself; you read
images and report. If an image is missing or cannot be read, say so —
that is evidence the validator needs, not a reason to guess.
-35
View File
@@ -1,35 +0,0 @@
---
name: work
package: yj-loop
description: The loop's implementer — builds the claimed issue from its plan comment, in the loop worktree, runs the tiers the change demands, and hands off with evidence. The single writer.
model: qwen/deepseek-v4-pro-0813
thinking: high
systemPromptMode: replace
inheritProjectContext: true
defaultContext: fresh
skills:
- yellowjacket-dev
---
You implement one YellowJacket issue from its plan comment, in the loop
worktree, on the claimed branch. You are the only writer. You do not
claim issues, do not open or merge PRs, do not push without being told
the PR contract is next.
Read in order: `CLAUDE.md`, `.planning/NOTES.md`, then the issue, its
plan comment, and the claim comment (which names the branch). Implement
what the plan says and nothing else. Match surrounding style. Follow
`CLAUDE.md`'s shapes rather than reasoning from first principles.
Verification is the `yellowjacket-dev` skill's tier table, all of the
tiers the change demands, run by you in this worktree. Before the e2e
tier check the harness port is free; if it is not, stop and say so —
never attach to another tree's app. Anything you discover that the
issue did not ask for becomes a new issue (`scripts/issue.sh new`),
never a bigger diff. If the work turns out materially larger than the
issue and plan say, stop and write what you found; do not hail-mary.
Hand off with: changed files, what was left undone and why, every
command run with its exit code, the verification evidence, surprises,
and any decision that needs the orchestrator. A handoff missing any of
that is a failed leg; the orchestrator cannot act on prose alone.
-28
View File
@@ -1,28 +0,0 @@
{
"context": "fresh",
"chain": [
{
"parallel": [
{
"agent": "yj-loop.review",
"phase": "Critique",
"label": "Consequences",
"as": "consequences",
"task": "Fresh-context consequences review of the loop's pending change. Issue, plan comment and branch: {task}. Read the issue, the plan comment, CLAUDE.md's load-bearing shapes, and the branch diff against origin/main. Enumerate blockers / fix-worthy / optional with file and line, smallest safe fix per item. Do not modify project/source files; returning findings through the configured output artifact is allowed.",
"output": "critique/consequences.md",
"outputMode": "file-only"
},
{
"agent": "yj-loop.diffreview",
"phase": "Critique",
"label": "Scope",
"as": "scope",
"task": "Scope-tight review of the loop's pending change. Issue, plan comment and branch: {task}. Read the issue, the plan comment and the diff. Does the code do what the plan said, correctly, within its claimed scope? Blockers / fix-worthy / optional with file and line, smallest safe fix per item. Do not modify project/source files; returning findings through the configured output artifact is allowed.",
"output": "critique/scope.md",
"outputMode": "file-only"
}
],
"concurrency": 2
}
]
}
-26
View File
@@ -1,26 +0,0 @@
---
description: One tick of the autonomous YellowJacket backlog loop
---
You are the orchestrator of the YellowJacket backlog loop, waking for
one tick. Work in this directory. Read `.pi/skills/yj-loop/SKILL.md`
first — it is the operating procedure and it binds you. The design
questions are answered in `.planning/plans/active/020-autonomous-backlog-loop.md`;
the skill is what you run.
One tick means:
1. Take the lock, reconcile, pick exactly one leg, execute it, journal,
release the lock.
2. Delegate every deliberative leg to its `yj-loop.*` agent by name —
the model is pinned in the agent file, never an argument. You hold
only claim, shipping polls, merge, housekeep.
3. Touch only what the loop created. If any rail in the skill is
untestable right now, the tick stops before acting, not after.
4. If the scheduler fires while you are mid-answer, finish this tick
only. Two ticks never overlap; the lock is yours.
Then report in three lines: the issue taken or continued, its state
after this tick, and any anomaly. Stop. Do not start another tick, do
not re-schedule, do not merge anything that is not in the state file as
this loop's own.
+1 -8
View File
@@ -158,7 +158,7 @@ only climb when it cannot.
| You changed | Run | Cost | | You changed | Run | Cost |
|---|---|---| |---|---|---|
| A Lit component, a store, the shortcut service | `make ui-test` | ~2 s, no app | | A Lit component, a store, the shortcut service | `make ui-test` | ~2 s, no app |
| …and it renders differently | `make ui-visual` | + 10 baselines, opt-in, never gates | | …and it renders differently | `make ui-visual` | + 6 baselines, opt-in |
| Any Go code | `make test` | 3 passes, ~2 min | | Any Go code | `make test` | 3 passes, ~2 min |
| A service that emits events | `make test` — assert on the payload, see `backend/queue/emit_test.go` | in-process, no app | | A service that emits events | `make test` — assert on the payload, see `backend/queue/emit_test.go` | in-process, no app |
| A bound method or a bound struct field | `make bindings` then `make ui-test` | ~1.5 s + 2 s | | A bound method or a bound struct field | `make bindings` then `make ui-test` | ~1.5 s + 2 s |
@@ -180,13 +180,6 @@ less than it looks.)
Two rules about climbing: Two rules about climbing:
- **If you moved a component's geometry, run `make ui-visual` and
refresh that component's baseline in the same commit.** Nothing else
will: it is the one tier in this repo no hook and no CI job runs, and
it cannot be one — its references are machine-specific, measured in
[references/ui-tier.md](references/ui-tier.md). Four of them drifted
across three merges before anyone noticed (#196). Read the image;
never bless a reference you did not cause.
- **A component test passing is not the app rendering.** If you touched - **A component test passing is not the app rendering.** If you touched
anything in `frontend/src`, verify it in the real app too — start it anything in `frontend/src`, verify it in the real app too — start it
headless, `screenshot --filename=/tmp/shot.png`, and *read the PNG*. headless, `screenshot --filename=/tmp/shot.png`, and *read the PNG*.
@@ -42,16 +42,11 @@ strings and identical specs produce different bytes on different builds.
playback and then clicks pause races the track ending and fails playback and then clicks pause races the track ending and fails
against a correct UI. Use `LONG_TRACK` (90 s, `edge-lengths`) exported against a correct UI. Use `LONG_TRACK` (90 s, `edge-lengths`) exported
from `e2e/support/fixtures.ts`. from `e2e/support/fixtures.ts`.
- **WAV tracks scan like every other format.** #104 added - **WAV tracks scan in untitled.** `backend/tagwriter` writes WAV tags
`backend/riff`, so the scan reads the `id3 ` chunk `backend/tagwriter` into a RIFF `id3 ` chunk and `dhowden/tag` has no RIFF parser, so
writes and both WAVs come in fully tagged: "Field Recordings" is an there is no "Field Recordings" artist in the Artists view. This is a
ordinary artist in the Artists view, with a "Test Tones" album and a known open bug pinned by `TestWAVTagsAreNotReadableYet`; do not
cover. They are therefore not an example of an untitled or albumless "fix" a spec by asserting the broken behaviour elsewhere.
track — the only two tracks with no album are
`unsorted/no-tags-at-all.mp3` and `unsorted/title-only.mp3`. Prose
written before #104 says the opposite and names
`TestWAVTagsAreNotReadableYet`, a test that change deleted; that is
dated history rather than a description of the app.
## Seeds ## Seeds
@@ -57,21 +57,6 @@ behind `YJ_TESTCTL=1`, which `scripts/dev-headless.sh` sets and
staging the work that would produce it — job progress, download staging the work that would produce it — job progress, download
progress, scan progress. It calls `events.Deliver`, which *errors* progress, scan progress. It calls `events.Deliver`, which *errors*
when the event reaches nobody, so a `200` means it really arrived. when the event reaches nobody, so a `200` means it really arrived.
- **State you stage, you own** (#168). Nothing resets those stores, so
clear yours in `test.afterEach` with the same event that staged it
(`emit('JobsChanged', [])`) — the store replaces its list from every
snapshot, so `testctl` needs no special case. **Measured: this does
not currently cross a spec boundary**, because every test gets a fresh
page and `JobStore.init()` refetches `GetJobs()` from a backend
registry that `/__test/emit` never writes to. Stated anyway, because
it costs one line and the leak needs only one spec that keeps a page
alive — but do not cite #168 for a symptom you have not reproduced.
- **Measure against the thing next to you, not an absolute
coordinate.** An absolute number in a shell measurement is also a
claim about everything above it — `contentTop === 0` quietly asserts
"and no background job is running", which is not what that spec was
about or could arrange, while `contentTop === jobBandBottom` is true
either way. This is the half of #168 that stands on its own.
- **`restore` is slow** (~40 s in the suite) because it copies every - **`restore` is slow** (~40 s in the suite) because it copies every
table. Prefer snapshotting once and restoring only when a spec table. Prefer snapshotting once and restoring only when a spec
genuinely mutates state. genuinely mutates state.
@@ -78,58 +78,9 @@ synchronously.
Microtasks and not a timer, deliberately: a timer hangs forever under Microtasks and not a timer, deliberately: a timer hangs forever under
the suites that install fake ones. the suites that install fake ones.
## The visual tier does not gate, and that is measured (#196) Visual baselines are font-hinting and compositing sensitive, which is
why they are opt-in: they only mean anything on the machine that
`make ui-visual` is the same suite with nine `toMatchScreenshot` recorded them.
baselines switched on. **Nothing runs it but a person**, deliberately,
and the reason is a number rather than a preference: the committed
baselines were recorded on Arch, and replayed in a bare `ubuntu:24.04`
container — CI's `check` image — three of them fail for reasons that
have nothing to do with any component.
| baseline | Arch | ubuntu:24.04 |
|---|---|---|
| `page-header` filtered-by-search | passes | ratio 0.03 differ, against a 0.02 allowance |
| `track-info` | passes | ratio 0.03 differ |
| `seek-bar` | 1152×18 | 1152×17 |
The two references that were genuinely stale did not even agree about
their *new* size — `now-playing` renders 1152×65 on Arch and 1152×64 in
the container. So moving CI's `check` job from `make ui-test` to
`make ui-visual` is not a one-line change: it needs a second,
container-recorded baseline set, which every local run would then fail
against. That is the same trap the other way round, and a pre-push hook
is the same fault again — one machine's baselines against everybody
else's renderer.
So the tier stays local and opt-in, and the rule that replaces the gate
is:
- **A change that moves a component's geometry refreshes that
component's reference in the same commit, having read the image.**
Look at the PNG; the dimensions in the failure message are the cheap
half of the answer.
- **Never refresh a reference you did not cause.** #196 exists because
four of them drifted across three unrelated merges, and every red run
made the next person likelier to stop running the tier than to read
it.
- **State the world the shot is taken in.** The stores are singletons,
so a visual case that sets nothing photographs whatever the previous
case left behind — which is how the sidebar's baseline came to have
Tracks lit and `now-playing`'s to be playing from a dynamic mix.
- **Record one file with `make ui-visual-update UI_ARGS=<path>`**, and
check `git status` before committing either way. That filter is only
honoured since #204: the recipe was a bare `--update`, and vitest
takes the following positional as the flag's value, so the path was
swallowed and *every* baseline was re-recorded — blessing any stale
one in silence.
What the tier is worth, for the record: it is a *layout* check, blind to
colour (the component tier has no `:root`, so it renders the fallbacks —
`make ui-visual` passed unchanged through a whole palette rewrite,
twice), and it has caught one thing nothing else could — swapping
`library-status-indicator`'s `<button>` for a `<span>` lost the UA
stylesheet's `box-sizing` and grew the badge 36→38px.
## Bindings ## Bindings
-306
View File
@@ -1,306 +0,0 @@
---
name: yj-loop
description: Operating the autonomous backlog loop — the crank that works the YellowJacket tracker one issue at a time (tick mechanics, the state machine in Gitea, which agent and model take each leg, the escalation ladder, merge authority and the rails that stop it doing damage). Use whenever a scheduled tick fires, and when piloting or debugging the loop.
---
# The YellowJacket backlog loop
Design and arguments: `.planning/plans/active/020-autonomous-backlog-loop.md`.
This skill is the **operating procedure**; the plan is the reasoning.
`yellowjacket-dev` is the harness doctrine (tiers, seeds, traps); this
skill is the loop doctrine (who acts, on what model, with what authority).
Read the plan first, once. Then this file every tick.
## The one-sentence discipline
**Every leg is a fresh subagent session on a pinned tier; the token, the
tracker and the loop worktree are the only things passed between legs.
Never switch a model mid-session, never let two writers exist at once,
never keep state in a conversation.**
## Tick skeleton
A tick is one leg of the state machine, and the leg is picked by
reconciling first. Execute in this order:
1. **Lock.** `/tmp/yj-loop.lock` holds `pid + start-iso`. If a live
process owns it and is younger than 2 h: exit immediately, report
"tick skipped (lock held)". If the PID is dead, take the lock.
Remove it before every exit.
2. **Reconcile.** Fresh reads, never cached: open issues
(`scripts/issue.sh list`), PRs and CI via the REST API, branches via
`git ls-remote --heads origin`, `.pi/loop/state.json`. GITEA_TOKEN
refusing = the tick reports and exits; the identity rails below are
not optional.
3. **Pick the leg.** See the state machine below; the leg follows the
issue's lifecycle (claim→plan→…→merge→…→housekeep). Exactly one leg.
4. **Execute** — the leg table below says who acts and what they must
return.
5. **Journal** — one line per tick in the state file (issue, leg, result,
tick cost if leg reports it).
6. **Report** — three lines: issue taken or continued, its state now,
anomalies. Then stop. A tick that reports is a tick that can leave a
conversation behind.
## The state machine
The tracker is the truth. The state file (`.pi/loop/state.json`,
gitignored) is an index plus flags (`emulator`, `drain`); the tracker
wins every disagreement.
| Stage | Where it lives | Leg → actor |
|---|---|---|
| selected | nothing written until claim is possible | select |
| in flight | `Status/In Progress`, assignee, comment with branch+approach | claim (orchestrator, `scripts/issue.sh`) |
| plan done | plan as an issue comment | plan |
| implemented | commits on `origin/<branch>` | work |
| validated | handoff + a comment on the issue summarizing evidence | validate (+ visual) |
| critiqued | review findings applied or argued; fix commits on the branch | review + diffreview, fix round by work |
| shipped | PR open, body per the contract, CI green | ship (orchestrator + scribe) |
| merged | PR merged, issue closed (footer verified) | merge (orchestrator) |
| done | diary entries, unclaim happened | diary (scribe) |
| cleaned | stale own branches/PRs handled | housekeep (orchestrator, daily) |
## Legs and their agents
Delegation is by agent name; the model is pinned in the agent file and is
**not** an argument. Every leg prompt names: the issue, the evidence so
far (plan comment, handoffs), what the leg must produce, and its stop
rules. Never "go fix it" — the leg contract is in this file.
| Leg | Agent | Model (tier) | Produces |
|---|---|---|---|
| gather/mechanical dump | `yj-loop.inspect` | go/mimo-v2.5 (T0) | tracker/PR/CI/branch digest, verbatim |
| select next issue | `yj-loop.select` | glm/glm-5.3 (T2) | one issue + reasons, or "nothing qualifies" |
| plan | `yj-loop.plan` | glm/glm-5.3 (T2) | a plan comment on the issue |
| implement | `yj-loop.work` | qwen/deepseek-v4-pro-0813 (T1) | commits + a handoff (see contract below) |
| validate | `yj-loop.validate` | glm/glm-5.3 (T2) | pass/fail with evidence per acceptance item |
| visual evidence | `yj-loop.visual` | glm/glm-5.3-flash (T2) | what the screenshot actually shows |
| consequences review | `yj-loop.review` | glm/glm-5.3 (T2) | blockers / fix-worthy / optional findings |
| understood-diff review | `yj-loop.diffreview` | qwen/deepseek-v4-pro-0813 (T1) | same shape, scope-tight |
| escalation | `yj-loop.escalate` | go/kimi-k3 (T3) | same leg re-run, seeded with failure summary |
| prose (PR body, commit msgs, journal) | `yj-loop.scribe` | go/mimo-v2.5 (T0) | text only, from supplied facts |
**Model fallback on quota exhaustion.** The pinned models are the
intent, not a guarantee. The qwen token plan is a weekly pool and has
run dry mid-tick (`429 … 1-week quota exhausted`). When a leg's launch
fails with a 429, re-run it with a per-run `model` override one rung
down and journal the substitution — never spend the T3 escalation
model on a quota substitution. The qwen-pinned legs (`work`,
`diffreview`) fall back `qwen/deepseek-v4-pro-0813` → `go/deepseek-v4-pro`
→ `go/glm-5.3-flash`. Do **not** use the `deepseek/...` provider: it has
no models, only catalog overrides, and fails silently (empty artifact,
no session) — the model lives on the `go` gateway.
**Launch legs in the foreground.** The async subagent runner has died
without persisting a child session (nothing to resume) and emits
spurious "needs attention" nudges on runs that are already complete.
Foreground `subagent` calls are the reliable mode here. A worker that
dies mid-leg leaves uncommitted work: inspect the tree, then relaunch
to *complete* — never to re-implement.
Orchestrator-only legs: **claim** (`issue.sh claim --branch` — atomic,
refuses if held), **ship's PR/CI polling** (REST API below — `gitea_ci`
job_logs 404s on this Gitea; the REST endpoints are the way), **merge**
(API below), **housekeep**.
## Selection rules (`select`)
The rules from `.pi/prompts/next-issue.md` stay — priority order, #73's
sequence overriding labels where it speaks, skipping `Status/*` states
that mean busy, branch-collision check, verifiability, flakes. The
emulator flag **adds** emulator-verifiable Android issues; it never
reaches device-only ones. A "nothing qualifies" answer is a correct
tick, not a failure — report it and stop.
## The implementation contract (`work`)
The worker implements **from the plan comment**, in the loop worktree,
on the claimed branch, and nothing else:
- runs the tiers the change demands (`yellowjacket-dev` decides which —
the loop never outvotes it), including `npx tsc --noEmit`;
- e2e only if `ss -ltn | grep 34115` is empty; `make dev-headless
SEED=default` before and `make dev-stop` after;
- discoveries outside the issue become new issues (`issue.sh new`), never
bigger diffs; a materially-larger-than-implied issue stops the leg with
a comment and a label removal, not a hail-mary;
- handoff must state: changed files, what was left undone, commands run
with exit codes, verification evidence, surprises, decisions needing
approval. A handoff without that list is a failed leg.
## Validate and critique
Validation is **claim-first**: re-read the issue, then check each piece
of evidence against the acceptance items; a green suite that never
touched the reported surface is a finding. Screenshots go to `visual`,
never to a text-only tier.
Critique is the standing fan-out (`subagent` parallel: `yj-loop.review`
consequences + `yj-loop.diffreview` scope-tight, both fresh). The
orchestrator synthesizes: blockers and fix-worthy findings go back to
`work` as one bounded fix round (maximum three rounds total; then the
issue gets a `⟦loop⟧` comment stating what will not be fixed and why,
and the ship leg proceeds unless a finding is a blocker). Reviewers do
not edit files.
## Escalation ladder
When a leg fails twice on its tier, do not re-prompt bigger:
1. The failing session writes its summary: what it tried, what failed,
what it observed.
2. A **new** session on the next tier up is seeded with that summary and
the original leg contract.
3. T3 is the ceiling: fresh session, never parallel, **once per day**.
A day's escalation is spent — the issue waits until tomorrow.
Routing down is free; routing up is the budget.
## Ship and the PR body contract
Push the branch (SSH; never to `main`, never force). The PR body —
written by `scribe` from the validator's and reviewers' output — states:
what the issue was, what changed and why, **which verification tiers ran
and their results**, what was deliberately not done, the commit-to-issue
table, and `Closes #n`. `Closes` also sits one-per-line in a commit body
**inside the branch** — both, regardless of merge strategy, because the
pairing was measured.
Poll CI until `check` and `e2e` finish. On failure: read the log via
`GET /api/v1/repos/yonlu/yellowjacket/actions/runs/<run>/jobs` (per-step)
and `…/actions/jobs/<id>/logs` (full). Fix on the branch. **Two
consecutive identical failures = stop**: comment what is known on the
PR and the issue, leave both, report. Do not burn ticks on a red wall.
## Merge authority
Merge when, and only when, **all** hold:
- the PR was opened by this loop (it is in the state file's index);
- the protection contexts `CI / check` and `CI / e2e` are green on the
PR's head, read from the API, not from the PR page's badge;
- the PR reports mergeable;
- the critique leg ran and no open blocker stands;
- the branch is **not behind `origin/main`** — the protection's
`block_on_outdated_branch: true` refuses it anyway; never
`force_manually_merged` around it.
**Refresh before every merge.** In the loop worktree: `git fetch origin`
in the same breath, then `git merge origin/main` on the PR branch,
push. The fetch must be immediate — a cached `origin/main` merges
against the wrong base, CI goes green on it, and the merge comes back
405 "behind base", one whole CI cycle wasted (measured on the adoption
wave). A textual conflict
stops the leg there — as diff text, not as a failed merge click: hunks
the loop authored are resolved by the loop; anything else is left with
`⟦loop⟧` comment for a human, never forced. After any refresh push,
re-poll the PR's own required contexts on the **new head** before
merging.
**Merges happen one at a time**, each re-reading state — the previous
merge moved `main`, and the next PR's mergeability is recomputed at
its own turn.
```
curl -sS -X POST -H "Authorization: token $GITEA_TOKEN" \
-H "Content-Type: application/json" \
https://git.ljones.me/api/v1/repos/yonlu/yellowjacket/pulls/<n>/merge \
-d '{"Do":"merge","merge_message_field":"default","force_manually_merged":false}'
```
**Afterwards watch the `push` run on `main`** — the CI the merge
started. A red main after a loop merge is a **halt**: comment what is
known on the offending PR, mark the state file, stop taking new issues.
That run is the only thing between a clean textual merge of
independently-written PRs and a self-contradicting main; no
mergeability check sees it. Only a green main lets the tick proceed (to
footer verification, below).
Footer verification: `scripts/issue.sh list --state open` and check
the footer took. Close stragglers with `issue.sh close`, naming the
merge commit. `unclaim.yml` handles the label; it is not instant;
reopening does not restore it. Merging fans out to nothing (releases
are the manual `release.yml`, which the loop never runs) — the
criticism stands before the merge because nothing stands after it.
## Rails — the loop's absolute rules
1. **Touch only its own.** Issues it claimed, branches it made, PRs it
opened. `issue.sh claim` enforces the front gate; never work around a
refusal.
2. **One writer, one issue.** The loop worktree is the only dirty tree.
3. **Never merge a PR it did not open.** Any merge that violates this is
a hard stop.
4. **Human work is holy.** Human branches, PRs, assignees: leave exactly
as found. Cleanup never names them.
5. **The token is identity.** If GITEA_TOKEN misbehaves, the tick stops.
6. **New findings are new issues**, never scope creep. The tracker
vocabulary (`Kind/`, `Area/`, `Priority/`) stays intact in one
taxonomy; use `scripts/issue.sh new` with correct labels.
7. **Conventional Commits**, enforced by `scripts/commit-check.sh`; the
type list and `.releaserc.yml`'s must agree — a loop commit is a
release grammar token even after months of no manual releases.
8. **Tiers over vibes.** `yellowjacket-dev`'s tier table decides what a
change must pass; a skipped tier is stated, never silent.
9. **Two strikes on CI, three rounds of critique, one kimi a day.** The
loop's patience is finite on purpose.
10. **Every leg writes its evidence.** A leg that leaves nothing behind
is indistinguishable from a leg that did not run — which is how the
next tick re-does it.
11. **The loop may not re-schedule itself** (the scheduler refuses it
anyway — treat as an invariant, not a limitation).
12. **Drain means drain.** `drain: true` = finish in flight, take
nothing new, then stop.
## Emulator mode
Flag `emulator: true` in the state file **and** an already-booted
emulator (`adb devices` answers) opts in: `make android` (build), `make
android-install`, `make android-smoke` (crash check — the same pid
surviving is the only signal that means started), `make
android-screenshot` and `make android-eval` as evidence for `visual`.
The loop never boots or stops an emulator; that is the user's machine.
Device-only issues stay open under either setting. One-time setup the
user performs: `make android-setup` (~3.5 GB, creates the `yj-test`
AVD), then `make android-emulator` per session.
## ON / OFF / drain
- **Worktree:** `git worktree add ~/.paseo/worktrees/loop/jumpy-hound
origin/main` (from any clone; branch from origin/main in the loop
tree, never `git checkout main`). **Provision it once before the
first push:** `make build-frontend` and `make testdata` — the pre-push
`go-test` hook needs `frontend/dist` (the `//go:embed` in `main.go`)
and the fixture library, and refuses the push without them.
- **Session:** pi in that worktree, `/name loop`. Add the job via
`/schedule-prompt` (name `yj-loop`, cron
`0 0 10-18 * * 1-5`, prompt: "Read `.pi/skills/yj-loop/SKILL.md` and
run exactly one tick. Stop.") — session-bound by default.
- **OFF:** toggle the job, or close the session. **ON:** `pi --resume
loop` in the worktree, job enabled. Courses of the tick appear in
that session's transcript.
- **Tune in:** the same resume. Talk to it only between; a tick is
atomic.
## Troubleshooting
- `issue.sh: GITEA_TOKEN is not set` or a 401 — the token is the whole
identity (rails 5). Stop, do not fall back to anything.
- `gitea_ci`'s job log 404s — the REST endpoints above answer; this is
a Gitea build, not a fault.
- A spec fails that the tier doc says can fail from stale backend state
— restart the app tier before believing it (`yellowjacket-dev`).
- A tick that "did nothing" — reconcile again; the tracker usually says
which leg it really is.
- The job did not fire — the scheduler fires only while a session is
open in its directory (documented); "the loop is off" is the correct
reading, not a bug.
- `error: object file … is empty` / `unpack-objects failed` / `bad
object refs/heads/…` during a fetch or checkout — the shared object
store was corrupted (a killed fetch leaves 0-byte object files, and a
local ref can end up pointing at the dead sha1). **Halt and report**;
do not retry, the churn only deepens it. Human repair: delete the
0-byte objects, `git fetch origin --prune`, delete any ref that
still dangles (`git update-ref -d refs/heads/<b>`), re-checkout the
worktree at `origin/main`, then `git fsck --full`.
-185
View File
@@ -4912,188 +4912,3 @@ bridge leaves the wizard up with its "Get Started" button correctly
disabled — it gates on a directory chosen *in the wizard*, and the disabled — it gates on a directory chosen *in the wizard*, and the
existing-library check runs once, on mount. A reload clears it. Nothing existing-library check runs once, on mount. A reload clears it. Nothing
is broken; it cost twenty minutes of believing a tap had been swallowed. is broken; it cost twenty minutes of believing a tap had been swallowed.
## The sheet's scroll fade, and where a scrim may not go (measured 2026-08-23, headless)
#207's answer. The affordance is two background layers on
`wa-dialog::part(body)` and the conditionality is
`background-attachment`, not a scroll listener: a cover of the sheet's
own colour painted at the end of the *content* (`local`) over a shadow
pinned to the box (`scroll`), so the cover scrolls up and hides the
shadow exactly when there is nothing more to see.
Measured at 424x360 (which is where a menu overflows on `main`, since
`main` does not yet carry #67's eighth item — at 424x439 the track
list's seven items are `scrollHeight` 364 against `clientHeight` 364,
fitting exactly). Pixel at x=300, dark ramp, `bgElevated` `#343a40`:
| y | before | more below | at the end of the list |
|---|---|---|---|
| 330 | 52,58,64 | 50,56,62 | 52,58,64 |
| 340 | 52,58,64 | 43,48,53 | 52,58,64 |
| 350 | 52,58,64 | 33,37,40 | 52,58,64 |
| 359 | 52,58,64 | 22,24,27 | 52,58,64 |
Three things worth keeping.
**A menu that fits draws nothing**, which is the same measurement: at
424x439 the sheet is flat 52,58,64 to its bottom edge, because with no
overflow the `local` layer's positioning area *is* the padding box and
the cover lands on top of the shadow.
**A scrim over a menu row is that row's text surface**, so the 4.5:1
rule reaches it and this is why the curve is steep rather than linear.
A row is 48px with its label centred; 32px of scrim already down to a
quarter strength at 14px puts about 0.06 at the label. Checked on the
light ramp (`bgElevated` `#e9ecef`, text `#212529`) by overriding the
two custom properties on `:root`: background at the label 205,207,210,
which is **9.9:1**. The first draft — a linear 48px at 0.8 — put ~0.375
on that label, 5.0:1, passing but visibly greyed. The bottom few pixels
go to ~2.4:1 in either draft and are deliberately below where any
label of a *fully visible* row sits; a label that lands there belongs
to the half-cut row, which is the thing being signalled.
**A dark scrim on a dark surface reads far worse in a shrunk screenshot
than on screen.** The first two probes (24px/0.45, then 32px/0.75) were
measurably present — 52,58,64 down to 30,33,37 — and invisible in the
inline preview. Crop the bottom 70px and scale it up before judging;
the pixel values are the honest answer either way.
## The phone's context sheet is now longer than the phone (measured 2026-08-23, headless at 424x439)
#67 moves two destinations into every row menu, and the track list's
menu is where that runs out of screen. Measured against the running
app at the reference viewport, one row selected:
| menu | items | first item top | last item bottom |
|---|---|---|---|
| queue panel | 7 | 95 | 431 |
| track list | 8 | 86 | **470** |
The viewport is 439. So the track list's last item — "Remove from
Library" — is below the fold. It is **not unreachable**: the sheet is a
`wa-dialog` whose body is `overflow-y: auto`, measured `scrollHeight`
412 against `clientHeight` 373, and scrolling it 39px brings that item
fully into view (383–431). What it has is no *affordance*: nothing on
screen says the list continues.
Two things worth knowing before adding a ninth item anywhere.
**The limit was already reached, and this is what crossed it.** Seven
48px rows in a 373px body is 364px — the queue's menu fits with 8px to
spare and the track list's fitted exactly. Any item added to any of the
fourteen menus after #60 was going to be the one that overflowed; the
first one simply happened to be this.
**The measurement has to be taken with a row selected**, since the
`Go to` items are drawn for a single selection only, and on the *first*
track of the fixture library — which has no album (`01 Tone A`,
`02 Tone B`) — only "Go to Artist" appears. That is the 8 above; an
ordinary track makes it 9.
Filed as its own issue rather than fixed in #67's diff: it is a
property of the shared sheet (`components/menu-surface/`), not of the
items.
## The tap highlight is one inherited declaration (measured 2026-08-24)
`-webkit-tap-highlight-color` is an **inherited** property, and an
inherited property crosses a shadow boundary — so `html { … :
transparent }` in `index.css` reaches every shadow root in the app and
no component needs a rule of its own. Measured in the running app
(Chromium, `app-sidebar`'s `li button`, which is three shadow roots
from the document): `rgba(0, 0, 0, 0)` with the rule, and
`rgba(0, 0, 0, 0.18)` with it removed. That 0.18 grey over the bounding
rect of whatever was tapped is what #54 reported.
The same argument was already spent once and is worth not
re-deriving: `index.css`'s first rule is `*, *::before, *::after {
user-select: none }`, which for the same reason already covers the
shadow roots — #54's Findings ask for `user-select` on interactive
surfaces and it has been done since before the issue was filed.
**What the highlight was, on the surfaces that had nothing else, is the
press feedback.** Measured on a track row with the press rule removed
and the button held down: `rgba(255, 255, 255, 0.05)` — the *hover*
tint, arriving because the pointer is over the row, which is a
synthesised hover on a phone and outlives the press. With the rule:
0.12 while held, and the neighbouring row unchanged. So the press state
is part of removing the highlight rather than a separate polish item,
and the hover tints on those same surfaces moved behind
`(hover: hover) and (pointer: fine)`, which is #68's gate applied to a
tint rather than to a revealed control.
**`touch-action: manipulation` was considered and not taken.** The
Findings offer it for the 300ms tap delay; this app's viewport is
`width=device-width`, which is what removes that delay in Chrome, so
the stated benefit is not there to win. What it would change is the
gesture stack #63 tuned by measurement on the device (`pan-y` plus a
non-passive `preventDefault`), and that is not measurable from here.
## Art pop-in is measurable in a browser, if you count frames rather than milliseconds (measured 2026-08-24)
#65 is an Android report ("scrolling through albums, the art pops in")
and the desktop harness can measure it, which was not obvious: the
first attempt waited 220 ms after each scroll jump and found **zero**
blank covers on either build. The metric only discriminates at one and
two animation frames after the jump, which is where a pop-in actually
lives.
Protocol, on `make dev-headless SEED=bulk` (4 988 albums), ten
2 400px jumps of `.grid-scroll-container`, counting covers whose rect
intersects the viewport with `naturalWidth === 0`:
| build | blank at frame 1 | at frame 2 | at 50 ms |
|---|---|---|---|
| `main` | 254 / 258 | 214 / 258 | 0 |
| `main`, second run | 254 / 258 | 190 / 258 | 0 |
| prefetch | 117 / 258 | 77 / 258 | 0 |
| prefetch, second run | 118 / 258 | 96 / 258 | 0 |
Two things this protocol gets wrong if repeated carelessly. **A second
run in the same browser session measures the HTTP cache**, not the
build — the skill already warns about this for `make perf`, and it
applies to any image measurement; every row above is a fresh
`playwright-cli close` + `open`. And **the frontend is embedded**, so
comparing builds is a `git stash` *and* a rebuild, not a stash.
**The bulk library's covers are 300x300 and ~3.7 kB**, which is why
both builds are clean by 50 ms here and why the phone's number cannot
be inferred from this one — same caveat the skill already records
about full-size artwork.
**`rangeChanged` and `visibilityChanged` are different ranges**, and
the difference is the whole of this fix's value.
`@lit-labs/virtualizer` reports `_first`/`_last` (rendered, including
the ~1000px overhang) on the former and `_firstVisible`/`_lastVisible`
on the latter. Both grids listen to `visibilityChanged` for scroll
persistence, which wants the visible range and is correct; a prefetch
window measured from it lands mostly on cards that already exist.
Anchored there, the component test could see only one row past the
last rendered card.
**`_overhang` is not configurable.** It is a `protected` field set to
1000 in `BaseLayout` and read by every layout; there is no option on
`grid()`/`flow()` and no property on the element. The issue's Direction
("ask the virtualizer for a larger overscan") is therefore not
available without patching a private, which is why the request is
issued ahead of the element instead.
## Declined, and recorded nowhere else (2026-09-25, #256)
When CLAUDE.md was cut down to rules, most of its "considered and
declined" paragraphs already had a home in a code or config comment
beside what they explain. These three did not:
- **`touch-action: manipulation` was declined** (#54). The 300ms tap
delay it is offered for is already absent on a `width=device-width`
viewport; what it would actually change is the gesture stack #63 tuned
by measurement on the reference device.
- **There is no "Go to Genre"** in the phone row menu (#67). That menu
replaces name links a phone cannot use, and there has never been a
genre link to replace — it would be new navigation, which wants its
own issue.
- **The overlaid queue has no tap-outside gutter on a phone** (#171).
The drawer-style gutter would buy the affordance by taking width off a
full-screen surface on a 424px viewport; back and a 44px close button
answer it instead.
@@ -1,244 +0,0 @@
# 020 — The autonomous backlog loop
**Issue:** #236 (`Kind/Enhancement`, `Priority/Low`)
**Status:** active — phase 0, supervised pilot
**Relates:** #73 (the roadmap the loop follows), plan 005 (the harness the
loop drives). Cost and model-tier doctrine is the `pi-session-reference`
card handed to the session that designed this; the loop's copies of it
are deliberate one-paragraph summaries, not the authority.
A pi coding-agent configuration that, toggled on, works the Gitea tracker
one issue at a time — triage, claim, plan, implement, validate, critique,
PR, CI, merge, verify-close, diary — and then does it again. The tracker is
the state machine: whoever reads Gitea sees exactly where the loop is,
which is the property this document's rails exist to protect.
---
## The shape: a crank, not a resident brain
Half the design is that **nothing lives in a conversation**. Each tick is a
fresh, bounded unit of work; every transition writes evidence to Gitea
(label, comment, branch, PR) or to the loop's own state file; a tick that
dies mid-leg loses nothing, because the next tick resumes from what Gitea
says.
The other half is that **no leg trusts the one before it**. The worker
implements from the plan, not from the issue alone; the validator checks
the *claim*, not the green CI row; the merger merges only after reading the
protection contexts itself; the diary leg is what makes the next issue's
triage cheaper.
One issue in flight at a time. That is a pacing decision, not a
concurrency limit of the tooling — CI has a capacity-1 runner and the e2e
tier owns one headless port on this machine, so two writers would serialize
on infrastructure they cannot see and appear to be doing fine.
## The state machine
| Leg | Writes | Actor / model |
|---|---|---|
| reconcile | — | orchestrator + `inspect` (mimo-v2.5) |
| select | nothing on the tracker; decision logged in the tick transcript | `select` (glm-5.3) |
| claim | assignee + `Status/In Progress` + comment naming branch & approach | `scripts/issue.sh claim` |
| plan | plan as an issue comment | `plan` (glm-5.3) |
| implement | commits on the issue branch, in the loop worktree | `work` (qwen/deepseek-v4-pro-0813) |
| validate | verification evidence in the handoff | `validate` (glm-5.3), `visual` (glm-5.3-flash) for screenshots |
| critique | review findings; fix commits | `review` (glm-5.3) + `diffreview` (qwen) + fix round by `work` |
| ship | push, PR with body contract, CI read + fixes | orchestrator + `scribe` (mimo-v2.5) |
| merge | the merge; post-merge issue verification | orchestrator |
| diary | `.pi/journal.md`, `CLAUDE.md` if structural | `scribe` |
| housekeep | stale-branch/PR cleanup, state-file prune | orchestrator |
### Legs that are the orchestrator's alone
The orchestrator (the loop session) delegates every deliberative leg and
keeps three for itself because they are script-shaped and must not be
re-implemented by a model: claim (`issue.sh claim`, which refuses when
someone else holds the issue — the backstop), merge (API calls below), and
housekeep (branch deletion). If a tick does nothing else, it reconciles.
## Model routing
The routing authority is the card's four tiers, reproduced here as the
loop's assignment, not as an argument:
- **T0 `go/mimo-v2.5`** — mechanical gathering, commit/PR/journal prose,
any fan-out. Effectively free; wrong only where wrongness costs a
debugging session, so nothing above takes its word for a *fact*.
- **T1 `qwen/deepseek-v4-pro-0813`** — implement-from-a-written-plan,
understood-diff review, the orchestrator itself. The default session
model; half price 10:00–20:00 EDT, which the cron is shaped around.
- **T2 `glm/glm-5.3`** — repo-scale reasoning: selection, planning,
consequences review, validation judgement. Weekly credits with no
rollover: the loop draws them every week by construction, which is the
correct posture. **`glm-5.3-flash`** for anything multimodal
(screenshots, UI inspection).
- **T3 `go/kimi-k3`** — escalation only: two lower tiers already failed,
or the issue is a named gnarly one. A fresh session seeded with the
failing tier's own summary, never a mid-session switch, never parallel,
at most once per day.
The invariant behind all four, from the card: **routing down is cheap,
routing up is expensive.** An implementation that stalls is escalated by
having the T1 session write *what it tried, what failed, what it observed*
and handing that to a new session one tier up. Escalating a session in
place is forbidden in both directions.
Fan-out is allowed on T0 and T1 only (the Go plan's $12/5 h constraint
makes T3 fan-out self-defeating). Critique is the one standing fan-out:
two reviewers, two angles, one synthesis.
## Scheduling
`0 0 10-18 * * 1-5` (local = EDT): hourly on weekdays inside Qwen's
half-price window, clear of the card's ⚠ 2–6am band (DeepSeek peaks, GLM
loses its off-peak discount — the window the old `yj-backlog` cron sat in,
which this replaces as the loop supersedes it).
- A tick takes a lock (`/tmp/yj-loop.lock`, PID + timestamp). An overrun
tick makes the next fire exit immediately; serialization survives
whatever the scheduler does with overlapping fires.
- ~9 ticks/day; an issue is 2–5 ticks; **one to two issues per day** is
the natural rate. That also paces the bills without a budget flag.
- The port check is part of reconcile: if `34115` is occupied, the tick
refuses any leg that needs the headless app and defers to the next
tick, without complaint. A human's interactive tier always wins.
## Runtime and ON/OFF
The scheduler (`pi-schedule-prompt`) fires only while a pi session is open
in the job's directory — that limitation is the switch:
- **Worktree:** `git worktree add` a dedicated clone at
`~/.paseo/worktrees/loop/jumpy-hound`. Loop edits happen only there; a
dirty tree there is the loop's business and nobody else's. **Provision
it once before its first push:** `make build-frontend` + `make testdata`
— the pre-push `go-test` hook needs both and refuses without them.
- **Session:** pi in that worktree, `/name loop`. The job is bound to that
session, so another pi elsewhere in the same directory does not
double-fire it.
- **ON:** resume the loop session (`pi --resume loop`) and enable the job.
**OFF:** toggle the job off in `/schedule-prompt`, or close the session.
**Drain** (stop taking new work, finish in flight): set `drain: true` in
the state file.
- **Tune in:** the same `pi --resume loop` — the chat transcript *is* the
loop's log, each tick's reasoning inline, each leg reporting in.
## Identity, claims, and what the loop may touch
The loop operates **as the owner** via `GITEA_TOKEN` (scopes: `read:user`,
`write:issue`, `write:pull`, `write:repository`); pushes ride SSH and need
no token. Every tracker comment the loop writes is prefixed `⟦loop⟧`, so
the collaborator reads it as the pump and not as a person.
It may only ever touch work it created: issues it claimed, branches it
made, PRs it opened. Two mechanisms make that enforced rather than
intentional: `issue.sh claim` refuses an issue somebody else holds, and
reconcile checks `git ls-remote --heads origin` so a branch name collision
from a concurrent session is caught before the first edit.
## Merge lifecycle
- **Only PRs the loop opened.** A collaborator's PR is never merged, never
commented on for pressure, never touched.
- **Every branch is refreshed against main before its merge**, in the
loop worktree — the refresh is where a textual conflict surfaces, as
diff text: hunks the loop authored are resolved there, anything else
is left to a human with a `⟦loop⟧` comment. The protection's
`block_on_outdated_branch` makes the refresh mandatory for adopted
(pre-loop) branches: behind `main`, a PR cannot merge at all.
Required contexts are re-polled on the refreshed head.
- **Merges are one at a time**, each re-reading state — the previous
merge moved `main`, and the next PR's mergeability is recomputed at
its own turn.
- **Post-merge, the `push` run on `main` is watched.** A red main after
a loop merge halts the loop. That run is the only guard against the
class no mergeability check sees: two PRs touching the same file,
merging cleanly, contradicting each other.
- The gate is the protection rule itself, read from the API: contexts
`CI / check*` and `CI / e2e*` green, PR mergeable. (Required approvals
is 0 today; if a second person changes protection rules, the merge
endpoint refuses and the tick stops and reports — human business.)
- `Closes #n` goes **in a commit body inside the branch, one line per
issue, and in the PR body**. Both, because a squash route and a merge
route parse different texts, and this pairing was measured: a comma
list partially matched, five of ten issues.
- After merging: verify against `issue.sh list --state open` that the
issue actually closed; close any straggler naming the merge commit.
`unclaim.yml` strips `Status/In Progress` automatically; it is not
instant, and a re-open does not restore it — the verification is
against the open list, not against the label.
- Merging to `main` fans out to nothing: releases are the manual
`release.yml`, which this loop never runs. The blast radius of a
merge is the main branch's CI, and the critique leg is what stands
before it.
## Verification contract
The tier table is `yellowjacket-dev`'s; the loop re-states nothing above
it except the *division of duty*: the worker runs the tiers the change
demands, and the validator re-reads the issue and checks that the tier
evidence actually answers the claim — a green suite that never touched
the reported surface is a finding, not a pass. Cosmetics are read by a
model that can see (`visual`, the multimodal tier); a change that moves
geometry refreshes its `ui-visual` baseline in the same commit.
`tsc --noEmit` is part of the gate and nothing else runs it. The e2e app
is seeded (`SEED=default`) and stopped after.
## Android / emulator mode
The loop is **device-free by default**: issues whose verification is
physical-device behaviour stay open for humans (the repo's own tags say
which those are). One step of the ladder exists for the rest:
- `{"emulator": true}` in `.pi/loop/state.json` **plus an already-booted
emulator** (`adb devices` answers) opts the loop into building the APK
and using `android-smoke` (crash verification), and `android-screenshot`
/ `android-eval` as rendering evidence for `visual`.
- The loop **never boots or stops an emulator** — that is the user's
machine and their gesture. Boot it with `make android-emulator`
(one-time `make android-setup`, ~3.5 GB, creates the AVD), and
`make android-emulator-stop` when done.
- Real-device-only issues are skipped under either setting.
## Budgets and pacing
Expected spend: dominated by the T1 implementation leg inside the
half-price window (pennies to tens of cents) and T2 on weekly credits;
T3 bounded at one fresh call per day. The card's numbers ($12 per rolling
5 h, $30/week as burst headroom not allowance, GLM reset weekly) are the
sanity cells; the loop's own weekly check compares against them rather
than against the month.
## Cleanup (housekeep leg, once per day)
- Loop-owned branches whose commits are in `origin/main`: deleted, local
and remote.
- Loop-owned PRs open >7 days or red on a second identical CI cause:
commented with what is known (`⟦loop⟧`), and left — never silently
deleted.
- Anything not the loop's (assignee, branch, PR): left exactly as found.
## Pilot phases
- **P0 — supervised.** One tick, user watching the transcript: reconcile,
select, claim, plan. No merge.
- **P1 — observed.** Two ticks ending in the loop's first merge, watched
through CI → merge → verify-close.
- **P2 — unattended.** The schedule left on. Weekly check against the
card's two-minute ritual.
- **Hard stops** (any of these halts the loop and leaves a comment, never
a silent retry): a tick dies twice with no explanation; a merge happens
for a PR the loop did not open; spend outside the cells above by 2×.
## Not now, on purpose
- **Parallel worktrees** — blocked on e2e's exclusive port; viable only
with per-worktree headless ports or CI-only e2e. The shape (
supervisor + per-issue worktrees) is the target, not the first cut.
- **Weekend batch refactors** — DeepSeek off-peak is real but is a
scheduling knob on top of a working pump.
- **More chain files** — the critique fan-out is a chain; the rest stay
orchestrator-legs until two weeks of unattended runs say which legs
are actually fixed-shape.
@@ -1,263 +0,0 @@
# 021 — Listening accounting: smart plays, skips, and a real history
**Issue:** none yet — open one before the first edit (tracker is the
source of truth; `./scripts/issue.sh search "skip play count"` comes
back empty as of this writing).
**Status:** plan — not started.
**Relates:** play-count rendering (`frontend/src/components/track-list/columns.ts`),
smart playlists (`backend/smartplaylist/`), the event contract
(`TrackPlayCountChanged`), and any future Wrapped / "minutes listened"
surface.
---
## What exists now
Three facts, all load-bearing.
**A "play" is recorded only on a natural finish.** `recordPlay`
(`backend/queue/playhistory.go:9`) is called from exactly one place —
`OnPlaybackFinished` (`backend/queue/handlers.go:14`), and only when
`srcErr == nil`. A track the user skips past at 90% is *not* a play;
neither is one they pause at 60% and abandon. `play_count` /
`last_played` on `audio_files` reflect "finished to the end," nothing
more.
**There is no skip concept at all.** Skipping is indistinguishable
from a natural finish, a pause, or a shutdown. Nothing records "the
user rejected this track," so no downstream feature (smart playlists,
shuffle, the revisit shelf, a future skip-rate heuristic) can ask
about it.
**`play_history` is a write-only log.** It holds
`(audio_file_id, played_at)` and nothing reads it — no sqlc query
touches it, no `PlayHistory` read path exists. Its only recorded
purpose is the timestamps a future "minutes listened over time"
feature would need. It is classified `Authored, Cascade` in
`backend/datamap/datamap.go:272` ("Listening history").
So the gaps are: (1) skips are invisible, and (2) "played" is
under-counted — the opposite of the usual over-counting fear. The
scrobble intuition (count a play once `min(50%, 4:00)` has been
*heard*, independent of how it ends) is the fix for both.
---
## What we're building
A single classification of every track *exit*, plus one row per exit in
a listening log, plus the existing denormalized `play_count` /
`last_played` updated to match the new meaning. Three exit kinds:
| kind | condition |
|---|---|
| `complete` | reached natural end, **or** abandoned with `remaining <= tail` |
| `play` | heard `>= playThreshold`, abandoned before the tail |
| `skip` | user moved to a *different* track before `playThreshold` |
Not counted, not any kind: decode failure, pause/stop/shutdown before
the threshold, and tracks shorter than `minTrackLength`.
### The thresholds — named judgements, one file
Follow the `PreviousRestartThreshold` precedent (`backend/queue/queue.go:28`,
a bare `const` with a comment). A new `backend/queue/listen.go` (or a
tiny `backend/listencount` package) declares:
```go
const (
// A track this short is deliberated jingle / interstitial and is
// never counted, either way.
minTrackLength = 30 * time.Second
// The scrobble rule: half the track, or four minutes, whichever
// comes first (Last.fm / ListenBrainz).
playThresholdMax = 4 * time.Minute
// "Finished enough": within 15s of the end, or the last 10%,
// whichever is larger. A 10:00 ambient track gets a 60s fade
// window; a 2:00 pop song gets 15s.
tailWindowFloor = 15 * time.Second
tailWindowFraction = 0.10
)
func playThreshold(d time.Duration) time.Duration {
return min(d/2, playThresholdMax)
}
func tailWindow(d time.Duration) time.Duration {
return max(d/10, tailWindowFloor)
}
```
Classification is a pure function of `(reason, position, duration)` and
*therefore unit-testable without a player*:
```go
func classify(reason ExitReason, pos, dur time.Duration) Kind
```
`ExitReason` is `finished | skipped | failed | abandoned`. `skipped`
means the queue moved to a different track by user action (Next,
Previous past the restart threshold, PlayIndex, queue replacement,
select-from-a-list). `failed` is the decode-error path. `abandoned` is
pause/stop/unload/shutdown — and in v1 is a no-op (see open question 3).
**"Heard" is approximated by the position at exit.** We read
`player.CurrentPositionSeconds()` at the moment of the transition, not
an accumulated listen-time ledger. A user who seeks to 80% and listens
5 seconds reads as "heard 80%." That is deliberately accepted for v1:
it is how most players actually behave, it is drastically simpler, and
the failure mode ("counted a track you skimmed as played") is mild and
exactly what the scrobble threshold already forgives. Written down
because "position is not listen time" is the one assumption that will
look like a bug if it is not.
**Fires once per listen.** Leaving a track already leaves it; the
`chainID` guard in `player.onPlaybackFinished` (`backend/player/player.go:633`)
already swallows a stale finish callback, and a transition advances
`currentIndex` past the finished track. The classifier needs the same
guard so a Next-then-stale-finish cannot produce two rows. Key it on the
`(audioFileID, chainID)` the transition was about.
---
## Schema — resolved: fresh design, no migration
The A/B migration agonizing is moot. This app has two users and both
are devs, and play counts are explicitly not worth preserving yet — so
the schema is written as if listening accounting had been designed in
from the start, and the existing two databases rebuild what they need
(see below). There is no migration step and none is re-introduced.
**`play_history` is renamed to `listening_events`** and grows the three
kinds, plus the raw position/duration the classification was made from:
```sql
CREATE TABLE IF NOT EXISTS listening_events (
id INTEGER PRIMARY KEY,
audio_file_id INTEGER NOT NULL,
kind TEXT NOT NULL DEFAULT 'complete'
CHECK (kind IN ('complete','play','skip')),
position_seconds INTEGER NOT NULL DEFAULT 0,
duration_seconds INTEGER NOT NULL DEFAULT 0,
occurred_at DATETIME NOT NULL DEFAULT (datetime('now')),
FOREIGN KEY(audio_file_id) REFERENCES audio_files(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_listening_events_audio_file_id
ON listening_events(audio_file_id);
CREATE INDEX IF NOT EXISTS idx_listening_events_occurred_at
ON listening_events(occurred_at);
```
`position_seconds`/`duration_seconds` are kept raw so a future re-tune
of the threshold does not force the events to be re-recorded. `kind`
stays the write-time classification; the raw reading is evidence, not
a second copy of the rule.
**The counters are denormalized onto `audio_files`** — `skip_count` /
`last_skipped` join the existing `play_count` / `last_played`, because
that is where the hot read path already lives and a log join per track
row is not acceptable. This does grow the MIXED-KIND wart (see the
survey below for the structural answer), but it is the *continuation* of
the existing design, not a new leak: play counts sat on `audio_files`
from before this feature existed.
**What happens to the two real databases on next launch.**
`listening_events` is a new table, created verbatim. `audio_files`
gains two columns, which `retireStaleTables` treats as a stale Owned
table and rebuilds by rescan — dropping `play_count` / `last_played` /
`tag_status` with it, which is the accepted cost stated in the issue.
`play_history` is gone from the schema and the datamap, so
`obsoleteTables` drops it; its (natural-finish-only) timestamp rows go
with it. Nothing here is wrong on a fresh install, and on the two dev
machines the answer is the documented "delete and rescan."
---
## Wiring: where the classifier is called
The risk is not the classifier — it is that **every track-replacement
path must classify the outgoing track**, and there are many: `Next`,
`Previous` (past the 3s restart threshold), `PlayIndex`, `playFromStart`,
`SetQueue` / clear-and-play, remove-current, and select-from-a-list.
Miss one and that path silently never records a skip.
So the classification is centralized in one queue method —
```go
// leaveCurrent(reason) classifies the track at currentIndex as it is
// about to be replaced, and records exactly one listening event.
// Must be called without q.mu held (it writes to SQLite).
func (q *Queue) leaveCurrent(reason ExitReason)
```
— which reads position/duration from the player, calls `classify`, and
emits the play/skip row + `TrackPlayCountChanged` when `kind != skip`.
`OnPlaybackFinished(nil)` routes through `leaveCurrent(finished)`, the
navigation methods route through `leaveCurrent(skipped)` before they
advance, and `recordPlay` becomes the "did a play happen" half of it.
Because "one path forgot to call it" is the failure mode, a **source
sweep** pins it, on the pattern of `TestNoDirectRuntimeEmits`
(`backend/events/noemit_test.go`) and `TestCatalogCoversSchema`: a test
walks `backend/queue` for assignments to `currentIndex` (and the
`SetQueue` / remove paths) and fails if a mutation site does not sit
adjacent to a `leaveCurrent` call. The sweep is the enforcement; the
central method is the convenience.
`recordPlay` keeps its existing contract *when a play happens* —
`TrackPlayCountChanged` with `{audioFileId, filePath, playCount,
lastPlayed}` — so the frontend patch path and
`playhistory_test.go` keep passing. A skip emits no per-track event in
v1 (open question 4).
---
## Phases
1. **The classifier.** `listen.go`: the constants, `playThreshold`,
`tailWindow`, `classify`. Table-driven unit tests covering every
cell of the tristate, the <30s exemption, the tail window on both a
10:00 and a 2:00 track, and the clip at the 4:00 cap. No I/O.
2. **Schema.** *Done in this session.* `listening_events` replaces
`play_history`; `skip_count` / `last_skipped` added to
`audio_files`; datamap entry and `TestAuthoredCascadesAreDeliberate`
allow-list renamed; `recordPlay` writes `listening_events
('complete')`. `make generate` run; database / datamap / queue
tests green.
3. **Wiring.** `leaveCurrent`, the navigation/finish/error call sites,
the `fires once per listen` guard, and the source sweep. Extend
`playhistory_test.go` for skip/complete classification through the
queue rather than the pure function.
4. **Smart-playlist field.** `skip_count` (and optionally
`days_since_skipped`) in `smartplaylist.go` field/numeric maps and
the editor's field list, via subquery. A frontend event for skip —
if a UI wants a skip column — follows separately.
## Verification
- **Go:** the classifier is pure and exhaustively unit-tested; the
queue wiring is tested in-process with `events.WithSink`
(`backend/queue/emit_test.go` is the model), asserting a Next at 90%
emits a *play*, a Next at 10% emits a *skip and no play*, a natural
finish emits a *complete*.
- **Database:** schema + datamap tests fail-loud on any new or
reclassified table; `database_test.go`'s listening-events round-trip
asserts the new table and the four denormalized counter columns.
- **e2e:** `e2e/specs/play-count.spec.ts` already awaits
`TrackPlayCountChanged`; add the skip case (advance early, assert no
`TrackPlayCountChanged` and a `skip` row via the `__/test/sql`
endpoint if convenient, or via the playlist effect).
- No visual/component tier needed unless a skip column ships (phase 4).
## Open questions / decisions needed
1. **Migration mechanism.** Resolved — fresh design, no migration (see the schema section). Play counts are not worth preserving, both users are devs, and `audio_files` / `play_history` rebuild-or-drop on next launch.
2. **"Position is not listen time."** Accept the approximation for v1,
or track accumulated listen seconds (a real ledger on the player) now?
3. **Abandon on shutdown.** A track paused at 70% and then app-killed:
count a `play` (scrobble says heard) or leave it unrecorded? v1
proposes *unrecorded* — same as today — to keep the write path off
the shutdown critical path.
4. **Skip event to the frontend.** Emit now (parallel to
`TrackPlayCountChanged`) or only when a surface consumes it?
+3668 -300
View File
File diff suppressed because it is too large Load Diff
-173
View File
@@ -1,173 +0,0 @@
# Contributing to YellowJacket
This is the contributor's half of the [README](README.md): how to build it, how
to check a change, and how a change gets in. [`CLAUDE.md`](CLAUDE.md) is the
deep reference — the architecture, and the reasons behind the shape of it —
and is worth reading before a change of any size, because most of this
codebase's traps are written down there and nowhere else.
## Building from source
YellowJacket is [Go](https://go.dev/) with a [Lit](https://lit.dev/)/TypeScript
frontend, bridged by [Wails v3](https://wails.io/).
| Tool | Version |
|------|---------|
| Go | 1.26+ |
| Node.js | 22+ |
| pnpm | 10+ |
| Wails CLI | v3 — vendored, no install needed (`go tool wails3`) |
The Wails v3 CLI resolves from the `tool` block in `go.mod`, so there is nothing
to install globally; `make setup` fetches it with the rest of the tooling.
On Linux, install the system libraries Wails needs. v3 builds against GTK4 +
WebKitGTK 6.0 by default:
```bash
sudo apt-get install libasound2-dev libgtk-4-dev libwebkitgtk-6.0-dev # Debian/Ubuntu
sudo pacman -S alsa-lib gtk4 webkitgtk-6.0 # Arch
```
A machine without `webkitgtk-6.0` can still build with `-tags gtk3` against the
older WebKit2GTK 4.1 stack, but that is an escape hatch, not what CI or a
release builds. macOS and Windows need no extra system packages. Run
`go tool wails3 doctor` to check your environment.
```bash
make setup # install tooling, frontend packages and the git hooks
make dev # run with hot-reload
make build-dev # debug build with symbols
make build-prod # production build (stripped and trimmed)
make android # the arm64 APK, into bin/
```
The `Makefile` is the front door and carries a one-line description against
each target; `Taskfile.yml` and `build/<platform>/Taskfile.yml` are the build
implementation behind it and are not called directly.
## Generated code
Two generators run from `go generate ./...`, which `make generate` wraps:
**sqlc** turns `backend/database/sql/queries/` into Go in
`backend/database/sql/sqlcgen/`, and **templ** turns `.templ` files into
`*_templ.go` beside them. Never edit either output by hand — run
`make generate` after touching a `.sql` or a `.templ` file.
The TypeScript bindings in `frontend/bindings/` are generated by `wails3`
rather than by `go generate`, so they are a separate step: `make bindings`
regenerates them and `make bindings-check` fails if they are stale.
`frontend/src/events.ts` is generated too, from `backend/events/events.go`.
A pre-commit hook checks that all of this is fresh, so the usual way to meet it
is a failing commit rather than a bug.
## Checking a change
Run the tier the change actually demands, not the cheapest one.
| Change | Command |
|---|---|
| Go | `make lint` and `make test` — both cover all three build configurations |
| A frontend component or store | `make ui-test` (Vitest in a real Chromium, no backend) |
| A user-visible flow | `make e2e`, against a running `make dev-headless` |
| CSS | `make css-check` — see the Chrome 113 note below |
| Anything cosmetic | look at a screenshot; several bugs here were invisible to every assertion and obvious in an image |
`make test` needs the fixture library, which is generated rather than
committed — it runs `make testdata` itself (about a second).
The end-to-end tier drives the real app with no display at all: `make
dev-headless` starts it in the background on `:34115` (add `SEED=<name>` for a
seeded library, built by `make sandbox-seed NAME=<name>`), `make dev-logs` tails
it and `make dev-stop` stops it. **Check the port before starting one** — if
`:34115` is already answering, someone else's app is there, and a green result
about their build is worse than no result.
Two smaller checks exist because the failure they catch is silent:
`make bindings-check` (stale generated bindings) and `make css-check`, which is
two passes — one fails on a `css` literal ended early by a backtick inside a
comment, the other on a nested CSS rule that begins with a bare element
selector. Chrome 113 is what the reference Android device renders with, and it
drops such a rule without a word.
`make vulncheck` runs govulncheck over the module.
## The issue tracker is the source of truth
Work is described by issues before it is described by branches, and the tracker
is shared with people who cannot see your terminal.
- **Search before starting**, closed issues included: `./scripts/issue.sh search
<terms>`. "That was fixed three weeks ago" is the cheapest possible answer.
- **Claim before the first edit**, not before the commit:
`./scripts/issue.sh claim <n>` sets the assignee, applies `Status/In Progress`
and comments with the branch, so the work is visibly taken *while it is being
done*. It refuses if somebody else holds it — talk to them rather than working
around it.
- **If no issue covers the work, open one first** (`./scripts/issue.sh new`).
- **Findings get filed.** A bug tripped over on the way to something else is an
issue with a reproduction, not a wider diff and not a sentence in a chat log.
- **#73 is the roadmap** and states the order the backlog should be worked in.
`scripts/issue.sh` is the whole interface (`list`, `mine`, `search`, `show`,
`new`, `claim`, `unclaim`, `comment`, `close`, `label`, `depends`, `labels`) and
wants a `GITEA_TOKEN` with `write:issue`. The labels are a taxonomy rather than
tags: `Kind/*`, `Area/*`, `Priority/*`, `Platform/*`, plus `Reviewed/*` and
`Status/*`, of which the last two are exclusive scopes.
## Commits and pull requests
`main` is protected, so a branch and a PR are the only way in. Branch from
`origin/main`, and name the branch after the issue (`fix/140-…`, `feat/25-…`).
Commit subjects are [Conventional Commits](https://www.conventionalcommits.org/)
— `type(scope): subject`, imperative, ≤72 characters — and are enforced by a
`commit-msg` hook and by CI (`make commit-check`). This is load-bearing rather
than decorative: semantic-release reads the **type** to decide the next version,
so a CI-only change is `ci:` and never `fix(ci):`, which would ship a patch
release. `make release-dry` prints what a release would cut right now.
**The closing keyword goes in the commit body**, one issue per line, because
Gitea parses commit messages that reach `main` and does not parse the PR body:
```
docs: rewrite the README as a landing page
<why>
Closes #50
```
A PR body carries a commit-to-issue table, the verification you actually ran
(with results), and a `Closes` list for whoever reads it.
## Style
- **Go** — golangci-lint v2, strict: `err113` (static errors), `nlreturn`,
`wsl_v5`, `godot`, `sloglint`, `perfsprint`, and imports grouped stdlib →
third-party → `yellowjacket/…` by gci.
- **TypeScript** — strict mode, no implicit `any`, no unused locals or
parameters.
- Match the surrounding code. Where `CLAUDE.md` explains why something is shaped
the way it is, that shape is load-bearing and there is usually a test pinning
it.
Hooks do most of the enforcing (`lefthook.yml`, installed by `make setup`):
pre-commit runs vet, lint, the codegen checks, the frontend typecheck and the
two CSS checks in parallel; pre-push runs the Go suite and the UI tier,
deliberately one after the other rather than together.
## Where the rest of the documentation is
- [`CLAUDE.md`](CLAUDE.md) — architecture and constraints, in depth.
- [`docs/PROFILING.md`](docs/PROFILING.md) — Go pprof and frontend profiling.
- [`docs/android-release.md`](docs/android-release.md) — the APK, its signing
key, and what the release workflow checks.
- [`docs/index-cache.md`](docs/index-cache.md) — the search-index build cache
and why it has a snapshot.
- [`packaging/arch/README.md`](packaging/arch/README.md),
[`packaging/homebrew/README.md`](packaging/homebrew/README.md) — the two
package channels.
- `.planning/` — design documents and measured history, not a queue. The queue
is the tracker.
+3 -6
View File
@@ -160,11 +160,8 @@ ui-watch: ## Same suite, in watch mode
ui-visual: ## Run the suite including screenshot comparisons ui-visual: ## Run the suite including screenshot comparisons
@cd frontend && YJ_VISUAL=1 npx vitest run $(UI_ARGS) @cd frontend && YJ_VISUAL=1 npx vitest run $(UI_ARGS)
# `--update=true`, never a bare `--update`: vitest takes the following ui-visual-update: ## Re-record the screenshot baselines
# positional as the flag's value, so `--update <path>` swallows the path @cd frontend && YJ_VISUAL=1 npx vitest run --update $(UI_ARGS)
# and re-records every baseline in the repo instead of the one named.
ui-visual-update: ## Re-record the screenshot baselines (UI_ARGS=<path> to filter)
@cd frontend && YJ_VISUAL=1 npx vitest run --update=true $(UI_ARGS)
ui-setup: ## Install the Vitest browser provider's own Chromium (once) ui-setup: ## Install the Vitest browser provider's own Chromium (once)
@cd frontend && pnpm install && npx playwright install chromium @cd frontend && pnpm install && npx playwright install chromium
@@ -192,7 +189,7 @@ css-check: ## Fail on a css`` literal ended early by a backtick, or a nested rul
# Every command in them is a make target on purpose, so this is # Every command in them is a make target on purpose, so this is
# checkable. It also asserts AGENTS.md is a symlink to CLAUDE.md, so the # checkable. It also asserts AGENTS.md is a symlink to CLAUDE.md, so the
# two harnesses cannot drift onto two descriptions of one project. # two harnesses cannot drift onto two descriptions of one project.
skill-check: ## Fail if the docs name a missing make target, or AGENTS.md is not a symlink skill-check: ## Fail if the agent docs name a missing make target, or AGENTS.md is not a symlink
@./scripts/skill-check.sh @./scripts/skill-check.sh
# Conventional Commits, which CLAUDE.md claimed CI enforced for a long # Conventional Commits, which CLAUDE.md claimed CI enforced for a long
+80 -107
View File
@@ -2,139 +2,112 @@
*Music how it was meant to bee.* *Music how it was meant to bee.*
YellowJacket plays the music you already own. Point it at your folders and it YellowJacket is a fast, cross-platform desktop music player for your local
scans them, reads the tags and the cover art, and gives you a library you can collection. It plays your files, keeps your library tidy, and helps you discover
browse, search, queue and tidy up — on your own machine, with no account, no and organize your music — all in a clean, responsive interface. No accounts, no
streaming service and no telemetry. streaming, no telemetry: just your music on your machine.
It plays **MP3**, **FLAC**, **OGG Vorbis** and **WAV**, on **Linux** and Runs on **Linux**, **macOS**, and **Windows**.
**Android**, and builds from source on **macOS**.
![The track list, with something playing](docs/images/library.png) ## Features
## What it does ### Play your music
- Plays **MP3, FLAC, OGG Vorbis, and WAV**
- Play, pause, seek, and volume control with a mute toggle
- Gapless, glitch-free seeking backed by a read-ahead buffer
- A queue you can add to, reorder, and shuffle, with play-next support
- Shuffle and repeat (off / all / one)
- Picks up right where you left off — remembers your track, position, and volume between sessions
- Media-key and MPRIS support on Linux, so your desktop's playback controls just work
**Plays your files.** Play, pause, seek and volume with a mute toggle; a ### Keep your library organized
read-ahead buffer so seeking is instant rather than gappy; a queue you can add - Point it at your music folders and it scans them automatically
to, reorder and shuffle, with play-next; shuffle and repeat (off / all / one). - Reads tags and embedded cover art, and de-duplicates artwork so it isn't stored twice
It remembers the track, the position and the queue between sessions, and it - Incremental sync — only new or changed files get reprocessed, and deleted files are cleaned up
answers your desktop's media keys — MPRIS on Linux, a media notification and - Browse by **album**, **artist**, or **genre**, or search across everything
lock-screen controls on Android. - Mark favorites and see what you've been listening to with play history
- Edit track tags directly when something's off
**Keeps the library tidy.** It scans the folders you give it and rescans only ### Playlists
what changed, so a big library costs its full scan once. It de-duplicates - Create playlists, drag tracks in, and reorder them
embedded cover art rather than storing the same image a hundred times, notices - **Smart playlists** that build themselves from rules (by genre, rating, play count, and more)
files that have gone away, and spots duplicate tracks. Browse by album, artist - Pin a default playlist and spot duplicate tracks at a glance
or genre, search across everything, mark favourites, and see what you have been
playing.
**Playlists, and playlists that write themselves.** Drag tracks in and reorder ### Discover and clean up (powered by MusicBrainz)
them, or describe what you want — genre, play count, how long since you played - **Explore** — browse artists, releases, and genres from the MusicBrainz catalog, not just what's already in your library
it — and let a smart playlist keep itself up to date. - **Auto-tag** — match your files against MusicBrainz to fill in correct artist, album, and track metadata, with a review step before anything is written
- **Lyrics search** — find a track by a line you remember
**Explore and auto-tag, from the MusicBrainz catalog.** Explore browses artists,
releases and genres from the catalog rather than only from what you own, so an
album page can tell you that you have nine of its twelve tracks. Auto-tag
matches your files against MusicBrainz and fills in the metadata that is
missing, with a review step before anything is written to disk. Lyrics search
finds a track from a line you remember.
Explore needs its catalog, which is a one-off ~0.6 GB download from
**Settings → Search Index**. It asks first on a metered connection, and
everything else in the app works without it.
## Install ## Install
Every download comes from the Download the latest build for your platform from the
[releases page](https://git.ljones.me/yonlu/yellowjacket/releases). [releases page](https://git.ljones.me/yonlu/yellowjacket/releases).
### Linux | Platform | Download |
|----------|----------|
| Linux | `yellowjacket-linux-amd64` |
| macOS | `yellowjacket-darwin-universal.app.zip` (Apple Silicon + Intel) |
| Windows | `yellowjacket-windows-amd64.exe` |
Download `yellowjacket-<version>-linux-amd64.tar.gz` from the latest release and Prefer to build it yourself? See [Building from source](#building-from-source).
unpack it. It holds the binary, a `.desktop` entry and an icon.
On **Arch**, install it from the package registry instead and get updates with ## Getting started
the rest of your system — the one-time key import and `pacman.conf` block are in
[`packaging/arch/README.md`](packaging/arch/README.md):
```bash
sudo pacman -Sy yellowjacket
```
### Android
Install the APK from the release page, or from the URL below, which always
points at the newest build:
```
https://git.ljones.me/api/packages/yonlu/generic/yellowjacket-android/latest/yellowjacket.apk
```
That URL needs no credentials, so [Obtainium](https://obtainium.imranr.dev/) can
poll it directly and keep the app up to date. The build is `arm64-v8a` only, and
[`docs/android-release.md`](docs/android-release.md) says why.
### macOS
Homebrew builds it from source on your own Mac — there is no prebuilt `.app`,
because a signed macOS bundle needs a macOS machine to produce it and the
release runner is a Linux container.
```bash
brew install shadow-puppet/yellowjacket/yellowjacket
```
See [`packaging/homebrew/README.md`](packaging/homebrew/README.md).
### Windows
Not published. It cross-compiles cleanly, but no Windows build of this app has
ever been *run*, and nothing here can exercise one — so shipping it would be a
promise that cannot be kept. You can still build it yourself: see
[`CONTRIBUTING.md`](CONTRIBUTING.md).
### Coming from a 1.x install?
Versions restarted at **0.0.1** when releases became automatic, which every
package manager reads as a downgrade. It costs one reinstall, once — the details
are with each channel: [Homebrew](packaging/homebrew/README.md#upgrading-from-1x-needs-a-reinstall-once),
[Android](docs/android-release.md#the-1x-installs-cannot-be-upgraded-to-00x).
## First run
1. Launch YellowJacket. 1. Launch YellowJacket.
2. Add the folder your music lives in — the first-run wizard asks, and 2. Open **Settings** and add the folder(s) where your music lives.
**Settings → Libraries** is where you add more later. 3. Let the initial scan finish — you'll see progress as it works.
3. Watch the scan finish. It reports progress, and you can browse while it runs. 4. Browse by album, artist, or genre, queue something up, and press play.
4. Queue something and press play.
Your library and settings stay on your machine: Your library and settings are stored locally:
| | Linux / macOS | Windows | | | Linux / macOS | Windows |
|---|---|---| |---|---|---|
| Config | `~/.config/yellowjacket/` | `%LOCALAPPDATA%\yellowjacket\config` | | Config | `~/.config/yellowjacket/` | `%LOCALAPPDATA%\yellowjacket\config` |
| Library data | `~/.local/share/yellowjacket/` | `%LOCALAPPDATA%\yellowjacket\data` | | Library data | `~/.local/share/yellowjacket/` | `%LOCALAPPDATA%\yellowjacket\data` |
Setting `YJ_HOME` moves both, which is how you keep a second library separate. ## Building from source
## More screenshots YellowJacket is built with [Go](https://go.dev/) and a
[Lit](https://lit.dev/)/TypeScript frontend, bridged by the
[Wails](https://wails.io/) framework.
An album page knows what you own, and says so: **Prerequisites**
![An album page, with two discs and the transport playing](docs/images/album.png) | Tool | Version |
|------|---------|
| Go | 1.25+ |
| Node.js | 22+ |
| pnpm | 10+ |
| Wails CLI | v3 — vendored, no install needed (`go tool wails3`) |
The home page suggests somewhere to start rather than opening on a wall of The Wails v3 CLI resolves from the `tool` block in `go.mod`, so there is nothing
everything: to install globally; `make setup` fetches it with the rest of the tooling.
![The home page's shelves](docs/images/home.png) On Linux, install the system libraries Wails needs. v3 builds against GTK4 +
WebKitGTK 6.0 by default:
## Contributing, and the rest of the documentation ```bash
sudo apt-get install libasound2-dev libgtk-4-dev libwebkitgtk-6.0-dev # Debian/Ubuntu
sudo pacman -S alsa-lib gtk4 webkitgtk-6.0 # Arch
```
- [`CONTRIBUTING.md`](CONTRIBUTING.md) — build it from source, run the tests, A machine without `webkitgtk-6.0` can still build with `-tags gtk3` against the
and how a change gets in. older WebKit2GTK 4.1 stack, but that is an escape hatch, not what CI or a
- [`CLAUDE.md`](CLAUDE.md) — the deep reference: the architecture and the reasons release builds.
behind the shape of it.
- [The issue tracker](https://git.ljones.me/yonlu/yellowjacket/issues) is what macOS and Windows need no extra system packages. Run `go tool wails3 doctor` to
is wanted and what is being worked on; **#73** is the roadmap. check your environment.
- [Releases](https://git.ljones.me/yonlu/yellowjacket/releases) double as the
changelog — every one is generated from the commits it contains. **Build**
```bash
make setup # install tooling and git hooks
make dev # run with hot-reload
make build-prod # produce a release binary
```
More detail for contributors lives in [`CLAUDE.md`](./CLAUDE.md) — the
architecture, the conventions and the reasons behind them. What is
being worked on is [the issue
tracker](https://git.ljones.me/yonlu/yellowjacket/issues); #73 is the
roadmap.
-6
View File
@@ -499,10 +499,6 @@ func (yj *YellowJacketApp) OnStartup(ctx context.Context) {
PostRemove: yj.explore.InvalidateLibrarySync, PostRemove: yj.explore.InvalidateLibrarySync,
}) })
// A deleted playlist must not leave the queue's "Playing from"
// label pointing at it.
yj.playlist.SetOnPlaylistDeleted(yj.queue.DropSourceForPlaylist)
// Register playback finished handler to drive queue auto-advance. // Register playback finished handler to drive queue auto-advance.
yj.player.SetPlaybackFinishedHandler(yj.queue.OnPlaybackFinished) yj.player.SetPlaybackFinishedHandler(yj.queue.OnPlaybackFinished)
@@ -779,8 +775,6 @@ func (yj *YellowJacketApp) startJanitor() {
} }
yj.janitor.Register(maintenance.ExpiredHTTPCacheJob(yj.database)) yj.janitor.Register(maintenance.ExpiredHTTPCacheJob(yj.database))
yj.janitor.Register(maintenance.StaleArtistMetadataJob(yj.database))
yj.janitor.Register(maintenance.StaleSearchClicksJob(yj.database))
yj.janitor.Register(maintenance.OrphanedCoverFilesJob( yj.janitor.Register(maintenance.OrphanedCoverFilesJob(
yj.database, coversDir, library.CoverArtFileSet, yj.database, coversDir, library.CoverArtFileSet,
)) ))
-39
View File
@@ -303,24 +303,6 @@ func (c *Config) GetLibraryDirectory() string {
return string(c.Library.DirectoryPath) return string(c.Library.DirectoryPath)
} }
// A rejected setter puts the old value back, and that is not tidiness
// (#231). Save validates the *whole* config, so a value left behind by
// a failed write does not merely fail its own call: it fails every
// later save, of every unrelated setting, silently and for the rest of
// the session. Nothing reaches disk, so a restart clears it -- which
// is exactly what makes the fault hard to see and impossible to report.
//
// The setters below that assign and then validate therefore snapshot
// the field first and restore it on the error path. SetLibraryDirectory
// is the other safe shape and the better one where the value can be
// built on its own: it validates a candidate *before* assigning
// anything, so there is nothing to undo.
//
// Not every setter needs either. A bool, an int64 and the shortcut
// bindings pass through no validation that can reject them, and
// SetViewVisible refuses an unknown, non-hideable or launch-page view
// up front, so GeneralConfig.Validate never sees one it would fail on.
// SetLibraryDirectory validates and saves a new library directory, // SetLibraryDirectory validates and saves a new library directory,
// then emits the LibraryConfigChanged event so listeners (e.g. the // then emits the LibraryConfigChanged event so listeners (e.g. the
// Library scanner) can react. // Library scanner) can react.
@@ -378,14 +360,11 @@ func (c *Config) SetScanConcurrency(mode string) error {
c.Library.ApplyDefaults() c.Library.ApplyDefaults()
} }
previous := c.Library.ScanConcurrency
c.Library.ScanConcurrency = library.ScanConcurrency( c.Library.ScanConcurrency = library.ScanConcurrency(
mode, mode,
) )
if err := c.Library.Validate(); err != nil { if err := c.Library.Validate(); err != nil {
c.Library.ScanConcurrency = previous
return fmt.Errorf( return fmt.Errorf(
"invalid scan concurrency mode: %w", err, "invalid scan concurrency mode: %w", err,
) )
@@ -476,12 +455,9 @@ func (c *Config) SetThemeAccentColor(
c.Theme.ApplyDefaults() c.Theme.ApplyDefaults()
} }
previous := c.Theme.AccentColor
c.Theme.AccentColor = color c.Theme.AccentColor = color
if err := c.Theme.Validate(); err != nil { if err := c.Theme.Validate(); err != nil {
c.Theme.AccentColor = previous
return fmt.Errorf( return fmt.Errorf(
"invalid theme accent color: %w", err, "invalid theme accent color: %w", err,
) )
@@ -512,12 +488,9 @@ func (c *Config) SetThemeBackgroundShade(
c.Theme.ApplyDefaults() c.Theme.ApplyDefaults()
} }
previous := c.Theme.BackgroundShade
c.Theme.BackgroundShade = theme.BackgroundShade(shade) c.Theme.BackgroundShade = theme.BackgroundShade(shade)
if err := c.Theme.Validate(); err != nil { if err := c.Theme.Validate(); err != nil {
c.Theme.BackgroundShade = previous
return fmt.Errorf( return fmt.Errorf(
"invalid theme background shade: %w", err, "invalid theme background shade: %w", err,
) )
@@ -571,12 +544,9 @@ func (c *Config) SetDefaultPage(page string) error {
c.General.ApplyDefaults() c.General.ApplyDefaults()
} }
previous := c.General.DefaultPage
c.General.DefaultPage = View(page) c.General.DefaultPage = View(page)
if err := c.General.Validate(); err != nil { if err := c.General.Validate(); err != nil {
c.General.DefaultPage = previous
return fmt.Errorf( return fmt.Errorf(
"invalid default page: %w", err, "invalid default page: %w", err,
) )
@@ -621,12 +591,9 @@ func (c *Config) SetQueueFallback(mode string) error {
c.General.ApplyDefaults() c.General.ApplyDefaults()
} }
previous := c.General.QueueFallback
c.General.QueueFallback = QueueFallback(mode) c.General.QueueFallback = QueueFallback(mode)
if err := c.General.Validate(); err != nil { if err := c.General.Validate(); err != nil {
c.General.QueueFallback = previous
return fmt.Errorf( return fmt.Errorf(
"invalid queue fallback: %w", err, "invalid queue fallback: %w", err,
) )
@@ -834,12 +801,9 @@ func (c *Config) SetTrackListColumns(
c.TrackList = &tracklist.Config{} c.TrackList = &tracklist.Config{}
} }
previous := c.TrackList.Columns
c.TrackList.Columns = columns c.TrackList.Columns = columns
if err := c.TrackList.Validate(); err != nil { if err := c.TrackList.Validate(); err != nil {
c.TrackList.Columns = previous
return fmt.Errorf( return fmt.Errorf(
"invalid track-list columns: %w", err, "invalid track-list columns: %w", err,
) )
@@ -937,12 +901,9 @@ func (c *Config) SetFavoritesIconStyle(
c.Favorites.ApplyDefaults() c.Favorites.ApplyDefaults()
} }
previous := c.Favorites.IconStyle
c.Favorites.IconStyle = favorites.IconStyle(style) c.Favorites.IconStyle = favorites.IconStyle(style)
if err := c.Favorites.Validate(); err != nil { if err := c.Favorites.Validate(); err != nil {
c.Favorites.IconStyle = previous
return fmt.Errorf( return fmt.Errorf(
"invalid favorites icon style: %w", err, "invalid favorites icon style: %w", err,
) )
-262
View File
@@ -1,262 +0,0 @@
package config
import (
"log/slog"
"path/filepath"
"testing"
"yellowjacket/backend/library"
"yellowjacket/backend/tracklist"
)
// newSavableConfig builds a loaded, valid config in a temp directory,
// so Save() writes rather than refusing with errSaveBeforeLoad.
//
// The library directory is real and set, because Config.Validate only
// validates the Library section when DirectoryPath is non-empty -- an
// empty one would hide a poisoned ScanConcurrency from the whole-config
// save that is the symptom under test.
func newSavableConfig(t *testing.T) *Config {
t.Helper()
c := &Config{
logger: slog.Default(),
filePath: filepath.Join(t.TempDir(), "config.toml"),
Library: &library.Config{
DirectoryPath: library.Directory(t.TempDir()),
},
}
c.applyDefaults()
if err := c.Load(); err != nil {
t.Fatalf("Load() error: %v", err)
}
if err := c.Save(); err != nil {
t.Fatalf("Save() on a fresh config error: %v", err)
}
return c
}
// TestSetterRejectionDoesNotPoisonTheConfig is the whole of #231.
//
// Every setter here assigns to the in-memory config and then validates.
// When the validation rejects the argument, the rejected value has to go
// back -- not because the caller sees it (it gets an error either way),
// but because Config.Save() validates the *whole* config. A value left
// behind by a failed setter therefore fails every later save, of every
// unrelated setting, silently and for the rest of the session.
//
// So each case asserts three things in order: the setter reports the
// error, the getter still reports the old value, and an unrelated save
// still works. The third is the one the user feels.
func TestSetterRejectionDoesNotPoisonTheConfig(t *testing.T) {
t.Parallel()
cases := []struct {
name string
// reject calls the setter with an argument its own Validate
// refuses.
reject func(*Config) error
// read reports the value the setter writes, so the rollback is
// asserted on the config rather than only on the save.
read func(*Config) string
}{
{
name: "scan concurrency",
reject: func(c *Config) error {
return c.SetScanConcurrency("telepathy")
},
read: (*Config).GetScanConcurrency,
},
{
name: "theme accent colour",
reject: func(c *Config) error {
return c.SetThemeAccentColor("not-a-hex")
},
read: (*Config).GetThemeAccentColor,
},
{
name: "theme background shade",
reject: func(c *Config) error {
return c.SetThemeBackgroundShade("chartreuse")
},
read: (*Config).GetThemeBackgroundShade,
},
{
name: "default page",
reject: func(c *Config) error {
return c.SetDefaultPage("nowhere")
},
read: (*Config).GetDefaultPage,
},
{
name: "queue fallback",
reject: func(c *Config) error {
return c.SetQueueFallback("improvise")
},
read: (*Config).GetQueueFallback,
},
{
name: "favorites icon style",
reject: func(c *Config) error {
return c.SetFavoritesIconStyle("asterisk")
},
read: (*Config).GetFavoritesIconStyle,
},
{
name: "track-list columns",
reject: func(c *Config) error {
// titleArtist is a drawing definition, not a
// configurable column (#197), so it is exactly what
// the frontend used to be able to send.
return c.SetTrackListColumns([]tracklist.Column{
{ID: "titleArtist"},
})
},
read: func(c *Config) string {
return columnIDs(c.GetTrackListColumns())
},
},
{
name: "track-list columns, duplicated",
reject: func(c *Config) error {
// The route #197 closed was one invalid id; a
// duplicate is the one still reachable from a client
// that assembles the list itself.
return c.SetTrackListColumns([]tracklist.Column{
{ID: tracklist.ColTrackName},
{ID: tracklist.ColTrackName},
})
},
read: func(c *Config) string {
return columnIDs(c.GetTrackListColumns())
},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
c := newSavableConfig(t)
before := tc.read(c)
if err := tc.reject(c); err == nil {
t.Fatal("setter accepted an invalid value, want an error")
}
if after := tc.read(c); after != before {
t.Errorf(
"value after a rejected write = %q, want the previous %q",
after, before,
)
}
// The symptom: an unrelated setting can no longer be saved.
if err := c.SetPopupVolume(true); err != nil {
t.Errorf("an unrelated setter failed after a rejected write: %v", err)
}
if err := c.Save(); err != nil {
t.Errorf("Save() failed after a rejected write: %v", err)
}
})
}
}
// TestRejectedSetterLeavesNothingOnDisk pairs with the sweep above: the
// rollback must not be undone by what the file already holds, so a
// config reloaded from disk after a rejected write agrees with memory.
func TestRejectedSetterLeavesNothingOnDisk(t *testing.T) {
t.Parallel()
c := newSavableConfig(t)
if err := c.SetThemeAccentColor("#123456"); err != nil {
t.Fatalf("SetThemeAccentColor() error: %v", err)
}
if err := c.SetThemeAccentColor("not-a-hex"); err == nil {
t.Fatal("SetThemeAccentColor accepted a non-colour, want an error")
}
reloaded := &Config{logger: slog.Default(), filePath: c.filePath}
reloaded.applyDefaults()
if err := reloaded.Load(); err != nil {
t.Fatalf("Load() error: %v", err)
}
if got := reloaded.GetThemeAccentColor(); got != "#123456" {
t.Errorf("accent colour on disk = %q, want %q", got, "#123456")
}
if c.GetThemeAccentColor() != reloaded.GetThemeAccentColor() {
t.Errorf(
"in-memory accent %q disagrees with disk %q after a rejected write",
c.GetThemeAccentColor(), reloaded.GetThemeAccentColor(),
)
}
}
// TestSetLibraryDirectoryValidatesBeforeAssigning pins the precedent the
// seven rolled-back setters follow: this one has always built and
// validated a candidate before assigning, so a bad path never reaches
// the config at all.
func TestSetLibraryDirectoryValidatesBeforeAssigning(t *testing.T) {
t.Parallel()
c := newSavableConfig(t)
before := c.GetLibraryDirectory()
if err := c.SetLibraryDirectory(filepath.Join(t.TempDir(), "no-such-dir")); err == nil {
t.Fatal("SetLibraryDirectory accepted a missing directory, want an error")
}
if after := c.GetLibraryDirectory(); after != before {
t.Errorf("library directory = %q, want the previous %q", after, before)
}
if err := c.Save(); err != nil {
t.Errorf("Save() failed after a rejected library directory: %v", err)
}
}
// TestSetViewVisibleRefusesBeforeAssigning covers the other setter left
// out of the rollback pass: it guards its own argument up front, so
// GeneralConfig.Validate never sees a view it would reject.
func TestSetViewVisibleRefusesBeforeAssigning(t *testing.T) {
t.Parallel()
c := newSavableConfig(t)
if err := c.SetViewVisible("no-such-view", false); err == nil {
t.Fatal("SetViewVisible accepted an unknown view, want an error")
}
if err := c.SetViewVisible(c.GetDefaultPage(), false); err == nil {
t.Fatal("SetViewVisible hid the launch page, want an error")
}
if err := c.Save(); err != nil {
t.Errorf("Save() failed after a refused view visibility change: %v", err)
}
}
// columnIDs renders a column list for comparison in the table above.
func columnIDs(cols []tracklist.Column) string {
ids := make([]byte, 0, len(cols)*8)
for i, col := range cols {
if i > 0 {
ids = append(ids, ',')
}
ids = append(ids, col.ID...)
}
return string(ids)
}
+10 -28
View File
@@ -662,19 +662,19 @@ func TestSmartPlaylistColumns(t *testing.T) {
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Listening events tracking // Migration 10 — play history tracking
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
func TestListeningEventsTable(t *testing.T) { func TestPlayHistoryTable(t *testing.T) {
t.Parallel() t.Parallel()
db := NewTestDB(t) db := NewTestDB(t)
// Verify listening_events table exists. // Verify play_history table exists.
var tableCount int64 var tableCount int64
tblRows, err := db.QueryContext( tblRows, err := db.QueryContext(
"SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='listening_events'", "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='play_history'",
) )
if err != nil { if err != nil {
t.Fatalf("query sqlite_master: %v", err) t.Fatalf("query sqlite_master: %v", err)
@@ -695,14 +695,12 @@ func TestListeningEventsTable(t *testing.T) {
_ = tblRows.Close() _ = tblRows.Close()
if tableCount != 1 { if tableCount != 1 {
t.Errorf("listening_events table count = %d, want 1", tableCount) t.Errorf("play_history table count = %d, want 1", tableCount)
} }
// Verify audio_files has the denormalized listening counters. // Verify audio_files has play_count and last_played columns.
hasPlayCount := false hasPlayCount := false
hasLastPlayed := false hasLastPlayed := false
hasSkipCount := false
hasLastSkipped := false
colRows, err := db.QueryContext("PRAGMA table_info(audio_files)") colRows, err := db.QueryContext("PRAGMA table_info(audio_files)")
if err != nil { if err != nil {
@@ -734,14 +732,6 @@ func TestListeningEventsTable(t *testing.T) {
if name == "last_played" { if name == "last_played" {
hasLastPlayed = true hasLastPlayed = true
} }
if name == "skip_count" {
hasSkipCount = true
}
if name == "last_skipped" {
hasLastSkipped = true
}
} }
_ = colRows.Close() _ = colRows.Close()
@@ -754,14 +744,6 @@ func TestListeningEventsTable(t *testing.T) {
t.Error("audio_files missing last_played column") t.Error("audio_files missing last_played column")
} }
if !hasSkipCount {
t.Error("audio_files missing skip_count column")
}
if !hasLastSkipped {
t.Error("audio_files missing last_skipped column")
}
// Verify track_metadata VIEW includes play_count and last_played. // Verify track_metadata VIEW includes play_count and last_played.
viewCols := map[string]bool{} viewCols := map[string]bool{}
@@ -801,7 +783,7 @@ func TestListeningEventsTable(t *testing.T) {
t.Error("track_metadata VIEW missing last_played column") t.Error("track_metadata VIEW missing last_played column")
} }
// Round-trip: insert a listening_events row and verify play_count update. // Round-trip: insert a play_history row and verify play_count update.
// First, set up test data. The test DB already has library id=0. // First, set up test data. The test DB already has library id=0.
InsertTestTrack(t, db, TestTrack{ InsertTestTrack(t, db, TestTrack{
FilePath: "/test/play_history.mp3", FilePath: "/test/play_history.mp3",
@@ -839,12 +821,12 @@ func TestListeningEventsTable(t *testing.T) {
t.Errorf("initial play_count = %d, want 0", playCount) t.Errorf("initial play_count = %d, want 0", playCount)
} }
// Insert a listening_events row (kind defaults to 'complete'). // Insert a play_history row and update play_count.
_, err = db.ExecContext( _, err = db.ExecContext(
"INSERT INTO listening_events (audio_file_id, kind) VALUES (1, 'complete')", "INSERT INTO play_history (audio_file_id) VALUES (1)",
) )
if err != nil { if err != nil {
t.Fatalf("insert listening_events: %v", err) t.Fatalf("insert play_history: %v", err)
} }
_, err = db.ExecContext( _, err = db.ExecContext(
+4 -16
View File
@@ -151,26 +151,14 @@ func (d *DB) SetLyrics(audioFileID int64, lyrics, source, recordingMBID string)
return d.upsertLyricsIndex(audioFileID, lyrics) return d.upsertLyricsIndex(audioFileID, lyrics)
} }
// DeleteLyricsIndex removes one file's entry from the contentless
// lyrics_index. It is called wherever a file row is deleted — the
// `lyrics` table cascades with its file, but the FTS entry does not and
// would otherwise accumulate for the life of the install (#249).
func (d *DB) DeleteLyricsIndex(audioFileID int64) error {
if _, err := d.db.ExecContext(d.Ctx,
"DELETE FROM lyrics_index WHERE rowid = ?", audioFileID,
); err != nil {
return fmt.Errorf("could not delete lyrics_index row: %w", err)
}
return nil
}
// upsertLyricsIndex refreshes a single file's entry in the contentless // upsertLyricsIndex refreshes a single file's entry in the contentless
// lyrics_index. contentless_delete=1 makes the DELETE valid; an empty // lyrics_index. contentless_delete=1 makes the DELETE valid; an empty
// lyrics string leaves the row deleted. // lyrics string leaves the row deleted.
func (d *DB) upsertLyricsIndex(audioFileID int64, lyrics string) error { func (d *DB) upsertLyricsIndex(audioFileID int64, lyrics string) error {
if err := d.DeleteLyricsIndex(audioFileID); err != nil { if _, err := d.db.ExecContext(d.Ctx,
return err "DELETE FROM lyrics_index WHERE rowid = ?", audioFileID,
); err != nil {
return fmt.Errorf("could not delete lyrics_index row: %w", err)
} }
if strings.TrimSpace(lyrics) == "" { if strings.TrimSpace(lyrics) == "" {
-177
View File
@@ -1,177 +0,0 @@
package database
import (
"context"
"database/sql"
"fmt"
"log/slog"
"strings"
)
// Preserving a playlist entry across the loss of its track is two
// statements, not one, and the split is not tidiness -- it is what
// makes the important half work in the situation that needs it most.
//
// `playlist_tracks.audio_file_id` is ON DELETE SET NULL, so an entry
// outlives its file as an id-less row that says nothing about what the
// user put in the playlist. The phantom_* columns carry the answer
// across and ResolvePhantomTracksAfterScan re-links them afterwards --
// but only if something fills them *before* the rows go.
//
// The two halves are not equally important and are not equally
// available:
//
// - **phantom_file_path is the one that matters.**
// ResolvePhantomTracksAfterScan matches it against
// `audio_files.file_path`, so without it an entry can never be
// re-linked and the playlist is empty for good. It comes straight
// off `audio_files`, whose `file_path` is the table's natural key
// and has been present in every shape it has ever had -- including
// the pre-013 stub of `(id, file_path, recording_id)`.
// - The rest is *display* for a phantom entry before a rescan
// re-links it, and it comes from the `track_metadata` view, which
// is the one definition of a track row and not worth restating.
//
// Reading the view is what cannot be relied on here, and that is the
// whole reason for the split. This runs *before* applySchema, which is
// precisely the moment the schema is inconsistent: the view is whatever
// the last launch's schema declared, while `audio_files` is whatever
// the launch before that left behind. A view over columns the table no
// longer has is not merely empty -- `pragma_table_info` on it *errors*,
// and so does selecting from it. `cmd/indexbuild`'s fixture is exactly
// that shape and is what caught this.
//
// COALESCE keeps an existing phantom value in both halves: an entry
// already phantom is one whose file went missing in an earlier pass,
// and its recorded metadata is the only copy left. Overwriting that
// from a NULL join erases the rows this exists to protect.
const (
preservePhantomPathSQL = `
UPDATE playlist_tracks
SET phantom_file_path = COALESCE(phantom_file_path, (
SELECT af.file_path FROM audio_files af
WHERE af.id = playlist_tracks.audio_file_id
))
WHERE audio_file_id IS NOT NULL
`
preservePhantomDisplaySQL = `
UPDATE playlist_tracks
SET
phantom_title = COALESCE(phantom_title, (
SELECT tm.title FROM track_metadata tm
WHERE tm.id = playlist_tracks.audio_file_id
)),
phantom_artist = COALESCE(phantom_artist, (
SELECT tm.artist_name FROM track_metadata tm
WHERE tm.id = playlist_tracks.audio_file_id
)),
phantom_album = COALESCE(phantom_album, (
SELECT tm.album FROM track_metadata tm
WHERE tm.id = playlist_tracks.audio_file_id
)),
phantom_duration_ms = COALESCE(phantom_duration_ms, (
SELECT af.length_milliseconds FROM audio_files af
WHERE af.id = playlist_tracks.audio_file_id
)),
phantom_genre = COALESCE(phantom_genre, (
SELECT tm.genre FROM track_metadata tm
WHERE tm.id = playlist_tracks.audio_file_id
)),
phantom_cover_art_path = COALESCE(phantom_cover_art_path, (
SELECT tm.cover_art_path FROM track_metadata tm
WHERE tm.id = playlist_tracks.audio_file_id
))
WHERE audio_file_id IS NOT NULL
`
)
// PreservePlaylistPhantoms records every linked playlist entry's track
// metadata on the entry itself, so the entry survives the rows being
// deleted underneath it.
//
// Every path that empties `audio_files` must call this (or the scoped
// variant below) first, inside the same transaction as the delete.
// These paths have drifted before: the full rescan in backend/library
// did this and the stale-shape retire in this package did not, so the
// *documented* repair ("delete and rescan") preserved playlists while
// the automatic one that exists to spare the user that work silently
// emptied them (#183). The incremental scan's orphan cleanup and
// RemoveFromLibrary drifted the same way and are #246.
//
// The display half is skipped, with a warning, when `track_metadata`
// cannot answer -- see the note above. Skipping it costs a phantom
// entry its title until a rescan re-links it; skipping the path half
// would cost the entry outright, so that one is an error.
func PreservePlaylistPhantoms(
ctx context.Context, tx *sql.Tx, logger *slog.Logger,
) error {
return preservePlaylistPhantoms(ctx, tx, nil, logger)
}
// PreservePlaylistPhantomsForFiles is PreservePlaylistPhantoms scoped to
// the given audio file ids, for the two removal paths that delete a
// known subset of the table rather than all of it: the incremental
// scan's orphan cleanup and RemoveFromLibrary. A bulk pass there would
// rewrite every linked playlist row on every scan for nothing.
func PreservePlaylistPhantomsForFiles(
ctx context.Context, tx *sql.Tx, ids []int64, logger *slog.Logger,
) error {
return preservePlaylistPhantoms(ctx, tx, ids, logger)
}
func preservePlaylistPhantoms(
ctx context.Context, tx *sql.Tx, ids []int64, logger *slog.Logger,
) error {
clause, args, skip := phantomIDFilter(ids)
if skip {
return nil
}
if _, err := tx.ExecContext(
ctx, preservePhantomPathSQL+clause, args...,
); err != nil {
return fmt.Errorf(
"could not preserve playlist track file paths: %w", err,
)
}
if _, err := tx.ExecContext(
ctx, preservePhantomDisplaySQL+clause, args...,
); err != nil {
// A failed statement does not roll back a SQLite transaction,
// so the path half above stands and the entries remain
// re-linkable.
logger.Warn(
"could not record display metadata for playlist entries; "+
"they will be re-linked by the next scan but read as "+
"unknown until then",
"err", err,
)
}
return nil
}
// phantomIDFilter builds the extra WHERE terms and arguments that scope
// a preservation pass to a set of audio file ids. A nil ids returns the
// empty clause (a bulk run over every linked entry); an empty slice
// reports skip, since there is nothing to preserve.
func phantomIDFilter(ids []int64) (clause string, args []any, skip bool) {
switch {
case ids == nil:
return "", nil, false
case len(ids) == 0:
return "", nil, true
}
clause = " AND audio_file_id IN (" +
strings.Repeat("?,", len(ids)-1) + "?)"
args = make([]any, len(ids))
for i, id := range ids {
args[i] = id
}
return clause, args, false
}
-94
View File
@@ -1,94 +0,0 @@
package database
import (
"context"
"database/sql"
"testing"
)
// TestPreservePlaylistPhantomsForFilesScopesToTheRequestedIDs is the
// scoping half of the scoped variant: a run over one file's id must
// fill that file's playlist entries and leave every other entry alone,
// because the incremental scan calls this once per orphan batch and a
// pass that rewrote the whole table would touch every playlist row on
// every scan.
func TestPreservePlaylistPhantomsForFilesScopesToTheRequestedIDs(t *testing.T) {
ctx := context.Background()
db := openRaw(t, t.TempDir())
if _, err := db.ExecContext(ctx, "PRAGMA foreign_keys = ON"); err != nil {
t.Fatalf("pragma: %v", err)
}
if err := applySchema(ctx, db); err != nil {
t.Fatalf("applySchema: %v", err)
}
if _, err := db.ExecContext(ctx, `
INSERT INTO playlists (id, name) VALUES (1, 'keepme');
INSERT INTO libraries (id, name, path) VALUES (0, 'test', '/music');
INSERT INTO artists (id, name) VALUES (3, 'Aurora Fields');
INSERT INTO cover_art (id, file_path, mime_type)
VALUES (9, 'covers/7.jpg', 'image/jpeg');
INSERT INTO albums (id, name, artist_id, cover_art_id)
VALUES (4, 'Tideline', 3, 9);
INSERT INTO audio_files
(id, file_path, file_type_id, length_milliseconds,
title, artist_credit, artist_id, album_id)
VALUES
(7, '/music/a.flac', 1, 1000,
'Slack Water', 'Aurora Fields', 3, 4),
(8, '/music/b.flac', 1, 2000,
'Second Tide', 'Aurora Fields', 3, 4);
INSERT INTO playlist_tracks (playlist_id, audio_file_id, position)
VALUES (1, 7, 0), (1, 8, 1);
`); err != nil {
t.Fatalf("seed: %v", err)
}
tx, err := db.BeginTx(ctx, nil)
if err != nil {
t.Fatalf("begin: %v", err)
}
defer func() { _ = tx.Rollback() }()
if err := PreservePlaylistPhantomsForFiles(
ctx, tx, []int64{7}, testLogger(),
); err != nil {
t.Fatalf("preserve: %v", err)
}
if err := tx.Commit(); err != nil {
t.Fatalf("commit: %v", err)
}
var filled, untouched sql.NullString
if err := db.QueryRowContext(ctx,
"SELECT phantom_file_path FROM playlist_tracks WHERE audio_file_id = 7",
).Scan(&filled); err != nil {
t.Fatalf("read the requested entry: %v", err)
}
if filled.String != "/music/a.flac" {
t.Errorf(
"requested entry phantom_file_path = %q, want %q",
filled.String, "/music/a.flac",
)
}
if err := db.QueryRowContext(ctx,
"SELECT phantom_file_path FROM playlist_tracks WHERE audio_file_id = 8",
).Scan(&untouched); err != nil {
t.Fatalf("read the untouched entry: %v", err)
}
if untouched.Valid {
t.Errorf(
"untouched entry got phantom_file_path = %q, want NULL "+
"(a scoped run must not rewrite the whole table)",
untouched.String,
)
}
}
@@ -68,14 +68,8 @@ CREATE TABLE IF NOT EXISTS audio_files (
-- compared against the on-disk mtime during a scan to detect files -- compared against the on-disk mtime during a scan to detect files
-- another application retagged in place. -- another application retagged in place.
modified_at INTEGER NOT NULL DEFAULT 0, modified_at INTEGER NOT NULL DEFAULT 0,
-- Listening counts, denormalized from listening_events so the hot
-- read path (track list sort, shelves, smart playlists) never joins
-- a log table. Authored: a rescan cannot rebuild them. This is the
-- "MIXED KIND" half of audio_files the datamap notes.
play_count INTEGER NOT NULL DEFAULT 0, play_count INTEGER NOT NULL DEFAULT 0,
last_played DATETIME, last_played DATETIME,
skip_count INTEGER NOT NULL DEFAULT 0,
last_skipped DATETIME,
tag_status TEXT NOT NULL DEFAULT 'untagged' tag_status TEXT NOT NULL DEFAULT 'untagged'
CHECK(tag_status IN ( CHECK(tag_status IN (
'untagged', 'auto_matched', 'user_confirmed', 'user_skipped_permanent' 'untagged', 'auto_matched', 'user_confirmed', 'user_skipped_permanent'
@@ -45,6 +45,8 @@ CREATE INDEX IF NOT EXISTS idx_download_items_live
CREATE INDEX IF NOT EXISTS idx_download_items_state CREATE INDEX IF NOT EXISTS idx_download_items_state
ON download_items(state); ON download_items(state);
-- ListDownloadItemsForDownload filters on the parent download. -- idx_download_items_download is deliberately NOT declared here: on an
CREATE INDEX IF NOT EXISTS idx_download_items_download -- existing database this table already exists at schema-pass time with
ON download_items(download_id); -- its old column still named request_id, so an inline CREATE INDEX on
-- download_id would fail outright. See ensureDownloadIndexes in
-- backend/database/download_rename_migration.go.
@@ -66,11 +66,14 @@ CREATE TABLE IF NOT EXISTS download_requests (
FOREIGN KEY(parent_id) REFERENCES download_requests(id) ON DELETE CASCADE FOREIGN KEY(parent_id) REFERENCES download_requests(id) ON DELETE CASCADE
); );
CREATE INDEX IF NOT EXISTS idx_download_requests_due -- idx_download_requests_{due,entity,parent} are deliberately NOT
ON download_requests(state, next_try_at); -- declared here. This table name is reused from the old one-shot
-- attempt table (also called download_requests before the Want/Request
CREATE INDEX IF NOT EXISTS idx_download_requests_entity -- rename), so on an existing database this CREATE TABLE is a no-op
ON download_requests(entity, state); -- against a table that, at schema-pass time, is still shaped like the
-- OLD attempts table and lacks these columns entirely — an inline
CREATE INDEX IF NOT EXISTS idx_download_requests_parent -- CREATE INDEX here would fail outright rather than just no-op. See
ON download_requests(parent_id) WHERE parent_id IS NOT NULL; -- migrateDownloadRename/ensureDownloadIndexes in
-- backend/database/download_rename_migration.go, which create these
-- once the rename has actually happened (or immediately, on a fresh
-- database where the columns exist from the start).
@@ -1,42 +0,0 @@
-- One row per track *exit*, three ways a listen can end: it reached
-- the end, it was heard enough to count and then skipped past, or it
-- was abandoned for another track before anyone had really listened.
--
-- This is the source of truth for listening behaviour. The
-- denormalized `play_count` / `last_played` / `skip_count` /
-- `last_skipped` on audio_files are materialized from it, because the
-- hot read path (track-list sort, the shelves, smart playlists) must
-- not join a log that grows by one row per song forever.
--
-- `kind` is the classification, applied at write time:
--
-- complete the track reached its natural end, or was skipped in
-- its tail window (the last few seconds of a long fade).
-- play the scrobble threshold was heard — half the track or
-- four minutes, whichever is less — and the user moved on
-- before the end.
-- skip the user moved to a different track before that.
--
-- `position_seconds` / `duration_seconds` are the raw reading the
-- classification was made from, kept so a future re-tune of the
-- threshold does not need the events re-recorded. 0/0 on a row means
-- "not captured for this event" (e.g. a natural finish recorded before
-- these columns existed), not "a zero-second track".
CREATE TABLE IF NOT EXISTS listening_events (
id INTEGER PRIMARY KEY,
audio_file_id INTEGER NOT NULL,
kind TEXT NOT NULL DEFAULT 'complete'
CHECK (kind IN ('complete', 'play', 'skip')),
position_seconds INTEGER NOT NULL DEFAULT 0,
duration_seconds INTEGER NOT NULL DEFAULT 0,
occurred_at DATETIME NOT NULL DEFAULT (datetime('now')),
FOREIGN KEY(audio_file_id) REFERENCES audio_files(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_listening_events_audio_file_id
ON listening_events(audio_file_id);
-- "What did I listen to this month" walks this, rather than the
-- per-track index above.
CREATE INDEX IF NOT EXISTS idx_listening_events_occurred_at
ON listening_events(occurred_at);
@@ -0,0 +1,9 @@
CREATE TABLE IF NOT EXISTS play_history (
id INTEGER PRIMARY KEY,
audio_file_id INTEGER NOT NULL,
played_at DATETIME NOT NULL DEFAULT (datetime('now')),
FOREIGN KEY(audio_file_id) REFERENCES audio_files(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_play_history_audio_file_id
ON play_history(audio_file_id);
+7 -4
View File
@@ -1,15 +1,18 @@
CREATE TABLE IF NOT EXISTS queue ( CREATE TABLE IF NOT EXISTS queue (
id INTEGER PRIMARY KEY CHECK(id = 1), id INTEGER PRIMARY KEY CHECK(id = 1),
source_playlist_id INTEGER,
current_position INTEGER NOT NULL DEFAULT 0, current_position INTEGER NOT NULL DEFAULT 0,
shuffle_mode BOOLEAN NOT NULL DEFAULT false, shuffle_mode BOOLEAN NOT NULL DEFAULT false,
repeat_mode TEXT NOT NULL DEFAULT 'off', repeat_mode TEXT NOT NULL DEFAULT 'off',
shuffle_order TEXT, shuffle_order TEXT,
-- What the queue was built from ("Playing from: X"): an album, -- source_playlist_id above is unused dead weight (nothing has ever
-- playlist, smart playlist, genre or artist, identified by the id -- written it a nonzero value); source_type/source_id/source_label
-- that source_type's namespace gives it. -- below are its generalized replacement, covering albums, playlists,
-- smart playlists, genres and artists rather than playlists alone.
source_type TEXT NOT NULL DEFAULT '', source_type TEXT NOT NULL DEFAULT '',
source_id INTEGER NOT NULL DEFAULT 0, source_id INTEGER NOT NULL DEFAULT 0,
source_label TEXT NOT NULL DEFAULT '' source_label TEXT NOT NULL DEFAULT '',
FOREIGN KEY(source_playlist_id) REFERENCES playlists(id) ON DELETE SET NULL
); );
-- Singleton row: there is exactly one playback queue. -- Singleton row: there is exactly one playback queue.
+18 -4
View File
@@ -26,10 +26,17 @@ CREATE TABLE IF NOT EXISTS tagging_items (
-- complete rip of their own directory. parent_group_key is the -- complete rip of their own directory. parent_group_key is the
-- original folder group they were split from. -- original folder group they were split from.
-- --
-- These columns are appended after created_at rather than grouped -- These two columns are declared LAST, after created_at, even
-- with the rest of the row: sqlc's `SELECT *` scans (GetTaggingItem) -- though that reads oddly next to the rest of the table: sql/
-- bind column order positionally, so new columns always go at the -- migrations/0001 brings a pre-existing tagging_items up to date
-- end. -- with `ALTER TABLE ADD COLUMN`, which SQLite always appends at
-- the end of the column list. A fresh install (this file) and an
-- upgraded database (this file + the migration) must end up with
-- IDENTICAL column order, because sqlc-generated `SELECT *` scans
-- (e.g. GetTaggingItem) bind columns positionally — see the
-- schema/migration column-order test in database_test.go. Put
-- new columns wherever reads best when adding a table for the
-- first time; append-only from the second migration on.
synthetic INTEGER NOT NULL DEFAULT 0, synthetic INTEGER NOT NULL DEFAULT 0,
parent_group_key TEXT NOT NULL DEFAULT '', parent_group_key TEXT NOT NULL DEFAULT '',
-- album_artist_conflict latches to 1 the first time two tracks -- album_artist_conflict latches to 1 the first time two tracks
@@ -51,3 +58,10 @@ CREATE INDEX IF NOT EXISTS idx_tagging_items_library_status
CREATE INDEX IF NOT EXISTS idx_tagging_items_status_pending CREATE INDEX IF NOT EXISTS idx_tagging_items_status_pending
ON tagging_items(library_id) WHERE status = 'pending'; ON tagging_items(library_id) WHERE status = 'pending';
-- idx_tagging_items_parent_group_key is NOT declared here on
-- purpose: this file runs unconditionally, before migrations, even
-- against a database that hasn't run 0001 yet — an index predicate
-- referencing parent_group_key would fail on that table. It lives
-- solely in sql/migrations/0001_tagging_items_synthetic.sql, which
-- runs after the column exists either way (see database.go).
@@ -39,7 +39,7 @@ INSERT INTO audio_files (
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ? ?, ?, ?, ?, ?
) )
RETURNING id, file_path, library_id, file_type_id, length_milliseconds, sample_rate, bit_depth, channels, bitrate, file_size, title, artist_credit, artist_id, album_id, track_number, disc_number, total_tracks, year, composer, comment, recording_mbid, basename, group_key, modified_at, play_count, last_played, skip_count, last_skipped, tag_status RETURNING id, file_path, library_id, file_type_id, length_milliseconds, sample_rate, bit_depth, channels, bitrate, file_size, title, artist_credit, artist_id, album_id, track_number, disc_number, total_tracks, year, composer, comment, recording_mbid, basename, group_key, modified_at, play_count, last_played, tag_status
` `
type CreateAudioFileParams struct { type CreateAudioFileParams struct {
@@ -135,8 +135,6 @@ func (q *Queries) CreateAudioFile(ctx context.Context, arg CreateAudioFileParams
&i.ModifiedAt, &i.ModifiedAt,
&i.PlayCount, &i.PlayCount,
&i.LastPlayed, &i.LastPlayed,
&i.SkipCount,
&i.LastSkipped,
&i.TagStatus, &i.TagStatus,
) )
return i, err return i, err
@@ -194,7 +192,7 @@ func (q *Queries) GetAllAudioFilePaths(ctx context.Context) ([]GetAllAudioFilePa
const getAudioFile = `-- name: GetAudioFile :one const getAudioFile = `-- name: GetAudioFile :one
SELECT id, file_path, library_id, file_type_id, length_milliseconds, sample_rate, bit_depth, channels, bitrate, file_size, title, artist_credit, artist_id, album_id, track_number, disc_number, total_tracks, year, composer, comment, recording_mbid, basename, group_key, modified_at, play_count, last_played, skip_count, last_skipped, tag_status FROM audio_files WHERE id = ? LIMIT 1 SELECT id, file_path, library_id, file_type_id, length_milliseconds, sample_rate, bit_depth, channels, bitrate, file_size, title, artist_credit, artist_id, album_id, track_number, disc_number, total_tracks, year, composer, comment, recording_mbid, basename, group_key, modified_at, play_count, last_played, tag_status FROM audio_files WHERE id = ? LIMIT 1
` `
// --------------------------------------------------------------------- // ---------------------------------------------------------------------
@@ -230,15 +228,13 @@ func (q *Queries) GetAudioFile(ctx context.Context, id int64) (AudioFile, error)
&i.ModifiedAt, &i.ModifiedAt,
&i.PlayCount, &i.PlayCount,
&i.LastPlayed, &i.LastPlayed,
&i.SkipCount,
&i.LastSkipped,
&i.TagStatus, &i.TagStatus,
) )
return i, err return i, err
} }
const getAudioFileByPath = `-- name: GetAudioFileByPath :one const getAudioFileByPath = `-- name: GetAudioFileByPath :one
SELECT id, file_path, library_id, file_type_id, length_milliseconds, sample_rate, bit_depth, channels, bitrate, file_size, title, artist_credit, artist_id, album_id, track_number, disc_number, total_tracks, year, composer, comment, recording_mbid, basename, group_key, modified_at, play_count, last_played, skip_count, last_skipped, tag_status FROM audio_files WHERE file_path = ? LIMIT 1 SELECT id, file_path, library_id, file_type_id, length_milliseconds, sample_rate, bit_depth, channels, bitrate, file_size, title, artist_credit, artist_id, album_id, track_number, disc_number, total_tracks, year, composer, comment, recording_mbid, basename, group_key, modified_at, play_count, last_played, tag_status FROM audio_files WHERE file_path = ? LIMIT 1
` `
func (q *Queries) GetAudioFileByPath(ctx context.Context, filePath string) (AudioFile, error) { func (q *Queries) GetAudioFileByPath(ctx context.Context, filePath string) (AudioFile, error) {
@@ -271,8 +267,6 @@ func (q *Queries) GetAudioFileByPath(ctx context.Context, filePath string) (Audi
&i.ModifiedAt, &i.ModifiedAt,
&i.PlayCount, &i.PlayCount,
&i.LastPlayed, &i.LastPlayed,
&i.SkipCount,
&i.LastSkipped,
&i.TagStatus, &i.TagStatus,
) )
return i, err return i, err
@@ -340,7 +334,7 @@ func (q *Queries) GetAudioFilesByPaths(ctx context.Context, paths []string) ([]G
} }
const getAudioFilesInLibrary = `-- name: GetAudioFilesInLibrary :many const getAudioFilesInLibrary = `-- name: GetAudioFilesInLibrary :many
SELECT id, file_path, library_id, file_type_id, length_milliseconds, sample_rate, bit_depth, channels, bitrate, file_size, title, artist_credit, artist_id, album_id, track_number, disc_number, total_tracks, year, composer, comment, recording_mbid, basename, group_key, modified_at, play_count, last_played, skip_count, last_skipped, tag_status FROM audio_files WHERE library_id = ? SELECT id, file_path, library_id, file_type_id, length_milliseconds, sample_rate, bit_depth, channels, bitrate, file_size, title, artist_credit, artist_id, album_id, track_number, disc_number, total_tracks, year, composer, comment, recording_mbid, basename, group_key, modified_at, play_count, last_played, tag_status FROM audio_files WHERE library_id = ?
` `
func (q *Queries) GetAudioFilesInLibrary(ctx context.Context, libraryID int64) ([]AudioFile, error) { func (q *Queries) GetAudioFilesInLibrary(ctx context.Context, libraryID int64) ([]AudioFile, error) {
@@ -379,8 +373,6 @@ func (q *Queries) GetAudioFilesInLibrary(ctx context.Context, libraryID int64) (
&i.ModifiedAt, &i.ModifiedAt,
&i.PlayCount, &i.PlayCount,
&i.LastPlayed, &i.LastPlayed,
&i.SkipCount,
&i.LastSkipped,
&i.TagStatus, &i.TagStatus,
); err != nil { ); err != nil {
return nil, err return nil, err
+15 -19
View File
@@ -94,8 +94,6 @@ type AudioFile struct {
ModifiedAt int64 ModifiedAt int64
PlayCount int64 PlayCount int64
LastPlayed sql.NullTime LastPlayed sql.NullTime
SkipCount int64
LastSkipped sql.NullTime
TagStatus string TagStatus string
} }
@@ -263,15 +261,6 @@ type Library struct {
AutotagWarningAcked int64 AutotagWarningAcked int64
} }
type ListeningEvent struct {
ID int64
AudioFileID int64
Kind string
PositionSeconds int64
DurationSeconds int64
OccurredAt time.Time
}
type Lyric struct { type Lyric struct {
AudioFileID int64 AudioFileID int64
Text string Text string
@@ -284,6 +273,12 @@ type LyricsIndex struct {
Lyrics string Lyrics string
} }
type PlayHistory struct {
ID int64
AudioFileID int64
PlayedAt time.Time
}
type PlayerState struct { type PlayerState struct {
ID int64 ID int64
Volume int64 Volume int64
@@ -317,14 +312,15 @@ type PlaylistTrack struct {
} }
type Queue struct { type Queue struct {
ID int64 ID int64
CurrentPosition int64 SourcePlaylistID sql.NullInt64
ShuffleMode bool CurrentPosition int64
RepeatMode string ShuffleMode bool
ShuffleOrder sql.NullString RepeatMode string
SourceType string ShuffleOrder sql.NullString
SourceID int64 SourceType string
SourceLabel string SourceID int64
SourceLabel string
} }
type QueueTrack struct { type QueueTrack struct {
-55
View File
@@ -245,13 +245,6 @@ func dropDeferred(
ctx context.Context, db *sql.DB, logger *slog.Logger, ctx context.Context, db *sql.DB, logger *slog.Logger,
drop map[string]string, drop map[string]string,
) error { ) error {
// Asked before the transaction opens, because the answer is about
// which tables are live and that cannot change underneath us here.
preserve, err := shouldPreservePhantoms(ctx, db, drop)
if err != nil {
return err
}
tx, err := db.BeginTx(ctx, nil) tx, err := db.BeginTx(ctx, nil)
if err != nil { if err != nil {
return fmt.Errorf("could not begin the retire transaction: %w", err) return fmt.Errorf("could not begin the retire transaction: %w", err)
@@ -263,22 +256,6 @@ func dropDeferred(
return fmt.Errorf("could not defer foreign keys: %w", err) return fmt.Errorf("could not defer foreign keys: %w", err)
} }
// Before any drop, so every entry still has a track to read. It is
// in this transaction rather than beside it because the preservation
// and the delete have to succeed or fail together: a commit that
// dropped the files without the phantoms is the bug, and a commit
// that wrote phantoms without dropping anything is a lie about rows
// that are still there.
if preserve {
logger.Info(
"preserving playlist entries across the retire of audio_files",
)
if err := PreservePlaylistPhantoms(ctx, tx, logger); err != nil {
return err
}
}
// Sorted, so a failure is reproducible. Map order is random, and a // Sorted, so a failure is reproducible. Map order is random, and a
// bug that depends on which table happens to go first reproduces on // bug that depends on which table happens to go first reproduces on
// one run in three and passes review on the other two -- which is // one run in three and passes review on the other two -- which is
@@ -306,38 +283,6 @@ func dropDeferred(
return nil return nil
} }
// shouldPreservePhantoms reports whether this retire is about to take
// `audio_files` out from under the playlists.
//
// The `playlist_tracks` check is not defensive padding. This runs
// *before* applySchema, which is the moment the schema is by definition
// mid-repair, and the preservation reads a table it does not drop. A
// database old enough not to have it would otherwise fail here, and
// failing here means the app does not open at all -- while nothing is
// lost by skipping, since an absent `playlist_tracks` holds no
// playlists to save.
//
// It deliberately does *not* ask after `track_metadata`. Whether that
// view can answer is PreservePlaylistPhantoms's own business, because a
// view broken against an older `audio_files` is a state this function
// cannot detect without hitting the same error it is trying to avoid:
// pragma_table_info on such a view errors rather than reporting no
// columns.
func shouldPreservePhantoms(
ctx context.Context, db *sql.DB, drop map[string]string,
) (bool, error) {
if _, going := drop["audio_files"]; !going {
return false, nil
}
cols, err := liveColumns(ctx, db, "playlist_tracks")
if err != nil {
return false, err
}
return len(cols) > 0, nil
}
// staleReason reports why a live table disagrees with its declaration, // staleReason reports why a live table disagrees with its declaration,
// or "" when it agrees. A column the live table does not have is the // or "" when it agrees. A column the live table does not have is the
// additive case; a column whose declared type changed is the one an // additive case; a column whose declared type changed is the one an
-177
View File
@@ -497,180 +497,3 @@ func TestParseCreateTablesReadsTheRealSchema(t *testing.T) {
} }
} }
} }
// TestRetiringAudioFilesKeepsPlaylistContents is the symptom this
// repair exists for: a playlist survived the retire as a row count and
// nothing else.
//
// TestRetiringOwnedTablesDoesNotDangle already asserts the entry does
// not keep a stale id, which is the *dangerous* half. It is satisfied
// just as well by an entry that says nothing at all, which is the
// half that quietly emptied every playlist -- so this asserts what the
// entry still knows, and specifically phantom_file_path, because that
// is the column ResolvePhantomTracksAfterScan matches back against
// audio_files.file_path.
//
// Note the seed drops `comment`, not `artist_credit`: the mutation has
// to leave `track_metadata` standing, since a real launch reaches the
// retire with the view the previous launch created. A test that drops
// the view first is testing the skip path, not this one.
func TestRetiringAudioFilesKeepsPlaylistContents(t *testing.T) {
ctx := context.Background()
db := openRaw(t, t.TempDir())
if _, err := db.ExecContext(ctx, "PRAGMA foreign_keys = ON"); err != nil {
t.Fatalf("pragma: %v", err)
}
if err := applySchema(ctx, db); err != nil {
t.Fatalf("applySchema: %v", err)
}
if _, err := db.ExecContext(ctx, `
INSERT INTO playlists (id, name) VALUES (1, 'keepme');
INSERT INTO libraries (id, name, path) VALUES (0, 'test', '/music');
INSERT INTO artists (id, name) VALUES (3, 'Aurora Fields');
INSERT INTO cover_art (id, file_path, mime_type)
VALUES (9, 'covers/7.jpg', 'image/jpeg');
INSERT INTO genres (id, name) VALUES (5, 'Ambient');
INSERT INTO albums (id, name, artist_id, cover_art_id)
VALUES (4, 'Tideline', 3, 9);
INSERT INTO audio_files
(id, file_path, file_type_id, length_milliseconds,
title, artist_credit, artist_id, album_id)
VALUES (7, '/music/a.flac', 1, 1000,
'Slack Water', 'Aurora Fields', 3, 4);
INSERT INTO file_genres (audio_file_id, genre_id) VALUES (7, 5);
INSERT INTO playlist_tracks (playlist_id, audio_file_id, position)
VALUES (1, 7, 0);
ALTER TABLE audio_files DROP COLUMN comment;
`); err != nil {
t.Fatalf("seed: %v", err)
}
if err := retireStaleTables(ctx, db, testLogger()); err != nil {
t.Fatalf("retire: %v", err)
}
if err := applySchema(ctx, db); err != nil {
t.Fatalf("applySchema: %v", err)
}
var (
path, title, artist, album, genre, cover sql.NullString
duration sql.NullInt64
)
if err := db.QueryRowContext(ctx, `
SELECT phantom_file_path, phantom_title, phantom_artist,
phantom_album, phantom_duration_ms, phantom_genre,
phantom_cover_art_path
FROM playlist_tracks WHERE playlist_id = 1
`).Scan(&path, &title, &artist, &album, &duration, &genre, &cover); err != nil {
t.Fatalf("read the surviving entry: %v", err)
}
// The one that matters: without it the entry can never be re-linked
// by the rescan the retire itself provokes.
if path.String != "/music/a.flac" {
t.Fatalf(
"phantom_file_path is %q, want %q -- the playlist entry "+
"cannot be re-linked and the playlist is empty for good",
path.String, "/music/a.flac",
)
}
if title.String != "Slack Water" {
t.Errorf("phantom_title is %q, want %q", title.String, "Slack Water")
}
if artist.String != "Aurora Fields" {
t.Errorf("phantom_artist is %q, want %q", artist.String, "Aurora Fields")
}
if album.String != "Tideline" {
t.Errorf("phantom_album is %q, want %q", album.String, "Tideline")
}
if duration.Int64 != 1000 {
t.Errorf("phantom_duration_ms is %d, want 1000", duration.Int64)
}
if genre.String != "Ambient" {
t.Errorf("phantom_genre is %q, want %q", genre.String, "Ambient")
}
if cover.String != "covers/7.jpg" {
t.Errorf("phantom_cover_art_path is %q, want %q", cover.String, "covers/7.jpg")
}
}
// TestRetiringAudioFilesKeepsPathsWhenTheViewCannotAnswer is the case
// that broke cmd/indexbuild: this repair runs *before* applySchema, so
// `track_metadata` is whatever the last launch declared while
// `audio_files` is whatever the launch before that left behind, and a
// view over columns the table no longer has does not read as empty --
// it errors.
//
// The pre-013 stub shape below is the real one that fixture carries.
// What must survive is phantom_file_path, because `file_path` is the
// table's natural key and has been in every shape it ever had; the
// display columns are allowed to be absent, and the open must not fail.
func TestRetiringAudioFilesKeepsPathsWhenTheViewCannotAnswer(t *testing.T) {
ctx := context.Background()
db := openRaw(t, t.TempDir())
if _, err := db.ExecContext(ctx, "PRAGMA foreign_keys = ON"); err != nil {
t.Fatalf("pragma: %v", err)
}
if err := applySchema(ctx, db); err != nil {
t.Fatalf("applySchema: %v", err)
}
// The rows go in *after* the reshape: dropping audio_files with
// foreign keys on would fire the ON DELETE SET NULL and null the
// entry this test is about, which would pass for the wrong reason.
if _, err := db.ExecContext(ctx, `
DROP TABLE audio_files;
CREATE TABLE audio_files (
id INTEGER PRIMARY KEY,
file_path TEXT NOT NULL UNIQUE,
recording_id INTEGER
);
INSERT INTO playlists (id, name) VALUES (1, 'keepme');
INSERT INTO audio_files (id, file_path) VALUES (7, '/music/a.flac');
INSERT INTO playlist_tracks (playlist_id, audio_file_id, position)
VALUES (1, 7, 0);
`); err != nil {
t.Fatalf("seed: %v", err)
}
// The symptom this guards: the repair must not turn a recoverable
// database into one the app refuses to open.
if err := retireStaleTables(ctx, db, testLogger()); err != nil {
t.Fatalf(
"the retire failed on a view it could not read, so the app "+
"would not open at all: %v", err,
)
}
if err := applySchema(ctx, db); err != nil {
t.Fatalf("applySchema: %v", err)
}
var path sql.NullString
if err := db.QueryRowContext(ctx,
"SELECT phantom_file_path FROM playlist_tracks WHERE playlist_id = 1",
).Scan(&path); err != nil {
t.Fatalf("read the surviving entry: %v", err)
}
if path.String != "/music/a.flac" {
t.Fatalf(
"phantom_file_path is %q, want %q -- the display half being "+
"unavailable must not cost the entry its one re-link key",
path.String, "/music/a.flac",
)
}
}
+4 -5
View File
@@ -269,11 +269,10 @@ var tables = []Table{
"from owned files plus the LRCLIB backfill.", "from owned files plus the LRCLIB backfill.",
}, },
{ {
Name: "listening_events", Kind: Authored, Lifetime: Cascade, Name: "play_history", Kind: Authored, Lifetime: Cascade,
Note: "Listening history, one row per track exit (complete, play " + Note: "Listening history. Authored, but intentionally cascades " +
"or skip). Authored, but intentionally cascades with its " + "with its track — history for a file no longer in the library " +
"track — history for a file no longer in the library has " + "has nothing to point at.",
"nothing to point at.",
}, },
{ {
Name: "player_state", Kind: Authored, Lifetime: Retained, Name: "player_state", Kind: Authored, Lifetime: Retained,
+3 -3
View File
@@ -212,14 +212,14 @@ func TestLifetimesMatchSchema(t *testing.T) {
// Authored data is unrecoverable, so it must never be removed as a side // Authored data is unrecoverable, so it must never be removed as a side
// effect of deleting owned data. Cascade is allowed only where the // effect of deleting owned data. Cascade is allowed only where the
// catalog explains why (listening_events, queue_tracks); this test pins the // catalog explains why (play_history, queue_tracks); this test pins the
// set so a new cascade onto authored data is a deliberate decision. // set so a new cascade onto authored data is a deliberate decision.
func TestAuthoredCascadesAreDeliberate(t *testing.T) { func TestAuthoredCascadesAreDeliberate(t *testing.T) {
t.Parallel() t.Parallel()
allowed := map[string]bool{ allowed := map[string]bool{
"listening_events": true, "play_history": true,
"queue_tracks": true, "queue_tracks": true,
// Download history is scoped to the library it imported into. // Download history is scoped to the library it imported into.
// When that library is removed the files it acquired go with // When that library is removed the files it acquired go with
-248
View File
@@ -1,248 +0,0 @@
package download
import (
"context"
"os"
"path/filepath"
"testing"
)
// Multi-disc rips, single-track results and coverage counted in tracks
// rather than files (#270).
func TestParsePathReadsTheDiscFromItsFolder(t *testing.T) {
t.Parallel()
cases := []struct {
path string
disc int
track int
folder string
}{
{`\share\Pink Floyd - The Wall (1979)\CD2\03 Hey You.flac`, 2, 3, "The Wall"},
{`\share\The Wall\Disc 1\01 In The Flesh.flac`, 1, 1, "The Wall"},
{`\share\The Wall\[Disk-2]\01 Hey You.flac`, 2, 1, "The Wall"},
{`\share\The Wall\CD1 - Live\04 Mother.flac`, 1, 4, "The Wall"},
// The filename's own disc number is more specific than the folder.
{`\share\The Wall\CD1\2-05 Comfortably Numb.flac`, 2, 5, "The Wall"},
// Not a disc folder: a number is required.
{`\share\CDs\The Wall\01 In The Flesh.flac`, 0, 1, "The Wall"},
}
for _, tc := range cases {
t.Run(tc.path, func(t *testing.T) {
t.Parallel()
got := ParsePath(tc.path)
if got.Disc != tc.disc || got.Track != tc.track || got.Folder != tc.folder {
t.Errorf(
"ParsePath = disc %d track %d folder %q, want %d %d %q",
got.Disc, got.Track, got.Folder, tc.disc, tc.track, tc.folder,
)
}
})
}
}
func TestAlbumDir(t *testing.T) {
t.Parallel()
cases := map[string]string{
`\share\Album\CD1\01 A.flac`: "/share/Album",
`\share\Album\01 A.flac`: "/share/Album",
`CD1\01 A.flac`: "CD1",
`\share\CD Collection\01.mp3`: "/share/CD Collection",
}
for in, want := range cases {
if got := AlbumDir(in); got != want {
t.Errorf("AlbumDir(%q) = %q, want %q", in, got, want)
}
}
}
// One album shared as CD1/CD2 is one candidate, named after the album.
func TestSlskdGroupsDiscFoldersIntoOneCandidate(t *testing.T) {
t.Parallel()
stub := newSlskdStub(t)
stub.responses = []slskdResponse{{
Username: "peer",
Files: []slskdFile{
{Filename: `\m\The Wall\CD1\01 In The Flesh.flac`, Size: 1},
{Filename: `\m\The Wall\CD1\02 The Thin Ice.flac`, Size: 1},
{Filename: `\m\The Wall\CD2\01 Hey You.flac`, Size: 1},
{Filename: `\m\The Wall\CD2\02 Is There Anybody Out There.flac`, Size: 1},
},
}}
s, _ := newStubSlskd(t, stub)
got, err := s.Search(context.Background(), Download{Query: "the wall"})
if err != nil {
t.Fatalf("Search: %v", err)
}
if len(got) != 1 {
t.Fatalf("got %d candidates, want the two discs as one", len(got))
}
if got[0].Title != "The Wall" || len(got[0].Files) != 4 {
t.Errorf(
"candidate = %q with %d files, want \"The Wall\" with 4",
got[0].Title, len(got[0].Files),
)
}
}
// A track search matches one file per folder, so a single-track request
// must accept a one-file folder that an album request rightly drops.
func TestSlskdKeepsASingleFileForATrackRequest(t *testing.T) {
t.Parallel()
stub := newSlskdStub(t)
stub.responses = []slskdResponse{{
Username: "peer",
Files: []slskdFile{
{Filename: `\m\OK Computer\02 Paranoid Android.flac`, Size: 1},
},
}}
s, _ := newStubSlskd(t, stub)
track, err := s.Search(context.Background(), Download{
RecordingMBID: "rec-1", Artist: "Radiohead", Album: "Paranoid Android",
})
if err != nil {
t.Fatalf("Search: %v", err)
}
if len(track) != 1 {
t.Errorf("track request: got %d candidates, want 1", len(track))
}
album, err := s.Search(context.Background(), Download{
ReleaseMBID: "rel-1", Artist: "Radiohead", Album: "OK Computer",
})
if err != nil {
t.Fatalf("Search: %v", err)
}
if len(album) != 0 {
t.Errorf("album request: got %d candidates, want the one-file folder dropped", len(album))
}
}
// Two discs with a file of the same name both reach staging, each under
// its disc folder, where the importer reads the disc number from.
func TestSlskdCollectKeepsDiscFolders(t *testing.T) {
t.Parallel()
stub := newSlskdStub(t)
s, downloads := newStubSlskd(t, stub)
for _, disc := range []string{"CD1", "CD2"} {
dir := filepath.Join(downloads, disc)
if err := os.MkdirAll(dir, 0o750); err != nil {
t.Fatalf("mkdir: %v", err)
}
if err := os.WriteFile(
filepath.Join(dir, "01 Intro.flac"), []byte(disc), 0o600,
); err != nil {
t.Fatalf("write: %v", err)
}
}
dst := t.TempDir()
got, err := s.collect(Candidate{Files: []CandidateFile{
{Path: `\m\Album\CD1\01 Intro.flac`, IsAudio: true},
{Path: `\m\Album\CD2\01 Intro.flac`, IsAudio: true},
}}, dst)
if err != nil {
t.Fatalf("collect: %v", err)
}
if len(got.Files) != 2 {
t.Fatalf("collected %d files, want 2", len(got.Files))
}
for _, disc := range []string{"CD1", "CD2"} {
data, err := os.ReadFile(filepath.Join(dst, disc, "01 Intro.flac"))
if err != nil || string(data) != disc {
t.Errorf("%s's file missing or overwritten: %q, %v", disc, data, err)
}
if hint := ParsePath(filepath.Join(dst, disc, "01 Intro.flac")); hint.Disc == 0 {
t.Errorf("staged %s file lost its disc number", disc)
}
}
}
// A two-disc release whose discs both number from 01 aligns completely
// once the disc comes from the folder; before, disc 2's 01 collided with
// disc 1's.
func TestMultiDiscCandidateAlignsEveryTrack(t *testing.T) {
t.Parallel()
dl := Download{
ReleaseMBID: "the-wall",
Artist: "Pink Floyd",
Album: "The Wall",
Expected: []ExpectedTrack{
{DiscNumber: 1, Position: 1, Title: "In the Flesh?"},
{DiscNumber: 1, Position: 2, Title: "The Thin Ice"},
{DiscNumber: 2, Position: 1, Title: "Hey You"},
{DiscNumber: 2, Position: 2, Title: "Is There Anybody Out There?"},
},
}
c := Candidate{
Title: "The Wall",
Files: []CandidateFile{
{Path: `\m\Pink Floyd - The Wall\CD1\01 In the Flesh.flac`, Size: 1},
{Path: `\m\Pink Floyd - The Wall\CD1\02 The Thin Ice.flac`, Size: 1},
{Path: `\m\Pink Floyd - The Wall\CD2\01 Hey You.flac`, Size: 1},
{Path: `\m\Pink Floyd - The Wall\CD2\02 Is There Anybody Out There.flac`, Size: 1},
},
}
got := Score(dl, c, 50, AutoDownloadPrefs{})
if got.Match.Completeness != 1 {
t.Errorf("completeness = %f, want 1", got.Match.Completeness)
}
if got.Match.AlbumFit < 0.99 {
t.Errorf("album fit = %f, want the album's own name to match", got.Match.AlbumFit)
}
if got.Match.Overall < minMatch {
t.Errorf("match = %f, want it to clear the auto-pick bar %f", got.Match.Overall, minMatch)
}
}
// Ten files against a ten-track album is not a complete album when only
// three of them are its tracks.
func TestCompletenessCountsTracksNotFiles(t *testing.T) {
t.Parallel()
dl := okComputer()
c := Candidate{Title: "OK Computer", Files: []CandidateFile{
{Path: `\m\Radiohead - OK Computer\Airbag.flac`, Size: 1},
{Path: `\m\Radiohead - OK Computer\Paranoid Android.flac`, Size: 1},
{Path: `\m\Radiohead - OK Computer\Exit Music (For a Film).flac`, Size: 1},
{Path: `\m\Radiohead - OK Computer\Creep.flac`, Size: 1},
}}
got := Score(dl, c, 50, AutoDownloadPrefs{})
if got.Match.Completeness > 0.76 {
t.Errorf(
"completeness = %f with 3 of 4 tracks present, want at most 0.75",
got.Match.Completeness,
)
}
}
+12 -13
View File
@@ -47,7 +47,7 @@ func grabAll(
go func() { go func() {
defer wg.Done() defer wg.Done()
f.manager.grab(ctx, dl, candidate, nil, false) f.manager.grab(ctx, dl, candidate, nil)
}() }()
} }
@@ -79,9 +79,9 @@ func TestConcurrencyForPrefersOverrideThenKind(t *testing.T) {
want int want int
}{ }{
{ {
name: "slskd defaults to a few peers", name: "slskd defaults to one",
cfg: Config{Kind: KindSlskd}, cfg: Config{Kind: KindSlskd},
want: 3, want: 1,
}, },
{ {
name: "usenet defaults higher", name: "usenet defaults higher",
@@ -92,9 +92,9 @@ func TestConcurrencyForPrefersOverrideThenKind(t *testing.T) {
name: "explicit override wins", name: "explicit override wins",
cfg: Config{ cfg: Config{
Kind: KindSlskd, Kind: KindSlskd,
Settings: map[string]string{concurrencyKey: "1"}, Settings: map[string]string{concurrencyKey: "3"},
}, },
want: 1, want: 3,
}, },
{ {
name: "nonsense override falls back", name: "nonsense override falls back",
@@ -102,7 +102,7 @@ func TestConcurrencyForPrefersOverrideThenKind(t *testing.T) {
Kind: KindSlskd, Kind: KindSlskd,
Settings: map[string]string{concurrencyKey: "not a number"}, Settings: map[string]string{concurrencyKey: "not a number"},
}, },
want: 3, want: 1,
}, },
{ {
name: "zero override falls back", name: "zero override falls back",
@@ -110,7 +110,7 @@ func TestConcurrencyForPrefersOverrideThenKind(t *testing.T) {
Kind: KindSlskd, Kind: KindSlskd,
Settings: map[string]string{concurrencyKey: "0"}, Settings: map[string]string{concurrencyKey: "0"},
}, },
want: 3, want: 1,
}, },
{ {
name: "unknown kind falls back to the global default", name: "unknown kind falls back to the global default",
@@ -126,9 +126,9 @@ func TestConcurrencyForPrefersOverrideThenKind(t *testing.T) {
} }
} }
// The reason the per-provider cap exists: a daemon capped at one // The reason the per-provider cap exists: a Soulseek daemon capped at
// transfer must serialize, even when the global cap would allow more and // one transfer must serialize, even when the global cap would allow
// the user has queued several albums at once. // more and the user has queued several albums at once.
func TestPerProviderCapSerializesTransfers(t *testing.T) { func TestPerProviderCapSerializesTransfers(t *testing.T) {
t.Parallel() t.Parallel()
@@ -142,7 +142,6 @@ func TestPerProviderCapSerializesTransfers(t *testing.T) {
ID: 1, ID: 1,
Kind: KindSlskd, Kind: KindSlskd,
Priority: 50, Priority: 50,
Settings: map[string]string{concurrencyKey: "1"},
}, slow) }, slow)
// Three requests against the same one-at-a-time provider. // Three requests against the same one-at-a-time provider.
@@ -211,8 +210,8 @@ func TestSyncSemaphoresReplacesChangedLimits(t *testing.T) {
f.manager.installProvider(Config{ID: 1, Kind: KindSlskd}, nil) f.manager.installProvider(Config{ID: 1, Kind: KindSlskd}, nil)
first := f.manager.semaphoreFor(1) first := f.manager.semaphoreFor(1)
if want := kindConcurrency[KindSlskd]; cap(first) != want { if cap(first) != 1 {
t.Fatalf("slskd semaphore cap = %d, want %d", cap(first), want) t.Fatalf("slskd semaphore cap = %d, want 1", cap(first))
} }
// Same limit: the semaphore is kept, so in-flight accounting is not // Same limit: the semaphore is kept, so in-flight accounting is not
-261
View File
@@ -1,261 +0,0 @@
package download
import (
"context"
"errors"
"os"
"testing"
)
// A transfer that fails on one copy of an album is not a failed
// download while another acceptable copy exists. On Soulseek the usual
// failure is one peer being offline, with several others offering the
// same folder.
var errPeerOffline = errors.New("peer went offline")
func TestManagerFallsBackToTheNextCandidate(t *testing.T) {
t.Parallel()
f := newManagerFixture(t)
// The failing source ranks first on priority, so the fallback is
// what reaches the one that works.
bad := fakeWithAlbum(1, "offline-peer", ".flac")
bad.GrabErr = errPeerOffline
good := fakeWithAlbum(2, "online-peer", ".flac")
f.manager.installProvider(Config{ID: 1, Priority: 90}, bad)
f.manager.installProvider(Config{ID: 2, Priority: 10}, good)
dl := fourTrackDownload()
if _, err := f.manager.Start(context.Background(), dl); err != nil {
t.Fatalf("Start: %v", err)
}
waitForDownloadState(t, f.store, dl.ID, StateComplete)
if bad.GrabCalls != 1 || good.GrabCalls != 1 {
t.Errorf(
"grabs: failing=%d working=%d, want 1 and 1",
bad.GrabCalls, good.GrabCalls,
)
}
// The abandoned attempt's staging goes with it; only a request that
// fails outright keeps its staging for inspection.
waitFor(t, func() bool {
entries, err := os.ReadDir(f.staging.Root())
return err == nil && len(entries) == 0
}, "the failed attempt's staging was never released")
}
// Falling back must not lower the bar. A second choice outside the
// user's guardrails is not a choice auto-pick may make, first or second.
func TestManagerFallbackRespectsTheGuardrails(t *testing.T) {
t.Parallel()
f := newManagerFixture(t)
f.manager.SetPreferences(AutoDownloadPrefs{MaxSizeMB: 50})
bad := fakeWithAlbum(1, "offline-peer", ".flac")
bad.GrabErr = errPeerOffline
bad.Candidates[0].TotalSize = 40 << 20
huge := fakeWithAlbum(2, "oversized", ".flac")
huge.Candidates[0].TotalSize = 900 << 20
f.manager.installProvider(Config{ID: 1, Priority: 90}, bad)
f.manager.installProvider(Config{ID: 2, Priority: 10}, huge)
dl := fourTrackDownload()
if _, err := f.manager.Start(context.Background(), dl); err != nil {
t.Fatalf("Start: %v", err)
}
waitForDownloadState(t, f.store, dl.ID, StateFailed)
if huge.GrabCalls != 0 {
t.Errorf("fell back to a candidate over the size ceiling")
}
}
// A copy the user picked by hand is the copy they asked for. Quietly
// substituting another is a decision they did not make.
func TestManagerPickDoesNotFallBack(t *testing.T) {
t.Parallel()
f := newManagerFixture(t)
bad := fakeWithAlbum(1, "offline-peer", ".flac")
bad.GrabErr = errPeerOffline
good := fakeWithAlbum(2, "online-peer", ".flac")
f.manager.installProvider(Config{ID: 1, Priority: 90}, bad)
f.manager.installProvider(Config{ID: 2, Priority: 10}, good)
// A ceiling below both copies parks the result set for the user.
f.manager.SetPreferences(AutoDownloadPrefs{MaxSizeMB: 1})
bad.Candidates[0].TotalSize = 30 << 20
good.Candidates[0].TotalSize = 30 << 20
dl := fourTrackDownload()
if _, err := f.manager.Start(context.Background(), dl); err != nil {
t.Fatalf("Start: %v", err)
}
if err := f.manager.Pick(
context.Background(), dl.ID, "offline-peer-cand",
); err != nil {
t.Fatalf("Pick: %v", err)
}
waitForDownloadState(t, f.store, dl.ID, StateFailed)
if good.GrabCalls != 0 {
t.Errorf("a hand-picked grab fell back to another candidate")
}
}
// Fallback is for surviving an offline peer or two, not for walking a
// forty-peer list for six hours.
func TestManagerFallbackIsBounded(t *testing.T) {
t.Parallel()
f := newManagerFixture(t)
var providers []*FakeProvider
for i := int64(1); i <= maxGrabAttempts+2; i++ {
p := fakeWithAlbum(i, "peer-"+itoa(int(i)), ".flac")
p.GrabErr = errPeerOffline
f.manager.installProvider(Config{ID: i, Priority: 50}, p)
providers = append(providers, p)
}
dl := fourTrackDownload()
if _, err := f.manager.Start(context.Background(), dl); err != nil {
t.Fatalf("Start: %v", err)
}
waitForDownloadState(t, f.store, dl.ID, StateFailed)
grabs := 0
for _, p := range providers {
grabs += p.GrabCalls
}
if grabs != maxGrabAttempts {
t.Errorf("grabs = %d, want %d", grabs, maxGrabAttempts)
}
}
// The veto judges the best candidate *inside* the guardrails, so the
// grab has to take that one — not the overall best, which may be the
// very copy the user said not to take unattended.
func TestManagerAutoPickTakesTheBestEligibleCandidate(t *testing.T) {
t.Parallel()
f := newManagerFixture(t)
f.manager.SetPreferences(AutoDownloadPrefs{MaxSizeMB: 50})
huge := fakeWithAlbum(1, "oversized", ".flac")
huge.Candidates[0].TotalSize = 900 << 20
fits := fakeWithAlbum(2, "fits", ".flac")
fits.Candidates[0].TotalSize = 40 << 20
f.manager.installProvider(Config{ID: 1, Priority: 90}, huge)
f.manager.installProvider(Config{ID: 2, Priority: 10}, fits)
dl := fourTrackDownload()
ranked, err := f.manager.Start(context.Background(), dl)
if err != nil {
t.Fatalf("Start: %v", err)
}
if ranked[0].ID != "oversized-cand" {
t.Fatalf("fixture: best overall is %s, want the oversized copy", ranked[0].ID)
}
waitForDownloadState(t, f.store, dl.ID, StateComplete)
if huge.GrabCalls != 0 || fits.GrabCalls != 1 {
t.Errorf(
"grabs: oversized=%d fits=%d, want 0 and 1",
huge.GrabCalls, fits.GrabCalls,
)
}
}
// On Soulseek a failure is the peer's, so every folder that peer offered
// goes with it. Elsewhere a failure is the release's, and one indexer's
// other releases are still worth trying.
func TestRuledOutBy(t *testing.T) {
t.Parallel()
failed := []Candidate{
{ID: "slskd:alice:Album", Kind: KindSlskd, ProviderID: 1, Origin: "alice"},
{ID: "tracker-1", Kind: KindProwlarr, ProviderID: 2, Origin: "indexer"},
}
cases := []struct {
name string
c Candidate
want bool
}{
{
name: "the same candidate",
c: Candidate{ID: "tracker-1", Kind: KindProwlarr, ProviderID: 2, Origin: "indexer"},
want: true,
},
{
name: "another folder from a failed peer",
c: Candidate{
ID: "slskd:alice:Album (2)",
Kind: KindSlskd,
ProviderID: 1,
Origin: "alice",
},
want: true,
},
{
name: "another peer",
c: Candidate{ID: "slskd:bob:Album", Kind: KindSlskd, ProviderID: 1, Origin: "bob"},
want: false,
},
{
name: "another release from the same indexer",
c: Candidate{ID: "tracker-2", Kind: KindProwlarr, ProviderID: 2, Origin: "indexer"},
want: false,
},
{
name: "a peer of the same name on a different daemon",
c: Candidate{
ID: "slskd:alice:Album",
Kind: KindSlskd,
ProviderID: 3,
Origin: "alice",
},
want: false,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
if got := ruledOutBy(tc.c, failed); got != tc.want {
t.Errorf("ruledOutBy = %v, want %v", got, tc.want)
}
})
}
}
-77
View File
@@ -1,77 +0,0 @@
package download
import (
"context"
"sync"
)
// keyedLock is a set of mutexes created on demand, one per key, that
// honour a context while waiting. An entry lives only while someone
// holds or waits on it, so a key per Soulseek peer or per folder name
// does not accumulate for the life of the process.
type keyedLock[K comparable] struct {
mu sync.Mutex
held map[K]*keyedEntry
}
type keyedEntry struct {
ch chan struct{}
// refs counts holders and waiters; the entry is dropped at zero.
refs int
}
// acquire blocks until k is free or ctx ends, and returns the function
// that frees it.
func (l *keyedLock[K]) acquire(ctx context.Context, k K) (func(), error) {
l.mu.Lock()
if l.held == nil {
l.held = map[K]*keyedEntry{}
}
e, ok := l.held[k]
if !ok {
e = &keyedEntry{ch: make(chan struct{}, 1)}
l.held[k] = e
}
e.refs++
l.mu.Unlock()
select {
case e.ch <- struct{}{}:
case <-ctx.Done():
l.drop(k, e)
return nil, ctx.Err()
}
var once sync.Once
return func() {
once.Do(func() {
<-e.ch
l.drop(k, e)
})
}, nil
}
func (l *keyedLock[K]) drop(k K, e *keyedEntry) {
l.mu.Lock()
defer l.mu.Unlock()
e.refs--
if e.refs == 0 {
delete(l.held, k)
}
}
// size reports how many keys are held or awaited, for tests.
func (l *keyedLock[K]) size() int {
l.mu.Lock()
defer l.mu.Unlock()
return len(l.held)
}
+65 -245
View File
@@ -59,15 +59,13 @@ const concurrencyKey = "maxConcurrent"
// A single global cap is the wrong shape here: usenet and torrent // A single global cap is the wrong shape here: usenet and torrent
// clients are built to run many transfers at once and are throttled by // clients are built to run many transfers at once and are throttled by
// bandwidth, while Soulseek transfers come from one person's home // bandwidth, while Soulseek transfers come from one person's home
// upload slot. Politeness there is per *peer* — asking one user for two // upload slot. Hitting the same peer with parallel requests gets you
// folders at once gets you queued behind everyone else at best and // queued behind everyone else at best and banned at worst, so slskd is
// banned at worst — and the manager holds that line separately, one // capped at one — the polite number, and the one that actually
// grab per peer (peerLocks). Two different users do not compete for // completes fastest, because a Soulseek peer serves one file at a time
// anyone's slot, so the daemon-wide number only bounds how many peers // regardless of how many you ask for.
// are asked at once, and one slow peer no longer serialises every other
// Soulseek download behind it.
var kindConcurrency = map[Kind]int{ var kindConcurrency = map[Kind]int{
KindSlskd: 3, KindSlskd: 1,
KindYtDlp: 2, KindYtDlp: 2,
KindQBittorrent: 4, KindQBittorrent: 4,
KindSABnzbd: 4, KindSABnzbd: 4,
@@ -157,11 +155,6 @@ type Manager struct {
semMu sync.Mutex semMu sync.Mutex
provSem map[int64]chan struct{} provSem map[int64]chan struct{}
// peerLocks holds one grab per Soulseek peer, taken before any
// slot: a grab waiting for a busy peer must not sit on a provider
// slot another peer could be using.
peerLocks keyedLock[peerKey]
// delegatePoll is how often delegating managers are asked for // delegatePoll is how often delegating managers are asked for
// status. A field rather than the constant so tests can drive the // status. A field rather than the constant so tests can drive the
// full delegate flow without sleeping through it. // full delegate flow without sleeping through it.
@@ -584,12 +577,12 @@ func (m *Manager) Start(
)) ))
} }
if pick, ok := autoPick(dl, ranked, m.preferences()); ok { if m.AutoPickable(dl, ranked) {
if job != nil { if job != nil {
job.Logf(jobs.LevelInfo, "Auto-selected best candidate") job.Logf(jobs.LevelInfo, "Auto-selected best candidate")
} }
go m.grab(context.WithoutCancel(ctx), dl, pick, job, true) go m.grab(context.WithoutCancel(ctx), dl, ranked[0], job)
return ranked, nil return ranked, nil
} }
@@ -629,13 +622,6 @@ func (m *Manager) Attempt(
return false, veto, nil return false, veto, nil
} }
pick, ok := autoPick(dl, ranked, m.preferences())
if !ok {
// Unreachable while autoPick and AutoPickVeto agree; kept so a
// future divergence refuses rather than grabbing blind.
return false, "no candidate clears the auto-download bar", nil
}
if err := m.store.CreateDownload(ctx, dl); err != nil { if err := m.store.CreateDownload(ctx, dl); err != nil {
return false, "", err return false, "", err
} }
@@ -657,7 +643,7 @@ func (m *Manager) Attempt(
)) ))
} }
go m.grab(context.WithoutCancel(ctx), dl, pick, job, true) go m.grab(context.WithoutCancel(ctx), dl, ranked[0], job)
return true, "", nil return true, "", nil
} }
@@ -692,7 +678,7 @@ func (m *Manager) Pick(
job := m.startJob(dl) job := m.startJob(dl)
go m.grab(context.WithoutCancel(ctx), dl, *chosen, job, false) go m.grab(context.WithoutCancel(ctx), dl, *chosen, job)
return nil return nil
} }
@@ -716,20 +702,13 @@ func (m *Manager) Cancel(ctx context.Context, downloadID string) error {
return nil return nil
} }
// grab drives one request all the way to the library. It runs on its // grab drives one candidate all the way to the library. It runs on its
// own goroutine and owns the job from here on. // own goroutine and owns the job from here on.
//
// When fallback is set and a candidate's transfer fails, the next
// candidate that auto-pick would itself have accepted is tried in its
// place (see nextCandidate). It is set for the two unattended routes
// and not for a candidate the user picked by hand: they chose that copy,
// and quietly substituting another is a decision they did not make.
func (m *Manager) grab( func (m *Manager) grab(
ctx context.Context, ctx context.Context,
dl Download, dl Download,
c Candidate, c Candidate,
job *jobs.Handle, job *jobs.Handle,
fallback bool,
) { ) {
ctx, cancel := context.WithTimeout(ctx, grabTimeout) ctx, cancel := context.WithTimeout(ctx, grabTimeout)
defer cancel() defer cancel()
@@ -744,99 +723,6 @@ func (m *Manager) grab(
m.actMu.Unlock() m.actMu.Unlock()
}() }()
var failed []Candidate
for {
out := m.attemptGrab(ctx, dl, c, job)
if out.err == nil {
m.finishGrab(ctx, dl, out.item, out.imported, job)
return
}
failed = append(failed, c)
next, ok := m.nextCandidate(ctx, dl, failed, out, fallback)
if !ok {
m.failDownload(ctx, job, dl.ID, out.err)
return
}
m.logger.Info(
"download candidate failed; trying the next",
"download", dl.ID,
"failed", c.ID,
"next", next.ID,
"error", out.err,
)
if job != nil {
job.Logf(jobs.LevelWarn, fmt.Sprintf(
"%s failed (%v); trying %s instead",
describeCandidate(c), out.err, describeCandidate(next),
))
}
// The failed attempt's staging holds at most a partial folder
// nobody is going to import, and the next attempt reserves its
// own. Only the final failure keeps its staging for inspection.
if out.item.StagingDir != "" {
if err := m.staging.Release(out.item.StagingDir); err != nil {
m.logger.Warn("could not release staging dir", "error", err)
}
}
c = next
}
}
// maxGrabAttempts bounds how many candidates one request will try. A
// popular album can have dozens of peers; the point of falling back is
// to survive the ordinary one or two that are offline, not to walk the
// whole list for six hours.
const maxGrabAttempts = 3
// peerKey names one Soulseek user on one daemon. The same username on
// two daemons is two logins and two queues.
type peerKey struct {
provider int64
peer string
}
// peerKeyFor returns the peer a candidate is fetched from, when the
// source is one where asking a peer for two things at once is rude.
func peerKeyFor(c Candidate) (peerKey, bool) {
if c.Kind != KindSlskd || c.Origin == "" {
return peerKey{}, false
}
return peerKey{provider: c.ProviderID, peer: c.Origin}, true
}
// grabOutcome is how one candidate's attempt ended.
type grabOutcome struct {
item DownloadItem
imported ImportResult
err error
// retryable reports whether another candidate might succeed where
// this one failed: the transfer failed, or delivered too little of
// the album. Anything else — no staging space, no library root, a
// tag write failing — would fail the next candidate identically.
retryable bool
}
// attemptGrab takes one candidate through transfer and import. It
// records the item's own failure, but not the download's: whether the
// download has failed is the caller's decision, since another candidate
// may yet succeed.
func (m *Manager) attemptGrab(
ctx context.Context,
dl Download,
c Candidate,
job *jobs.Handle,
) grabOutcome {
// Who will move the bytes is decided before any slot is taken, so // Who will move the bytes is decided before any slot is taken, so
// the transfer waits in its own provider's queue rather than in a // the transfer waits in its own provider's queue rather than in a
// global one. A delegate takes no slot at all: the transfer is // global one. A delegate takes no slot at all: the transfer is
@@ -845,26 +731,21 @@ func (m *Manager) attemptGrab(
// work against our budget. // work against our budget.
plan, err := m.planTransfer(dl, c) plan, err := m.planTransfer(dl, c)
if err != nil { if err != nil {
return grabOutcome{err: err} m.failDownload(ctx, job, dl.ID, err)
return
} }
if !plan.delegated() { if !plan.delegated() {
if key, ok := peerKeyFor(c); ok {
release, err := m.peerLocks.acquire(ctx, key)
if err != nil {
return grabOutcome{err: err}
}
defer release()
}
provSem := m.semaphoreFor(plan.transportID) provSem := m.semaphoreFor(plan.transportID)
select { select {
case provSem <- struct{}{}: case provSem <- struct{}{}:
defer func() { <-provSem }() defer func() { <-provSem }()
case <-ctx.Done(): case <-ctx.Done():
return grabOutcome{err: ctx.Err()} m.failDownload(ctx, job, dl.ID, ctx.Err())
return
} }
globalSem := m.globalSem() globalSem := m.globalSem()
@@ -873,7 +754,9 @@ func (m *Manager) attemptGrab(
case globalSem <- struct{}{}: case globalSem <- struct{}{}:
defer func() { <-globalSem }() defer func() { <-globalSem }()
case <-ctx.Done(): case <-ctx.Done():
return grabOutcome{err: ctx.Err()} m.failDownload(ctx, job, dl.ID, ctx.Err())
return
} }
} }
@@ -888,30 +771,24 @@ func (m *Manager) attemptGrab(
dir, err := m.staging.Reserve(item.ID) dir, err := m.staging.Reserve(item.ID)
if err != nil { if err != nil {
return grabOutcome{err: err} m.failDownload(ctx, job, dl.ID, err)
return
} }
item.StagingDir = dir item.StagingDir = dir
if err := m.store.CreateItem(ctx, item); err != nil { if err := m.store.CreateItem(ctx, item); err != nil {
return grabOutcome{item: item, err: err} m.failDownload(ctx, job, dl.ID, err)
}
fail := func(err error, retryable bool) grabOutcome { return
if serr := m.store.SetItemState(
ctx, item.ID, StateFailed, err.Error(),
); serr != nil {
m.logger.Warn("could not record item failure", "error", serr)
}
return grabOutcome{item: item, err: err, retryable: retryable}
} }
result, err := m.transfer(ctx, dl, item, plan, job) result, err := m.transfer(ctx, dl, item, plan, job)
if err != nil { if err != nil {
// A delegate's failure is the external manager's verdict on the m.failItem(ctx, job, item, dl.ID, err)
// whole request, not on one copy of it.
return fail(err, !plan.delegated()) return
} }
m.setStates(ctx, dl.ID, item.ID, StateImporting) m.setStates(ctx, dl.ID, item.ID, StateImporting)
@@ -921,116 +798,42 @@ func (m *Manager) attemptGrab(
job.SetStages(importStages(2)) job.SetStages(importStages(2))
} }
var imported ImportResult
if result.Delegated { if result.Delegated {
// The external manager already placed and tagged these files in // The external manager already placed and tagged these files in
// its own library. Moving them out from under a system that is // its own library. Moving them out from under a system that is
// still managing them would be worse than useless, so the files // still managing them would be worse than useless, so the files
// are recorded where they are and the library scan picks them // are recorded where they are and the library scan picks them
// up in place. // up in place.
imported = ImportResult{Paths: result.Files}
if job != nil { if job != nil {
job.Logf(jobs.LevelInfo, fmt.Sprintf( job.Logf(jobs.LevelInfo, fmt.Sprintf(
"External manager imported %d files; recording them in place", "External manager imported %d files; recording them in place",
len(result.Files), len(result.Files),
)) ))
} }
} else {
opts := m.importOptions()
opts.WriteTags = true
return grabOutcome{ opts.LibraryRoot, err = m.library.LibraryPath(dl.LibraryID)
item: item, if err != nil {
imported: ImportResult{Paths: result.Files}, m.failItem(ctx, job, item, dl.ID,
fmt.Errorf("resolve library root: %w", err))
return
}
imported, err = m.importer.Import(ctx, dl, result, opts)
if err != nil {
m.failItem(ctx, job, item, dl.ID, err)
return
} }
} }
opts := m.importOptions()
opts.WriteTags = true
opts.LibraryRoot, err = m.library.LibraryPath(dl.LibraryID)
if err != nil {
return fail(fmt.Errorf("resolve library root: %w", err), false)
}
imported, err := m.importer.Import(ctx, dl, result, opts)
if err != nil {
return fail(err, errors.Is(err, ErrTooIncomplete))
}
return grabOutcome{item: item, imported: imported}
}
// nextCandidate picks the candidate to try after the ones in failed.
//
// It only ever offers a candidate auto-pick would have taken on its own
// (autoAcceptable), so falling back cannot lower the bar an unattended
// download is held to: the second choice has to clear the same gates
// the first did.
//
// On Soulseek a failure belongs to the *peer* — offline, refusing, or
// holding us in a queue — so every folder that peer offered is skipped
// with it. Elsewhere a failure belongs to the release, and only that
// candidate is.
func (m *Manager) nextCandidate(
ctx context.Context,
dl Download,
failed []Candidate,
out grabOutcome,
fallback bool,
) (Candidate, bool) {
if !fallback || !out.retryable || ctx.Err() != nil ||
len(failed) >= maxGrabAttempts {
return Candidate{}, false
}
m.resMu.RLock()
ranked := m.results[dl.ID]
m.resMu.RUnlock()
prefs := m.preferences()
for _, c := range ranked {
if ruledOutBy(c, failed) || !autoAcceptable(dl, c, prefs) {
continue
}
return c, true
}
return Candidate{}, false
}
// ruledOutBy reports whether a failure among failed also rules out c.
func ruledOutBy(c Candidate, failed []Candidate) bool {
for _, f := range failed {
if c.ID == f.ID && c.ProviderID == f.ProviderID {
return true
}
if c.Kind == KindSlskd && f.Kind == KindSlskd &&
c.ProviderID == f.ProviderID && c.Origin != "" &&
c.Origin == f.Origin {
return true
}
}
return false
}
// describeCandidate names a candidate for the job log.
func describeCandidate(c Candidate) string {
if c.Origin != "" {
return fmt.Sprintf("%q from %s", c.Title, c.Origin)
}
return fmt.Sprintf("%q", c.Title)
}
// finishGrab records a successful import and retires what the request
// was holding.
func (m *Manager) finishGrab(
ctx context.Context,
dl Download,
item DownloadItem,
imported ImportResult,
job *jobs.Handle,
) {
if err := m.store.SetItemImported( if err := m.store.SetItemImported(
ctx, item.ID, imported.Paths, ctx, item.ID, imported.Paths,
); err != nil { ); err != nil {
@@ -1374,6 +1177,23 @@ func (m *Manager) failDownload(
} }
} }
// failItem records an item-level failure and fails its download.
func (m *Manager) failItem(
ctx context.Context,
job *jobs.Handle,
item DownloadItem,
downloadID string,
err error,
) {
if serr := m.store.SetItemState(
ctx, item.ID, StateFailed, err.Error(),
); serr != nil {
m.logger.Warn("could not record item failure", "error", serr)
}
m.failDownload(ctx, job, downloadID, err)
}
// startJob registers the request in the background jobs panel. // startJob registers the request in the background jobs panel.
func (m *Manager) startJob(dl Download) *jobs.Handle { func (m *Manager) startJob(dl Download) *jobs.Handle {
if m.jobsReg == nil { if m.jobsReg == nil {
+8 -140
View File
@@ -66,14 +66,6 @@ var (
// separatorPattern splits "Artist - Album" style folder names. // separatorPattern splits "Artist - Album" style folder names.
separatorPattern = regexp.MustCompile(`\s+[-–—]\s+`) separatorPattern = regexp.MustCompile(`\s+[-–—]\s+`)
// discFolderPattern matches a directory that holds one disc of an
// album rather than the album: "CD1", "CD 2", "Disc 3", "Disk-1",
// "[Disc 2]", "CD1 - The Early Years". A number is required, so a
// folder merely called "CDs" is not one.
discFolderPattern = regexp.MustCompile(
`(?i)^\s*[\[(]?\s*(?:cd|disc|disk)\s*[-_.#]?\s*(\d{1,2})\b`,
)
) )
// FormatForPath returns the audio format implied by a path's extension, // FormatForPath returns the audio format implied by a path's extension,
@@ -102,63 +94,16 @@ type TrackHint struct {
Folder string Folder string
} }
// discFolder reports whether a directory name is one disc of an album,
// and which.
func discFolder(name string) (int, bool) {
m := discFolderPattern.FindStringSubmatch(name)
if m == nil {
return 0, false
}
n, err := strconv.Atoi(m[1])
if err != nil || n == 0 {
return 0, false
}
return n, true
}
// AlbumDir is the directory that holds a file's *album*: its parent,
// or its grandparent when the parent is a disc folder.
//
// Multi-disc rips are shared as `Album/CD1/…` and `Album/CD2/…`, and
// grouping candidates by the immediate parent split one album into two
// half-albums, each titled "CD1". Neither could clear the completeness
// or album-title bars, so a multi-disc release could not be auto-picked
// at all. A disc folder at the root has no album above it and is
// returned as it is.
func AlbumDir(p string) string {
dir := path.Dir(strings.ReplaceAll(p, `\`, "/"))
if _, ok := discFolder(path.Base(dir)); !ok {
return dir
}
parent := path.Dir(dir)
if parent == "." || parent == "/" || parent == "" {
return dir
}
return parent
}
// ParsePath extracts what it can from one candidate file path. // ParsePath extracts what it can from one candidate file path.
func ParsePath(p string) TrackHint { func ParsePath(p string) TrackHint {
// Soulseek paths are Windows-style; normalize before splitting. // Soulseek paths are Windows-style; normalize before splitting.
norm := strings.ReplaceAll(p, `\`, "/") norm := strings.ReplaceAll(p, `\`, "/")
base := path.Base(norm) base := path.Base(norm)
folder := path.Base(path.Dir(norm))
name := strings.TrimSuffix(base, path.Ext(base)) name := strings.TrimSuffix(base, path.Ext(base))
// The album's name is the album directory's, not a disc folder's, hint := TrackHint{Folder: cleanAlbumName(folder)}
// and the disc folder is where a multi-disc rip says which disc a
// file is on. A disc number in the filename ("2-01 …") is more
// specific and overrides it below.
hint := TrackHint{Folder: cleanAlbumName(path.Base(AlbumDir(norm)))}
if disc, ok := discFolder(path.Base(path.Dir(norm))); ok {
hint.Disc = disc
}
if m := trackNumPattern.FindStringSubmatch(name); m != nil { if m := trackNumPattern.FindStringSubmatch(name); m != nil {
if m[1] != "" { if m[1] != "" {
@@ -258,72 +203,21 @@ func AnnotateFiles(files []CandidateFile) []CandidateFile {
// matchFiles aligns a candidate's audio files to the expected tracklist // matchFiles aligns a candidate's audio files to the expected tracklist
// and returns the per-file assignment plus the mean title similarity of // and returns the per-file assignment plus the mean title similarity of
// the aligned pairs. alignFiles is the same alignment with the // the aligned pairs.
// duration evidence as well.
func matchFiles(
files []CandidateFile,
expected []ExpectedTrack,
) ([]CandidateFile, float64) {
a := alignFiles(files, expected)
return a.files, a.titleFit
}
// alignment is what aligning a candidate to a tracklist found.
type alignment struct {
files []CandidateFile
// titleFit is the mean title similarity over aligned pairs.
titleFit float64
// durationFit is the mean duration agreement over aligned pairs
// where both sides state a length, and timedPairs is how many such
// pairs there were.
durationFit float64
timedPairs int
aligned int
}
// durationAgreement scores how well a file's length matches the
// expected track's, in 0..1. Rips of the same master differ by a
// second or two of silence; a different edit, a live take or a
// truncated file differs by tens of seconds.
func durationAgreement(got, want int64) float64 {
const (
exactMillis = 3_000
wrongMillis = 30_000
)
d := got - want
if d < 0 {
d = -d
}
switch {
case d <= exactMillis:
return 1
case d >= wrongMillis:
return 0
default:
return 1 - float64(d-exactMillis)/float64(wrongMillis-exactMillis)
}
}
// alignFiles aligns a candidate's audio files to the expected tracklist.
// //
// Alignment is greedy by score rather than optimal: candidate folders // Alignment is greedy by score rather than optimal: candidate folders
// are small (a few dozen files at most) and the common cases — correct // are small (a few dozen files at most) and the common cases — correct
// track numbers, or clean "NN Title" names — are unambiguous, so the // track numbers, or clean "NN Title" names — are unambiguous, so the
// extra machinery of Hungarian assignment buys nothing here. // extra machinery of Hungarian assignment buys nothing here.
func alignFiles( func matchFiles(
files []CandidateFile, files []CandidateFile,
expected []ExpectedTrack, expected []ExpectedTrack,
) alignment { ) ([]CandidateFile, float64) {
annotated := make([]CandidateFile, len(files)) annotated := make([]CandidateFile, len(files))
copy(annotated, files) copy(annotated, files)
if len(expected) == 0 { if len(expected) == 0 {
return alignment{files: annotated} return annotated, 0
} }
hints := make([]TrackHint, len(annotated)) hints := make([]TrackHint, len(annotated))
@@ -336,19 +230,8 @@ func alignFiles(
var ( var (
total float64 total float64
matched int matched int
durTotal float64
timed int
) )
// timing adds a pair's duration evidence when both sides state one.
timing := func(f CandidateFile, e ExpectedTrack) {
if f.LengthMillis > 0 && e.LengthMillis > 0 {
durTotal += durationAgreement(f.LengthMillis, e.LengthMillis)
timed++
}
}
// Pass 1: trust explicit track numbers when they are unique and in // Pass 1: trust explicit track numbers when they are unique and in
// range. A folder that numbers its files correctly is the strong // range. A folder that numbers its files correctly is the strong
// case, and title comparison only adds noise there. // case, and title comparison only adds noise there.
@@ -367,8 +250,6 @@ func alignFiles(
total += autotag.TitleSimilarity(hints[i].Title, expected[idx].Title) total += autotag.TitleSimilarity(hints[i].Title, expected[idx].Title)
matched++ matched++
timing(annotated[i], expected[idx])
} }
// Pass 2: title similarity for whatever is left. // Pass 2: title similarity for whatever is left.
@@ -403,26 +284,13 @@ func alignFiles(
total += bestSim total += bestSim
matched++ matched++
timing(annotated[i], expected[bestIdx])
} }
if matched == 0 { if matched == 0 {
return alignment{files: annotated} return annotated, 0
} }
a := alignment{ return annotated, total / float64(matched)
files: annotated,
titleFit: total / float64(matched),
timedPairs: timed,
aligned: matched,
}
if timed > 0 {
a.durationFit = durTotal / float64(timed)
}
return a
} }
// indexForPosition finds the expected track at a disc/track position. // indexForPosition finds the expected track at a disc/track position.
-224
View File
@@ -1,224 +0,0 @@
package download
import (
"context"
"errors"
"path/filepath"
"sync"
"testing"
"time"
)
// Soulseek politeness is per peer, not per daemon (#272).
func TestKeyedLockSerialisesOneKeyOnly(t *testing.T) {
t.Parallel()
var l keyedLock[string]
ctx := context.Background()
releaseA, err := l.acquire(ctx, "a")
if err != nil {
t.Fatalf("acquire a: %v", err)
}
// Another key is free while "a" is held.
releaseB, err := l.acquire(ctx, "b")
if err != nil {
t.Fatalf("acquire b: %v", err)
}
releaseB()
// The same key waits, and gives up with its context.
short, cancel := context.WithTimeout(ctx, 20*time.Millisecond)
defer cancel()
if _, err := l.acquire(short, "a"); !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("second acquire of a held key = %v, want the deadline", err)
}
releaseA()
releaseA() // Idempotent: a second call must not free someone else's hold.
if n := l.size(); n != 0 {
t.Errorf("%d keys left behind, want none once nobody holds or waits", n)
}
}
// grabEach runs one grab per candidate and returns a function that waits
// for all of them; grabAll's reasons for waiting apply.
func grabEach(t *testing.T, f managerFixture, cands []Candidate) func() {
t.Helper()
ctx := context.Background()
var wg sync.WaitGroup
for i, c := range cands {
dl := fourTrackDownload()
dl.ID = "dl-" + string(rune('a'+i))
if err := f.store.CreateDownload(ctx, dl); err != nil {
t.Fatalf("CreateDownload: %v", err)
}
wg.Add(1)
go func() {
defer wg.Done()
f.manager.grab(ctx, dl, c, nil, false)
}()
}
return func() {
done := make(chan struct{})
go func() {
wg.Wait()
close(done)
}()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Error("transfers did not finish")
}
}
}
func slskdCandidates(p *FakeProvider, peers ...string) []Candidate {
out := make([]Candidate, 0, len(peers))
for i, peer := range peers {
c := p.Candidates[0]
c.ID = c.ID + "-" + itoa(i)
c.Kind = KindSlskd
c.ProviderID = 1
c.Origin = peer
out = append(out, c)
}
return out
}
// Three albums from one user are asked for one at a time, even though
// the daemon would allow three transfers.
func TestOnePeerIsAskedForOneThingAtATime(t *testing.T) {
t.Parallel()
f := newManagerFixture(t)
f.manager.SetMaxConcurrent(4)
p := fakeWithAlbum(1, "slskd", ".flac")
p.GrabGate = make(chan struct{})
f.manager.installProvider(Config{ID: 1, Kind: KindSlskd, Priority: 50}, p)
wait := grabEach(t, f, slskdCandidates(p, "alice", "alice", "alice"))
waitFor(t, func() bool { return p.GrabCallCount() >= 1 }, "no grab started")
time.Sleep(150 * time.Millisecond)
if got := p.MaxParallelGrabs(); got != 1 {
t.Errorf("%d simultaneous grabs from one peer, want 1", got)
}
close(p.GrabGate)
waitFor(t, func() bool { return p.GrabCallCount() == 3 }, "queued grabs never ran")
wait()
if n := f.manager.peerLocks.size(); n != 0 {
t.Errorf("%d peer locks left behind", n)
}
}
// Different users run at once, up to the daemon's cap — the point of
// the change: one slow peer no longer holds up every other.
func TestDifferentPeersRunTogether(t *testing.T) {
t.Parallel()
f := newManagerFixture(t)
f.manager.SetMaxConcurrent(8)
p := fakeWithAlbum(1, "slskd", ".flac")
p.GrabGate = make(chan struct{})
f.manager.installProvider(Config{ID: 1, Kind: KindSlskd, Priority: 50}, p)
wait := grabEach(t, f, slskdCandidates(p, "alice", "bob", "carol", "dave"))
waitFor(
t,
func() bool { return p.MaxParallelGrabs() >= kindConcurrency[KindSlskd] },
"different peers were serialised",
)
time.Sleep(100 * time.Millisecond)
if got := p.MaxParallelGrabs(); got != kindConcurrency[KindSlskd] {
t.Errorf("%d simultaneous grabs, want the daemon cap %d", got, kindConcurrency[KindSlskd])
}
close(p.GrabGate)
wait()
}
func TestSlskdLocalFolders(t *testing.T) {
t.Parallel()
s := &slskd{downloadsPath: "/dl"}
got := s.localFolders(Candidate{Files: []CandidateFile{
{Path: `\m\The Wall\CD2\01 Hey You.flac`},
{Path: `\m\The Wall\CD1\01 In The Flesh.flac`},
{Path: `\m\The Wall\CD1\02 The Thin Ice.flac`},
}})
want := []string{filepath.Join("/dl", "CD1"), filepath.Join("/dl", "CD2")}
if len(got) != len(want) || got[0] != want[0] || got[1] != want[1] {
t.Errorf("localFolders = %q, want %q", got, want)
}
}
// Two peers' "Greatest Hits" land in one slskd directory, so the second
// grab does not enqueue until the first has collected its files.
func TestSlskdSameFolderNameWaits(t *testing.T) {
t.Parallel()
stub := newSlskdStub(t)
s, downloads := newStubSlskd(t, stub)
c := Candidate{
Payload: map[string]string{"username": "bob"},
Files: []CandidateFile{
{Path: `\music\Greatest Hits\01 Intro.flac`, Size: 1, IsAudio: true},
},
}
release, err := lockSlskdFolders(
context.Background(), []string{filepath.Join(downloads, "Greatest Hits")},
)
if err != nil {
t.Fatalf("lock: %v", err)
}
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
defer cancel()
if _, err := s.Grab(ctx, c, t.TempDir(), nil); !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("Grab = %v, want it to wait on the held folder", err)
}
release()
stub.mu.Lock()
posted := stub.posted
stub.mu.Unlock()
if posted {
t.Error("enqueued transfers into a folder another grab held")
}
}
+7 -8
View File
@@ -236,18 +236,17 @@ func Register(d Descriptor, c Constructor) {
} }
// concurrencyField describes the per-provider transfer limit, with help // concurrencyField describes the per-provider transfer limit, with help
// text explaining what the number means where it means something // text explaining why the default is what it is — a user who raises
// unusual: on slskd it counts peers, since each peer is only ever asked // slskd from 1 to 8 and gets themselves queued behind every other
// for one folder at a time whatever it is set to. // Soulseek user deserves to have been warned.
func concurrencyField(k Kind) Field { func concurrencyField(k Kind) Field {
help := "Maximum simultaneous transfers from this client." help := "Maximum simultaneous transfers from this client."
if k == KindSlskd { if k == KindSlskd {
help = "How many Soulseek users to download from at once. " + help = "Maximum simultaneous transfers. Soulseek peers serve " +
"Each user is only ever asked for one album at a time, " + "one file at a time and queue or ban clients that ask for " +
"since peers queue or ban clients that ask for more; " + "more, so 1 is both the polite setting and usually the " +
"this bounds how many different users are asked in " + "fastest."
"parallel."
} }
return Field{ return Field{
+86 -598
View File
@@ -5,12 +5,9 @@ import (
"errors" "errors"
"fmt" "fmt"
"log/slog" "log/slog"
"net/url"
"os" "os"
"path" "path"
"path/filepath" "path/filepath"
"regexp"
"slices"
"strings" "strings"
"time" "time"
@@ -72,34 +69,12 @@ const (
slskdTransferPoll = 3 * time.Second slskdTransferPoll = 3 * time.Second
// slskdMinFiles is the fewest audio files a folder needs before it // slskdMinFiles is the fewest audio files a folder needs before it
// is offered as a candidate for an album. Soulseek returns a lot of // is offered as a candidate. Soulseek returns a lot of one-file
// one-file noise for common queries. A single-track request takes // noise for common queries.
// one (see minFilesFor).
slskdMinFiles = 2 slskdMinFiles = 2
// slskdHTTPTimeout bounds one API call. // slskdHTTPTimeout bounds one API call.
slskdHTTPTimeout = 20 * time.Second slskdHTTPTimeout = 20 * time.Second
// millisPerSecond converts slskd's whole-second file lengths.
millisPerSecond = 1000
// slskdStallAfter is how long a grab may go without a byte arriving
// before the peer is given up on. It is measured from enqueue, so
// it covers a peer that queues us and never starts as well as one
// that starts and stops. Ten minutes is long enough for a short
// queue ahead of us to clear and short enough that one unresponsive
// peer does not hold slskd's single transfer slot for an evening.
slskdStallAfter = 10 * time.Minute
// slskdAbsentGrace is how long a requested file may be missing from
// slskd's transfer list before it is counted as failed. slskd lists
// a transfer as soon as it accepts it, so a file still absent after
// a few polls was refused.
slskdAbsentGrace = 30 * time.Second
// slskdCancelTimeout bounds the cleanup that cancels abandoned
// transfers.
slskdCancelTimeout = 15 * time.Second
) )
func init() { func init() {
@@ -159,8 +134,6 @@ type slskd struct {
searchPoll time.Duration searchPoll time.Duration
searchWait time.Duration searchWait time.Duration
transferPoll time.Duration transferPoll time.Duration
stallAfter time.Duration
absentGrace time.Duration
} }
// newSlskd builds the provider from config. // newSlskd builds the provider from config.
@@ -215,8 +188,6 @@ func newSlskd(
searchPoll: slskdSearchPoll, searchPoll: slskdSearchPoll,
searchWait: slskdSearchWait, searchWait: slskdSearchWait,
transferPoll: slskdTransferPoll, transferPoll: slskdTransferPoll,
stallAfter: slskdStallAfter,
absentGrace: slskdAbsentGrace,
}, nil }, nil
} }
@@ -260,13 +231,14 @@ type slskdSearch struct {
// slskdResponse is one peer's answer to a search. // slskdResponse is one peer's answer to a search.
type slskdResponse struct { type slskdResponse struct {
Username string `json:"username"` Username string `json:"username"`
HasFreeUploadSlot bool `json:"hasFreeUploadSlot"` HasFreeUploadSlot bool `json:"hasFreeUploadSlot"`
QueueLength int `json:"queueLength"` QueueLength int `json:"queueLength"`
UploadSpeed int64 `json:"uploadSpeed"` UploadSpeed int64 `json:"uploadSpeed"`
Files []slskdFile `json:"files"` Files []slskdFile `json:"files"`
LockedFileCount int `json:"lockedFileCount"` LockedFileCount int `json:"lockedFileCount"`
FileCount int `json:"fileCount"` FileCount int `json:"fileCount"`
FreeUploadSlotFlag bool `json:"freeUploadSlots"`
} }
// slskdFile is one file a peer is offering. // slskdFile is one file a peer is offering.
@@ -274,9 +246,7 @@ type slskdFile struct {
Filename string `json:"filename"` Filename string `json:"filename"`
Size int64 `json:"size"` Size int64 `json:"size"`
BitRate int `json:"bitRate"` BitRate int `json:"bitRate"`
Length int `json:"length"`
// Length is the duration in whole seconds.
Length int `json:"length"`
} }
// slskdTransfer is one download's state. // slskdTransfer is one download's state.
@@ -308,89 +278,24 @@ func (t slskdTransfer) done() (finished, ok bool) {
// per-folder candidates. A folder from one peer is the unit a user // per-folder candidates. A folder from one peer is the unit a user
// actually wants: Soulseek has no album concept, but people organise // actually wants: Soulseek has no album concept, but people organise
// their shares by album directory. // their shares by album directory.
//
// Up to two queries run at once — the request as written and a
// normalised form of it (see slskdQueries) — and their candidates are
// merged. They run concurrently rather than as a fallback because the
// manager gives a provider one search budget, and a Soulseek search
// spends most of it waiting for peers to answer; a second query after
// the first would not fit.
func (s *slskd) Search(ctx context.Context, dl Download) ([]Candidate, error) { func (s *slskd) Search(ctx context.Context, dl Download) ([]Candidate, error) {
queries := slskdQueries(dl)
if len(queries) == 0 {
return nil, nil
}
type found struct {
candidates []Candidate
err error
}
results := make(chan found, len(queries))
for _, q := range queries {
go func(q string) {
c, err := s.searchOnce(ctx, q, minFilesFor(dl))
results <- found{candidates: c, err: err}
}(q)
}
var (
out []Candidate
seen = map[string]bool{}
firstErr error
answered int
)
for range queries {
r := <-results
if r.err != nil {
s.logger.Debug("slskd search failed", "error", r.err)
if firstErr == nil {
firstErr = r.err
}
continue
}
answered++
// The same peer's folder turns up under both queries; the ID is
// peer and folder, so it is the same candidate.
for _, c := range r.candidates {
if seen[c.ID] {
continue
}
seen[c.ID] = true
out = append(out, c)
}
}
if answered == 0 {
return nil, firstErr
}
return out, nil
}
// searchOnce runs one query to completion and returns its candidates.
func (s *slskd) searchOnce(
ctx context.Context,
text string,
minFiles int,
) ([]Candidate, error) {
// slskd's search endpoint deserializes id as a .NET Guid server-side, // slskd's search endpoint deserializes id as a .NET Guid server-side,
// so it must be a dashed UUID — the app's own newID() (a plain hex // so it must be a dashed UUID — the app's own newID() (a plain hex
// string, used for request/item IDs elsewhere) is rejected with an // string, used for request/item IDs elsewhere) is rejected with an
// HTTP 400 before any search happens. // HTTP 400 before any search happens.
searchID := uuid.NewString() searchID := uuid.NewString()
if err := s.client.post( body := map[string]any{
ctx, "/api/v0/searches", s.searchRequest(searchID, text, minFiles), nil, "id": searchID,
); err != nil { "searchText": dl.SearchText(),
}
if err := s.client.post(ctx, "/api/v0/searches", body, nil); err != nil {
return nil, err
}
search, err := s.awaitSearch(ctx, searchID)
if err != nil {
return nil, err return nil, err
} }
@@ -402,222 +307,52 @@ func (s *slskd) searchOnce(
) )
}() }()
if err := s.awaitSearch(ctx, searchID); err != nil { return s.candidatesFrom(search), nil
return nil, err
}
responses, err := s.searchResponses(ctx, searchID)
if err != nil {
return nil, err
}
return s.candidatesFrom(responses, minFiles), nil
}
// searchRequest is the body that starts a search.
//
// Every option is stated rather than left to the daemon, because
// slskd's defaults are its own and not ours. Its search timeout in
// particular has to finish inside our wait: a search that slskd is still
// running when we stop polling is results we asked for and discarded.
// The response and file limits are raised well above what a popular
// album produces, and the peer filters let slskd drop answers this
// provider would only score down to nothing — a folder too small to be
// a candidate, a peer with a queue it will not reach today.
func (s *slskd) searchRequest(id, text string, minFiles int) map[string]any {
const (
responseLimit = 500
fileLimit = 20_000
maximumPeerQueueLength = 100
)
// A tenth of the wait is left for the last poll and the responses
// fetch.
timeout := s.searchWait - s.searchWait/10
return map[string]any{
"id": id,
"searchText": text,
"searchTimeout": timeout.Milliseconds(),
"responseLimit": responseLimit,
"fileLimit": fileLimit,
"filterResponses": true,
"minimumResponseFileCount": minFiles,
"maximumPeerQueueLength": maximumPeerQueueLength,
}
}
// slskdQueries is what is searched for a request: the request's own
// search text, and a normalised form of it when that differs.
//
// Soulseek matches every term against the file's full path, so each
// extra word is a filter, and some words filter wrongly:
//
// - edition qualifiers — "(Deluxe Edition)", "[2011 Remaster]" — are
// in the catalog's title and rarely in anyone's folder name;
// - punctuation splits a term oddly, and a term that starts with "-"
// is an *exclusion*, so an album called "-ism" searches for
// everything without it;
// - "Various Artists" is in no one's path for a compilation.
//
// A query the user typed is theirs and is searched exactly as written.
func slskdQueries(dl Download) []string {
primary := strings.TrimSpace(dl.SearchText())
if primary == "" {
return nil
}
out := []string{primary}
if dl.Query != "" {
return out
}
artist := dl.Artist
if isVariousArtists(artist) {
artist = ""
}
normal := Download{
Artist: normalizeSearchTerms(artist),
Album: normalizeSearchTerms(editionPattern.ReplaceAllString(dl.Album, " ")),
}
if alt := strings.TrimSpace(normal.SearchText()); alt != "" &&
!strings.EqualFold(alt, primary) {
out = append(out, alt)
}
return out
}
var (
// editionPattern finds an edition qualifier: a bracketed group that
// names an edition, or a trailing " - 2011 Remaster".
editionPattern = regexp.MustCompile(
`(?i)\s*[(\[][^)\]]*\b(?:deluxe|edition|remaster(?:ed)?|expanded|` +
`anniversary|bonus|explicit|reissue|special|collector'?s?|` +
`version|mono|stereo)\b[^)\]]*[)\]]` +
`|\s+-\s+(?:\d{4}\s+)?remaster(?:ed)?\b.*$`,
)
// nonWordPattern is everything that is not a letter or a digit.
nonWordPattern = regexp.MustCompile(`[^\p{L}\p{N}]+`)
)
// normalizeSearchTerms reduces text to plain words.
func normalizeSearchTerms(s string) string {
return strings.Join(strings.Fields(nonWordPattern.ReplaceAllString(s, " ")), " ")
}
// isVariousArtists reports whether an artist credit is a compilation's
// placeholder rather than an artist.
func isVariousArtists(artist string) bool {
switch strings.ToLower(strings.TrimSpace(artist)) {
case "various artists", "various", "va":
return true
default:
return false
}
}
// minFilesFor is the fewest audio files a folder must offer to be a
// candidate for this request.
//
// Soulseek answers a search with the files that match it, not with the
// folders they sit in. An album query matches every file in the album's
// folder, because the folder name carries the terms; a *track* query
// usually matches one file per folder. The two-file floor that filters
// out one-file noise for an album therefore filtered out every result
// for a track, and a single-track request could never be served here.
func minFilesFor(dl Download) int {
if dl.RecordingMBID != "" {
return 1
}
return slskdMinFiles
} }
// awaitSearch polls until the search completes or the budget runs out. // awaitSearch polls until the search completes or the budget runs out.
// A timeout is not an error: partial Soulseek results are normal and // A timeout is not an error: partial Soulseek results are normal and
// often good enough. // often good enough.
// func (s *slskd) awaitSearch(
// The poll asks for the search's state only. It used to ask for every ctx context.Context,
// response on every one-second tick, which for a popular album is the searchID string,
// same few thousand file entries serialised twenty times to be read ) (slskdSearch, error) {
// once; searchResponses fetches them once at the end.
func (s *slskd) awaitSearch(ctx context.Context, searchID string) error {
deadline := time.Now().Add(s.searchWait) deadline := time.Now().Add(s.searchWait)
var last slskdSearch
for time.Now().Before(deadline) { for time.Now().Before(deadline) {
select { select {
case <-ctx.Done(): case <-ctx.Done():
return fmt.Errorf("%w: search cancelled", ErrSlskdTimeout) return last, fmt.Errorf("%w: search cancelled", ErrSlskdTimeout)
case <-time.After(s.searchPoll): case <-time.After(s.searchPoll):
} }
var search slskdSearch var search slskdSearch
if err := s.client.get( if err := s.client.get(
ctx, "/api/v0/searches/"+searchID, &search, ctx,
"/api/v0/searches/"+searchID+"?includeResponses=true",
&search,
); err != nil { ); err != nil {
return err return last, err
} }
last = search
if search.IsComplete { if search.IsComplete {
return nil return search, nil
} }
} }
return nil return last, nil
} }
// searchResponses fetches a search's responses once. // candidatesFrom groups a search's responses into candidates.
// func (s *slskd) candidatesFrom(search slskdSearch) []Candidate {
// `/searches/{id}/responses` is the endpoint for that; a daemon that out := make([]Candidate, 0, len(search.Responses))
// does not answer it is asked the older way, with the search itself
// carrying its responses, so an older slskd degrades to the previous
// behaviour rather than to no results at all.
func (s *slskd) searchResponses(
ctx context.Context,
searchID string,
) ([]slskdResponse, error) {
var responses []slskdResponse
err := s.client.get( for _, resp := range search.Responses {
ctx, "/api/v0/searches/"+searchID+"/responses", &responses,
)
if err == nil {
return responses, nil
}
s.logger.Debug(
"slskd responses endpoint failed; asking with the search",
"error", err,
)
var search slskdSearch
if err := s.client.get(
ctx,
"/api/v0/searches/"+searchID+"?includeResponses=true",
&search,
); err != nil {
return nil, err
}
return search.Responses, nil
}
// candidatesFrom groups a search's responses into candidates, dropping
// folders with fewer than minFiles audio files.
func (s *slskd) candidatesFrom(
responses []slskdResponse,
minFiles int,
) []Candidate {
out := make([]Candidate, 0, len(responses))
for _, resp := range responses {
for folder, files := range groupByFolder(resp.Files) { for folder, files := range groupByFolder(resp.Files) {
audio := 0 audio := 0
@@ -637,14 +372,12 @@ func (s *slskd) candidatesFrom(
Format: format, Format: format,
Bitrate: f.BitRate, Bitrate: f.BitRate,
IsAudio: isAudio, IsAudio: isAudio,
LengthMillis: int64(f.Length) * millisPerSecond,
}) })
total += f.Size total += f.Size
} }
if audio < minFiles { if audio < slskdMinFiles {
continue continue
} }
@@ -665,15 +398,13 @@ func (s *slskd) candidatesFrom(
return out return out
} }
// groupByFolder buckets a peer's files by the album directory they sit // groupByFolder buckets a peer's files by their containing directory.
// in — the containing directory, or the one above it for a disc folder
// (see AlbumDir), so a multi-disc rip is one candidate and not two.
func groupByFolder(files []slskdFile) map[string][]slskdFile { func groupByFolder(files []slskdFile) map[string][]slskdFile {
out := map[string][]slskdFile{} out := map[string][]slskdFile{}
for _, f := range files { for _, f := range files {
dir := AlbumDir(f.Filename) norm := strings.ReplaceAll(f.Filename, `\`, "/")
out[dir] = append(out[dir], f) out[path.Dir(norm)] = append(out[path.Dir(norm)], f)
} }
return out return out
@@ -687,7 +418,7 @@ func groupByFolder(files []slskdFile) map[string][]slskdFile {
func peerHealth(r slskdResponse) float64 { func peerHealth(r slskdResponse) float64 {
score := 0.35 score := 0.35
if r.HasFreeUploadSlot { if r.HasFreeUploadSlot || r.FreeUploadSlotFlag {
score += 0.4 score += 0.4
} }
@@ -736,21 +467,6 @@ func (s *slskd) Grab(
) )
} }
// slskd keeps finished transfers listed until someone removes them,
// and a transfer is matched to the request by filename. A record
// left by an earlier attempt at the same file from the same peer
// would otherwise be read as this attempt's answer the moment the
// first poll came back — an old failure failing a transfer that has
// not started. So what is already terminal is noted before enqueueing
// and ignored after.
release, err := lockSlskdFolders(ctx, s.localFolders(c))
if err != nil {
return Result{}, err
}
defer release()
stale := s.terminalTransferIDs(ctx, username)
wanted := make([]map[string]any, 0, len(c.Files)) wanted := make([]map[string]any, 0, len(c.Files))
for _, f := range c.Files { for _, f := range c.Files {
wanted = append(wanted, map[string]any{ wanted = append(wanted, map[string]any{
@@ -760,122 +476,24 @@ func (s *slskd) Grab(
} }
if err := s.client.post( if err := s.client.post(
ctx, slskdDownloadsPath(username), wanted, nil, ctx, "/api/v0/transfers/downloads/"+username, wanted, nil,
); err != nil { ); err != nil {
return Result{}, err return Result{}, err
} }
if err := s.awaitTransfers( if err := s.awaitTransfers(ctx, username, c, onProgress); err != nil {
ctx, username, stale, c, onProgress,
); err != nil {
return Result{}, err return Result{}, err
} }
return s.collect(c, dst) return s.collect(c, dst)
} }
// slskdFolders serialises grabs that land in the same local folder.
//
// slskd names a download's directory after the remote *leaf* folder, so
// two different albums both shared as "Greatest Hits" — or any two
// multi-disc rips, whose leaves are "CD1" and "CD2" — are written into
// one directory, and collect finds files by name there. Run at once,
// a file one peer never sent is filled by the other peer's file of the
// same name. One grab per peer made that impossible; several peers at
// once makes it likely. It is package-level and keyed on the full
// path because two configured clients can share one daemon.
var slskdFolders keyedLock[string]
// localFolders returns the directories under downloadsPath a candidate's
// files will be written to, sorted so every grab takes them in the same
// order and two cannot each hold what the other waits for.
func (s *slskd) localFolders(c Candidate) []string {
var out []string
for _, f := range c.Files {
norm := strings.ReplaceAll(f.Path, `\`, "/")
out = append(out, filepath.Join(s.downloadsPath, path.Base(path.Dir(norm))))
}
slices.Sort(out)
return slices.Compact(out)
}
// lockSlskdFolders takes every folder in order, releasing what it holds
// if the context ends part way.
func lockSlskdFolders(ctx context.Context, folders []string) (func(), error) {
releases := make([]func(), 0, len(folders))
releaseAll := func() {
for _, r := range slices.Backward(releases) {
r()
}
}
for _, f := range folders {
r, err := slskdFolders.acquire(ctx, f)
if err != nil {
releaseAll()
return nil, err
}
releases = append(releases, r)
}
return releaseAll, nil
}
// slskdDownloadsPath is the transfers endpoint for one peer. Soulseek
// usernames may contain spaces and punctuation, so the name is escaped
// rather than spliced into the path.
func slskdDownloadsPath(username string) string {
return "/api/v0/transfers/downloads/" + url.PathEscape(username)
}
// terminalTransferIDs returns the ids of this peer's transfers that are
// already finished. Best effort: slskd answers 404 for a peer it has no
// transfers with, and any failure here means only that there is nothing
// to ignore.
func (s *slskd) terminalTransferIDs(
ctx context.Context,
username string,
) map[string]bool {
transfers, err := s.transfersFor(ctx, username)
if err != nil {
return nil
}
out := make(map[string]bool, len(transfers))
for _, t := range transfers {
if finished, _ := t.done(); finished && t.ID != "" {
out[t.ID] = true
}
}
return out
}
// awaitTransfers polls until every requested file reaches a terminal // awaitTransfers polls until every requested file reaches a terminal
// state, the transfer stalls, or the caller gives up. // state. Soulseek queues are measured in hours, so the only deadline
// // is the caller's context.
// Soulseek queues are measured in hours, so there is no deadline on the
// transfer as a whole — but there is one on *progress*. slskd's
// transfer limit is one, so a peer that holds us in its queue without
// sending a byte is not only failing this download, it is holding every
// other Soulseek download behind it. After stallAfter with nothing
// moving the peer is given up on, and the manager tries another.
//
// Whatever way this ends short of every file finishing, the transfers
// still live in slskd are cancelled there. Returning without doing so
// leaves the daemon downloading into its own folder for a request
// nobody is waiting on any more.
func (s *slskd) awaitTransfers( func (s *slskd) awaitTransfers(
ctx context.Context, ctx context.Context,
username string, username string,
stale map[string]bool,
c Candidate, c Candidate,
onProgress ProgressFunc, onProgress ProgressFunc,
) error { ) error {
@@ -884,18 +502,9 @@ func (s *slskd) awaitTransfers(
wanted[f.Path] = true wanted[f.Path] = true
} }
var (
started = time.Now()
lastProgress = started
lastBytes int64
live []slskdTransfer
)
for { for {
select { select {
case <-ctx.Done(): case <-ctx.Done():
s.cancelTransfers(username, live)
return fmt.Errorf("%w: transfer cancelled", ErrSlskdTimeout) return fmt.Errorf("%w: transfer cancelled", ErrSlskdTimeout)
case <-time.After(s.transferPoll): case <-time.After(s.transferPoll):
} }
@@ -903,177 +512,60 @@ func (s *slskd) awaitTransfers(
transfers, err := s.transfersFor(ctx, username) transfers, err := s.transfersFor(ctx, username)
if err != nil { if err != nil {
// A blip talking to the daemon should not abandon a // A blip talking to the daemon should not abandon a
// transfer that may be hours in — but a daemon that stays // transfer that may be hours in.
// away is a stall like any other.
s.logger.Debug("slskd transfer poll failed", "error", err) s.logger.Debug("slskd transfer poll failed", "error", err)
if time.Since(lastProgress) >= s.stallAfter {
s.cancelTransfers(username, live)
return fmt.Errorf(
"%w: slskd has not answered for %s: %w",
ErrSlskdTimeout, s.stallAfter, err,
)
}
continue continue
} }
tally := tallyTransfers( var (
transfers, wanted, stale, done, failed int
time.Since(started) >= s.absentGrace, current int64
) )
live = tally.live
if tally.bytes > lastBytes { for _, t := range transfers {
lastBytes = tally.bytes if !wanted[t.Filename] {
lastProgress = time.Now() continue
}
current += t.BytesTransferred
finished, ok := t.done()
if !finished {
continue
}
if ok {
done++
} else {
failed++
}
} }
if onProgress != nil { if onProgress != nil {
onProgress(Progress{ onProgress(Progress{
Current: tally.bytes, Current: current,
Total: c.TotalSize, Total: c.TotalSize,
Phase: fmt.Sprintf( Phase: fmt.Sprintf(
"Transferring from %s (%d/%d)", "Transferring from %s (%d/%d)", username, done, len(wanted),
username, tally.done, len(wanted),
), ),
}) })
} }
if tally.done+tally.failed >= len(wanted) { if done+failed < len(wanted) {
// Some files failing is normal — a peer goes offline
// mid-folder. Let the importer's completeness check decide
// whether what arrived is enough, rather than discarding it
// here.
if tally.done == 0 {
return fmt.Errorf(
"%w: all %d files failed",
ErrSlskdTransferFailed, tally.failed,
)
}
return nil
}
if time.Since(lastProgress) < s.stallAfter {
continue continue
} }
s.cancelTransfers(username, live) // Some files failing is normal — a peer goes offline mid-folder.
// Let the importer's completeness check decide whether what
// A folder that stalls on its last track is the same shape as // arrived is enough, rather than discarding it here.
// one whose last track failed, and goes forward the same way. if done == 0 {
if tally.done > 0 { return fmt.Errorf(
s.logger.Info( "%w: all %d files failed", ErrSlskdTransferFailed, failed,
"slskd transfer stalled; keeping what arrived",
"peer", username,
"done", tally.done,
"wanted", len(wanted),
)
return nil
}
return fmt.Errorf(
"%w: %s sent nothing in %s",
ErrSlskdTimeout, username, s.stallAfter,
)
}
}
// transferTally is one poll's reading of the files a grab asked for.
type transferTally struct {
done, failed int
bytes int64
// live are the requested transfers slskd is still working on,
// which are what has to be cancelled if the grab is abandoned.
live []slskdTransfer
}
// tallyTransfers reads a peer's transfer list against the files a grab
// asked for.
//
// A requested file slskd does not list at all is one it never accepted
// — refused at enqueue, or dropped — and it will never reach a terminal
// state to be counted by. Once absentExpired, such a file counts as
// failed, or the grab would wait on it until the six-hour ceiling.
func tallyTransfers(
transfers []slskdTransfer,
wanted map[string]bool,
stale map[string]bool,
absentExpired bool,
) transferTally {
seen := make(map[string]slskdTransfer, len(wanted))
for _, t := range transfers {
if !wanted[t.Filename] || stale[t.ID] {
continue
}
seen[t.Filename] = t
}
var out transferTally
for name := range wanted {
t, ok := seen[name]
if !ok {
if absentExpired {
out.failed++
}
continue
}
out.bytes += t.BytesTransferred
finished, succeeded := t.done()
switch {
case !finished:
out.live = append(out.live, t)
case succeeded:
out.done++
default:
out.failed++
}
}
return out
}
// cancelTransfers asks slskd to cancel and forget transfers this grab
// is abandoning. It runs on a context of its own: the usual reason to
// be here is that the caller's context has just been cancelled, and a
// cleanup that inherited it would never be sent.
func (s *slskd) cancelTransfers(username string, live []slskdTransfer) {
if len(live) == 0 {
return
}
ctx, cancel := context.WithTimeout(
context.Background(), slskdCancelTimeout,
)
defer cancel()
for _, t := range live {
if t.ID == "" {
continue
}
endpoint := slskdDownloadsPath(username) + "/" +
url.PathEscape(t.ID) + "?remove=true"
if err := s.client.delete(ctx, endpoint); err != nil {
s.logger.Warn(
"could not cancel slskd transfer",
"peer", username,
"file", t.Filename,
"error", err,
) )
} }
return nil
} }
} }
@@ -1089,7 +581,9 @@ func (s *slskd) transfersFor(
} `json:"directories"` } `json:"directories"`
} }
if err := s.client.get(ctx, slskdDownloadsPath(username), &raw); err != nil { if err := s.client.get(
ctx, "/api/v0/transfers/downloads/"+username, &raw,
); err != nil {
return nil, err return nil, err
} }
@@ -1122,13 +616,7 @@ func (s *slskd) collect(c Candidate, dst string) (Result, error) {
continue continue
} }
// A multi-disc candidate keeps its disc folders in staging.
// Flattened, disc 2's "01 Intro.flac" overwrites disc 1's, and
// the importer loses the folder it reads the disc number from.
target := filepath.Join(dst, base) target := filepath.Join(dst, base)
if _, ok := discFolder(folder); ok {
target = filepath.Join(dst, folder, base)
}
if err := movePath(src, target); err != nil { if err := movePath(src, target); err != nil {
return Result{}, fmt.Errorf("collect %s: %w", base, err) return Result{}, fmt.Errorf("collect %s: %w", base, err)
+1 -368
View File
@@ -31,30 +31,11 @@ type slskdStub struct {
transfers [][]slskdTransfer transfers [][]slskdTransfer
pollCount int pollCount int
// before is what the downloads endpoint reports until something is
// enqueued: records slskd already held from earlier attempts.
before []slskdTransfer
// enqueued records what was requested for download. // enqueued records what was requested for download.
enqueued []map[string]any enqueued []map[string]any
posted bool
// paths records the escaped path of every transfers call, and
// cancelled the escaped request URI of every DELETE.
paths []string
cancelled []string
// unauthorized makes every call return 401. // unauthorized makes every call return 401.
unauthorized bool unauthorized bool
// searches records every search request body, and searchGets the
// request URI of every search GET.
searches []map[string]any
searchGets []string
// noResponsesEndpoint makes /searches/{id}/responses 404, as an
// older daemon would.
noResponsesEndpoint bool
} }
func newSlskdStub(t *testing.T) *slskdStub { func newSlskdStub(t *testing.T) *slskdStub {
@@ -76,16 +57,6 @@ func newSlskdStub(t *testing.T) *slskdStub {
return return
} }
var body map[string]any
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
t.Errorf("decode search body: %v", err)
}
s.mu.Lock()
s.searches = append(s.searches, body)
s.mu.Unlock()
w.WriteHeader(http.StatusCreated) w.WriteHeader(http.StatusCreated)
}) })
@@ -102,26 +73,8 @@ func newSlskdStub(t *testing.T) *slskdStub {
s.mu.Lock() s.mu.Lock()
responses := s.responses responses := s.responses
noEndpoint := s.noResponsesEndpoint
s.searchGets = append(s.searchGets, r.URL.RequestURI())
s.mu.Unlock() s.mu.Unlock()
if strings.HasSuffix(r.URL.Path, "/responses") {
if noEndpoint {
w.WriteHeader(http.StatusNotFound)
return
}
writeJSON(t, w, responses)
return
}
if r.URL.Query().Get("includeResponses") != "true" {
responses = nil
}
writeJSON(t, w, slskdSearch{ writeJSON(t, w, slskdSearch{
ID: "search-1", ID: "search-1",
IsComplete: true, IsComplete: true,
@@ -134,12 +87,7 @@ func newSlskdStub(t *testing.T) *slskdStub {
return return
} }
s.mu.Lock() if r.Method == http.MethodPost {
s.paths = append(s.paths, r.URL.EscapedPath())
s.mu.Unlock()
switch r.Method {
case http.MethodPost:
var body []map[string]any var body []map[string]any
if err := json.NewDecoder(r.Body).Decode(&body); err != nil { if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
@@ -148,35 +96,15 @@ func newSlskdStub(t *testing.T) *slskdStub {
s.mu.Lock() s.mu.Lock()
s.enqueued = body s.enqueued = body
s.posted = true
s.mu.Unlock() s.mu.Unlock()
w.WriteHeader(http.StatusCreated) w.WriteHeader(http.StatusCreated)
return
case http.MethodDelete:
s.mu.Lock()
s.cancelled = append(s.cancelled, r.URL.RequestURI())
s.mu.Unlock()
w.WriteHeader(http.StatusNoContent)
return return
} }
s.mu.Lock() s.mu.Lock()
if !s.posted {
before := s.before
s.mu.Unlock()
writeJSON(t, w, map[string]any{
"directories": []map[string]any{{"files": before}},
})
return
}
idx := s.pollCount idx := s.pollCount
if idx >= len(s.transfers) { if idx >= len(s.transfers) {
idx = len(s.transfers) - 1 idx = len(s.transfers) - 1
@@ -263,11 +191,6 @@ func newStubSlskd(t *testing.T, stub *slskdStub) (*slskd, string) {
s.searchWait = 200 * time.Millisecond s.searchWait = 200 * time.Millisecond
s.transferPoll = time.Millisecond s.transferPoll = time.Millisecond
// Long enough that no existing test trips them by accident; the
// tests about stalls and absences set their own.
s.stallAfter = time.Minute
s.absentGrace = time.Minute
return s, downloads return s, downloads
} }
@@ -642,293 +565,3 @@ func TestSlskdRequiresConfiguration(t *testing.T) {
}) })
} }
} }
// slskdAlbum is a two-file candidate from peer, with the files slskd
// would have written already in place under downloads.
func slskdAlbum(t *testing.T, downloads, peer string, arrived ...string) Candidate {
t.Helper()
folder := filepath.Join(downloads, "Album")
if err := os.MkdirAll(folder, 0o750); err != nil {
t.Fatalf("mkdir: %v", err)
}
for _, name := range arrived {
if err := os.WriteFile(
filepath.Join(folder, name), []byte("audio"), 0o600,
); err != nil {
t.Fatalf("write: %v", err)
}
}
return Candidate{
Files: []CandidateFile{
{Path: `\s\Album\01 A.flac`, Size: 500, IsAudio: true},
{Path: `\s\Album\02 B.flac`, Size: 500, IsAudio: true},
},
TotalSize: 1000,
Payload: map[string]string{"username": peer},
}
}
// cancelledURIs returns what the stub was asked to cancel.
func (s *slskdStub) cancelledURIs() []string {
s.mu.Lock()
defer s.mu.Unlock()
return append([]string(nil), s.cancelled...)
}
// A peer that queues us and never sends a byte is given up on, and the
// queued transfers are cancelled in slskd rather than left to start
// hours later for a request nobody is waiting on.
func TestSlskdGrabGivesUpOnAStalledPeer(t *testing.T) {
t.Parallel()
stub := newSlskdStub(t)
stub.transfers = [][]slskdTransfer{{
{ID: "t1", Filename: `\s\Album\01 A.flac`, State: "Queued, Remotely"},
{ID: "t2", Filename: `\s\Album\02 B.flac`, State: "Queued, Remotely"},
}}
s, downloads := newStubSlskd(t, stub)
s.stallAfter = 30 * time.Millisecond
_, err := s.Grab(
context.Background(), slskdAlbum(t, downloads, "peer"), t.TempDir(), nil,
)
if !errors.Is(err, ErrSlskdTimeout) {
t.Fatalf("error = %v, want ErrSlskdTimeout", err)
}
got := stub.cancelledURIs()
if len(got) != 2 {
t.Fatalf("cancelled %v, want both queued transfers", got)
}
for _, uri := range got {
if !strings.HasSuffix(uri, "?remove=true") {
t.Errorf("cancel %s does not remove the record", uri)
}
}
}
// A folder that stalls on its last track goes forward with what
// arrived, the same as one whose last track failed; the importer's
// completeness check decides whether that is enough.
func TestSlskdGrabKeepsWhatArrivedBeforeAStall(t *testing.T) {
t.Parallel()
stub := newSlskdStub(t)
stub.transfers = [][]slskdTransfer{{
{
ID: "t1", Filename: `\s\Album\01 A.flac`,
State: "Completed, Succeeded", BytesTransferred: 500,
},
{ID: "t2", Filename: `\s\Album\02 B.flac`, State: "Queued, Remotely"},
}}
s, downloads := newStubSlskd(t, stub)
s.stallAfter = 30 * time.Millisecond
got, err := s.Grab(
context.Background(),
slskdAlbum(t, downloads, "peer", "01 A.flac"),
t.TempDir(), nil,
)
if err != nil {
t.Fatalf("Grab: %v", err)
}
if len(got.Files) != 1 {
t.Errorf("collected %d files, want the 1 that arrived", len(got.Files))
}
if cancelled := stub.cancelledURIs(); len(cancelled) != 1 ||
!strings.Contains(cancelled[0], "/t2") {
t.Errorf("cancelled %v, want only the stalled t2", cancelled)
}
}
// Progress is what holds the stall timer off. A transfer that keeps
// moving bytes is never abandoned, however long it takes.
func TestSlskdGrabWaitsOnATransferThatIsMoving(t *testing.T) {
t.Parallel()
stub := newSlskdStub(t)
for b := int64(1); b <= 100; b++ {
stub.transfers = append(stub.transfers, []slskdTransfer{
{ID: "t1", Filename: `\s\Album\01 A.flac`, State: "InProgress", BytesTransferred: b},
{ID: "t2", Filename: `\s\Album\02 B.flac`, State: "Queued, Remotely"},
})
}
stub.transfers = append(stub.transfers, []slskdTransfer{
{
ID: "t1",
Filename: `\s\Album\01 A.flac`,
State: "Completed, Succeeded",
BytesTransferred: 500,
},
{
ID: "t2",
Filename: `\s\Album\02 B.flac`,
State: "Completed, Succeeded",
BytesTransferred: 500,
},
})
s, downloads := newStubSlskd(t, stub)
// A hundred polls take several times the stall window; each one
// moves a byte. The window is kept well above one poll so a
// descheduled test runner does not read as a stall.
s.transferPoll = 5 * time.Millisecond
s.stallAfter = 150 * time.Millisecond
got, err := s.Grab(
context.Background(),
slskdAlbum(t, downloads, "peer", "01 A.flac", "02 B.flac"),
t.TempDir(), nil,
)
if err != nil {
t.Fatalf("Grab: %v", err)
}
if len(got.Files) != 2 {
t.Errorf("collected %d files, want 2", len(got.Files))
}
}
// A file slskd never lists was refused at enqueue and will never reach
// a terminal state. It counts as failed once the grace period is up,
// rather than being waited on until the six-hour ceiling.
func TestSlskdGrabCountsAnUnlistedFileAsFailed(t *testing.T) {
t.Parallel()
stub := newSlskdStub(t)
stub.transfers = [][]slskdTransfer{{
{
ID: "t1", Filename: `\s\Album\01 A.flac`,
State: "Completed, Succeeded", BytesTransferred: 500,
},
}}
s, downloads := newStubSlskd(t, stub)
s.absentGrace = 20 * time.Millisecond
got, err := s.Grab(
context.Background(),
slskdAlbum(t, downloads, "peer", "01 A.flac"),
t.TempDir(), nil,
)
if err != nil {
t.Fatalf("Grab: %v", err)
}
if len(got.Files) != 1 {
t.Errorf("collected %d files, want 1", len(got.Files))
}
}
// A finished record left by an earlier attempt at the same file is not
// this attempt's answer. Without the snapshot it would fail the grab on
// the first poll, before the new transfer had started.
func TestSlskdGrabIgnoresAnEarlierAttemptsRecord(t *testing.T) {
t.Parallel()
stale := slskdTransfer{
ID: "old", Filename: `\s\Album\01 A.flac`, State: "Completed, Errored",
}
stub := newSlskdStub(t)
stub.before = []slskdTransfer{stale}
stub.transfers = [][]slskdTransfer{
{stale},
{
stale,
{
ID: "new", Filename: `\s\Album\01 A.flac`,
State: "Completed, Succeeded", BytesTransferred: 500,
},
},
}
s, downloads := newStubSlskd(t, stub)
c := slskdAlbum(t, downloads, "peer", "01 A.flac")
c.Files = c.Files[:1]
got, err := s.Grab(context.Background(), c, t.TempDir(), nil)
if err != nil {
t.Fatalf("Grab: %v", err)
}
if len(got.Files) != 1 {
t.Errorf("collected %d files, want 1", len(got.Files))
}
}
// Cancelling the download cancels the transfer in slskd too. The
// cleanup must not inherit the cancelled context, or it is never sent.
func TestSlskdGrabCancelsTransfersWhenTheCallerGivesUp(t *testing.T) {
t.Parallel()
stub := newSlskdStub(t)
stub.transfers = [][]slskdTransfer{{
{ID: "t1", Filename: `\s\Album\01 A.flac`, State: "InProgress", BytesTransferred: 10},
{ID: "t2", Filename: `\s\Album\02 B.flac`, State: "Queued, Remotely"},
}}
s, downloads := newStubSlskd(t, stub)
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
defer cancel()
_, err := s.Grab(ctx, slskdAlbum(t, downloads, "peer"), t.TempDir(), nil)
if !errors.Is(err, ErrSlskdTimeout) {
t.Fatalf("error = %v, want ErrSlskdTimeout", err)
}
if got := stub.cancelledURIs(); len(got) != 2 {
t.Errorf("cancelled %v, want both live transfers", got)
}
}
// Soulseek usernames carry spaces and punctuation; spliced raw into the
// path, a name with a slash addresses a different endpoint entirely.
func TestSlskdEscapesTheUsername(t *testing.T) {
t.Parallel()
stub := newSlskdStub(t)
stub.transfers = [][]slskdTransfer{{
{
ID: "t1", Filename: `\s\Album\01 A.flac`,
State: "Completed, Succeeded", BytesTransferred: 500,
},
{
ID: "t2", Filename: `\s\Album\02 B.flac`,
State: "Completed, Succeeded", BytesTransferred: 500,
},
}}
s, downloads := newStubSlskd(t, stub)
if _, err := s.Grab(
context.Background(),
slskdAlbum(t, downloads, "dj a/b", "01 A.flac", "02 B.flac"),
t.TempDir(), nil,
); err != nil {
t.Fatalf("Grab: %v", err)
}
stub.mu.Lock()
paths := append([]string(nil), stub.paths...)
stub.mu.Unlock()
for _, p := range paths {
if p != "/api/v0/transfers/downloads/dj%20a%2Fb" {
t.Errorf("transfers call went to %s", p)
}
}
}
+3 -21
View File
@@ -7,7 +7,6 @@ import (
"path/filepath" "path/filepath"
"runtime" "runtime"
"strings" "strings"
"syscall"
"testing" "testing"
) )
@@ -18,21 +17,6 @@ import (
// stubYtDlp writes an executable script that echoes the given stdout // stubYtDlp writes an executable script that echoes the given stdout
// and returns it as a provider config binary path. // and returns it as a provider config binary path.
//
// The write is held under syscall.ForkLock, and that is not tidiness:
// the kernel refuses to exec a file that is open for writing anywhere
// in the process, and these tests are parallel, so a *sibling* test's
// fork can duplicate this descriptor in the moment it is open and
// carry it past our close — the exec a moment later then fails with
// ETXTBSY, "text file busy". That is #146, seen once in CI and once
// locally, on trees containing no Go at all. Closing sooner is not
// available (os.WriteFile has already closed the file before anything
// execs it) and O_CLOEXEC does not help, because the window is between
// another goroutine's fork and its own exec. ForkLock is the lock
// syscall.forkExec takes across that fork, so holding it here means no
// child can exist while the descriptor does. Measured on this helper
// under 12 concurrent writers: 176-189 of 2400 execs refused without
// it, 0 of 2400 with it.
func stubYtDlp(t *testing.T, script string) string { func stubYtDlp(t *testing.T, script string) string {
t.Helper() t.Helper()
@@ -42,11 +26,9 @@ func stubYtDlp(t *testing.T, script string) string {
path := filepath.Join(t.TempDir(), "yt-dlp") path := filepath.Join(t.TempDir(), "yt-dlp")
syscall.ForkLock.Lock() if err := os.WriteFile(
err := os.WriteFile(path, []byte("#!/bin/sh\n"+script), 0o700) path, []byte("#!/bin/sh\n"+script), 0o700,
syscall.ForkLock.Unlock() ); err != nil {
if err != nil {
t.Fatalf("write stub: %v", err) t.Fatalf("write stub: %v", err)
} }
+18 -104
View File
@@ -35,15 +35,6 @@ const (
weightArtistFit = 0.12 weightArtistFit = 0.12
) )
// Match sub-weights when the candidate's durations are known. Duration
// takes its weight from title fit, the signal it corroborates: a title
// says which song a file claims to be, a length says whether it is that
// recording — the right edit, the whole file, not the live take.
const (
timedWeightTitleFit = 0.25
timedWeightDurationFit = 0.15
)
// Quality sub-weights. Each set sums to 1.0. // Quality sub-weights. Each set sums to 1.0.
// //
// There are two of them because a stated preference changes what the // There are two of them because a stated preference changes what the
@@ -328,13 +319,13 @@ func Score(dl Download, c Candidate, priority int, prefs AutoDownloadPrefs) Cand
audio := c.AudioFiles() audio := c.AudioFiles()
a := alignFiles(audio, dl.Expected) matched, titleFit := matchFiles(audio, dl.Expected)
// Write the alignment back so the picker can show which file maps // Write the alignment back so the picker can show which file maps
// to which track. // to which track.
c.Files = mergeMatched(c.Files, a.files) c.Files = mergeMatched(c.Files, matched)
c.Match = scoreMatch(dl, c, audio, a) c.Match = scoreMatch(dl, c, audio, titleFit)
c.Quality = scoreQuality( c.Quality = scoreQuality(
c, audio, priority, prefs, dl.runtimeMillis(), c, audio, priority, prefs, dl.runtimeMillis(),
) )
@@ -349,21 +340,14 @@ func scoreMatch(
dl Download, dl Download,
c Candidate, c Candidate,
audio []CandidateFile, audio []CandidateFile,
a alignment, titleFit float64,
) MatchScore { ) MatchScore {
m := MatchScore{ m := MatchScore{
Anchored: dl.Anchored(), Anchored: dl.Anchored(),
TitleFit: a.titleFit, TitleFit: titleFit,
DurationFit: a.durationFit,
// Durations count once at least half the aligned pairs state
// one; a single timed pair would be a coin toss carrying 15%.
DurationKnown: a.timedPairs > 0 && a.timedPairs*2 >= a.aligned,
} }
m.Completeness = completeness( m.Completeness = completeness(len(audio), len(dl.Expected))
alignedCount(c.Files), len(audio), len(dl.Expected),
)
// The candidate's own title, and the folder its files sit in, are // The candidate's own title, and the folder its files sit in, are
// two independent guesses at the album name. Take the better one: // two independent guesses at the album name. Take the better one:
@@ -383,16 +367,9 @@ func scoreMatch(
// With no expected tracklist there is no title signal at all, so // With no expected tracklist there is no title signal at all, so
// redistribute its weight onto the album/artist evidence rather // redistribute its weight onto the album/artist evidence rather
// than scoring every free-text result as half-wrong. // than scoring every free-text result as half-wrong.
switch { if len(dl.Expected) == 0 {
case len(dl.Expected) == 0:
m.Overall = 0.55*m.AlbumFit + 0.45*m.ArtistFit m.Overall = 0.55*m.AlbumFit + 0.45*m.ArtistFit
case m.DurationKnown: } else {
m.Overall = timedWeightTitleFit*m.TitleFit +
timedWeightDurationFit*m.DurationFit +
weightCompleteness*m.Completeness +
weightAlbumFit*m.AlbumFit +
weightArtistFit*m.ArtistFit
default:
m.Overall = weightTitleFit*m.TitleFit + m.Overall = weightTitleFit*m.TitleFit +
weightCompleteness*m.Completeness + weightCompleteness*m.Completeness +
weightAlbumFit*m.AlbumFit + weightAlbumFit*m.AlbumFit +
@@ -438,54 +415,30 @@ func artistFit(want string, c Candidate) float64 {
return best return best
} }
// completeness scores how much of the expected tracklist a candidate // completeness scores audio file count against the expected track
// covers. Extra files are penalized far more gently than missing ones: // count. Extra files are penalized far more gently than missing ones:
// a folder with bonus tracks or a stray intro is still the album, while // a folder with bonus tracks or a stray intro is still the album, while
// a folder missing half the tracks is not. // a folder missing half the tracks is not.
// func completeness(got, want int) float64 {
// **Coverage is counted in aligned tracks, not in files.** It used to
// be the audio file count, so any ten files scored full marks against
// a ten-track album whether or not they were its tracks — and since
// title fit is the mean over the files that *did* align, a folder where
// three titles matched read as a near-perfect candidate on both counts.
// `aligned` is how many files matchFiles assigned to an expected track;
// `audio` still sets the penalty for extras, because a folder of thirty
// files holding the ten wanted is a worse copy than one holding ten.
func completeness(aligned, audio, want int) float64 {
if want == 0 { if want == 0 {
if audio > 0 { if got > 0 {
return 0.5 return 0.5
} }
return 0 return 0
} }
if aligned == 0 { if got == 0 {
return 0 return 0
} }
cover := float64(min(aligned, want)) / float64(want) if got >= want {
extra := float64(got-want) / float64(want)
if audio > want { return math.Max(0.75, 1.0-0.25*extra)
extra := float64(audio-want) / float64(want)
cover *= math.Max(0.75, 1.0-0.25*extra)
} }
return cover return float64(got) / float64(want)
}
// alignedCount is how many audio files were assigned to an expected
// track.
func alignedCount(files []CandidateFile) int {
n := 0
for _, f := range files {
if f.IsAudio && f.MatchedTo != 0 {
n++
}
}
return n
} }
// scoreQuality answers whether this is a good copy. // scoreQuality answers whether this is a good copy.
@@ -782,45 +735,6 @@ func AutoPickVeto(
return "" return ""
} }
// autoAcceptable reports whether auto-pick may take this one candidate
// without asking: the request is anchored to a tracklist, and the
// candidate is inside the user's guardrails and clears the match and
// quality bars. It is AutoPickVeto's test applied to a single
// candidate, which is what falling back to a second choice needs.
func autoAcceptable(dl Download, c Candidate, prefs AutoDownloadPrefs) bool {
return dl.Anchored() &&
len(dl.Expected) > 0 &&
prefs.eligible(c, dl.runtimeMillis()) &&
c.Match.Overall >= minMatch &&
c.Quality.Overall >= minQuality
}
// autoPick returns the candidate auto-pick takes: the best-ranked one
// it may take at all.
//
// That is not `ranked[0]`. AutoPickVeto judges the best candidate
// *inside* the guardrails, so when the overall best is outside them —
// over the size ceiling, say — the veto passes on the strength of the
// second, and grabbing the first would download exactly the copy the
// user said not to take unattended.
func autoPick(
dl Download,
ranked []Candidate,
prefs AutoDownloadPrefs,
) (Candidate, bool) {
if AutoPickVeto(dl, ranked, prefs) != "" {
return Candidate{}, false
}
for _, c := range ranked {
if autoAcceptable(dl, c, prefs) {
return c, true
}
}
return Candidate{}, false
}
// mergeMatched copies MatchedTo assignments from the audio-only slice // mergeMatched copies MatchedTo assignments from the audio-only slice
// back onto the full file list. // back onto the full file list.
func mergeMatched(all, matched []CandidateFile) []CandidateFile { func mergeMatched(all, matched []CandidateFile) []CandidateFile {
+10 -14
View File
@@ -282,32 +282,28 @@ func TestCompleteness(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
aligned int got int
audio int
want int want int
minScore float64 minScore float64
maxScore float64 maxScore float64
}{ }{
{"exact", 10, 10, 10, 1.0, 1.0}, {"exact", 10, 10, 1.0, 1.0},
{"half missing", 5, 5, 10, 0.49, 0.51}, {"half missing", 5, 10, 0.49, 0.51},
{"one bonus track", 10, 11, 10, 0.95, 1.0}, {"one bonus track", 11, 10, 0.95, 1.0},
{"double", 10, 20, 10, 0.74, 0.76}, {"double", 20, 10, 0.74, 0.76},
{"nothing", 0, 0, 10, 0, 0}, {"nothing", 0, 10, 0, 0},
{"no expectation", 0, 5, 0, 0.5, 0.5}, {"no expectation", 5, 0, 0.5, 0.5},
// Ten files are not ten tracks: three that align are three.
{"right count, wrong tracks", 3, 10, 10, 0.29, 0.31},
{"files that align to nothing", 0, 10, 10, 0, 0},
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
t.Parallel() t.Parallel()
got := completeness(tt.aligned, tt.audio, tt.want) got := completeness(tt.got, tt.want)
if got < tt.minScore || got > tt.maxScore { if got < tt.minScore || got > tt.maxScore {
t.Errorf( t.Errorf(
"completeness(%d, %d, %d) = %f, want in [%f, %f]", "completeness(%d, %d) = %f, want in [%f, %f]",
tt.aligned, tt.audio, tt.want, got, tt.minScore, tt.maxScore, tt.got, tt.want, got, tt.minScore, tt.maxScore,
) )
} }
}) })
-238
View File
@@ -1,238 +0,0 @@
package download
import (
"context"
"strings"
"testing"
)
// What Soulseek is asked, how, and what is kept from the answer (#271).
func TestSlskdQueries(t *testing.T) {
t.Parallel()
cases := []struct {
name string
dl Download
want []string
}{
{
name: "a plain request is searched once",
dl: Download{Artist: "Radiohead", Album: "OK Computer"},
want: []string{"Radiohead OK Computer"},
},
{
name: "an edition qualifier gets a second query without it",
dl: Download{Artist: "Radiohead", Album: "OK Computer (Collector's Edition)"},
want: []string{
"Radiohead OK Computer (Collector's Edition)",
"Radiohead OK Computer",
},
},
{
name: "a trailing remaster note",
dl: Download{Artist: "Pink Floyd", Album: "Animals - 2018 Remaster"},
want: []string{
"Pink Floyd Animals - 2018 Remaster",
"Pink Floyd Animals",
},
},
{
name: "a leading dash would be an exclusion",
dl: Download{Artist: "Mocky", Album: "-ism"},
want: []string{"Mocky -ism", "Mocky ism"},
},
{
name: "a compilation is not searched by its placeholder artist",
dl: Download{Artist: "Various Artists", Album: "Pulp Fiction"},
want: []string{"Various Artists Pulp Fiction", "Pulp Fiction"},
},
{
name: "what the user typed is searched as written",
dl: Download{Query: "ok computer (deluxe)", Album: "OK Computer (Deluxe)"},
want: []string{"ok computer (deluxe)"},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
got := slskdQueries(tc.dl)
if strings.Join(got, "|") != strings.Join(tc.want, "|") {
t.Errorf("slskdQueries = %q, want %q", got, tc.want)
}
})
}
}
// Both queries run, the options are stated rather than left to the
// daemon's defaults, and a folder both queries found is one candidate.
func TestSlskdSearchRunsBothQueriesAndMerges(t *testing.T) {
t.Parallel()
stub := newSlskdStub(t)
stub.responses = []slskdResponse{{
Username: "peer",
Files: []slskdFile{
{Filename: `\m\Radiohead - OK Computer\01 Airbag.flac`, Size: 1, Length: 284},
{Filename: `\m\Radiohead - OK Computer\02 Paranoid Android.flac`, Size: 1, Length: 383},
},
}}
s, _ := newStubSlskd(t, stub)
got, err := s.Search(context.Background(), Download{
ReleaseMBID: "rel", Artist: "Radiohead", Album: "OK Computer (Deluxe Edition)",
})
if err != nil {
t.Fatalf("Search: %v", err)
}
if len(got) != 1 {
t.Fatalf("got %d candidates, want the one folder once", len(got))
}
if got[0].Files[0].LengthMillis != 284_000 {
t.Errorf("length = %d ms, want 284000 from slskd's seconds", got[0].Files[0].LengthMillis)
}
stub.mu.Lock()
searches := append([]map[string]any(nil), stub.searches...)
gets := append([]string(nil), stub.searchGets...)
stub.mu.Unlock()
if len(searches) != 2 {
t.Fatalf("ran %d searches, want 2", len(searches))
}
for _, body := range searches {
for _, key := range []string{
"searchTimeout", "responseLimit", "fileLimit",
"minimumResponseFileCount", "maximumPeerQueueLength",
} {
if _, ok := body[key]; !ok {
t.Errorf("search %q does not state %s", body["searchText"], key)
}
}
}
// The responses are fetched once at the end, not with every poll.
for _, uri := range gets {
if strings.Contains(uri, "includeResponses") {
t.Errorf("poll %s asked for every response", uri)
}
}
}
// A daemon without the responses endpoint still returns results.
func TestSlskdSearchFallsBackForAnOlderDaemon(t *testing.T) {
t.Parallel()
stub := newSlskdStub(t)
stub.noResponsesEndpoint = true
stub.responses = []slskdResponse{{
Username: "peer",
Files: []slskdFile{
{Filename: `\m\Album\01 A.flac`, Size: 1},
{Filename: `\m\Album\02 B.flac`, Size: 1},
},
}}
s, _ := newStubSlskd(t, stub)
got, err := s.Search(context.Background(), Download{Query: "album"})
if err != nil {
t.Fatalf("Search: %v", err)
}
if len(got) != 1 {
t.Errorf("got %d candidates, want 1 through the fallback", len(got))
}
}
func TestDurationAgreement(t *testing.T) {
t.Parallel()
cases := []struct {
got, want int64
score float64
}{
{300_000, 300_000, 1},
{301_500, 300_000, 1}, // a second of silence
{300_000, 316_500, 0.5},
{300_000, 345_000, 0}, // a different edit
}
for _, tc := range cases {
if got := durationAgreement(tc.got, tc.want); got < tc.score-0.01 || got > tc.score+0.01 {
t.Errorf("durationAgreement(%d, %d) = %f, want %f", tc.got, tc.want, got, tc.score)
}
}
}
// Two folders with the same track names are told apart by their
// lengths: one is the album, the other a live record of the same songs.
func TestDurationsSeparateTheRightRecording(t *testing.T) {
t.Parallel()
dl := okComputer()
timed := func(id string, lengths ...int64) Candidate {
c := candidateFor(id, allTitles(), ".flac", 30_000_000)
for i := range c.Files {
c.Files[i].LengthMillis = lengths[i]
}
return c
}
studio := timed("studio", trackMillis, trackMillis+1_000, trackMillis, trackMillis-500)
live := timed(
"live",
trackMillis+60_000,
trackMillis+75_000,
trackMillis+50_000,
trackMillis+90_000,
)
ranked := Rank(dl, []Candidate{live, studio}, nil, AutoDownloadPrefs{})
if ranked[0].ID != "studio" {
t.Fatalf("winner = %s, want the recording whose lengths match", ranked[0].ID)
}
if !ranked[0].Match.DurationKnown || ranked[0].Match.DurationFit < 0.99 {
t.Errorf(
"studio duration fit = %f known=%v",
ranked[0].Match.DurationFit,
ranked[0].Match.DurationKnown,
)
}
if ranked[1].Match.DurationFit != 0 {
t.Errorf("live duration fit = %f, want 0", ranked[1].Match.DurationFit)
}
}
// Without lengths the score is exactly what it was before durations
// were read, so a provider that reports none is not penalised.
func TestUnknownDurationsLeaveTheScoreAlone(t *testing.T) {
t.Parallel()
dl := okComputer()
c := Score(dl, candidateFor("c", allTitles(), ".flac", 30_000_000), 50, AutoDownloadPrefs{})
if c.Match.DurationKnown {
t.Fatal("no file states a length, yet durations are known")
}
want := weightTitleFit*c.Match.TitleFit +
weightCompleteness*c.Match.Completeness +
weightAlbumFit*c.Match.AlbumFit +
weightArtistFit*c.Match.ArtistFit
if c.Match.Overall != want {
t.Errorf("match = %f, want the untimed formula's %f", c.Match.Overall, want)
}
}
+7 -19
View File
@@ -232,17 +232,12 @@ type Candidate struct {
// results give paths and sizes but no tags, so Format and duration are // results give paths and sizes but no tags, so Format and duration are
// inferred from the path and size where possible. // inferred from the path and size where possible.
type CandidateFile struct { type CandidateFile struct {
Path string `json:"path"` Path string `json:"path"`
Size int64 `json:"size"` Size int64 `json:"size"`
Format Format `json:"format"` Format Format `json:"format"`
Bitrate int `json:"bitrate,omitempty"` // kbps, 0 when unknown Bitrate int `json:"bitrate,omitempty"` // kbps, 0 when unknown
IsAudio bool `json:"isAudio"` IsAudio bool `json:"isAudio"`
MatchedTo int `json:"matchedTo,omitempty"` // expected track position
// LengthMillis is the file's duration as the source reports it, or
// 0 when it does not. Soulseek reports it for most audio files.
LengthMillis int64 `json:"lengthMillis,omitempty"`
MatchedTo int `json:"matchedTo,omitempty"` // expected track position
} }
// Format is a normalized audio container/codec name. // Format is a normalized audio container/codec name.
@@ -291,14 +286,7 @@ type MatchScore struct {
TitleFit float64 `json:"titleFit"` // filenames vs expected titles TitleFit float64 `json:"titleFit"` // filenames vs expected titles
ArtistFit float64 `json:"artistFit"` // path/origin vs expected artist ArtistFit float64 `json:"artistFit"` // path/origin vs expected artist
AlbumFit float64 `json:"albumFit"` // folder name vs album title AlbumFit float64 `json:"albumFit"` // folder name vs album title
Completeness float64 `json:"completeness"` // aligned tracks vs expected count Completeness float64 `json:"completeness"` // audio files vs expected count
// DurationFit is how well the aligned files' lengths agree with the
// expected tracks', and DurationKnown whether enough of them stated
// a length for that to count. When it does not, the score is the
// four text signals alone, exactly as before durations were read.
DurationFit float64 `json:"durationFit"`
DurationKnown bool `json:"durationKnown"`
// Anchored records whether an MBID drove this score. Unanchored // Anchored records whether an MBID drove this score. Unanchored
// matches are capped, because there is nothing to be right about. // matches are capped, because there is nothing to be right about.
+11 -88
View File
@@ -1,10 +1,8 @@
package explore package explore
import ( import (
"bytes"
"context" "context"
"database/sql" "database/sql"
"database/sql/driver"
"errors" "errors"
"fmt" "fmt"
"os" "os"
@@ -285,25 +283,6 @@ func (si *SearchIndex) importCoreArtifact(ctx context.Context, path string) erro
} }
merged, mergeErr := si.mergeArtifactRows(ctx, info.rows) merged, mergeErr := si.mergeArtifactRows(ctx, info.rows)
// Every row the artifact declares has to land. The walk partitions
// the artifact's key space, so a total short of info.rows does not
// mean the artifact was smaller than it said -- it means a predicate
// filtered rows out, and the catalog is quietly partial. Equality
// rather than a lower bound because RowsAffected counts an upsert
// that changes nothing, and a row already merged locally is counted
// again here.
//
// One reachable case, so this is not merely a tripwire: a row whose
// mbid is empty is excluded by `mbid > ?` in both encodings, and an
// artifact carrying one would otherwise import as complete.
if mergeErr == nil && merged != info.rows {
mergeErr = fmt.Errorf(
"%w: merged %d of %d rows — a row the artifact holds was not selected",
ErrArtifactUnusable, merged, info.rows,
)
}
if mergeErr == nil { if mergeErr == nil {
si.mergeArtifactCredits(ctx) si.mergeArtifactCredits(ctx)
} }
@@ -369,12 +348,8 @@ func (si *SearchIndex) analyzeIndex() {
// is an index range scan and a cancelled import leaves committed work // is an index range scan and a cancelled import leaves committed work
// behind rather than rolling it all back. // behind rather than rolling it all back.
func (si *SearchIndex) mergeArtifactRows(ctx context.Context, total int) (int, error) { func (si *SearchIndex) mergeArtifactRows(ctx context.Context, total int) (int, error) {
// Asked once, because it is a property of the file and it decides
// how the walk's own comparisons are typed. See artifactKey.
storesText := si.artifactStoresText()
selectColumns := artifactSelectColumns( selectColumns := artifactSelectColumns(
storesText, si.artifactHasTotals(), si.artifactStoresText(), si.artifactHasTotals(),
) )
insertSQL := ` insertSQL := `
@@ -392,7 +367,7 @@ func (si *SearchIndex) mergeArtifactRows(ctx context.Context, total int) (int, e
WHERE mbid > ? AND mbid <= ?` + upsertIndexConflictSQL WHERE mbid > ? AND mbid <= ?` + upsertIndexConflictSQL
var ( var (
cursor artifactKey cursor string
merged int merged int
) )
@@ -401,30 +376,17 @@ func (si *SearchIndex) mergeArtifactRows(ctx context.Context, total int) (int, e
return merged, err return merged, err
} }
upper, hasUpper, err := si.artifactBatchBound(storesText, cursor) upper, hasUpper, err := si.artifactBatchBound(cursor)
if err != nil { if err != nil {
return merged, err return merged, err
} }
if hasUpper && bytes.Compare(upper, cursor) <= 0 {
// The predicate matched the cursor itself, so the walk can
// never advance. SQLite says nothing when a comparison is
// made between types it will not coerce - the query simply
// answers wrongly - so a mismatch here would otherwise spin
// forever behind an unmoving progress bar. Fail instead.
return merged, fmt.Errorf(
"%w: artifact walk did not advance past %x",
ErrArtifactUnusable, []byte(cursor),
)
}
var res sql.Result var res sql.Result
if hasUpper { if hasUpper {
res, err = si.db.ExecContext(insertRangeSQL, res, err = si.db.ExecContext(insertRangeSQL, cursor, upper)
cursor.bind(storesText), upper.bind(storesText))
} else { } else {
res, err = si.db.ExecContext(insertSQL, cursor.bind(storesText)) res, err = si.db.ExecContext(insertSQL, cursor)
} }
if err != nil { if err != nil {
@@ -451,65 +413,26 @@ func (si *SearchIndex) mergeArtifactRows(ctx context.Context, total int) (int, e
} }
} }
// artifactKey is one MBID as the attached artifact stores it: 16 raw
// bytes in a compact artifact, the dashed 36-character form in one
// published before that storage change.
//
// It is a type with a bind method rather than a string because the
// comparison it feeds is typed, and the wrong type is silent. SQLite
// does not coerce between TEXT and BLOB and orders every blob after
// every text value, so a cursor bound as text against a byte column
// makes `mbid > ?` true of the whole table - the walk rediscovers the
// same batch bound forever, and `mbid <= ?` false of the whole table,
// so no batch merges at all. Nothing errors; the import simply never
// finishes. bind is the one place that knows which form the column is
// in, decided by artifactStoresText, which asks the artifact rather than
// trusting a version number.
type artifactKey []byte
// bind renders the key as a statement argument in the artifact's own
// encoding.
func (k artifactKey) bind(storesText bool) driver.Value {
if storesText {
return string(k)
}
// Never nil. database/sql converts a nil []byte to SQL NULL, and
// `mbid > NULL` is NULL for every row - so an unset cursor would
// agree with nothing and import nothing, which is the same silently
// empty merge this type exists to prevent, one type over.
if k == nil {
return []byte{}
}
return []byte(k)
}
// artifactBatchBound returns the MBID that ends the next batch, and // artifactBatchBound returns the MBID that ends the next batch, and
// whether one exists — no bound means the remainder is the last batch. // whether one exists — no bound means the remainder is the last batch.
// func (si *SearchIndex) artifactBatchBound(cursor string) (string, bool, error) {
// The bound is read out of the artifact and handed back as an var bound string
// artifactKey, because it becomes the next comparison the walk makes.
func (si *SearchIndex) artifactBatchBound(
storesText bool, cursor artifactKey,
) (artifactKey, bool, error) {
var bound []byte
err := si.db.QueryRowWriter( err := si.db.QueryRowWriter(
`SELECT mbid FROM core.explore_index `SELECT mbid FROM core.explore_index
WHERE mbid > ? ORDER BY mbid LIMIT 1 OFFSET ?`, WHERE mbid > ? ORDER BY mbid LIMIT 1 OFFSET ?`,
cursor.bind(storesText), artifactMergeBatch-1, cursor, artifactMergeBatch-1,
).Scan(&bound) ).Scan(&bound)
if errors.Is(err, sql.ErrNoRows) { if errors.Is(err, sql.ErrNoRows) {
return nil, false, nil return "", false, nil
} }
if err != nil { if err != nil {
return nil, false, fmt.Errorf("%w: batch bound: %w", ErrArtifactUnusable, err) return "", false, fmt.Errorf("%w: batch bound: %w", ErrArtifactUnusable, err)
} }
return artifactKey(bound), true, nil return bound, true, nil
} }
// stampArtifactMeta records what the merge established: the catalog half // stampArtifactMeta records what the merge established: the catalog half
+62 -250
View File
@@ -1,7 +1,6 @@
package explore package explore
import ( import (
"bytes"
"context" "context"
"database/sql" "database/sql"
"encoding/hex" "encoding/hex"
@@ -72,7 +71,13 @@ func writeTestArtifact(
} }
} }
stampArtifactMeta(t, db, meta) for k, v := range meta {
if _, err := db.Exec(
`INSERT INTO artifact_meta (key, value) VALUES (?, ?)`, k, v,
); err != nil {
t.Fatalf("stamp artifact meta: %v", err)
}
}
for _, r := range rows { for _, r := range rows {
if _, err := db.Exec(` if _, err := db.Exec(`
@@ -88,101 +93,6 @@ func writeTestArtifact(
return path return path
} }
// compactArtifactSchema is the artifact cmd/indexexport publishes: the
// catalog's ids as 16 raw bytes, its entity types as codes, and the
// per-release-group total_tracks the exporter added after the first
// artifact was shipped.
//
// It matters that a fixture carries this encoding and not the older
// text one, because SQLite does not coerce between TEXT and BLOB and
// every comparison the importer makes against an mbid is therefore
// encoding-sensitive. writeTestArtifact above is the *other* fixture:
// it still writes the text form, which is what the first published
// artifact carries and what the importer must keep reading.
var compactArtifactSchema = []string{
`CREATE TABLE explore_index (
entity_type INTEGER NOT NULL,
mbid BLOB NOT NULL,
title TEXT NOT NULL,
artist_name TEXT NOT NULL,
artist_mbid BLOB NOT NULL,
aliases TEXT NOT NULL DEFAULT '',
popularity INTEGER NOT NULL DEFAULT 0,
listener_count INTEGER NOT NULL DEFAULT 0,
duration INTEGER NOT NULL DEFAULT 0,
caa_release_mbid BLOB NOT NULL DEFAULT x'',
release_name TEXT NOT NULL DEFAULT '',
primary_type TEXT NOT NULL DEFAULT '',
secondary_types TEXT NOT NULL DEFAULT '',
release_date TEXT NOT NULL DEFAULT '',
total_tracks INTEGER NOT NULL DEFAULT 0,
artist_type TEXT NOT NULL DEFAULT '',
country TEXT NOT NULL DEFAULT '',
disambiguation TEXT NOT NULL DEFAULT '',
sort_name TEXT NOT NULL DEFAULT '',
discog_fetched INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY (mbid)
) WITHOUT ROWID`,
`CREATE TABLE artifact_meta (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
)`,
}
// writeCompactTestArtifact builds the artifact the exporter publishes
// today, in its own encoding, so the importer is exercised against what
// a client actually downloads rather than against what it was written
// for.
func writeCompactTestArtifact(
t *testing.T, meta map[string]string, rows []artifactRow,
) string {
t.Helper()
path := filepath.Join(t.TempDir(), "core-index.db")
db, err := sql.Open("sqlite", "file:"+path)
if err != nil {
t.Fatalf("open artifact: %v", err)
}
defer func() { _ = db.Close() }()
for _, stmt := range compactArtifactSchema {
if _, err := db.Exec(stmt); err != nil {
t.Fatalf("create artifact schema: %v", err)
}
}
stampArtifactMeta(t, db, meta)
for _, r := range rows {
if _, err := db.Exec(`
INSERT INTO explore_index
(entity_type, mbid, title, artist_name, artist_mbid, popularity)
VALUES (?, ?, ?, ?, ?, ?)`,
entityCode(r.entityType), mbidBytes(r.mbid), r.title,
r.artistName, mbidBytes(r.artistMBID), r.popularity,
); err != nil {
t.Fatalf("insert artifact row: %v", err)
}
}
return path
}
// stampArtifactMeta writes the artifact_meta rows a fixture declares.
func stampArtifactMeta(t *testing.T, db *sql.DB, meta map[string]string) {
t.Helper()
for k, v := range meta {
if _, err := db.Exec(
`INSERT INTO artifact_meta (key, value) VALUES (?, ?)`, k, v,
); err != nil {
t.Fatalf("stamp artifact meta: %v", err)
}
}
}
// validMeta is the artifact_meta a well-formed artifact carries. // validMeta is the artifact_meta a well-formed artifact carries.
func validMeta() map[string]string { func validMeta() map[string]string {
return map[string]string{ return map[string]string{
@@ -360,71 +270,6 @@ func TestImportCoreArtifactBatchWalkCoversAllRows(t *testing.T) {
} }
} }
// TestImportCoreArtifactBatchWalkCoversAllRowsCompact is the batch walk
// on the encoding the exporter actually publishes.
//
// The walk positions itself by comparing the artifact's own mbid column
// against the last id it reached, and that column holds 16 raw bytes.
// SQLite does not coerce between TEXT and BLOB, and a blob sorts after
// every text value, so a cursor bound as text is a predicate that either
// matches every row or none: `mbid > ?` with an empty text key is true
// of the whole table, so
// the 100th row is always the 100th row and the bound never advances,
// while `mbid <= <text>` is false of the whole table, so no batch ever
// merges. The result is not a wrong import but an unbounded loop that
// merges nothing and never fails.
//
// Both encodings are covered on purpose. The walk was only ever tested
// against the text fixture above, which is why it shipped broken on the
// one the clients download.
func TestImportCoreArtifactBatchWalkCoversAllRowsCompact(t *testing.T) {
db := database.NewTestDB(t)
si := NewSearchIndex(db, nil, nil, testLogger())
original := artifactMergeBatch
artifactMergeBatch = 100
t.Cleanup(func() { artifactMergeBatch = original })
const total = 337
rows := make([]artifactRow, 0, total)
for i := range total {
rows = append(rows, artifactRow{
entityType: EntityRecording,
mbid: syntheticMBID(i),
title: "Song",
artistName: "Artist",
artistMBID: artA,
popularity: i,
})
}
path := writeCompactTestArtifact(t, validMeta(), rows)
if err := si.importCoreArtifact(context.Background(), path); err != nil {
t.Fatalf("importCoreArtifact: %v", err)
}
var got, top int
if err := db.QueryRowWriter(
"SELECT COUNT(*), MAX(popularity) FROM explore_index",
).Scan(&got, &top); err != nil {
t.Fatalf("count rows: %v", err)
}
if got != total {
t.Errorf("merged %d rows, want %d", got, total)
}
// A count alone would pass if the walk re-merged the same first
// batch forever, so the far end of the artifact is checked too.
if top != total-1 {
t.Errorf("highest popularity = %d, want %d", top, total-1)
}
}
func TestImportCoreArtifactRejectsBadArtifacts(t *testing.T) { func TestImportCoreArtifactRejectsBadArtifacts(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
@@ -609,9 +454,61 @@ func TestArtifactColumnsMatchExporter(t *testing.T) {
// the importer decides by asking the artifact, not by trusting a // the importer decides by asking the artifact, not by trusting a
// version number, and both must land identically. // version number, and both must land identically.
func TestImportCoreArtifactAcceptsBothEncodings(t *testing.T) { func TestImportCoreArtifactAcceptsBothEncodings(t *testing.T) {
compact := writeCompactTestArtifact(t, validMeta(), []artifactRow{ compact := filepath.Join(t.TempDir(), "core-index.db")
{EntityArtist, artA, "Artist A", "Artist A", artA, 5000},
}) db, err := sql.Open("sqlite", "file:"+compact)
if err != nil {
t.Fatalf("open artifact: %v", err)
}
if _, err := db.Exec(`CREATE TABLE explore_index (
entity_type INTEGER NOT NULL,
mbid BLOB NOT NULL,
title TEXT NOT NULL,
artist_name TEXT NOT NULL,
artist_mbid BLOB NOT NULL,
aliases TEXT NOT NULL DEFAULT '',
popularity INTEGER NOT NULL DEFAULT 0,
listener_count INTEGER NOT NULL DEFAULT 0,
duration INTEGER NOT NULL DEFAULT 0,
caa_release_mbid BLOB NOT NULL DEFAULT x'',
release_name TEXT NOT NULL DEFAULT '',
primary_type TEXT NOT NULL DEFAULT '',
secondary_types TEXT NOT NULL DEFAULT '',
release_date TEXT NOT NULL DEFAULT '',
artist_type TEXT NOT NULL DEFAULT '',
country TEXT NOT NULL DEFAULT '',
disambiguation TEXT NOT NULL DEFAULT '',
sort_name TEXT NOT NULL DEFAULT '',
discog_fetched INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY (mbid)
)`); err != nil {
t.Fatalf("create artifact table: %v", err)
}
if _, err := db.Exec(
`CREATE TABLE artifact_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)`,
); err != nil {
t.Fatalf("create artifact meta: %v", err)
}
for k, v := range validMeta() {
if _, err := db.Exec(
"INSERT INTO artifact_meta (key, value) VALUES (?, ?)", k, v,
); err != nil {
t.Fatalf("write artifact meta: %v", err)
}
}
if _, err := db.Exec(`
INSERT INTO explore_index (entity_type, mbid, title, artist_name, artist_mbid, popularity)
VALUES (1, ?, 'Artist A', 'Artist A', ?, 5000)`,
mbidBytes(artA), mbidBytes(artA),
); err != nil {
t.Fatalf("write artifact row: %v", err)
}
_ = db.Close()
live := database.NewTestDB(t) live := database.NewTestDB(t)
si := NewSearchIndex(live, nil, nil, testLogger()) si := NewSearchIndex(live, nil, nil, testLogger())
@@ -851,88 +748,3 @@ func TestImportCoreArtifactWithoutCredits(t *testing.T) {
t.Errorf("credit refs = %d, want 0", refs) t.Errorf("credit refs = %d, want 0", refs)
} }
} }
// TestImportCoreArtifactRefusesAMergeThatLosesRows is the count guard's
// positive case.
//
// The walk's predicates partition the artifact's key space, so a merge
// that lands fewer rows than the artifact declares means a predicate
// dropped some — and the failure is a catalog that looks populated and
// is missing things nobody can name. An empty mbid is the reachable
// way to get there: `mbid > ?` is false of it in both encodings, so it
// is never selected, and nothing else in the import would notice.
func TestImportCoreArtifactRefusesAMergeThatLosesRows(t *testing.T) {
db := database.NewTestDB(t)
si := NewSearchIndex(db, nil, nil, testLogger())
path := writeCompactTestArtifact(t, validMeta(), []artifactRow{
{EntityArtist, artA, "Artist A", "Artist A", artA, 5000},
{EntityArtist, "", "Nameless", "Artist A", artA, 4000},
})
err := si.importCoreArtifact(context.Background(), path)
if err == nil {
t.Fatal("a merge that lost a row was reported as a complete import")
}
if !strings.Contains(err.Error(), "merged 1 of 2 rows") {
t.Errorf("error = %v, want it to name the shortfall", err)
}
// And the same rule as every other rejection: a failed merge must not
// leave the index claiming it has a catalog, or the real build would
// never run again.
if si.hasMeta(dumpImportDoneKey) {
t.Error("a failed import still stamped dump_import_done")
}
}
// TestArtifactKeyBindsInTheArtifactsOwnEncoding pins the one place the
// batch walk's comparison type is decided.
//
// Every wrong answer is silent, which is why it is worth pinning all
// four. SQLite does not coerce TEXT to BLOB and orders every blob after
// every text value, so a text key against a byte column makes
// `mbid > ?` true of the whole artifact - the cursor never advances and
// the walk spins forever without merging a row - while a byte key
// against a text column makes it false of the whole artifact, so every
// batch merges nothing and the import "succeeds" empty. An unset cursor
// is the same fault once more: database/sql converts a nil []byte to
// SQL NULL, and `mbid > NULL` matches no row at all.
func TestArtifactKeyBindsInTheArtifactsOwnEncoding(t *testing.T) {
raw := mbidBytes(artA)
for _, tt := range []struct {
name string
key artifactKey
want []byte
}{
{"unset", nil, []byte{}},
{"set", artifactKey(raw), raw},
} {
t.Run("bytes/"+tt.name, func(t *testing.T) {
got, ok := tt.key.bind(false).([]byte)
if !ok {
t.Fatalf("bind(false) = %T, want []byte", tt.key.bind(false))
}
if got == nil {
t.Fatal("bound to SQL NULL, which matches no row")
}
if !bytes.Equal(got, tt.want) {
t.Errorf("bind(false) = %x, want %x", got, tt.want)
}
})
}
// The dashed form is what an artifact published before the storage
// change carries, and it has to compare as text against text.
if got := artifactKey(nil).bind(true); got != "" {
t.Errorf("bind(true) on an unset cursor = %#v, want an empty string", got)
}
if got := artifactKey(artA).bind(true); got != artA {
t.Errorf("bind(true) = %#v, want %q", got, artA)
}
}
-266
View File
@@ -1,266 +0,0 @@
package explore
import (
"context"
"database/sql"
"io"
"os"
"path/filepath"
"strings"
"testing"
"yellowjacket/backend/database"
)
// Import of the artifact we actually publish, as a client imports it.
//
// Every other test here builds a fixture, and a fixture is a second
// description of the storage format that can be wrong in the same
// direction as the code reading it. That is how #258 shipped: the
// importer positioned its batch walk with a Go `string` cursor against
// the artifact's 16-byte `mbid` column, and SQLite neither coerces
// between TEXT and BLOB nor complains about the comparison — so the walk
// merged nothing and never advanced, and no install could finish its
// first index build. The fixture that guards the walk writes the old
// text encoding; the only compact fixture is one row, below the batch
// size, so the bound query never ran. Both passed throughout.
//
// So this one takes the published file and runs the client's own path
// over it — checksum, decompress, merge — and asserts that what the
// artifact holds is what the client ends up with.
//
// It skips without the path, so an ordinary test run pays nothing for
// it, and the publish job is where it is meant to run:
//
// YJ_CORE_INDEX_ARTIFACT=/tmp/core-index.db.zst \
// go test -tags indexbuild -run TestImportPublishedArtifact \
// ./backend/explore/
//
// The indexbuild tag is not incidental: that job's container has no GTK,
// and the default tag set links the app through Wails.
// publishedArtifactEnv points at the published artifact: the compressed
// core-index.db.zst, or the unpacked core-index.db.
const publishedArtifactEnv = "YJ_CORE_INDEX_ARTIFACT"
// artifactTotals is the pair this test compares across the boundary.
//
// Rows is the whole point — a merge that lands fewer of them than the
// artifact declares is a catalog that looks populated and is missing
// things nobody can name — and popularity is the half whose absence was
// reported when it happened, because it arrives only through the merge.
type artifactTotals struct {
rows int
withListen int
}
func TestImportPublishedArtifact(t *testing.T) {
published := strings.TrimSpace(os.Getenv(publishedArtifactEnv))
if published == "" {
t.Skipf("set %s=<core-index.db.zst> to import the published artifact",
publishedArtifactEnv)
}
if _, err := os.Stat(published); err != nil {
t.Fatalf("%s: %v", publishedArtifactEnv, err)
}
// A file-backed database rather than NewTestDB's in-memory one: the
// artifact is ~135MB and a million rows, which is not a thing to hold
// in RAM inside a test. YJ_HOME is how NewDB is pointed somewhere
// disposable, and going through NewDB means this is the constructor,
// the schema and the read pool the app itself opens.
//
// Nothing closes it, because nothing can: `DB` has no Close and the
// app's handles are process-lifetime by design. The directory is
// unlinked at cleanup and the file goes with it.
t.Setenv("YJ_HOME", t.TempDir())
db, err := database.NewDB(testLogger())
if err != nil {
t.Fatalf("open database: %v", err)
}
si := NewSearchIndex(db, nil, nil, testLogger())
// The checksum the publisher shipped, if it shipped one. Every
// client verifies it and refuses the artifact when it does not
// match, so a wrong one breaks Explore for everyone who has not
// already imported — and nothing else would see it, because the
// comparison is between two files only the publisher has.
if want, ok := publishedChecksum(published); ok {
got, err := fileSHA256(published)
if err != nil {
t.Fatalf("checksum the artifact: %v", err)
}
if got != want {
t.Errorf("published artifact hashes to %s, but its .sha256 says %s",
got, want)
}
}
unpacked := unpackPublishedArtifact(t, si, published)
want, err := artifactTotalsOf(unpacked)
if err != nil {
t.Fatalf("count the artifact's rows: %v", err)
}
if err := si.importCoreArtifact(context.Background(), unpacked); err != nil {
t.Fatalf("importCoreArtifact: %v", err)
}
got, err := indexTotalsOf(db)
if err != nil {
t.Fatalf("count the index's rows: %v", err)
}
if got.rows != want.rows {
t.Errorf("merged %d rows, but the artifact holds %d",
got.rows, want.rows)
}
if got.withListen != want.withListen {
t.Errorf("%d rows carry a listen count, but the artifact holds %d of them",
got.withListen, want.withListen)
}
// The FTS index is rebuilt from the table once the merge is done, and
// it is what search actually reads: a merge that lands without it
// leaves Explore silently matching nothing, which is the state #258
// produced by a different route.
var indexed int
if err := db.QueryRowWriter(
"SELECT COUNT(*) FROM explore_index_fts",
).Scan(&indexed); err != nil {
t.Fatalf("count the FTS index: %v", err)
}
if indexed != got.rows {
t.Errorf("FTS index holds %d rows against the table's %d",
indexed, got.rows)
}
// And one row read back through the app's own path, which is the
// other direction of every conversion the merge makes: a byte MBID
// out of the table, the app's dashed form, and back in as a lookup.
var raw []byte
if err := db.QueryRowWriter(`
SELECT mbid FROM explore_index
WHERE entity_type = 1 /* artist */ AND popularity > 0
ORDER BY popularity DESC LIMIT 1`).Scan(&raw); err != nil {
t.Fatalf("read a stored mbid: %v", err)
}
dashed, err := mbidFromBytes(raw)
if err != nil {
t.Fatalf("the stored mbid is not one: %v", err)
}
artist := si.LookupArtistByMBID(dashed)
if artist == nil {
t.Fatalf("the artifact's most popular artist %s does not look up", dashed)
}
if artist.Popularity == 0 {
t.Errorf("artist %s came back with no popularity", dashed)
}
}
// publishedChecksum reads the sha256 the publisher wrote beside the
// artifact, in `sha256sum` output form. A missing file is not a
// failure: it is only there when the artifact came from the publish job.
func publishedChecksum(path string) (string, bool) {
body, err := os.ReadFile(path + ".sha256")
if err != nil {
return "", false
}
sum := strings.TrimSpace(string(body))
if i := strings.IndexAny(sum, " \t"); i > 0 {
sum = sum[:i]
}
if len(sum) != 64 {
return "", false
}
return strings.ToLower(sum), true
}
// unpackPublishedArtifact returns a path to the unpacked database,
// going through the client's own decompression when it is handed the
// compressed file that is actually published.
func unpackPublishedArtifact(t *testing.T, si *SearchIndex, path string) string {
t.Helper()
if strings.HasSuffix(path, ".db") {
return path
}
// Copied into the test's own directory first: decompress writes
// beside the compressed file, and the publisher's directory is not
// this test's to write in.
staging := t.TempDir()
dst := filepath.Join(staging, coreArtifactFile)
src, err := os.Open(path)
if err != nil {
t.Fatalf("open the published artifact: %v", err)
}
defer func() { _ = src.Close() }()
out, err := os.Create(dst)
if err != nil {
t.Fatalf("create a staging copy: %v", err)
}
if _, err := io.Copy(out, src); err != nil {
t.Fatalf("copy the published artifact: %v", err)
}
if err := out.Close(); err != nil {
t.Fatalf("close the staging copy: %v", err)
}
fetcher := &artifactFetcher{si: si, stagingDir: staging}
if err := fetcher.decompress(context.Background()); err != nil {
t.Fatalf("decompress the published artifact: %v", err)
}
return fetcher.unpackedPath()
}
// artifactTotalsOf counts what an artifact file holds, read directly so
// the numbers do not depend on anything the client does.
func artifactTotalsOf(path string) (artifactTotals, error) {
db, err := sql.Open("sqlite", "file:"+path+"?mode=ro")
if err != nil {
return artifactTotals{}, err
}
defer func() { _ = db.Close() }()
var totals artifactTotals
err = db.QueryRow(`SELECT COUNT(*), COALESCE(SUM(popularity > 0), 0)
FROM explore_index`).Scan(&totals.rows, &totals.withListen)
if err != nil {
return artifactTotals{}, err
}
return totals, nil
}
// indexTotalsOf counts what the client ended up with.
func indexTotalsOf(db *database.DB) (artifactTotals, error) {
var totals artifactTotals
err := db.QueryRowWriter(`SELECT COUNT(*), COALESCE(SUM(popularity > 0), 0)
FROM explore_index`).Scan(&totals.rows, &totals.withListen)
return totals, err
}
+11 -17
View File
@@ -383,13 +383,9 @@ func (si *SearchIndex) topByPopularity(
// MusicBrainz IDs, so this never touches the library tables and asks // MusicBrainz IDs, so this never touches the library tables and asks
// one query rather than one per artist. // one query rather than one per artist.
// //
// **The row is drawn at random, and that is the whole point of it.** // The artists are drawn most-popular-owned-album first, so a large
// Ordered by popularity it was a second leaderboard: the same handful of // library's pool is the part of it the user is likeliest to recognise
// big names appeared every time the page opened, which is not what "you // rather than whichever artists sort first.
// own one album by these artists" is saying. The pool is still bounded
// to `pool` artists — a 4 000-artist library does not need all of them
// ranked — but which of them, and which of their albums, is `RANDOM()`,
// so the shelf is a different sample each visit.
func (si *SearchIndex) unownedAlbumsBySinglyOwnedArtists( func (si *SearchIndex) unownedAlbumsBySinglyOwnedArtists(
ctx context.Context, ctx context.Context,
pool, limit int, pool, limit int,
@@ -400,17 +396,15 @@ func (si *SearchIndex) unownedAlbumsBySinglyOwnedArtists(
WHERE entity_type = 2 /* release_group */ WHERE entity_type = 2 /* release_group */
AND in_library = 0 AND in_library = 0
AND artist_mbid IN ( AND artist_mbid IN (
SELECT artist_mbid FROM ( SELECT artist_mbid FROM explore_index
SELECT artist_mbid FROM explore_index WHERE entity_type = 2 /* release_group */
WHERE entity_type = 2 /* release_group */ AND in_library = 1
AND in_library = 1 AND artist_mbid != x''
AND artist_mbid != x'' GROUP BY artist_mbid
GROUP BY artist_mbid HAVING COUNT(*) = 1
HAVING COUNT(*) = 1 ORDER BY MAX(popularity) DESC
)
ORDER BY RANDOM()
LIMIT ?) LIMIT ?)
ORDER BY RANDOM() ORDER BY popularity DESC
LIMIT ?`, LIMIT ?`,
pool, limit, pool, limit,
)) ))
-6
View File
@@ -3,7 +3,6 @@ package explore
import ( import (
"context" "context"
"log/slog" "log/slog"
"sort"
"testing" "testing"
"yellowjacket/backend/database" "yellowjacket/backend/database"
@@ -203,11 +202,6 @@ func TestShelves_MoreFromOwnedNeedsExactlyOneOwnedAlbum(t *testing.T) {
titles = append(titles, album.Title) titles = append(titles, album.Title)
} }
// The row is a random sample, so the *set* is what is asserted and
// not the order — see `unownedAlbumsBySinglyOwnedArtists` for why
// the ordering was given up.
sort.Strings(titles)
if len(titles) != 2 || titles[0] != "Second" || titles[1] != "Third" { if len(titles) != 2 || titles[0] != "Second" || titles[1] != "Third" {
t.Fatalf("albums = %v, want [Second Third]", titles) t.Fatalf("albums = %v, want [Second Third]", titles)
} }
-72
View File
@@ -3,7 +3,6 @@ package library
import ( import (
"bytes" "bytes"
"crypto/sha256" "crypto/sha256"
"database/sql"
"encoding/hex" "encoding/hex"
"fmt" "fmt"
"image" "image"
@@ -70,77 +69,6 @@ func CoverArtFileSet(coverPath string) []string {
return paths return paths
} }
// sweepOrphanedCoverArt deletes the cover_art rows no album references
// and returns their file paths, for the caller to remove from disk
// after the transaction commits. Cover art is referenced only by
// albums.cover_art_id, so an orphan is a cover whose album is gone —
// which is every album the caller just swept.
//
// One implementation because the scan path, RemoveFromLibrary and
// RemoveLibrary all reach this state, and the scan side used to skip it
// entirely while RemoveLibrary did it inline (#247).
func (l *Library) sweepOrphanedCoverArt(tx *sql.Tx) ([]string, error) {
const orphanSQL = `
SELECT file_path FROM cover_art WHERE id NOT IN (
SELECT DISTINCT cover_art_id FROM albums
WHERE cover_art_id IS NOT NULL
)`
rows, err := tx.QueryContext(l.ctx, orphanSQL)
if err != nil {
return nil, fmt.Errorf("could not query orphaned cover art: %w", err)
}
var paths []string
for rows.Next() {
var filePath string
if err := rows.Scan(&filePath); err != nil {
l.logger.Warn("could not scan cover art path", "err", err)
continue
}
paths = append(paths, filePath)
}
// Close before the DELETE: the two run on the one writer connection.
if err := rows.Close(); err != nil {
l.logger.Warn("could not close cover art rows", "err", err)
}
if len(paths) > 0 {
if _, err := tx.ExecContext(l.ctx, `
DELETE FROM cover_art WHERE id NOT IN (
SELECT DISTINCT cover_art_id FROM albums
WHERE cover_art_id IS NOT NULL
)`); err != nil {
return nil, fmt.Errorf("could not delete orphaned cover_art: %w", err)
}
}
return paths, nil
}
// removeCoverArtFiles removes a cover original and its derived size
// variants. Only the original is stored in cover_art.file_path; the
// _sm/_md/_lg tiers are derived filenames beside it, so they have to be
// removed by name or they accumulate forever.
func (l *Library) removeCoverArtFiles(coverPaths []string) {
for _, coverPath := range coverPaths {
for _, path := range CoverArtFileSet(coverPath) {
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
l.logger.Warn(
"could not remove orphaned cover art file",
"path", path,
"err", err,
)
}
}
}
}
// saveCoverArt saves embedded cover art to the cache directory. // saveCoverArt saves embedded cover art to the cache directory.
// Returns the file path where the art was saved, or empty string // Returns the file path where the art was saved, or empty string
// if no picture data. Timing is recorded in the provided metrics. // if no picture data. Timing is recorded in the provided metrics.
+4 -4
View File
@@ -8,7 +8,6 @@ import (
"yellowjacket/backend/coverart" "yellowjacket/backend/coverart"
"yellowjacket/backend/database/sql/sqlcgen" "yellowjacket/backend/database/sql/sqlcgen"
"yellowjacket/internal/testfixtures"
) )
// TestScan_StoresOnlyCoverTiers pins the size decision: a scan writes // TestScan_StoresOnlyCoverTiers pins the size decision: a scan writes
@@ -27,9 +26,10 @@ func TestScan_StoresOnlyCoverTiers(t *testing.T) {
lib, db := setupTestLibrary(t) lib, db := setupTestLibrary(t)
// Load skips when the fixture library has not been generated, as root, err := filepath.Abs("../../test_data/music_library_test")
// every other fixture test does. if err != nil {
root := testfixtures.Load(t).Root() t.Fatalf("resolve fixture path: %v", err)
}
library, err := db.Queries.CreateLibrary(lib.ctx, sqlcgen.CreateLibraryParams{ library, err := db.Queries.CreateLibrary(lib.ctx, sqlcgen.CreateLibraryParams{
Name: "Fixtures", Name: "Fixtures",
+50 -8
View File
@@ -320,12 +320,43 @@ func (l *Library) RemoveLibrary(id int64) (*RemovalSummary, error) {
genresRemoved, _ := result.RowsAffected() genresRemoved, _ := result.RowsAffected()
// Collect and delete orphaned cover_art rows before the commit. The // 15. Collect orphaned cover_art file paths for post-commit cleanup.
// shared helper is the one place this sweep lives, so the scan path, // SAFETY: Hand-crafted SELECT for orphaned cover art identification.
// RemoveFromLibrary and this removal cannot drift (#247). // Parameterless.
orphanedCoverArtPaths, err := l.sweepOrphanedCoverArt(tx) rows, err := tx.QueryContext(l.ctx,
`SELECT file_path FROM cover_art WHERE id NOT IN (
SELECT DISTINCT cover_art_id FROM albums
WHERE cover_art_id IS NOT NULL
)`)
if err != nil { if err != nil {
return nil, err return nil, fmt.Errorf("could not query orphaned cover art: %w", err)
}
var orphanedCoverArtPaths []string
for rows.Next() {
var filePath string
if err := rows.Scan(&filePath); err != nil {
l.logger.Warn("could not scan cover art path", "err", err)
continue
}
orphanedCoverArtPaths = append(orphanedCoverArtPaths, filePath)
}
if err := rows.Close(); err != nil {
l.logger.Warn("could not close cover art rows", "err", err)
}
// 16. Delete orphaned cover_art rows.
// SAFETY: Hand-crafted orphan cleanup SQL. Parameterless.
if _, err := tx.ExecContext(l.ctx,
`DELETE FROM cover_art WHERE id NOT IN (
SELECT DISTINCT cover_art_id FROM albums
WHERE cover_art_id IS NOT NULL
)`); err != nil {
return nil, fmt.Errorf("could not delete orphaned cover_art: %w", err)
} }
// 17. Delete the library's tagging queue. tagging_items holds a // 17. Delete the library's tagging queue. tagging_items holds a
@@ -361,9 +392,20 @@ func (l *Library) RemoveLibrary(id int64) (*RemovalSummary, error) {
// avoids a costly full re-index of all remaining tracks (~10s for // avoids a costly full re-index of all remaining tracks (~10s for
// 25K tracks). // 25K tracks).
// Post-commit: remove the orphaned cover art files and their sized // 21. Post-commit: Delete orphaned cover art files and their sized
// variants. // variants. Only the original is stored in cover_art.file_path; the
l.removeCoverArtFiles(orphanedCoverArtPaths) // _sm/_md/_lg thumbnails are derived filenames beside it, so they
// have to be removed by name or they accumulate forever.
for _, coverPath := range orphanedCoverArtPaths {
for _, path := range CoverArtFileSet(coverPath) {
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
l.logger.Warn("could not remove orphaned cover art file",
"path", path,
"err", err,
)
}
}
}
// 22. Post-commit: Compact queue. // 22. Post-commit: Compact queue.
if l.removalHooks.CompactQueue != nil { if l.removalHooks.CompactQueue != nil {
+32 -119
View File
@@ -911,96 +911,30 @@ func (l *Library) scanInternal(
orphanStart := time.Now() orphanStart := time.Now()
// Snapshot the orphan set first. The playlist-phantom
// preservation and the deletes are one transaction (the
// preservation has to land before the ON DELETE SET NULL, and
// both have to succeed or neither does), and the ids are what
// scope that preservation to just these files instead of
// rewriting every playlist row on a routine scan.
var (
orphans []sqlcgen.AudioFile
orphanPaths []string
)
existingPaths.Range(func(key, value any) bool { existingPaths.Range(func(key, value any) bool {
orphanPaths = append(orphanPaths, key.(string)) path := key.(string)
orphans = append(orphans, value.(sqlcgen.AudioFile)) audioFile := value.(sqlcgen.AudioFile)
return true l.logger.Debug(
}) "removing orphaned database entry",
"path", path, "id", audioFile.ID,
)
deleted := make([]bool, len(orphans)) if err := l.db.Queries.DeleteAudioFile(
l.ctx, audioFile.ID,
); err != nil {
l.logger.Warn(
"failed to delete orphaned audio file",
"path", path,
"id", audioFile.ID,
"err", err,
)
if len(orphans) > 0 { metrics.addWarning(path, "orphan", err)
orphanIDs := make([]int64, len(orphans))
for i, f := range orphans { return true
orphanIDs[i] = f.ID
} }
tx, beginErr := l.db.BeginTx()
if beginErr != nil {
metrics.addWarning("", "orphan", beginErr)
} else {
defer func() { _ = tx.Rollback() }() // no-op after commit
if err := database.PreservePlaylistPhantomsForFiles(
l.ctx, tx, orphanIDs, l.logger,
); err != nil {
// Deleting without the phantoms is exactly the
// playlist-emptying bug the preservation exists to
// prevent, so leave the rows for the next scan
// rather than empty the playlists now.
l.logger.Error(
"skipping orphan deletion: could not preserve "+
"playlist entries",
"err", err,
)
metrics.addWarning("", "orphan", err)
_ = tx.Rollback()
} else {
txq := l.db.Queries.WithTx(tx)
for i, f := range orphans {
if err := txq.DeleteAudioFile(
l.ctx, f.ID,
); err != nil {
l.logger.Warn(
"failed to delete orphaned audio file",
"path", orphanPaths[i],
"id", f.ID,
"err", err,
)
metrics.addWarning(orphanPaths[i], "orphan", err)
continue
}
deleted[i] = true
}
if err := tx.Commit(); err != nil {
l.logger.Error(
"could not commit orphan deletion",
"err", err,
)
metrics.addWarning("", "orphan", err)
}
}
}
}
// Post-commit bookkeeping for the files that actually went.
for i, f := range orphans {
if !deleted[i] {
continue
}
path := orphanPaths[i]
// Keep the file's tagging group in sync: drop the group's // Keep the file's tagging group in sync: drop the group's
// track count and clear it out once empty, mirroring the // track count and clear it out once empty, mirroring the
// bookkeeping maybeRebindTaggingGroup does for a group_key // bookkeeping maybeRebindTaggingGroup does for a group_key
@@ -1008,25 +942,25 @@ func (l *Library) scanInternal(
// and replaced leaves a stale tagging_items row behind — // and replaced leaves a stale tagging_items row behind —
// its track_count still counts the deleted files, and it // its track_count still counts the deleted files, and it
// never clears from the autotag queue. // never clears from the autotag queue.
if f.GroupKey != "" { if audioFile.GroupKey != "" {
if err := l.db.Queries.DecrementTaggingItemTrackCount( if err := l.db.Queries.DecrementTaggingItemTrackCount(
l.ctx, f.GroupKey, l.ctx, audioFile.GroupKey,
); err != nil { ); err != nil {
l.logger.Warn( l.logger.Warn(
"failed to decrement tagging group for orphan", "failed to decrement tagging group for orphan",
"path", path, "path", path,
"group_key", f.GroupKey, "group_key", audioFile.GroupKey,
"err", err, "err", err,
) )
metrics.addWarning(path, "orphan", err) metrics.addWarning(path, "orphan", err)
} else if err := l.db.Queries.DeleteTaggingItemIfEmpty( } else if err := l.db.Queries.DeleteTaggingItemIfEmpty(
l.ctx, f.GroupKey, l.ctx, audioFile.GroupKey,
); err != nil { ); err != nil {
l.logger.Warn( l.logger.Warn(
"failed to clean up emptied tagging group for orphan", "failed to clean up emptied tagging group for orphan",
"path", path, "path", path,
"group_key", f.GroupKey, "group_key", audioFile.GroupKey,
"err", err, "err", err,
) )
@@ -1034,21 +968,13 @@ func (l *Library) scanInternal(
} }
} }
// Remove from FTS5 search index and the lyrics index. // Remove from FTS5 search index.
if err := l.db.DeleteSearchIndex(f.ID); err != nil { if err := l.db.DeleteSearchIndex(
audioFile.ID,
); err != nil {
l.logger.Warn( l.logger.Warn(
"failed to delete FTS entry for orphan", "failed to delete FTS entry for orphan",
"id", f.ID, "id", audioFile.ID,
"err", err,
)
metrics.addWarning(path, "orphan", err)
}
if err := l.db.DeleteLyricsIndex(f.ID); err != nil {
l.logger.Warn(
"failed to delete lyrics index entry for orphan",
"id", f.ID,
"err", err, "err", err,
) )
@@ -1056,7 +982,9 @@ func (l *Library) scanInternal(
} }
removed.Add(1) removed.Add(1)
}
return true
})
metrics.OrphanCleanup = time.Since(orphanStart) metrics.OrphanCleanup = time.Since(orphanStart)
@@ -1265,32 +1193,17 @@ func (l *Library) pruneEmptyEntities() {
} }
} }
// Cover art after albums: a cover whose album just went is
// unreferenced, and leaving the row behind keeps its files exempt
// from the janitor's covers sweep forever (#247).
orphanedCovers, err := l.sweepOrphanedCoverArt(tx)
if err != nil {
l.logger.Warn("could not sweep orphaned cover art", "err", err)
return
}
if err := tx.Commit(); err != nil { if err := tx.Commit(); err != nil {
l.logger.Warn("could not commit entity cleanup", "err", err) l.logger.Warn("could not commit entity cleanup", "err", err)
return return
} }
// Post-commit: the rows are gone, so their files can go too. if len(albumIDs) > 0 || len(artistIDs) > 0 || len(genreIDs) > 0 {
l.removeCoverArtFiles(orphanedCovers)
if len(albumIDs) > 0 || len(artistIDs) > 0 || len(genreIDs) > 0 ||
len(orphanedCovers) > 0 {
l.logger.Info("pruned empty library entities", l.logger.Info("pruned empty library entities",
"albums", len(albumIDs), "albums", len(albumIDs),
"artists", len(artistIDs), "artists", len(artistIDs),
"genres", len(genreIDs), "genres", len(genreIDs),
"covers", len(orphanedCovers),
) )
} }
} }
-101
View File
@@ -94,57 +94,6 @@ func countRows(
return n return n
} }
// RemoveFromLibrary is the one path that empties a track *deliberately*:
// the file stays on disk but is excluded, so nothing re-imports it. The
// playlist entry must still survive as a re-linkable phantom rather than
// an empty row, because a later full rescan clears the exclusion and is
// what re-links the entry then (#246).
func TestRemoveFromLibrary_PreservesPlaylistPhantoms(t *testing.T) {
t.Parallel()
lib, db := setupTestLibrary(t)
seedRemovableLibrary(t, lib, "/nonexistent/cover.jpg")
if _, err := lib.db.ExecContext(
`INSERT INTO playlists (name) VALUES ('keepme')`,
); err != nil {
t.Fatalf("seed playlist: %v", err)
}
playlistID := queryInt(
t, db, `SELECT id FROM playlists WHERE name = 'keepme'`,
)
if _, err := lib.db.ExecContext(
`INSERT INTO playlist_tracks (playlist_id, audio_file_id, position)
SELECT ?, id, 0 FROM audio_files
WHERE file_path = '/music/song.mp3'`,
playlistID,
); err != nil {
t.Fatalf("seed playlist_tracks: %v", err)
}
if _, err := lib.RemoveFromLibrary([]string{"/music/song.mp3"}); err != nil {
t.Fatalf("RemoveFromLibrary: %v", err)
}
phantomPath := queryString(
t, db,
`SELECT phantom_file_path FROM playlist_tracks
WHERE playlist_id = ?`,
playlistID,
)
if phantomPath != "/music/song.mp3" {
t.Fatalf(
"phantom_file_path is %q, want %q -- the entry cannot be "+
"re-linked after a later full rescan",
phantomPath, "/music/song.mp3",
)
}
}
// A library with tagging_items must still be removable. tagging_items // A library with tagging_items must still be removable. tagging_items
// FK-references libraries with no ON DELETE clause, so leaving those // FK-references libraries with no ON DELETE clause, so leaving those
// rows behind fails the DELETE and rolls back the entire removal. // rows behind fails the DELETE and rolls back the entire removal.
@@ -249,53 +198,3 @@ func TestCoverArtFileSet(t *testing.T) {
} }
} }
} }
// Removing the last track of an album must take the album's cover art
// with it — both the row and every derived file — or the row keeps its
// files exempt from the janitor's covers sweep forever (#247).
func TestRemoveFromLibrary_DeletesOrphanedCoverArt(t *testing.T) {
t.Parallel()
lib, _ := setupTestLibrary(t)
dir := t.TempDir()
// The largest tier is what cover_art.file_path names; write every
// variant so the sweep has a real set to remove.
for _, tier := range thumbnailTiers {
p := filepath.Join(dir, coverart.SizedFilename("abc123.jpg", tier.Suffix))
if err := os.WriteFile(p, []byte("img"), 0o600); err != nil {
t.Fatalf("write %s: %v", p, err)
}
}
cover := filepath.Join(dir, coverart.SizedFilename("abc123.jpg", "_lg"))
seedRemovableLibrary(t, lib, cover)
// Link the album to the cover so it is not orphaned until the track
// (and with it the album) goes.
if _, err := lib.db.ExecContext(
`UPDATE albums SET cover_art_id =
(SELECT id FROM cover_art WHERE file_path = ?)
WHERE name = 'Test Album'`,
cover,
); err != nil {
t.Fatalf("link cover art: %v", err)
}
if _, err := lib.RemoveFromLibrary([]string{"/music/song.mp3"}); err != nil {
t.Fatalf("RemoveFromLibrary: %v", err)
}
if n := countRows(t, lib, "cover_art"); n != 0 {
t.Errorf("cover_art has %d rows after removal, want 0", n)
}
for _, tier := range thumbnailTiers {
p := filepath.Join(dir, coverart.SizedFilename("abc123.jpg", tier.Suffix))
if _, err := os.Stat(p); !os.IsNotExist(err) {
t.Errorf("cover art file still present: %s", filepath.Base(p))
}
}
}
-25
View File
@@ -4,7 +4,6 @@ import (
"errors" "errors"
"fmt" "fmt"
"yellowjacket/backend/database"
"yellowjacket/backend/events" "yellowjacket/backend/events"
) )
@@ -58,25 +57,6 @@ func (l *Library) RemoveFromLibrary(filePaths []string) (*RemovalResult, error)
var result RemovalResult var result RemovalResult
// Preserve the playlist entries before the rows go, so they survive
// as re-linkable phantoms rather than empty rows. The track is
// excluded and will not be re-imported on its own, but a later full
// rescan clears the exclusion and this is what lets the entry
// re-link then — the same preservation every other path that empties
// audio_files performs (#246).
rowIDs := make([]int64, len(rows))
for i, row := range rows {
rowIDs[i] = row.ID
}
if err := database.PreservePlaylistPhantomsForFiles(
l.ctx, tx, rowIDs, l.logger,
); err != nil {
return nil, fmt.Errorf(
"could not preserve playlist entries for removal: %w", err,
)
}
// Exclude every path the caller named, including one whose row has // Exclude every path the caller named, including one whose row has
// already gone: the user asked for that file to stay out, and a row // already gone: the user asked for that file to stay out, and a row
// that disappeared between the click and the commit is not a reason // that disappeared between the click and the commit is not a reason
@@ -147,11 +127,6 @@ func (l *Library) RemoveFromLibrary(filePaths []string) (*RemovalResult, error)
l.logger.Warn("could not delete FTS entry for removed track", l.logger.Warn("could not delete FTS entry for removed track",
"path", row.FilePath, "id", row.ID, "err", err) "path", row.FilePath, "id", row.ID, "err", err)
} }
if err := l.db.DeleteLyricsIndex(row.ID); err != nil {
l.logger.Warn("could not delete lyrics index entry for removed track",
"path", row.FilePath, "id", row.ID, "err", err)
}
} }
// Deleting an audio_files row cascades to queue_tracks, so the // Deleting an audio_files row cascades to queue_tracks, so the
+28 -7
View File
@@ -8,7 +8,6 @@ import (
"time" "time"
"yellowjacket/backend/coverart" "yellowjacket/backend/coverart"
"yellowjacket/backend/database"
) )
var errNoLibrariesConfigured = errors.New( var errNoLibrariesConfigured = errors.New(
@@ -138,12 +137,34 @@ func (l *Library) clearLibraryTables() error {
// metadata for all linked tracks before audio_files are deleted. // metadata for all linked tracks before audio_files are deleted.
// ON DELETE SET NULL will null out audio_file_id, converting them // ON DELETE SET NULL will null out audio_file_id, converting them
// to phantoms that ResolvePhantomTracksAfterScan can re-link. // to phantoms that ResolvePhantomTracksAfterScan can re-link.
// if _, err := tx.ExecContext(l.ctx, `
// Shared with the stale-shape retire in backend/database, which is UPDATE playlist_tracks
// the other path that empties this table and which did not do this SET
// (#183): the statement lives there so the two cannot drift again. phantom_title = COALESCE(phantom_title, (
if err := database.PreservePlaylistPhantoms(l.ctx, tx, l.logger); err != nil { SELECT tm.title FROM track_metadata tm
return err WHERE tm.id = playlist_tracks.audio_file_id
)),
phantom_artist = COALESCE(phantom_artist, (
SELECT tm.artist_name FROM track_metadata tm
WHERE tm.id = playlist_tracks.audio_file_id
)),
phantom_album = COALESCE(phantom_album, (
SELECT tm.album FROM track_metadata tm
WHERE tm.id = playlist_tracks.audio_file_id
)),
phantom_duration_ms = COALESCE(phantom_duration_ms, (
SELECT af.length_milliseconds FROM audio_files af
WHERE af.id = playlist_tracks.audio_file_id
)),
phantom_file_path = COALESCE(phantom_file_path, (
SELECT af.file_path FROM audio_files af
WHERE af.id = playlist_tracks.audio_file_id
))
WHERE audio_file_id IS NOT NULL
`); err != nil {
return fmt.Errorf(
"could not preserve playlist track metadata: %w", err,
)
} }
// Phase 2: the files. file_genres cascades with them. // Phase 2: the files. file_genres cascades with them.
-95
View File
@@ -231,98 +231,3 @@ func TestScan_MultipleDirectoriesDoNotCrossContaminate(t *testing.T) {
t.Errorf("Album A and Album B must not share a group_key: %+v", keys) t.Errorf("Album A and Album B must not share a group_key: %+v", keys)
} }
} }
// TestScan_OrphanCleanupPreservesPlaylistPhantoms guards #246: a file
// deleted from the library folder *outside* YellowJacket is discovered
// as an orphan by the next scan, and its playlist entry must survive as
// a re-linkable phantom — the same preservation the full rescan and
// stale-retire paths already perform, scoped here to just the orphaned
// file. Before the fix the entry became an empty row (audio_file_id
// NULL and no phantom_file_path), which nothing can ever re-link.
func TestScan_OrphanCleanupPreservesPlaylistPhantoms(t *testing.T) {
t.Parallel()
lib, db := setupTestLibrary(t)
root := t.TempDir()
track := filepath.Join(root, "gone.mp3")
writeTestTrack(t, track, 0)
library, err := db.Queries.CreateLibrary(lib.ctx, sqlcgen.CreateLibraryParams{
Name: "orphans",
Path: root,
})
if err != nil {
t.Fatalf("create library: %v", err)
}
if metrics := lib.scanInternal(library.ID, library.Name, library.Path); metrics == nil {
t.Fatal("first scan returned nil metrics")
}
trackID := queryInt(
t, db, "SELECT id FROM audio_files WHERE file_path = ?", track,
)
if trackID == 0 {
t.Fatal("first scan did not import the track")
}
if _, err := db.ExecContext(
`INSERT INTO playlists (name) VALUES ('keepme')`,
); err != nil {
t.Fatalf("seed playlist: %v", err)
}
playlistID := queryInt(
t, db, "SELECT id FROM playlists WHERE name = 'keepme'",
)
if _, err := db.ExecContext(
`INSERT INTO playlist_tracks (playlist_id, audio_file_id, position)
VALUES (?, ?, 0)`,
playlistID, trackID,
); err != nil {
t.Fatalf("seed playlist_tracks: %v", err)
}
// The file goes away outside the app.
if err := os.Remove(track); err != nil {
t.Fatalf("remove track: %v", err)
}
if metrics := lib.scanInternal(library.ID, library.Name, library.Path); metrics == nil {
t.Fatal("second scan returned nil metrics")
}
if n := queryInt(
t, db, "SELECT COUNT(*) FROM audio_files WHERE file_path = ?", track,
); n != 0 {
t.Fatalf("audio_files still holds the removed path: %d rows", n)
}
if n := queryInt(
t, db,
"SELECT COUNT(*) FROM playlist_tracks WHERE playlist_id = ? "+
"AND audio_file_id IS NULL",
playlistID,
); n != 1 {
t.Fatalf(
"playlist entry did not become a phantom: %d null-id rows, want 1",
n,
)
}
phantomPath := queryString(
t, db,
"SELECT phantom_file_path FROM playlist_tracks WHERE playlist_id = ?",
playlistID,
)
if phantomPath != track {
t.Fatalf(
"phantom_file_path = %q, want %q -- the entry cannot be "+
"re-linked if the file comes back",
phantomPath, track,
)
}
}
-117
View File
@@ -666,120 +666,3 @@ func TestExpiredHTTPCacheJob_TrimsToBudget(t *testing.T) {
t.Errorf("kept %q, want the longest-lived row", kept) t.Errorf("kept %q, want the longest-lived row", kept)
} }
} }
// TestStaleArtistMetadataJob pins the sweep's two keep rules: an owned
// artist's metadata survives, a browsed artist's survives while it still
// holds cached artwork, and everything else goes (#248).
func TestStaleArtistMetadataJob(t *testing.T) {
t.Parallel()
db := database.NewTestDB(t)
const (
ownedMBID = "11111111-1111-1111-1111-111111111111"
browsedMBID = "22222222-2222-2222-2222-222222222222"
staleMBID = "33333333-3333-3333-3333-333333333333"
)
// The owned artist is in the library - which means a *file* says
// so. An artists row on its own is not ownership.
database.InsertTestTrack(t, db, database.TestTrack{
FilePath: "/music/owned.mp3",
Artist: "Owned",
ArtistMBID: ownedMBID,
})
for _, mbid := range []string{ownedMBID, browsedMBID, staleMBID} {
if _, err := db.ExecContext(
`INSERT INTO artist_metadata (mbid, source, data, fetched_at)
VALUES (?, 'wikidata-p18', x'00', CURRENT_TIMESTAMP)`,
mbid,
); err != nil {
t.Fatalf("seed artist_metadata for %s: %v", mbid, err)
}
}
// The browsed artist holds cached artwork, so its metadata is still
// referenced and must survive.
if _, err := db.ExecContext(
`INSERT INTO artist_images
(artist_mbid, source, source_url, file_path)
VALUES (?, 'test', 'http://x', '/art/primary.jpg')`,
browsedMBID,
); err != nil {
t.Fatalf("seed artist_images: %v", err)
}
if _, err := StaleArtistMetadataJob(db).Run(context.Background()); err != nil {
t.Fatalf("run job: %v", err)
}
for _, tc := range []struct {
mbid string
want int
}{
{ownedMBID, 1},
{browsedMBID, 1},
{staleMBID, 0},
} {
var n int
if err := db.QueryRowWriter(
"SELECT COUNT(*) FROM artist_metadata WHERE mbid = ?", tc.mbid,
).Scan(&n); err != nil {
t.Fatalf("count %s: %v", tc.mbid, err)
}
if n != tc.want {
t.Errorf("artist_metadata rows for %s = %d, want %d", tc.mbid, n, tc.want)
}
}
}
// TestStaleSearchClicksJob deletes only the clicks old enough to have
// left the retention window (#249).
func TestStaleSearchClicksJob(t *testing.T) {
t.Parallel()
db := database.NewTestDB(t)
count := func(mbid string) int {
t.Helper()
var n int
if err := db.QueryRowWriter(
"SELECT COUNT(*) FROM search_clicks WHERE entity_mbid = ?", mbid,
).Scan(&n); err != nil {
t.Fatalf("count %s: %v", mbid, err)
}
return n
}
seed := func(query, mbid, lastClicked string) {
t.Helper()
if _, err := db.ExecContext(
`INSERT INTO search_clicks
(query, entity_mbid, entity_type, click_count, last_clicked)
VALUES (?, ?, 'recording', 1, ?)`,
query, mbid, lastClicked,
); err != nil {
t.Fatalf("seed search_clicks: %v", err)
}
}
seed("tide", "aaaa", "2024-01-01 00:00:00") // stale
seed("tide", "bbbb", "2999-01-01 00:00:00") // recent
if _, err := StaleSearchClicksJob(db).Run(context.Background()); err != nil {
t.Fatalf("run job: %v", err)
}
if n := count("bbbb"); n != 1 {
t.Errorf("recent click was deleted: %d rows, want 1", n)
}
if n := count("aaaa"); n != 0 {
t.Errorf("stale click survived: %d rows, want 0", n)
}
}
-66
View File
@@ -628,69 +628,3 @@ func dirSize(dir string) (bytes, files int64) {
return bytes, files return bytes, files
} }
// StaleArtistMetadataJob evicts long-lived artist metadata (bios, wiki
// leads, relationships) for artists the user no longer has any reason
// to keep around: not owned and holding no cached artwork.
//
// artist_metadata has no TTL by design — entity data changes rarely and
// re-fetching spends someone else's rate limit — so without a sweep it
// grows for the life of the install. This is the "swept when the
// artist is no longer referenced" contract the datamap always declared
// for it and nothing ever performed (#248).
func StaleArtistMetadataJob(db *database.DB) Job {
return Job{
Name: "artist-metadata-sweep",
MinInterval: dailyInterval,
Run: func(_ context.Context) (Result, error) {
res, err := db.ExecContext(
`DELETE FROM artist_metadata
WHERE mbid NOT IN (` + ownedArtistMBIDs + `)
AND mbid NOT IN (
SELECT artist_mbid FROM artist_images
)`,
)
if err != nil {
return Result{}, fmt.Errorf(
"delete stale artist_metadata rows: %w", err,
)
}
rows, _ := res.RowsAffected()
return Result{RowsDeleted: rows}, nil
},
}
}
// searchClicksRetention is how long a search-click ranking signal stays
// useful. search_clicks is authored behavioural data — nothing that
// owns a row ever drops it — so age is the ceiling that keeps the table
// from growing without bound for the life of the install (#249).
const searchClicksRetention = "-180 days"
// StaleSearchClicksJob deletes search-click ranking rows older than the
// retention window. Rows are small and the table grows slowly, so this
// runs daily and does almost nothing most runs.
func StaleSearchClicksJob(db *database.DB) Job {
return Job{
Name: "search-clicks-sweep",
MinInterval: dailyInterval,
Run: func(_ context.Context) (Result, error) {
res, err := db.ExecContext(
`DELETE FROM search_clicks
WHERE last_clicked < datetime('now', ?)`,
searchClicksRetention,
)
if err != nil {
return Result{}, fmt.Errorf(
"delete stale search_clicks rows: %w", err,
)
}
rows, _ := res.RowsAffected()
return Result{RowsDeleted: rows}, nil
},
}
}
-8
View File
@@ -74,14 +74,6 @@ func ExtractTags(path string) (*TrackMetadata, error) {
// ExtractTagsFromReader reads metadata from an io.ReadSeeker. // ExtractTagsFromReader reads metadata from an io.ReadSeeker.
func ExtractTagsFromReader(r io.ReadSeeker) (*TrackMetadata, error) { func ExtractTagsFromReader(r io.ReadSeeker) (*TrackMetadata, error) {
// The container decides, so this is asked before tag.ReadFrom and
// not after its failure: a WAV's tags live in a RIFF chunk that
// dhowden/tag cannot see, and its fallback -- an ID3v1 trailer --
// would otherwise outrank them.
if meta, ok := wavTags(r); ok {
return meta, nil
}
m, err := tag.ReadFrom(r) m, err := tag.ReadFrom(r)
if err != nil { if err != nil {
// No tags found is not necessarily an error - return empty metadata // No tags found is not necessarily an error - return empty metadata
-68
View File
@@ -1,68 +0,0 @@
package metadata
import (
"bytes"
"errors"
"fmt"
"io"
"strings"
"yellowjacket/backend/riff"
)
// wavTags reads the ID3v2 tag a WAV carries in its RIFF "id3 " chunk,
// which is where backend/tagwriter puts it and where dhowden/tag --
// having no RIFF reader at all -- cannot look. Without this a WAV
// scans as an untagged file however carefully it was tagged.
//
// ok is false when r is not a RIFF/WAVE container, and the read
// position is restored either way so the caller can carry on.
func wavTags(r io.ReadSeeker) (*TrackMetadata, bool) {
start, err := r.Seek(0, io.SeekCurrent)
if err != nil {
return nil, false
}
id3Data, chunkErr := riff.ID3Chunk(r)
if _, err := r.Seek(start, io.SeekStart); err != nil {
return nil, false
}
switch {
case chunkErr == nil:
return wavTagsFrom(id3Data), true
// Not ours to read: let the ordinary dispatch have the file.
case errors.Is(chunkErr, riff.ErrNotRIFF), errors.Is(chunkErr, riff.ErrNotWAVE):
return nil, false
// A RIFF container we cannot get a tag out of -- no chunk, an RF64
// file, a truncated header. That is a file with no readable tags,
// which is what the scanner's filename fallback is for.
default:
return &TrackMetadata{}, true
}
}
// wavTagsFrom parses the bytes of a WAV's ID3v2 chunk.
func wavTagsFrom(id3Data []byte) *TrackMetadata {
meta, err := extractID3v2Lenient(bytes.NewReader(id3Data))
if err != nil {
// A tag holding no frames is not a damaged tag: writing every
// field back out empty leaves one, and warning about it would
// put a fault on a file that has none.
if errors.Is(err, ErrTagsUnreadable) {
return &TrackMetadata{}
}
return &TrackMetadata{
TagReadWarning: fmt.Errorf("%w: %w", ErrTagsUnreadable, err),
}
}
// extractID3v2Lenient names MP3, being the recovery path for one.
meta.FileFormat = strings.ToUpper(strings.TrimPrefix(string(WAV), "."))
return meta
}
-28
View File
@@ -134,12 +134,6 @@ type Service struct {
libraryDir LibraryDirProvider libraryDir LibraryDirProvider
favoritesConf FavoritesConfigProvider favoritesConf FavoritesConfigProvider
// onDeleted, when set, is called after a playlist is deleted so
// cross-cutting state that points at it (the queue's "Playing
// from" label) can stop pointing at a playlist that no longer
// exists. Wired from app.go, like Library.SetRemovalHooks.
onDeleted func(playlistID int64)
// dataDirOverride, when non-empty, replaces the OS user data // dataDirOverride, when non-empty, replaces the OS user data
// directory as the base for the playlists folder. Set by tests to // directory as the base for the playlists folder. Set by tests to
// keep M3U writes out of the real user data directory. // keep M3U writes out of the real user data directory.
@@ -172,17 +166,6 @@ func (s *Service) SetFavoritesConfig(
s.favoritesConf = provider s.favoritesConf = provider
} }
// SetOnPlaylistDeleted registers a callback invoked after a playlist is
// deleted, for cross-cutting invalidation.
//
//wails:ignore // internal wiring, not part of the app's IPC surface.
func (s *Service) SetOnPlaylistDeleted(onDeleted func(playlistID int64)) {
s.mu.Lock()
defer s.mu.Unlock()
s.onDeleted = onDeleted
}
// ServiceStartup is v3's service lifecycle hook: it runs once the // ServiceStartup is v3's service lifecycle hook: it runs once the
// runtime exists, and ctx is cancelled when the app shuts down. It // runtime exists, and ctx is cancelled when the app shuts down. It
// replaces v2's SetContext, which had to be called by hand from // replaces v2's SetContext, which had to be called by hand from
@@ -783,17 +766,6 @@ func (s *Service) DeletePlaylist(playlistID int64) error {
s.emitEvent(events.PlaylistDeleted, playlistID) s.emitEvent(events.PlaylistDeleted, playlistID)
// Cross-cutting invalidation: the queue's "Playing from" label may
// point at this playlist, and a link to a playlist that no longer
// exists is worse than none.
s.mu.Lock()
onDeleted := s.onDeleted
s.mu.Unlock()
if onDeleted != nil {
onDeleted(playlistID)
}
// Recreate the default playlist if we just deleted it. // Recreate the default playlist if we just deleted it.
if s.defaultPlaylistID() == playlistID { if s.defaultPlaylistID() == playlistID {
s.EnsureDefaultPlaylist() s.EnsureDefaultPlaylist()
+4 -6
View File
@@ -6,7 +6,7 @@ import (
"yellowjacket/backend/events" "yellowjacket/backend/events"
) )
// recordPlay inserts a listening_events row and updates the denormalized // recordPlay inserts a play_history row and updates the denormalized
// play_count / last_played columns on audio_files. Called from // play_count / last_played columns on audio_files. Called from
// OnPlaybackFinished for the track that just finished. // OnPlaybackFinished for the track that just finished.
// //
@@ -20,12 +20,10 @@ func (q *Queue) recordPlay(audioFileID int64) {
now := time.Now().UTC().Format(time.DateTime) now := time.Now().UTC().Format(time.DateTime)
// Insert the listening event. A natural finish is a 'complete' by // Insert play_history row.
// construction; position/duration are the classifier's to fill once
// skips are recorded (see .planning/plans/active/021).
_, err := q.db.ExecContext( _, err := q.db.ExecContext(
`INSERT INTO listening_events (audio_file_id, kind, occurred_at) `INSERT INTO play_history (audio_file_id, played_at)
VALUES (?, 'complete', ?)`, VALUES (?, ?)`,
audioFileID, now, audioFileID, now,
) )
if err != nil { if err != nil {
-15
View File
@@ -1581,21 +1581,6 @@ func (q *Queue) dropSource() {
q.source = Source{} q.source = Source{}
} }
// DropSourceForPlaylist clears the queue's "Playing from" label when
// its source playlist is deleted. A link back to a playlist that no
// longer exists is worse than none, and the label otherwise survives
// the deletion until the next SetQueue (#249).
func (q *Queue) DropSourceForPlaylist(playlistID int64) {
q.mu.Lock()
defer q.mu.Unlock()
if (q.source.Type == "playlist" || q.source.Type == "smartPlaylist") &&
q.source.ID == playlistID {
q.dropSource()
q.persistState()
}
}
// commitMutation persists the current queue state after a mutation. // commitMutation persists the current queue state after a mutation.
// When reindex is true, track positions are renumbered first. // When reindex is true, track positions are renumbered first.
// The caller must hold q.mu. // The caller must hold q.mu.
-32
View File
@@ -535,35 +535,3 @@ func TestCycleRepeat_CyclesThroughModes(t *testing.T) {
t.Errorf("after third cycle: got %q, want %q", state.RepeatMode, RepeatOff) t.Errorf("after third cycle: got %q, want %q", state.RepeatMode, RepeatOff)
} }
} }
// TestDropSourceForPlaylist clears the "Playing from" label when the
// queue's source playlist is deleted, and leaves it alone otherwise
// (#249).
func TestDropSourceForPlaylist(t *testing.T) {
t.Parallel()
q, db := setupTestQueue(t)
paths := seedAudioFiles(t, db, 2)
q.SetQueue(paths, 0, false, Source{Type: "playlist", ID: 42, Label: "Road Trip"})
q.DropSourceForPlaylist(42)
if got := q.GetState().Source; got != (Source{}) {
t.Errorf("source = %+v, want empty after playlist 42 deleted", got)
}
}
func TestDropSourceForPlaylistIgnoresOtherPlaylists(t *testing.T) {
t.Parallel()
q, db := setupTestQueue(t)
paths := seedAudioFiles(t, db, 2)
source := Source{Type: "smartPlaylist", ID: 42, Label: "Road Trip"}
q.SetQueue(paths, 0, false, source)
q.DropSourceForPlaylist(7)
if got := q.GetState().Source; got != source {
t.Errorf("source = %+v, want %+v unchanged for a different playlist", got, source)
}
}
-186
View File
@@ -1,186 +0,0 @@
// Package riff reads the chunk layout of a RIFF/WAVE container.
//
// It exists because both halves of WAV tagging need it and neither can
// import the other: backend/tagwriter writes a WAV's tags into a RIFF
// "id3 " chunk and already imports backend/metadata, which is what has
// to read them back out. backend/tagtotals is the precedent.
//
// The two readers here are deliberately different. Parse holds every
// chunk's data in memory, which is what rewriting a file needs; a WAV's
// audio *is* the "data" chunk, so doing that on the scan path would
// read every library file in full. ID3Chunk seeks over what it is not
// looking for instead. Both walk the same headers.
package riff
import (
"bytes"
"encoding/binary"
"errors"
"fmt"
"io"
"strings"
)
// Sentinel errors describing a container this package will not read.
var (
ErrRF64NotSupported = errors.New("RF64 files are not yet supported")
ErrNotRIFF = errors.New("not a RIFF file")
ErrNotWAVE = errors.New("not a WAVE file")
ErrNoID3Chunk = errors.New("no ID3 chunk in RIFF file")
)
// Chunk holds a single RIFF sub-chunk (ID + raw data).
type Chunk struct {
ID [4]byte
Data []byte
}
// IsID3 reports whether id is that of an ID3v2 RIFF chunk. Both
// lowercase "id3 " and uppercase "ID3 " are accepted.
func IsID3(id [4]byte) bool {
return strings.ToLower(string(id[:3])) == "id3"
}
// Parse reads every RIFF sub-chunk from r, in order, starting at the
// reader's current position. It rejects RF64 files and non-WAVE
// containers with descriptive errors. The parser is lenient: it
// tolerates a missing final padding byte and ignores the declared
// RIFF size.
func Parse(r io.Reader) ([]Chunk, error) {
if err := readContainer(r); err != nil {
return nil, err
}
var chunks []Chunk
for {
id, size, err := nextHeader(r)
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return nil, err
}
// Copied rather than allocated up front, as ID3Chunk does: the
// size is four bytes off the file, so a truncated one is free to
// declare a chunk larger than the whole of itself.
var data bytes.Buffer
if _, err := io.CopyN(&data, r, int64(size)); err != nil {
return nil, fmt.Errorf("read chunk data for %q: %w", id, err)
}
chunks = append(chunks, Chunk{ID: id, Data: data.Bytes()})
// Odd-length chunks have a padding byte. Lenient: if the
// read fails (e.g. EOF), just break rather than error.
if size%2 != 0 {
var pad [1]byte
if _, err := r.Read(pad[:]); err != nil {
break
}
}
}
return chunks, nil
}
// ID3Chunk returns the payload of the ID3v2 chunk of the RIFF/WAVE
// container at the reader's current position, seeking over every other
// chunk rather than reading it. It returns ErrNoID3Chunk when the
// container carries no such chunk, and leaves the read position
// unspecified either way.
func ID3Chunk(r io.ReadSeeker) ([]byte, error) {
if err := readContainer(r); err != nil {
return nil, err
}
for {
id, size, err := nextHeader(r)
if errors.Is(err, io.EOF) {
return nil, ErrNoID3Chunk
}
if err != nil {
return nil, err
}
if !IsID3(id) {
// Odd-length chunks carry a padding byte. Seeking past
// the end of the file is not an error; the next header
// read is what reports the end.
if _, err := r.Seek(int64(size)+int64(size%2), io.SeekCurrent); err != nil {
return nil, fmt.Errorf("skip chunk %q: %w", id, err)
}
continue
}
// Copied rather than allocated up front: a truncated file is
// free to declare a chunk larger than the whole of itself.
var data bytes.Buffer
if _, err := io.CopyN(&data, r, int64(size)); err != nil {
return nil, fmt.Errorf("read chunk data for %q: %w", id, err)
}
return data.Bytes(), nil
}
}
// readContainer consumes the 12-byte RIFF/WAVE header at the reader's
// current position.
func readContainer(r io.Reader) error {
var magic [4]byte
if _, err := io.ReadFull(r, magic[:]); err != nil {
return fmt.Errorf("read RIFF magic: %w", err)
}
if string(magic[:]) == "RF64" {
return ErrRF64NotSupported
}
if string(magic[:]) != "RIFF" {
return fmt.Errorf("%w: got %q", ErrNotRIFF, magic)
}
// Read (and discard) RIFF size — lenient, do not enforce.
var riffSize uint32
if err := binary.Read(r, binary.LittleEndian, &riffSize); err != nil {
return fmt.Errorf("read RIFF size: %w", err)
}
var form [4]byte
if _, err := io.ReadFull(r, form[:]); err != nil {
return fmt.Errorf("read WAVE form type: %w", err)
}
if string(form[:]) != "WAVE" {
return fmt.Errorf("%w: got %q", ErrNotWAVE, form)
}
return nil
}
// nextHeader reads one sub-chunk header. It returns io.EOF once the
// chunks are exhausted, including for a header cut short.
func nextHeader(r io.Reader) ([4]byte, uint32, error) {
var id [4]byte
_, err := io.ReadFull(r, id[:])
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
return id, 0, io.EOF
}
if err != nil {
return id, 0, fmt.Errorf("read chunk ID: %w", err)
}
var size uint32
if err := binary.Read(r, binary.LittleEndian, &size); err != nil {
return id, 0, fmt.Errorf("read chunk size for %q: %w", id, err)
}
return id, size, nil
}
-254
View File
@@ -1,254 +0,0 @@
package riff_test
import (
"bytes"
"encoding/binary"
"errors"
"runtime"
"testing"
"yellowjacket/backend/riff"
)
// chunk is one sub-chunk to put in a test container.
type chunk struct {
id string
data []byte
}
// buildRIFF assembles a container from magic, form type and chunks,
// padding odd-length chunks the way a writer must.
func buildRIFF(magic, form string, chunks []chunk) []byte {
var body bytes.Buffer
body.WriteString(form)
for _, c := range chunks {
body.WriteString(c.id)
_ = binary.Write(&body, binary.LittleEndian, uint32(len(c.data)))
body.Write(c.data)
if len(c.data)%2 != 0 {
body.WriteByte(0)
}
}
var out bytes.Buffer
out.WriteString(magic)
_ = binary.Write(&out, binary.LittleEndian, uint32(body.Len()))
out.Write(body.Bytes())
return out.Bytes()
}
func TestID3Chunk_FindsTheTagPastTheAudio(t *testing.T) {
t.Parallel()
tests := []struct {
name string
chunks []chunk
want string
}{
{
name: "after an odd-length chunk",
chunks: []chunk{
{id: "fmt ", data: make([]byte, 16)},
{id: "LIST", data: []byte("INFOodd")},
{id: "data", data: make([]byte, 200)},
{id: "id3 ", data: []byte("ID3vTAG")},
},
want: "ID3vTAG",
},
{
// The chunk ID is written both ways in the wild, and the
// writer accepts either, so the reader must too.
name: "uppercase ID3",
chunks: []chunk{
{id: "data", data: make([]byte, 8)},
{id: "ID3 ", data: []byte("upper")},
},
want: "upper",
},
{
name: "first chunk",
chunks: []chunk{
{id: "id3 ", data: []byte("first")},
{id: "data", data: make([]byte, 8)},
},
want: "first",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
r := bytes.NewReader(buildRIFF("RIFF", "WAVE", tc.chunks))
got, err := riff.ID3Chunk(r)
if err != nil {
t.Fatalf("ID3Chunk: %v", err)
}
if string(got) != tc.want {
t.Errorf("chunk data: got %q, want %q", got, tc.want)
}
})
}
}
func TestID3Chunk_RejectsWhatItCannotRead(t *testing.T) {
t.Parallel()
tests := []struct {
name string
bytes []byte
want error
}{
{
name: "no ID3 chunk",
bytes: buildRIFF("RIFF", "WAVE", []chunk{{id: "data", data: []byte{1, 2}}}),
want: riff.ErrNoID3Chunk,
},
{
name: "no chunks at all",
bytes: buildRIFF("RIFF", "WAVE", nil),
want: riff.ErrNoID3Chunk,
},
{
name: "not RIFF",
bytes: []byte("ID3\x03\x00\x00\x00\x00\x00\x00\x00\x00"),
want: riff.ErrNotRIFF,
},
{
name: "not WAVE",
bytes: buildRIFF("RIFF", "AVI ", []chunk{{id: "id3 ", data: []byte("x")}}),
want: riff.ErrNotWAVE,
},
{
name: "RF64",
bytes: buildRIFF("RF64", "WAVE", []chunk{{id: "id3 ", data: []byte("x")}}),
want: riff.ErrRF64NotSupported,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
_, err := riff.ID3Chunk(bytes.NewReader(tc.bytes))
if !errors.Is(err, tc.want) {
t.Errorf("ID3Chunk error: got %v, want %v", err, tc.want)
}
})
}
}
// A file cut short mid-chunk is a file with no tag, not a reason to
// allocate the size it claims: the declared size is four bytes any
// truncation can leave saying 4 GB.
func TestID3Chunk_ToleratesATruncatedFile(t *testing.T) {
t.Parallel()
full := buildRIFF("RIFF", "WAVE", []chunk{
{id: "data", data: make([]byte, 64)},
{id: "id3 ", data: []byte("tag")},
})
t.Run("cut inside the audio", func(t *testing.T) {
t.Parallel()
_, err := riff.ID3Chunk(bytes.NewReader(full[:32]))
if !errors.Is(err, riff.ErrNoID3Chunk) {
t.Errorf("ID3Chunk error: got %v, want %v", err, riff.ErrNoID3Chunk)
}
})
t.Run("cut inside the tag", func(t *testing.T) {
t.Parallel()
if _, err := riff.ID3Chunk(bytes.NewReader(full[:len(full)-2])); err == nil {
t.Error("ID3Chunk: got nil error for a truncated tag chunk")
}
})
}
// Parse is the writer's half and reads every chunk into memory, which
// is what preserving them needs.
func TestParse_ReadsEveryChunkInOrder(t *testing.T) {
t.Parallel()
raw := buildRIFF("RIFF", "WAVE", []chunk{
{id: "fmt ", data: make([]byte, 16)},
{id: "LIST", data: []byte("INFOodd")},
{id: "id3 ", data: []byte("tag")},
})
chunks, err := riff.Parse(bytes.NewReader(raw))
if err != nil {
t.Fatalf("Parse: %v", err)
}
want := []string{"fmt ", "LIST", "id3 "}
if len(chunks) != len(want) {
t.Fatalf("chunk count: got %d, want %d", len(chunks), len(want))
}
for i, id := range want {
if got := string(chunks[i].ID[:]); got != id {
t.Errorf("chunk %d: got %q, want %q", i, got, id)
}
}
if !riff.IsID3(chunks[2].ID) || string(chunks[2].Data) != "tag" {
t.Errorf("id3 chunk: got %q", chunks[2].Data)
}
// The padding byte after an odd chunk is not part of its data.
if string(chunks[1].Data) != "INFOodd" {
t.Errorf("odd chunk data: got %q, want %q", chunks[1].Data, "INFOodd")
}
}
// A chunk size is four bytes read off the file, so a truncated or
// malformed WAV is free to declare a chunk larger than the whole of
// itself. Parse must grow with what arrives rather than with what was
// claimed.
//
// This measures the allocation instead of the error because the error
// is the same either way: a build sizing its buffer from the header
// reports the truncation correctly, having asked the allocator for a
// gigabyte on the way. Deliberately not parallel — TotalAlloc is
// process-wide, and a test paused beside another one is measuring it
// too.
func TestParse_DoesNotAllocateWhatAChunkClaims(t *testing.T) {
// Large enough that a header-sized buffer is unmistakable, in a
// container of a few dozen bytes.
const declared = 1 << 30
var raw bytes.Buffer
raw.WriteString("RIFF")
_ = binary.Write(&raw, binary.LittleEndian, uint32(declared+12))
raw.WriteString("WAVE")
raw.WriteString("data")
_ = binary.Write(&raw, binary.LittleEndian, uint32(declared))
raw.WriteString("and then the file ends")
var before, after runtime.MemStats
runtime.GC()
runtime.ReadMemStats(&before)
if _, err := riff.Parse(bytes.NewReader(raw.Bytes())); err == nil {
t.Fatal("Parse: got nil error for a chunk larger than the file holding it")
}
runtime.ReadMemStats(&after)
if grew := after.TotalAlloc - before.TotalAlloc; grew > 1<<20 {
t.Errorf("Parse allocated %d bytes reading a %d-byte file whose chunk header claimed %d",
grew, raw.Len(), declared)
}
}
+4 -5
View File
@@ -13,10 +13,9 @@ import (
// indistinguishable from never having written one. So these assert the // indistinguishable from never having written one. So these assert the
// round trip through the *reader the scan uses*, not the bytes. // round trip through the *reader the scan uses*, not the bytes.
// //
// WAV was the exception until #104 -- dhowden/tag has no RIFF reader, // WAV is the exception and it is not this change's: dhowden/tag has no
// so metadata.ExtractTags could not see a WAV's ID3 chunk and this // RIFF reader at all, so metadata.ExtractTags cannot see a WAV's ID3
// case read the chunk itself, which is a test of the writer wearing // chunk -- which is why every other test here reads that chunk itself.
// the shape of a round trip. All four go through the scanner now.
func TestWriteTotals_RoundTripsInEveryFormat(t *testing.T) { func TestWriteTotals_RoundTripsInEveryFormat(t *testing.T) {
t.Parallel() t.Parallel()
@@ -92,7 +91,7 @@ func TestWriteTotals_RoundTripsInEveryFormat(t *testing.T) {
}, },
{ {
name: "wav", name: "wav",
read: viaScanner, read: readWavID3Tags,
write: func(t *testing.T, dir string) string { write: func(t *testing.T, dir string) string {
t.Helper() t.Helper()
+105 -11
View File
@@ -8,22 +8,116 @@ import (
"io" "io"
"log/slog" "log/slog"
"os" "os"
"strings"
id3v2 "github.com/bogem/id3v2/v2" id3v2 "github.com/bogem/id3v2/v2"
"yellowjacket/backend/fileutil" "yellowjacket/backend/fileutil"
"yellowjacket/backend/riff"
) )
// errFileTooLargeForWAV is the one RIFF error that belongs to the // Sentinel errors for WAV RIFF operations.
// writer; reading rejects a container in backend/riff. var (
var errFileTooLargeForWAV = errors.New("file too large for WAV format (>4GB)") errRF64NotSupported = errors.New("RF64 files are not yet supported")
errNotRIFF = errors.New("not a RIFF file")
errNotWAVE = errors.New("not a WAVE file")
errFileTooLargeForWAV = errors.New("file too large for WAV format (>4GB)")
)
// riffChunk holds a single RIFF sub-chunk (ID + raw data).
type riffChunk struct {
id [4]byte
data []byte
}
// parseRIFF reads all RIFF sub-chunks from r. It rejects RF64 files
// and non-WAVE containers with descriptive errors. The parser is
// lenient on read: it tolerates missing padding bytes and ignores
// the declared RIFF size.
func parseRIFF(r io.ReadSeeker) ([]riffChunk, error) {
// Read 4-byte container magic.
var magic [4]byte
if _, err := io.ReadFull(r, magic[:]); err != nil {
return nil, fmt.Errorf("read RIFF magic: %w", err)
}
if string(magic[:]) == "RF64" {
return nil, errRF64NotSupported
}
if string(magic[:]) != "RIFF" {
return nil, fmt.Errorf("%w: got %q", errNotRIFF, magic)
}
// Read (and discard) RIFF size — lenient, do not enforce.
var riffSize uint32
if err := binary.Read(r, binary.LittleEndian, &riffSize); err != nil {
return nil, fmt.Errorf("read RIFF size: %w", err)
}
// Read 4-byte form type.
var form [4]byte
if _, err := io.ReadFull(r, form[:]); err != nil {
return nil, fmt.Errorf("read WAVE form type: %w", err)
}
if string(form[:]) != "WAVE" {
return nil, fmt.Errorf("%w: got %q", errNotWAVE, form)
}
// Read sub-chunks until EOF.
var chunks []riffChunk
for {
var chunkID [4]byte
_, err := io.ReadFull(r, chunkID[:])
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
break
}
if err != nil {
return nil, fmt.Errorf("read chunk ID: %w", err)
}
var chunkSize uint32
if err := binary.Read(r, binary.LittleEndian, &chunkSize); err != nil {
return nil, fmt.Errorf("read chunk size for %q: %w", chunkID, err)
}
data := make([]byte, chunkSize)
if _, err := io.ReadFull(r, data); err != nil {
return nil, fmt.Errorf("read chunk data for %q: %w", chunkID, err)
}
chunks = append(chunks, riffChunk{id: chunkID, data: data})
// Odd-length chunks have a padding byte. Lenient: if the
// read fails (e.g. EOF), just break rather than error.
if chunkSize%2 != 0 {
var pad [1]byte
if _, err := r.Read(pad[:]); err != nil {
break
}
}
}
return chunks, nil
}
// isID3ChunkID returns true if id represents an ID3v2 RIFF chunk.
// Both lowercase "id3 " and uppercase "ID3 " are accepted.
func isID3ChunkID(id [4]byte) bool {
s := strings.ToLower(string(id[:3]))
return s == "id3"
}
// writeRIFF writes a complete RIFF/WAVE container to w, preserving // writeRIFF writes a complete RIFF/WAVE container to w, preserving
// the given chunks in order and appending the id3Data as the final // the given chunks in order and appending the id3Data as the final
// "id3 " chunk. Returns errFileTooLargeForWAV if the result would // "id3 " chunk. Returns errFileTooLargeForWAV if the result would
// exceed the 4 GB RIFF limit. // exceed the 4 GB RIFF limit.
func writeRIFF(w io.Writer, chunks []riff.Chunk, id3Data []byte) error { func writeRIFF(w io.Writer, chunks []riffChunk, id3Data []byte) error {
// Calculate total RIFF payload size: // Calculate total RIFF payload size:
// 4 bytes (WAVE form type) // 4 bytes (WAVE form type)
// + for each preserved chunk: 8 (header) + len(data) + padding // + for each preserved chunk: 8 (header) + len(data) + padding
@@ -31,7 +125,7 @@ func writeRIFF(w io.Writer, chunks []riff.Chunk, id3Data []byte) error {
riffPayload := uint64(4) riffPayload := uint64(4)
for _, c := range chunks { for _, c := range chunks {
sz := uint64(len(c.Data)) sz := uint64(len(c.data))
riffPayload += 8 + sz riffPayload += 8 + sz
if sz%2 != 0 { if sz%2 != 0 {
@@ -68,7 +162,7 @@ func writeRIFF(w io.Writer, chunks []riff.Chunk, id3Data []byte) error {
// Write each preserved chunk. // Write each preserved chunk.
for _, c := range chunks { for _, c := range chunks {
if err := writeChunk(w, c.ID, c.Data); err != nil { if err := writeChunk(w, c.id, c.data); err != nil {
return err return err
} }
} }
@@ -131,7 +225,7 @@ func writeWavTags(
return fmt.Errorf("open wav for reading: %w", err) return fmt.Errorf("open wav for reading: %w", err)
} }
allChunks, err := riff.Parse(f) allChunks, err := parseRIFF(f)
// Close immediately — we need the handle released before // Close immediately — we need the handle released before
// AtomicWrite creates the replacement file. // AtomicWrite creates the replacement file.
@@ -143,13 +237,13 @@ func writeWavTags(
// Separate preserved chunks from existing ID3 data. // Separate preserved chunks from existing ID3 data.
var ( var (
preserved []riff.Chunk preserved []riffChunk
existingID3 []byte existingID3 []byte
) )
for _, c := range allChunks { for _, c := range allChunks {
if riff.IsID3(c.ID) { if isID3ChunkID(c.id) {
existingID3 = c.Data existingID3 = c.data
} else { } else {
preserved = append(preserved, c) preserved = append(preserved, c)
} }
+17 -103
View File
@@ -12,7 +12,6 @@ import (
id3v2 "github.com/bogem/id3v2/v2" id3v2 "github.com/bogem/id3v2/v2"
"yellowjacket/backend/metadata" "yellowjacket/backend/metadata"
"yellowjacket/backend/riff"
) )
// createTestWAV builds a minimal valid WAV file with an optional // createTestWAV builds a minimal valid WAV file with an optional
@@ -271,88 +270,6 @@ func TestWriteWavTags_PartialUpdate(t *testing.T) {
assertStrField(t, "Composer", meta.Composer, "Original Composer") assertStrField(t, "Composer", meta.Composer, "Original Composer")
} }
// The writer has always been correct and the reader could not see it:
// a WAV tagged by this app scanned as an untagged file, so editing
// tags, autotagging a folder or importing a WAV download all appeared
// to work and changed nothing the library could show (#104). So this
// asserts the write through metadata.ExtractTags -- the reader the
// scan uses -- rather than through the id3 chunk.
func TestWriteWavTags_ReadBackByTheScanner(t *testing.T) {
t.Parallel()
dir := t.TempDir()
path := createTestWAV(t, dir, "scanner.wav", nil)
art := tinyJPEG(t)
changes := TagChanges{
FieldTitle: "Some Song",
FieldArtist: "Some Artist",
FieldAlbum: "Some Album",
FieldAlbumArtist: "Some Album Artist",
FieldGenre: "Rock",
FieldYear: 2024,
FieldTrackNumber: 3,
FieldComposer: "Some Composer",
FieldCoverArt: art,
}
if err := writeWavTags(testLogger(), path, changes); err != nil {
t.Fatalf("writeWavTags: %v", err)
}
meta, err := metadata.ExtractTags(path)
if err != nil {
t.Fatalf("ExtractTags: %v", err)
}
if meta.TagReadWarning != nil {
t.Errorf("TagReadWarning: %v", meta.TagReadWarning)
}
assertStrField(t, "Title", meta.Title, "Some Song")
assertStrField(t, "Artist", meta.Artist, "Some Artist")
assertStrField(t, "Album", meta.Album, "Some Album")
assertStrField(t, "AlbumArtist", meta.AlbumArtist, "Some Album Artist")
assertStrField(t, "Genre", meta.Genre, "Rock")
assertStrField(t, "Composer", meta.Composer, "Some Composer")
assertStrField(t, "FileFormat", meta.FileFormat, "WAV")
assertIntField(t, "Year", meta.Year, 2024)
assertIntField(t, "TrackNumber", meta.TrackNumber, 3)
if !strings.HasPrefix(meta.TagFormat, "ID3v2") {
t.Errorf("TagFormat: got %q, want an ID3v2 version", meta.TagFormat)
}
if meta.Picture == nil {
t.Fatal("expected cover art, got nil")
}
if !bytes.Equal(meta.Picture.Data, art) {
t.Errorf("picture data mismatch: got %d bytes, want %d",
len(meta.Picture.Data), len(art))
}
}
// An untagged WAV is a file with no tags, not a file with a problem:
// the scanner falls back to the filename and must not be handed a
// warning to surface about it.
func TestUntaggedWav_ReadsAsEmptyWithoutAWarning(t *testing.T) {
t.Parallel()
path := createTestWAV(t, t.TempDir(), "bare.wav", nil)
meta, err := metadata.ExtractTags(path)
if err != nil {
t.Fatalf("ExtractTags: %v", err)
}
if meta.TagReadWarning != nil {
t.Errorf("TagReadWarning: %v", meta.TagReadWarning)
}
assertStrField(t, "Title", meta.Title, "")
}
func TestWriteWavTags_ChunkPreservation(t *testing.T) { func TestWriteWavTags_ChunkPreservation(t *testing.T) {
t.Parallel() t.Parallel()
@@ -365,7 +282,7 @@ func TestWriteWavTags_ChunkPreservation(t *testing.T) {
t.Fatalf("open original: %v", err) t.Fatalf("open original: %v", err)
} }
origChunks, err := riff.Parse(origFile) origChunks, err := parseRIFF(origFile)
_ = origFile.Close() _ = origFile.Close()
if err != nil { if err != nil {
@@ -375,7 +292,7 @@ func TestWriteWavTags_ChunkPreservation(t *testing.T) {
// Record original chunk data by ID string. // Record original chunk data by ID string.
origData := map[string][]byte{} origData := map[string][]byte{}
for _, c := range origChunks { for _, c := range origChunks {
origData[string(c.ID[:])] = c.Data origData[string(c.id[:])] = c.data
} }
// Write a tag to trigger RIFF rewrite. // Write a tag to trigger RIFF rewrite.
@@ -392,7 +309,7 @@ func TestWriteWavTags_ChunkPreservation(t *testing.T) {
t.Fatalf("open after write: %v", err) t.Fatalf("open after write: %v", err)
} }
newChunks, err := riff.Parse(newFile) newChunks, err := parseRIFF(newFile)
_ = newFile.Close() _ = newFile.Close()
if err != nil { if err != nil {
@@ -403,7 +320,7 @@ func TestWriteWavTags_ChunkPreservation(t *testing.T) {
origNonID3 := 0 origNonID3 := 0
for _, c := range origChunks { for _, c := range origChunks {
if !riff.IsID3(c.ID) { if !isID3ChunkID(c.id) {
origNonID3++ origNonID3++
} }
} }
@@ -411,7 +328,7 @@ func TestWriteWavTags_ChunkPreservation(t *testing.T) {
newNonID3 := 0 newNonID3 := 0
for _, c := range newChunks { for _, c := range newChunks {
if !riff.IsID3(c.ID) { if !isID3ChunkID(c.id) {
newNonID3++ newNonID3++
} }
} }
@@ -442,17 +359,17 @@ func TestWriteWavTags_ChunkPreservation(t *testing.T) {
// in chunks and its data matches want byte-for-byte. // in chunks and its data matches want byte-for-byte.
func checkChunkPreserved( func checkChunkPreserved(
t *testing.T, t *testing.T,
chunks []riff.Chunk, chunks []riffChunk,
idStr string, idStr string,
want []byte, want []byte,
) { ) {
t.Helper() t.Helper()
for _, c := range chunks { for _, c := range chunks {
if string(c.ID[:]) == idStr { if string(c.id[:]) == idStr {
if !bytes.Equal(c.Data, want) { if !bytes.Equal(c.data, want) {
t.Errorf("chunk %q data changed: got %d bytes, want %d", t.Errorf("chunk %q data changed: got %d bytes, want %d",
idStr, len(c.Data), len(want)) idStr, len(c.data), len(want))
} }
return return
@@ -513,7 +430,7 @@ func TestWriteWavTags_RejectsRF64(t *testing.T) {
buf.WriteString("WAVE") buf.WriteString("WAVE")
// Minimal ds64 chunk (required for RF64 but we just need // Minimal ds64 chunk (required for RF64 but we just need
// enough bytes for riff.Parse to hit the RF64 rejection). // enough bytes for parseRIFF to hit the RF64 rejection).
buf.WriteString("ds64") buf.WriteString("ds64")
_ = binary.Write(&buf, binary.LittleEndian, uint32(28)) //nolint:mnd _ = binary.Write(&buf, binary.LittleEndian, uint32(28)) //nolint:mnd
buf.Write(make([]byte, 28)) //nolint:mnd buf.Write(make([]byte, 28)) //nolint:mnd
@@ -537,12 +454,9 @@ func TestWriteWavTags_RejectsRF64(t *testing.T) {
// readWavID3Tags extracts ID3v2 metadata from a WAV file by parsing // readWavID3Tags extracts ID3v2 metadata from a WAV file by parsing
// the RIFF structure and reading the id3 chunk with bogem/id3v2. // the RIFF structure and reading the id3 chunk with bogem/id3v2.
// // dhowden/tag's ReadFrom does not support WAV files, and its
// metadata.ExtractTags reads a WAV since #104 and is what the round // ReadID3v2Tags fails on empty tags (after clearing all frames).
// trips assert through. This stays for the two cases that are about // Using bogem/id3v2.ParseReader handles all cases correctly.
// the bytes rather than about the scan: a tag with every frame
// cleared, which no reader reports as anything, and the chunk
// preservation test, which is already parsing the container itself.
func readWavID3Tags( func readWavID3Tags(
t *testing.T, t *testing.T,
path string, path string,
@@ -556,17 +470,17 @@ func readWavID3Tags(
defer func() { _ = f.Close() }() defer func() { _ = f.Close() }()
chunks, err := riff.Parse(f) chunks, err := parseRIFF(f)
if err != nil { if err != nil {
t.Fatalf("riff.Parse: %v", err) t.Fatalf("parseRIFF: %v", err)
} }
// Find the id3 chunk. // Find the id3 chunk.
var id3Data []byte var id3Data []byte
for _, c := range chunks { for _, c := range chunks {
if riff.IsID3(c.ID) { if isID3ChunkID(c.id) {
id3Data = c.Data id3Data = c.data
break break
} }
Binary file not shown.

Before

Width:  |  Height:  |  Size: 83 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 285 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 128 KiB

+4 -59
View File
@@ -42,10 +42,10 @@ const ACTIONS = ['Import', 'New Playlist', 'New Smart Playlist'];
* because the number this issue is about (a button 48px wider than the * because the number this issue is about (a button 48px wider than the
* box holding it) is not in the accessibility tree at all. * box holding it) is not in the accessibility tree at all.
*/ */
const headerFit = (page: import('@playwright/test').Page, view = 'playlist-view') => const headerFit = (page: import('@playwright/test').Page) =>
page.evaluate((tag) => { page.evaluate(() => {
const root = document const root = document
.querySelector(`[data-testid="main-content"] ${tag}`) .querySelector('[data-testid="main-content"] playlist-view')
?.shadowRoot?.querySelector('page-header')?.shadowRoot; ?.shadowRoot?.querySelector('page-header')?.shadowRoot;
if (!root) return null; if (!root) return null;
@@ -76,7 +76,7 @@ const headerFit = (page: import('@playwright/test').Page, view = 'playlist-view'
...root.querySelectorAll('#page-header-overflow wa-dropdown-item'), ...root.querySelectorAll('#page-header-overflow wa-dropdown-item'),
].map((i) => i.textContent?.trim() ?? ''), ].map((i) => i.textContent?.trim() ?? ''),
}; };
}, view); });
test.describe('the page header never clips an action', () => { test.describe('the page header never clips an action', () => {
test.beforeEach(async ({ app }) => { test.beforeEach(async ({ app }) => {
@@ -316,58 +316,3 @@ test.describe('the page header never clips an action', () => {
await expect.poll(async () => (await headerFit(app))?.menu).toEqual([]); await expect.poll(async () => (await headerFit(app))?.menu).toEqual([]);
}); });
}); });
/**
* The Tracks header carries the play-all/shuffle-all pair (#31), so
* the promise above has to hold for it too — the same per-button
* measurement, one view over. Its two actions are the whole of the
* header's declared set, and the pair is what plays the list the row
* is in, so a button rendered 20px of its 90px is a queue of nothing.
*/
const TRACK_ACTIONS = ['Play all', 'Shuffle all'];
test.describe('the Tracks header never clips an action', () => {
test.beforeEach(async ({ app }) => {
await app.getByTestId('nav-tracks').click();
await expect(app.getByTestId('main-content')).toHaveAttribute(
'data-active-view',
'tracks',
);
});
test.afterEach(async ({ app }) => {
await app.setViewportSize({ width: 1280, height: 800 });
});
for (const vp of VIEWPORTS) {
test(`every action is reachable at ${vp.name}`, async ({ app }) => {
await app.setViewportSize({ width: vp.width, height: vp.height });
await expect
.poll(async () => (await headerFit(app, 'track-list'))?.clipped)
.toEqual([]);
const fit = (await headerFit(app, 'track-list'))!;
expect(fit.overflow).toBeLessThanOrEqual(0);
// Between them, buttons and menu account for both actions —
// not "it fits" but "nothing was dropped to make it fit".
expect([...fit.buttons, ...fit.menu].sort()).toEqual(
[...TRACK_ACTIONS].sort(),
);
});
}
/**
* The pair's names, through the accessibility tree — a shadow query
* measures, but it cannot say what a screen reader is offered.
*/
test('both actions are named controls', async ({ app }) => {
for (const label of TRACK_ACTIONS) {
await expect(
app.getByRole('button', { name: label, exact: true }),
).toBeVisible();
}
});
});
-17
View File
@@ -60,23 +60,6 @@ const PHONE = { width: 424, height: 439 };
const DESKTOP = { width: 1100, height: 800 }; const DESKTOP = { width: 1100, height: 800 };
test.describe('background jobs on a phone', () => { test.describe('background jobs on a phone', () => {
/**
* **State a spec stages is the spec's to clear.** `/__test/emit` writes
* to a store nothing resets, so the event that staged a job is the
* event that clears it — `JobStore` replaces its whole list from every
* snapshot, so `testctl` needs no special case.
*
* **Measured on #168: this does not currently outlive the page.** Every
* test gets a fresh page, and `JobStore.init()` refetches `GetJobs()`
* from a backend registry that `/__test/emit` never writes to, so the
* staged job is gone before the next spec starts. Ownership is stated
* rather than a live leak repaired — the leak needs a page that
* survives its own spec, and there is none today.
*/
test.afterEach(async ({ testctl }) => {
await testctl.emit('JobsChanged', []);
});
test('are shown in the band, without opening anything', async ({ test('are shown in the band, without opening anything', async ({
app, app,
testctl, testctl,
-140
View File
@@ -1,140 +0,0 @@
import { test, expect } from '../support/fixtures.js';
/**
* The web view's own tap highlight, and what replaced it (#54).
*
* Two halves, and each is here because no other tier can see it.
*
* **The highlight is killed by one declaration on `html`**, which
* reaches the app's shadow roots because `-webkit-tap-highlight-color`
* is inherited and inheritance crosses a shadow boundary. That is a
* property of `index.css`, and `index.css` is loaded by the real app
* and by nothing else — the component tier mounts a component with no
* page stylesheet at all, which is the same reason the theme's ramps
* are invisible to it.
*
* **The press state is measured rather than read.** The component tier
* asserts the shape of the stylesheet (which rule is inside which
* query, and that the press selector carries a state class), because
* `:active` cannot be forced there. Here there is a real pointer: hold
* the button down on a real row of the real list and read what the row
* became. That is the assertion that would fail if the rule were
* hoisted, renamed, or lost to `.selected`.
*
* What neither half is, is the device. Chrome 113's WebView is where
* the grey box was reported and where a finger is; the numbers from it
* are on the PR.
*/
type Page = import('@playwright/test').Page;
/** The phone this work was measured against, in CSS pixels. */
const DEVICE = { width: 424, height: 439 };
/** The computed tap-highlight colour of a node inside a shadow root. */
const tapHighlight = (page: Page, host: string, inner: string) =>
page.evaluate(
([hostSel, innerSel]) => {
const el = document
.querySelector(hostSel!)
?.shadowRoot?.querySelector(innerSel!);
if (!el) return null;
return getComputedStyle(el).getPropertyValue(
'-webkit-tap-highlight-color',
);
},
[host, inner],
);
test.describe('the tap highlight', () => {
test('is transparent inside a shadow root, from one rule on html', async ({
app,
browserName,
}) => {
await app.getByTestId('nav-tracks').click();
await expect(app.getByTestId('main-content')).toHaveAttribute(
'data-active-view',
'tracks',
);
const row = await tapHighlight(app, 'track-list', '.track-row');
expect(row).not.toBeNull();
// The property is a WebKit extension that only iOS honours, so an
// engine is free not to report one at all. Chromium always does —
// measured at rgba(0, 0, 0, 0.18) with the rule removed, which is
// the grey box the report describes — so the assertion is not
// skippable there, and nothing this app can do makes the property
// disappear on an engine that has it.
test.skip(
row === '',
`${browserName} reports no -webkit-tap-highlight-color to read`,
);
expect(row).toBe('rgba(0, 0, 0, 0)');
});
});
test.describe('the press state that replaced it', () => {
test.beforeEach(async ({ app }) => {
await app.setViewportSize(DEVICE);
await app.getByTestId('tab-tracks').click();
await expect(app.getByTestId('main-content')).toHaveAttribute(
'data-active-view',
'tracks',
);
await expect(app.locator('track-list').first()).toBeVisible();
});
test.afterEach(async ({ app }) => {
await app.mouse.up();
await app.setViewportSize({ width: 1440, height: 900 });
});
test('shows on the row being pressed, and on that row only', async ({
app,
}) => {
const rows = await app.evaluate(() => {
const found = document
.querySelector('track-list')
?.shadowRoot?.querySelectorAll('.track-row');
if (!found || found.length < 2) return null;
const rect = found[1]!.getBoundingClientRect();
return {
x: Math.round(rect.x + rect.width / 2),
y: Math.round(rect.y + rect.height / 2),
};
});
expect(rows).not.toBeNull();
const backgrounds = () =>
app.evaluate(() => {
const found = document
.querySelector('track-list')!
.shadowRoot!.querySelectorAll('.track-row');
return {
pressed: getComputedStyle(found[1]!).backgroundColor,
neighbour: getComputedStyle(found[2]!).backgroundColor,
};
});
await app.mouse.move(rows!.x, rows!.y);
await app.mouse.down();
const held = await backgrounds();
// The press overlay, from the theme rather than from a literal in
// a component: rgba(255, 255, 255, 0.12) on both dark ramps.
expect(held.pressed).toBe('rgba(255, 255, 255, 0.12)');
expect(held.neighbour).not.toBe(held.pressed);
await app.mouse.up();
});
});
-135
View File
@@ -1,135 +0,0 @@
import {
test,
expect,
callBinding,
openTheQueue,
NO_QUEUE_SOURCE,
} from '../support/fixtures.js';
import type { Page } from '@playwright/test';
/**
* #67 — a name is not a link on a phone, and the menu is where it went.
*
* The queue panel is the surface this is visible on: its rows draw a
* track title and an artist credit as `explore-link`s at every width,
* unlike `track-list`, whose phone column set stacks title over artist
* as plain text already.
*
* **The pair is what makes either assertion mean anything.** A link
* that is gone and a menu item that never arrived is not a smaller
* affordance — it is a destination the phone cannot reach, which is
* what plan 018's "no action is unreachable at any supported size"
* refuses. So each test asserts the phone and the desktop in the same
* breath: text *and* an item here, a link *and* no item there.
*
* The desktop half is also the regression guard for the change: menus
* above the breakpoint must be exactly what they were, because the name
* beside them is still a link and a menu that repeats the row is
* furniture.
*/
/** The reference device's real viewport, not a resized desktop. */
const DEVICE = { width: 424, height: 439 };
/** Wide enough that the queue is a column beside the content. */
const DESKTOP = { width: 1280, height: 800 };
const row = (app: Page, index: number) =>
app.locator(`queue-panel .track-item[data-index="${index}"]`);
/** The queue panel's own context menu, as a list of item labels. */
async function menuLabels(app: Page): Promise<string[]> {
return app.evaluate(() =>
[
...document
.querySelector('queue-panel')!
.shadowRoot!.querySelectorAll('wa-dropdown-item'),
].map((item) => item.textContent?.replace(/\s+/g, ' ').trim() ?? ''),
);
}
/**
* Queue three tracks that have an album, for the reason
* `queue-selection.spec.ts` states at length: `explore-link` routes a
* title to its *album's* page and renders plain text where it cannot
* route, so a track with no album answers this file's question with
* the wrong "no link".
*/
async function queueThree(app: Page): Promise<void> {
const paths = await app.evaluate(async () => {
const tracks = (await window.__yjEvents.call(
'library.Library.GetTracks',
[0],
10_000,
)) as { FilePath: string; Album: string; ArtistName: string }[];
return tracks
.filter((t) => t.Album !== '' && t.ArtistName !== '')
.slice(0, 3)
.map((t) => t.FilePath);
});
await callBinding(app, 'queue.Queue.SetQueue', [
paths,
0,
false,
NO_QUEUE_SOURCE,
]);
}
/** Open the row's context menu and read the items back. */
async function openRowMenu(app: Page, index: number): Promise<string[]> {
await row(app, index).click({ button: 'right' });
await expect
.poll(async () => (await menuLabels(app)).length)
.toBeGreaterThan(0);
return menuLabels(app);
}
test.describe('an inline name and the menu that replaces it', () => {
test.afterEach(async ({ app }) => {
await app.keyboard.press('Escape');
await callBinding(app, 'queue.Queue.Clear').catch(() => {
/* an empty queue is the state we were asking for */
});
await app.setViewportSize(DESKTOP);
});
test('a queue row is plain text on a phone and carries the destination', async ({
app,
}) => {
await app.setViewportSize(DEVICE);
await queueThree(app);
await openTheQueue(app);
await expect(row(app, 0)).toBeVisible();
// The name is text: nothing in the row is a link at all.
await expect(app.locator('queue-panel .track-item .explore-link')).toHaveCount(
0,
);
const labels = await openRowMenu(app, 0);
expect(labels).toContain('Go to Artist');
expect(labels).toContain('Go to Album');
});
test('the same row on a desktop is a link, and its menu is untouched', async ({
app,
}) => {
await app.setViewportSize(DESKTOP);
await queueThree(app);
await openTheQueue(app);
await expect(row(app, 0)).toBeVisible();
await expect(
row(app, 0).locator('.track-title .explore-link'),
).toHaveCount(1);
const labels = await openRowMenu(app, 0);
expect(labels).not.toContain('Go to Artist');
expect(labels).not.toContain('Go to Album');
});
});
+2 -2
View File
@@ -171,8 +171,8 @@ test.describe('search on a phone', () => {
// Attached, not visible: `wa-dialog`'s host is `display: contents`, // Attached, not visible: `wa-dialog`'s host is `display: contents`,
// so the element carrying the testid always reports hidden — what // so the element carrying the testid always reports hidden — what
// is visible is the native `<dialog>` inside it. That awkwardness // is visible is the native `<dialog>` inside it. That awkwardness
// is why the assertion that this is really up is the role query // is written down in CLAUDE.md and is why the assertion that this
// below. // is really up is the role query below.
await expect(dialog).toBeAttached(); await expect(dialog).toBeAttached();
// Named, which `getByRole` can answer and the a11y snapshot cannot // Named, which `getByRole` can answer and the a11y snapshot cannot
-220
View File
@@ -1,220 +0,0 @@
import { test, expect, callBinding, resetEvents, waitForEvent } from '../support/fixtures.js';
type Page = import('@playwright/test').Page;
/**
* Play-all/Shuffle-all, asserted on what the backend queued rather than
* on playback pixels.
*
* `SetQueue` reports the queue through `QueueChanged`, and `GetState`
* says exactly what it holds: the tracks in order, whether shuffle is
* on, and the `Source` the "Playing from" link is built from. That is
* the honest contract here — the buttons are only as good as the queue
* they build, and the queue is only as good as the source it names.
*/
interface QueueState {
tracks: { filePath: string; title: string }[];
currentIndex: number;
shuffleMode: boolean;
source: { type: string; id: number; label: string };
}
const TRACKS_SOURCE = { type: 'tracks', id: 0, label: 'All Tracks' };
const getQueue = (app: Page) =>
callBinding<QueueState>(app, 'queue.Queue.GetState');
/** The track paths a rendered track list shows, in row order. */
function displayedPaths(app: Page, scope: string): Promise<string[]> {
return app
.locator(`${scope} [data-testid="track-row"]`)
.evaluateAll((els) =>
els.map((el) => el.getAttribute('data-file-path') ?? ''),
);
}
/** Leave shuffle in a known state. The mode persists across specs in
* one backend process, so a test that asserts on it has to set it. */
async function setShuffleMode(app: Page, on: boolean): Promise<void> {
const state = await getQueue(app);
if (state.shuffleMode !== on) {
await resetEvents(app);
await callBinding(app, 'queue.Queue.ToggleShuffle');
await waitForEvent(app, 'QueueModeChanged');
}
}
test.describe('play-all/shuffle-all on the track list', () => {
test.beforeEach(async ({ app }) => {
await callBinding(app, 'queue.Queue.Clear').catch(() => {
/* the queue is clearable on every build these specs run against */
});
await setShuffleMode(app, false);
});
test('Tracks Play all queues the displayed list with an honest source', async ({
app,
}) => {
await app.getByTestId('nav-tracks').click();
await expect(app.getByTestId('main-content')).toHaveAttribute(
'data-active-view',
'tracks',
);
await expect(
app.locator('track-list [data-testid="track-row"]').first(),
).toBeVisible();
const paths = await displayedPaths(app, 'track-list');
await resetEvents(app);
await app.getByTestId('page-action-play-all').click();
await waitForEvent(app, 'QueueChanged');
const state = await getQueue(app);
expect(state.tracks.map((t) => t.filePath)).toEqual(paths);
expect(state.currentIndex).toBe(0);
expect(state.shuffleMode).toBe(false);
expect(state.source).toEqual(TRACKS_SOURCE);
});
test('Tracks Shuffle all turns shuffle on and keeps the source', async ({
app,
}) => {
await app.getByTestId('nav-tracks').click();
await expect(
app.locator('track-list [data-testid="track-row"]').first(),
).toBeVisible();
const paths = await displayedPaths(app, 'track-list');
await resetEvents(app);
await app.getByTestId('page-action-shuffle-all').click();
await waitForEvent(app, 'QueueChanged');
const state = await getQueue(app);
expect(state.tracks.map((t) => t.filePath)).toEqual(paths);
expect(state.shuffleMode).toBe(true);
expect(state.source).toEqual(TRACKS_SOURCE);
});
});
test.describe('play-all on an embedded track list', () => {
test.beforeEach(async ({ app }) => {
await callBinding(app, 'queue.Queue.Clear').catch(() => {});
await setShuffleMode(app, false);
});
test('a genre page queues the genre with its name as the source', async ({
app,
}) => {
await app.getByTestId('nav-genres').click();
await expect(app.getByTestId('main-content')).toHaveAttribute(
'data-active-view',
'genres',
);
const first = app.locator('genres-view .genre-card').first();
await expect(first).toBeVisible();
await first.click();
await expect(app.getByTestId('main-content')).toHaveAttribute(
'data-active-view',
'genre-details',
);
await expect(
app.locator('genre-details [data-testid="track-row"]').first(),
).toBeVisible();
const genreName = (await app
.locator('genre-details .genre-title')
.textContent())?.trim();
const paths = await displayedPaths(app, 'genre-details');
await resetEvents(app);
await app
.locator('genre-details [data-testid="page-action-play-all"]')
.click();
await waitForEvent(app, 'QueueChanged');
const state = await getQueue(app);
expect(state.tracks.map((t) => t.filePath)).toEqual(paths);
expect(state.currentIndex).toBe(0);
expect(state.source).toEqual({ type: 'genre', id: 0, label: genreName });
});
});
test.describe('play-all on the library artist page', () => {
test.beforeEach(async ({ app }) => {
await callBinding(app, 'queue.Queue.Clear').catch(() => {});
await setShuffleMode(app, false);
});
test('an artist page queues album paths in album order with the artist source', async ({
app,
}) => {
const artists = await callBinding<{ ID: number; Name: string }[]>(
app,
'library.Library.GetArtists',
[0],
);
const first = artists[0]!;
await app.evaluate(
([id, name]) => {
document.dispatchEvent(
new CustomEvent('navigate', {
detail: {
view: 'artist-details',
artistId: id,
artistName: name,
},
bubbles: true,
composed: true,
}),
);
},
[first.ID, first.Name] as const,
);
await expect(app.getByTestId('main-content')).toHaveAttribute(
'data-active-view',
'artist-details',
);
await expect(app.getByTestId('artist-play-all')).toBeEnabled();
const albums = await callBinding<{ ID: number }[]>(
app,
'library.Library.GetAlbumsByArtist',
[first.Name, 0],
);
const byAlbum = await callBinding<Record<string, string[]>>(
app,
'library.Library.GetFilePathsByAlbums',
[albums.map((a) => a.ID), 0],
);
const expected: string[] = [];
for (const album of albums) {
expected.push(...(byAlbum[String(album.ID)] ?? []));
}
await resetEvents(app);
await app.getByTestId('artist-play-all').click();
await waitForEvent(app, 'QueueChanged');
const state = await getQueue(app);
expect(state.tracks.map((t) => t.filePath)).toEqual(expected);
expect(state.source).toEqual({
type: 'artist',
id: first.ID,
label: first.Name,
});
});
});
+1 -1
View File
@@ -137,7 +137,7 @@ test.describe('queue', () => {
}); });
test('shuffle and repeat toggles report their state', async ({ app }) => { test('shuffle and repeat toggles report their state', async ({ app }) => {
const shuffle = app.getByRole('button', { name: 'Shuffle', exact: true }); const shuffle = app.getByRole('button', { name: 'Shuffle' });
await resetEvents(app); await resetEvents(app);
await shuffle.click(); await shuffle.click();

Some files were not shown because too many files have changed in this diff Show More