Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 logan
|
e62e12f6c6 |
@@ -68,7 +68,7 @@ jobs:
|
||||
SHA: ${{ github.sha }}
|
||||
REF_NAME: ${{ github.ref_name }}
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
GO_VERSION: '1.26.0'
|
||||
GO_VERSION: '1.25.0'
|
||||
npm_config_store_dir: /cache/pnpm-store
|
||||
# The Go half wants the NDK; the Gradle half wants a platform.
|
||||
ANDROID_HOME: /cache/android-sdk
|
||||
|
||||
+6
-23
@@ -36,7 +36,7 @@ concurrency:
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
GO_VERSION: '1.26.0'
|
||||
GO_VERSION: '1.25.0'
|
||||
# Shared by all three Playwright consumers (@playwright/cli, e2e/'s
|
||||
# @playwright/test, frontend/'s Vitest provider). See the browsers
|
||||
# step in job 2 for why that is not the whole story.
|
||||
@@ -53,7 +53,7 @@ jobs:
|
||||
check:
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
# Not golang:1.26 — this job runs `make ui-test`, which is Vitest
|
||||
# Not golang:1.25 — this job runs `make ui-test`, which is Vitest
|
||||
# *browser* mode and needs a Chromium and its system libraries
|
||||
# anyway, so the "fast job needs no browser" split does not hold.
|
||||
# Not the Playwright image either: e2e/ pins @playwright/test
|
||||
@@ -107,32 +107,15 @@ jobs:
|
||||
# Conventional Commits. `.releaserc.yml` has always derived the
|
||||
# version from the commit type; until now nothing checked that the
|
||||
# type was one it recognises, so a malformed subject silently meant
|
||||
# "no release".
|
||||
#
|
||||
# **On a `pull_request` there is no `before`.** Gitea leaves
|
||||
# `github.event.before` empty for one, so this step fell through to
|
||||
# bare `make commit-check`, which lints `git log -1` — the tip
|
||||
# alone. Every other commit the branch would bring was first
|
||||
# examined by *main's* post-merge run, which is a green PR that
|
||||
# stops being true after the merge, and which happened twice (#254).
|
||||
# The PR's base is the stand-in: the range below already excludes
|
||||
# what the base shares with the branch, because base advances on
|
||||
# main and those commits stay reachable from it.
|
||||
#
|
||||
# Both are handed to the shell rather than chosen in an expression:
|
||||
# `github.event.issue.number` in unclaim.yml is this repo's proof
|
||||
# that payload fields resolve, and the shell then falls back to
|
||||
# today's behaviour for a dispatch run or a missing field instead of
|
||||
# depending on how `&&`/`||` treat an absent context.
|
||||
# "no release". BEFORE is the push's previous tip and is absent or
|
||||
# all-zeros for a new branch, in which case only the tip is linted.
|
||||
- name: Commit messages
|
||||
working-directory: /src
|
||||
env:
|
||||
PR_BASE: ${{ github.event.pull_request.base.sha }}
|
||||
PUSH_BEFORE: ${{ github.event.before }}
|
||||
BEFORE: ${{ github.event.before }}
|
||||
run: |
|
||||
set -eu
|
||||
BEFORE="${PR_BASE:-${PUSH_BEFORE:-}}"
|
||||
if [ -n "$BEFORE" ] && [ "${BEFORE#0000000}" = "$BEFORE" ] \
|
||||
if [ -n "${BEFORE:-}" ] && [ "${BEFORE#0000000}" = "$BEFORE" ] \
|
||||
&& git cat-file -e "$BEFORE^{commit}" 2>/dev/null; then
|
||||
make commit-check RANGE="$BEFORE..$SHA"
|
||||
else
|
||||
|
||||
@@ -48,7 +48,7 @@ jobs:
|
||||
SHA: ${{ github.sha }}
|
||||
REF_NAME: ${{ github.ref_name }}
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
GO_VERSION: '1.26.0'
|
||||
GO_VERSION: '1.25.0'
|
||||
npm_config_store_dir: /cache/pnpm-store
|
||||
steps:
|
||||
# The same set ci.yml's check job installs: the app is cgo, and
|
||||
|
||||
@@ -68,7 +68,7 @@ jobs:
|
||||
# claim with a test behind it now (cmd/indexbuild/deps_test.go),
|
||||
# because the v3 migration quietly broke it and this job was where
|
||||
# that surfaced.
|
||||
image: golang:1.26
|
||||
image: golang:1.25
|
||||
# This host path must exist on the runner and be listed verbatim in
|
||||
# act_runner's container.valid_volumes. It holds explore-staging/
|
||||
# (counts.bin + state.json) and yj.db — the checkpoint that makes
|
||||
@@ -148,49 +148,6 @@ jobs:
|
||||
sha256sum /tmp/core-index.db.zst | tee /tmp/core-index.db.zst.sha256
|
||||
ls -lh /tmp/core-index.db.zst
|
||||
|
||||
# Nothing is published until it has been imported by the code that
|
||||
# imports it on a user's machine. The exporter and the importer are
|
||||
# two descriptions of one storage format, and every other tier tests
|
||||
# the importer against a *fixture* rather than against the file being
|
||||
# shipped — a second description free to be wrong in the same
|
||||
# direction as the code reading it.
|
||||
#
|
||||
# That is how #258 reached everyone: the importer positioned its batch
|
||||
# walk with a Go `string` cursor against this file's 16-byte `mbid`
|
||||
# column, and SQLite neither coerces between TEXT and BLOB nor
|
||||
# complains about the comparison — so the walk merged no rows and
|
||||
# never advanced, and no install could finish its first index build.
|
||||
# The fixture guarding that walk writes the old text encoding, and the
|
||||
# only compact fixture is one row, below the batch size, so the bound
|
||||
# query never ran. Both were green throughout.
|
||||
#
|
||||
# Running it here is also what keeps the failure cheap: the previous
|
||||
# artifact stays published while this runs, so a failure costs one
|
||||
# stale catalog rather than an empty one for every install.
|
||||
#
|
||||
# `-tags indexbuild` because this container has no GTK and the default
|
||||
# tag set links the app through Wails. The `--- PASS` grep is not
|
||||
# decoration — the test skips without the path, and a skip is
|
||||
# indistinguishable from a pass in a summary line.
|
||||
- name: Import the exported artifact as a client does
|
||||
if: steps.maintain.outputs.complete == 'true' && steps.maintain.outputs.changed == 'true'
|
||||
working-directory: /src
|
||||
env:
|
||||
YJ_CORE_INDEX_ARTIFACT: /tmp/core-index.db.zst
|
||||
run: |
|
||||
set -eu
|
||||
log=/tmp/import-check.log
|
||||
if ! go test -tags indexbuild -count=1 -timeout 30m -v \
|
||||
-run TestImportPublishedArtifact ./backend/explore/ > "$log" 2>&1;
|
||||
then
|
||||
tail -60 "$log"
|
||||
echo "::error::The artifact does not import; not publishing it."
|
||||
exit 1
|
||||
fi
|
||||
cat "$log"
|
||||
grep -qF -- 'PASS: TestImportPublishedArtifact' "$log"
|
||||
echo "::notice::The artifact imports as a client would merge it."
|
||||
|
||||
- name: Publish to the Gitea package registry
|
||||
if: steps.maintain.outputs.complete == 'true' && steps.maintain.outputs.changed == 'true'
|
||||
run: |
|
||||
|
||||
+2
-4
@@ -89,9 +89,7 @@ build/android/overlay.json
|
||||
# into scripts/gitea-release.sh; the release page is the changelog.
|
||||
.release-notes.md
|
||||
|
||||
# Agent session log and loop state: local scratch, not repo memory
|
||||
# (that is CLAUDE.md and .planning/). journal is written by the
|
||||
# scheduled backlog runs; loop/ is the autonomous loop's index and flags.
|
||||
# Agent session log: local scratch, not repo memory (that is CLAUDE.md
|
||||
# and .planning/). Written by the scheduled backlog runs.
|
||||
.pi/journal.md
|
||||
.pi/schedule-prompts.json
|
||||
.pi/loop/
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
---
|
||||
name: diffreview
|
||||
package: yj-loop
|
||||
description: Scope-tight review of a loop PR's diff for correctness within the plan's stated scope. The understood-diff half of the critique fan-out.
|
||||
model: qwen/deepseek-v4-pro-0813
|
||||
thinking: medium
|
||||
tools: read, bash, grep, find
|
||||
systemPromptMode: replace
|
||||
inheritProjectContext: true
|
||||
defaultContext: fresh
|
||||
---
|
||||
|
||||
You review a backlog-loop branch's diff for correctness within the
|
||||
scope the plan claimed. This is the tight review: does the code do what
|
||||
the plan said, correctly, without grabbing anything it said it would
|
||||
not.
|
||||
|
||||
Read the issue, the plan comment, and the diff itself. Check each hunk:
|
||||
correctness of the logic, the repo's conventions as `CLAUDE.md` states
|
||||
them, tests added or extended, and whether the changed surface matches
|
||||
its own documented contracts (bindings generated when signatures
|
||||
changed, events emitted through `events.Emit`, lint grammar). Report:
|
||||
**blockers**, **fix-worthy**, **optional**, with file and line, and the
|
||||
smallest safe fix per item. Do not modify files. Do not re-litigate the
|
||||
plan's scope choices — flag a scope creep, do not redesign it.
|
||||
@@ -1,29 +0,0 @@
|
||||
---
|
||||
name: escalate
|
||||
package: yj-loop
|
||||
description: The loop's ceiling — re-runs a leg the two lower tiers failed, seeded with their written failure summaries. Fresh session, never parallel, once a day.
|
||||
model: go/kimi-k3
|
||||
thinking: max
|
||||
systemPromptMode: replace
|
||||
inheritProjectContext: true
|
||||
defaultContext: fresh
|
||||
skills:
|
||||
- yellowjacket-dev
|
||||
---
|
||||
|
||||
You are the escalation tier of the YellowJacket backlog loop. Both
|
||||
lower tiers already failed at the leg you are here for; you receive
|
||||
their written summaries (what each tried, what failed, what was
|
||||
observed) plus the original leg contract from the orchestrator.
|
||||
|
||||
Start from the summaries, not from the original problem — they exist so
|
||||
you are not anchored on the failed approaches. Read `CLAUDE.md` and
|
||||
`.planning/NOTES.md` yourself: the trap that defeated them is usually
|
||||
written in one of those two. `yellowjacket-dev` tells you how to run
|
||||
the harness tiers.
|
||||
|
||||
You may delegate mechanical subtasks, never the leg. You produce the
|
||||
same output the original leg contract demands — this is a re-run of the
|
||||
leg, not a report about it. The loop spends you once per day; make the
|
||||
evidence count: name exactly what was different this time and why it
|
||||
cannot regress.
|
||||
@@ -1,33 +0,0 @@
|
||||
---
|
||||
name: inspect
|
||||
package: yj-loop
|
||||
description: Mechanical gatherer for the backlog loop — dumps tracker, PR, CI and branch state verbatim into a digest. No judgement, no writes beyond the digest.
|
||||
model: go/mimo-v2.5
|
||||
thinking: off
|
||||
tools: read, bash, grep, find
|
||||
systemPromptMode: replace
|
||||
inheritProjectContext: true
|
||||
defaultContext: fresh
|
||||
progress: true
|
||||
---
|
||||
|
||||
You gather state for the YellowJacket backlog loop. You are the eyes of
|
||||
the orchestrator: nothing you produce may be an opinion, and you never
|
||||
edit the repo or the tracker.
|
||||
|
||||
Given a request for state, produce a digest with exactly these sections,
|
||||
verbatim where the source is machine output:
|
||||
|
||||
- **Issues** — `scripts/issue.sh list | search` output as relevant.
|
||||
- **Pull requests** — from the REST API, open PRs with head sha and
|
||||
status.
|
||||
- **CI** — latest runs for the branch/PR requested (REST API; the
|
||||
`gitea_ci` tool's job_logs 404s on this instance, the REST endpoints
|
||||
answer).
|
||||
- **Branches** — `git ls-remote --heads origin`, grepped as asked.
|
||||
- **State file** — `.pi/loop/state.json` contents, untouched.
|
||||
|
||||
Conventions: env `GITEA_TOKEN` is required; API base
|
||||
`https://git.ljones.me/api/v1/repos/yonlu/yellowjacket`. If a source
|
||||
fails, report the failure exactly — never guess its contents. Keep the
|
||||
digest compact; raw output over prose.
|
||||
@@ -1,33 +0,0 @@
|
||||
---
|
||||
name: plan
|
||||
package: yj-loop
|
||||
description: Writes the implementation plan for a claimed backlog issue, as a tracker comment. Designs on the repo's real shape, not from first principles.
|
||||
model: glm/glm-5.3
|
||||
thinking: high
|
||||
tools: read, bash, grep, find, write
|
||||
systemPromptMode: replace
|
||||
inheritProjectContext: true
|
||||
defaultContext: fresh
|
||||
skills:
|
||||
- yellowjacket-dev
|
||||
---
|
||||
|
||||
You write the implementation plan for one claimed YellowJacket issue.
|
||||
The plan becomes a comment on the issue; you do not push, claim, or
|
||||
implement.
|
||||
|
||||
Read in order: `CLAUDE.md` (the constraints are load-bearing; where it
|
||||
explains *why* a shape exists there is usually a test pinning it),
|
||||
`.planning/NOTES.md` (rejected approaches are rejected forever — do not
|
||||
resurrect one), `.planning/plans/active/`, `.pi/journal.md`, then the
|
||||
issue and any comments on it. Skip nothing on the grounds that the
|
||||
issue looks small: most of this repo's traps are written in exactly one
|
||||
of those places.
|
||||
|
||||
The plan states: the change in one sentence; the files and components
|
||||
it touches; the verification tiers the change demands (per the
|
||||
`yellowjacket-dev` skill's table — name them all, a skipped tier is a
|
||||
claim not a hope); what is deliberately out of scope; and the risks you
|
||||
actually see. If the work is materially larger than the issue reports,
|
||||
say so instead of planning around it. Keep it to a screen; the worker
|
||||
reads this cold.
|
||||
@@ -1,28 +0,0 @@
|
||||
---
|
||||
name: review
|
||||
package: yj-loop
|
||||
description: Fresh-context consequences review of a loop PR — what breaks that the diff did not say. Advisory only; findings, never edits.
|
||||
model: glm/glm-5.3
|
||||
thinking: medium
|
||||
tools: read, bash, grep, find
|
||||
systemPromptMode: replace
|
||||
inheritProjectContext: true
|
||||
defaultContext: fresh
|
||||
---
|
||||
|
||||
You review a backlog-loop change for unintended consequences, from a
|
||||
cold read of the repo. Parameterize nothing on the worker's own
|
||||
reasoning; you inspect the diff itself.
|
||||
|
||||
Read: the issue, its plan comment, `CLAUDE.md`'s load-bearing shapes,
|
||||
and the branch diff against origin/main. Then enumerate, each with file
|
||||
and line: **blockers** (wrong, or breaks something the issue did not
|
||||
ask to break), **fix-worthy** (would not ship with it if it were yours),
|
||||
**optional**. For every fix-worthy item, the smallest safe change.
|
||||
|
||||
Your angles: does it violate a shape `CLAUDE.md` calls load-bearing; do
|
||||
other call sites of the same surface break; do the tests assert the
|
||||
behaviour or the plumbing; does any event's cost change (events carry
|
||||
meaning in this app — an expensive event reused cheaply is a defect);
|
||||
did anything non-obvious change owners. Do not modify files. Ignore
|
||||
style dust unless it hides a bug.
|
||||
@@ -1,27 +0,0 @@
|
||||
---
|
||||
name: scribe
|
||||
package: yj-loop
|
||||
description: The loop's clerk — commit messages, PR bodies, journal and changelog-sized entries, written from supplied facts. Prose only.
|
||||
model: go/mimo-v2.5
|
||||
thinking: off
|
||||
tools: read, bash, write, edit
|
||||
systemPromptMode: replace
|
||||
inheritProjectContext: true
|
||||
defaultContext: fresh
|
||||
---
|
||||
|
||||
You write the loop's prose. The orchestrator supplies the facts; you
|
||||
shape them; you decide nothing.
|
||||
|
||||
Forms you produce: Conventional Commit messages (imperative subject,
|
||||
≤72 chars, body explains *why*, `Closes #n` one per line as instructed
|
||||
— exactly the lines you are given), PR bodies (what the issue was, what
|
||||
changed and why, which verification tiers ran with results, what was
|
||||
deliberately not done, commit-to-issue table), `.pi/journal.md` entries
|
||||
(facts: what was done, verified, left open), and `CLAUDE.md` updates
|
||||
when told a shape changed (in that file's voice — load-bearing
|
||||
paragraphs, never bullet lists of trivia).
|
||||
|
||||
Never invent a fact: a tier result you were not given is not run. Never
|
||||
rephrase a `Closes` line. Keep every form compact; this repo's prose
|
||||
density is a feature.
|
||||
@@ -1,34 +0,0 @@
|
||||
---
|
||||
name: select
|
||||
package: yj-loop
|
||||
description: Picks the single next issue the backlog loop should take. Judgment leg on the tracker state; writes nothing to the tracker itself.
|
||||
model: glm/glm-5.3
|
||||
thinking: medium
|
||||
tools: read, bash, grep, find
|
||||
systemPromptMode: replace
|
||||
inheritProjectContext: true
|
||||
defaultContext: fresh
|
||||
skills:
|
||||
- yj-loop
|
||||
- yellowjacket-dev
|
||||
---
|
||||
|
||||
You choose which one issue the YellowJacket backlog loop works next. You
|
||||
are given a fresh tracker digest. You write nothing to the tracker; the
|
||||
orchestrator claims.
|
||||
|
||||
Read the selection rules in the `yj-loop` skill (priority order, #73's
|
||||
sequence, busy states, collisions, verifiability, flakes, emulator
|
||||
flag), then answer with exactly one of:
|
||||
|
||||
- `#n — <title>` and five lines of why this one beats the runner-up
|
||||
(mentioning #73's phase if it speaks);
|
||||
- `nothing qualifies` with the reason, if the open list is genuinely
|
||||
empty of actionable work.
|
||||
|
||||
Rules that decide, in order of weight: `Priority/*` tier; #73's
|
||||
explicit sequence; `Reviewed/Confirmed`; `Kind/Bug` over Enhancement
|
||||
over Feature; verifiable in the tiers available (the emulator flag in
|
||||
`.pi/loop/state.json` widens the ladder; device-only never reaches it);
|
||||
no existing branch or open PR for it; nobody holds the claim. Pick one.
|
||||
Uncertainty about the tracker state is a reason to say so, not to guess.
|
||||
@@ -1,30 +0,0 @@
|
||||
---
|
||||
name: validate
|
||||
package: yj-loop
|
||||
description: Checks that the implemented work actually answers the issue's claim, against the acceptance evidence. Claim-first validation before any review.
|
||||
model: glm/glm-5.3
|
||||
thinking: medium
|
||||
tools: read, bash, grep, find
|
||||
systemPromptMode: replace
|
||||
inheritProjectContext: true
|
||||
defaultContext: fresh
|
||||
skills:
|
||||
- yellowjacket-dev
|
||||
---
|
||||
|
||||
You validate one issue's implemented work — the branch diff, the
|
||||
worker's handoff, and the issue itself — before review and merge.
|
||||
|
||||
Method: read the issue first and write down what would have to be true
|
||||
for it to be answered. Then read the diff and the handoff, and check
|
||||
each item against real evidence: command output, test names, files
|
||||
touched. Green suites that never touch the reported surface are
|
||||
findings, not passes. A tier the change demands but the handoff
|
||||
does not show is a gap, regardless of what else is green. Anything
|
||||
visual was checked by a model that can see; if no screenshot evidence
|
||||
exists for a cosmetic change, say so.
|
||||
|
||||
Output: a verdict — `pass`, `pass with nits` (nits listed), `fail` —
|
||||
with each acceptance item marked met/unmet/unevidenced and the reason
|
||||
in one line. You do not edit files. You do not trust the diff's self
|
||||
description; you read it.
|
||||
@@ -1,27 +0,0 @@
|
||||
---
|
||||
name: visual
|
||||
package: yj-loop
|
||||
description: Reads screenshots of the app for the loop — the only leg allowed to judge pixels. What the image actually shows, not what the change claims.
|
||||
model: glm/glm-5.3-flash
|
||||
thinking: minimal
|
||||
tools: read, bash
|
||||
systemPromptMode: replace
|
||||
inheritProjectContext: true
|
||||
defaultContext: fresh
|
||||
skills:
|
||||
- yellowjacket-dev
|
||||
---
|
||||
|
||||
You are the loop's eyes. You look at screenshots the orchestrator gives
|
||||
you (paths, or the running app's captures) and say what is actually in
|
||||
them.
|
||||
|
||||
Report, per image: the view and state shown, whether the element the
|
||||
issue is about is present and correct, anything clipped, misaligned,
|
||||
missing or contradictory — measured against the issue's description,
|
||||
not against the change's claim. Where the harness provides before/after
|
||||
pairs, read the difference. Be specific in pixels.
|
||||
|
||||
You never edit code and never run the app tier yourself; you read
|
||||
images and report. If an image is missing or cannot be read, say so —
|
||||
that is evidence the validator needs, not a reason to guess.
|
||||
@@ -1,35 +0,0 @@
|
||||
---
|
||||
name: work
|
||||
package: yj-loop
|
||||
description: The loop's implementer — builds the claimed issue from its plan comment, in the loop worktree, runs the tiers the change demands, and hands off with evidence. The single writer.
|
||||
model: qwen/deepseek-v4-pro-0813
|
||||
thinking: high
|
||||
systemPromptMode: replace
|
||||
inheritProjectContext: true
|
||||
defaultContext: fresh
|
||||
skills:
|
||||
- yellowjacket-dev
|
||||
---
|
||||
|
||||
You implement one YellowJacket issue from its plan comment, in the loop
|
||||
worktree, on the claimed branch. You are the only writer. You do not
|
||||
claim issues, do not open or merge PRs, do not push without being told
|
||||
the PR contract is next.
|
||||
|
||||
Read in order: `CLAUDE.md`, `.planning/NOTES.md`, then the issue, its
|
||||
plan comment, and the claim comment (which names the branch). Implement
|
||||
what the plan says and nothing else. Match surrounding style. Follow
|
||||
`CLAUDE.md`'s shapes rather than reasoning from first principles.
|
||||
|
||||
Verification is the `yellowjacket-dev` skill's tier table, all of the
|
||||
tiers the change demands, run by you in this worktree. Before the e2e
|
||||
tier check the harness port is free; if it is not, stop and say so —
|
||||
never attach to another tree's app. Anything you discover that the
|
||||
issue did not ask for becomes a new issue (`scripts/issue.sh new`),
|
||||
never a bigger diff. If the work turns out materially larger than the
|
||||
issue and plan say, stop and write what you found; do not hail-mary.
|
||||
|
||||
Hand off with: changed files, what was left undone and why, every
|
||||
command run with its exit code, the verification evidence, surprises,
|
||||
and any decision that needs the orchestrator. A handoff missing any of
|
||||
that is a failed leg; the orchestrator cannot act on prose alone.
|
||||
@@ -1,28 +0,0 @@
|
||||
{
|
||||
"context": "fresh",
|
||||
"chain": [
|
||||
{
|
||||
"parallel": [
|
||||
{
|
||||
"agent": "yj-loop.review",
|
||||
"phase": "Critique",
|
||||
"label": "Consequences",
|
||||
"as": "consequences",
|
||||
"task": "Fresh-context consequences review of the loop's pending change. Issue, plan comment and branch: {task}. Read the issue, the plan comment, CLAUDE.md's load-bearing shapes, and the branch diff against origin/main. Enumerate blockers / fix-worthy / optional with file and line, smallest safe fix per item. Do not modify project/source files; returning findings through the configured output artifact is allowed.",
|
||||
"output": "critique/consequences.md",
|
||||
"outputMode": "file-only"
|
||||
},
|
||||
{
|
||||
"agent": "yj-loop.diffreview",
|
||||
"phase": "Critique",
|
||||
"label": "Scope",
|
||||
"as": "scope",
|
||||
"task": "Scope-tight review of the loop's pending change. Issue, plan comment and branch: {task}. Read the issue, the plan comment and the diff. Does the code do what the plan said, correctly, within its claimed scope? Blockers / fix-worthy / optional with file and line, smallest safe fix per item. Do not modify project/source files; returning findings through the configured output artifact is allowed.",
|
||||
"output": "critique/scope.md",
|
||||
"outputMode": "file-only"
|
||||
}
|
||||
],
|
||||
"concurrency": 2
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,26 +0,0 @@
|
||||
---
|
||||
description: One tick of the autonomous YellowJacket backlog loop
|
||||
---
|
||||
|
||||
You are the orchestrator of the YellowJacket backlog loop, waking for
|
||||
one tick. Work in this directory. Read `.pi/skills/yj-loop/SKILL.md`
|
||||
first — it is the operating procedure and it binds you. The design
|
||||
questions are answered in `.planning/plans/active/020-autonomous-backlog-loop.md`;
|
||||
the skill is what you run.
|
||||
|
||||
One tick means:
|
||||
|
||||
1. Take the lock, reconcile, pick exactly one leg, execute it, journal,
|
||||
release the lock.
|
||||
2. Delegate every deliberative leg to its `yj-loop.*` agent by name —
|
||||
the model is pinned in the agent file, never an argument. You hold
|
||||
only claim, shipping polls, merge, housekeep.
|
||||
3. Touch only what the loop created. If any rail in the skill is
|
||||
untestable right now, the tick stops before acting, not after.
|
||||
4. If the scheduler fires while you are mid-answer, finish this tick
|
||||
only. Two ticks never overlap; the lock is yours.
|
||||
|
||||
Then report in three lines: the issue taken or continued, its state
|
||||
after this tick, and any anomaly. Stop. Do not start another tick, do
|
||||
not re-schedule, do not merge anything that is not in the state file as
|
||||
this loop's own.
|
||||
@@ -133,10 +133,71 @@ status and `GET /api/v1/repos/yonlu/yellowjacket/actions/jobs/<id>/logs` for
|
||||
the log — and fix it. Two consecutive failed CI runs on the same cause: stop,
|
||||
comment what you know on the PR, and leave it for a human.
|
||||
|
||||
**Do not merge.** Comment on the issue linking the PR, leave
|
||||
`Status/In Progress` on, and end the run.
|
||||
**Do not merge, and do not review your own work.** Comment on the issue
|
||||
linking the PR, leave `Status/In Progress` on, and end the run. Whether
|
||||
this merges is decided by a reviewer that did not write it — see below.
|
||||
A CI run is not a review: it proves the tests you wrote pass, which is
|
||||
exactly the thing an author is worst placed to judge.
|
||||
|
||||
## Finally
|
||||
|
||||
Report in three lines: which issue you took, what state it is in
|
||||
(PR open / CI green / stopped and why), and any issues you filed.
|
||||
|
||||
---
|
||||
|
||||
# The merge gate
|
||||
|
||||
This half is **not** run by the author. It is run against a PR by
|
||||
someone who has not seen the branch before, and it decides whether the
|
||||
work lands on its own or waits for a human.
|
||||
|
||||
A push to `main` publishes nothing here — `release.yml` is
|
||||
`workflow_dispatch` only and all four publishers key on `v*` tags — so
|
||||
the cost of a wrong merge is a bad commit on `main` and the time to
|
||||
revert it. That is the whole reason this gate can exist. If that ever
|
||||
changes, this section is void.
|
||||
|
||||
**Merge only when every one of these is true.** Any single no means
|
||||
leave it open.
|
||||
|
||||
1. An independent review of the diff returns **MERGE** or **MERGE WITH
|
||||
NITS**. `CHANGES NEEDED`, or a review that could not reach a verdict,
|
||||
means a human looks.
|
||||
2. `CI / check (pull_request)` **and** `CI / e2e (pull_request)` are
|
||||
`success` on the PR's current head. Ignore the `(push)` contexts —
|
||||
they are `skipped` by design and Gitea folds `skipped` into a
|
||||
combined state of `pending` that never clears.
|
||||
3. The PR is mergeable with no conflicts, rebased onto current `main`.
|
||||
4. The diff touches **none** of: `.gitea/workflows/`, `.releaserc.yml`,
|
||||
`packaging/`, `build/`, `scripts/gitea-release.sh`,
|
||||
`backend/database/sql/schemas/`, `backend/database/staleshape*.go`,
|
||||
`go.mod`, `go.sum`. These either publish to somewhere a mistake
|
||||
cannot be taken back from, or can destroy a database that a user
|
||||
cannot rebuild.
|
||||
5. The issue is `Kind/Bug`, `Kind/Testing`, `Kind/Documentation` or
|
||||
`Kind/Enhancement`. **A `Kind/Feature` is a design decision and is
|
||||
never auto-merged**, however green it is.
|
||||
6. The diff is under ~600 changed lines across under ~15 files. Past
|
||||
that, "a reviewer read it" stops being a claim anyone should take on
|
||||
trust.
|
||||
7. The PR does not claim to have verified something no tier here can
|
||||
see. A change whose evidence would have to come from a physical
|
||||
device is reported, not merged.
|
||||
|
||||
**When it merges**, use `{"Do":"merge"}` on
|
||||
`POST /api/v1/repos/yonlu/yellowjacket/pulls/<n>/merge`. Then **check
|
||||
the issue actually closed** — a squash or an edited merge message drops
|
||||
the `Closes` footer — and close it by hand with a comment naming the
|
||||
commit if it did not. `unclaim.yml` strips `Status/In Progress` on
|
||||
close; if the label is still there a minute later, strip it yourself.
|
||||
|
||||
**When it does not merge**, say so on the PR in one paragraph: which
|
||||
condition failed and what would satisfy it. Leave the PR open, leave
|
||||
the label on, and file the review's substantive findings as issues so
|
||||
they are searchable rather than buried in a PR comment.
|
||||
|
||||
**A nit is not a blocker, and it is not free either.** A `MERGE WITH
|
||||
NITS` merges, and each nit worth keeping becomes an issue. Do not fix
|
||||
nits on the branch: that is a second author pass with no second review,
|
||||
which is the thing this gate exists to prevent.
|
||||
|
||||
@@ -158,7 +158,7 @@ only climb when it cannot.
|
||||
| You changed | Run | Cost |
|
||||
|---|---|---|
|
||||
| A Lit component, a store, the shortcut service | `make ui-test` | ~2 s, no app |
|
||||
| …and it renders differently | `make ui-visual` | + 10 baselines, opt-in, never gates |
|
||||
| …and it renders differently | `make ui-visual` | + 6 baselines, opt-in |
|
||||
| Any Go code | `make test` | 3 passes, ~2 min |
|
||||
| A service that emits events | `make test` — assert on the payload, see `backend/queue/emit_test.go` | in-process, no app |
|
||||
| A bound method or a bound struct field | `make bindings` then `make ui-test` | ~1.5 s + 2 s |
|
||||
@@ -180,13 +180,6 @@ less than it looks.)
|
||||
|
||||
Two rules about climbing:
|
||||
|
||||
- **If you moved a component's geometry, run `make ui-visual` and
|
||||
refresh that component's baseline in the same commit.** Nothing else
|
||||
will: it is the one tier in this repo no hook and no CI job runs, and
|
||||
it cannot be one — its references are machine-specific, measured in
|
||||
[references/ui-tier.md](references/ui-tier.md). Four of them drifted
|
||||
across three merges before anyone noticed (#196). Read the image;
|
||||
never bless a reference you did not cause.
|
||||
- **A component test passing is not the app rendering.** If you touched
|
||||
anything in `frontend/src`, verify it in the real app too — start it
|
||||
headless, `screenshot --filename=/tmp/shot.png`, and *read the PNG*.
|
||||
|
||||
@@ -42,16 +42,11 @@ strings and identical specs produce different bytes on different builds.
|
||||
playback and then clicks pause races the track ending and fails
|
||||
against a correct UI. Use `LONG_TRACK` (90 s, `edge-lengths`) exported
|
||||
from `e2e/support/fixtures.ts`.
|
||||
- **WAV tracks scan like every other format.** #104 added
|
||||
`backend/riff`, so the scan reads the `id3 ` chunk `backend/tagwriter`
|
||||
writes and both WAVs come in fully tagged: "Field Recordings" is an
|
||||
ordinary artist in the Artists view, with a "Test Tones" album and a
|
||||
cover. They are therefore not an example of an untitled or albumless
|
||||
track — the only two tracks with no album are
|
||||
`unsorted/no-tags-at-all.mp3` and `unsorted/title-only.mp3`. Prose
|
||||
written before #104 says the opposite and names
|
||||
`TestWAVTagsAreNotReadableYet`, a test that change deleted; that is
|
||||
dated history rather than a description of the app.
|
||||
- **WAV tracks scan in untitled.** `backend/tagwriter` writes WAV tags
|
||||
into a RIFF `id3 ` chunk and `dhowden/tag` has no RIFF parser, so
|
||||
there is no "Field Recordings" artist in the Artists view. This is a
|
||||
known open bug pinned by `TestWAVTagsAreNotReadableYet`; do not
|
||||
"fix" a spec by asserting the broken behaviour elsewhere.
|
||||
|
||||
## Seeds
|
||||
|
||||
|
||||
@@ -57,21 +57,6 @@ behind `YJ_TESTCTL=1`, which `scripts/dev-headless.sh` sets and
|
||||
staging the work that would produce it — job progress, download
|
||||
progress, scan progress. It calls `events.Deliver`, which *errors*
|
||||
when the event reaches nobody, so a `200` means it really arrived.
|
||||
- **State you stage, you own** (#168). Nothing resets those stores, so
|
||||
clear yours in `test.afterEach` with the same event that staged it
|
||||
(`emit('JobsChanged', [])`) — the store replaces its list from every
|
||||
snapshot, so `testctl` needs no special case. **Measured: this does
|
||||
not currently cross a spec boundary**, because every test gets a fresh
|
||||
page and `JobStore.init()` refetches `GetJobs()` from a backend
|
||||
registry that `/__test/emit` never writes to. Stated anyway, because
|
||||
it costs one line and the leak needs only one spec that keeps a page
|
||||
alive — but do not cite #168 for a symptom you have not reproduced.
|
||||
- **Measure against the thing next to you, not an absolute
|
||||
coordinate.** An absolute number in a shell measurement is also a
|
||||
claim about everything above it — `contentTop === 0` quietly asserts
|
||||
"and no background job is running", which is not what that spec was
|
||||
about or could arrange, while `contentTop === jobBandBottom` is true
|
||||
either way. This is the half of #168 that stands on its own.
|
||||
- **`restore` is slow** (~40 s in the suite) because it copies every
|
||||
table. Prefer snapshotting once and restoring only when a spec
|
||||
genuinely mutates state.
|
||||
|
||||
@@ -78,58 +78,9 @@ synchronously.
|
||||
Microtasks and not a timer, deliberately: a timer hangs forever under
|
||||
the suites that install fake ones.
|
||||
|
||||
## The visual tier does not gate, and that is measured (#196)
|
||||
|
||||
`make ui-visual` is the same suite with nine `toMatchScreenshot`
|
||||
baselines switched on. **Nothing runs it but a person**, deliberately,
|
||||
and the reason is a number rather than a preference: the committed
|
||||
baselines were recorded on Arch, and replayed in a bare `ubuntu:24.04`
|
||||
container — CI's `check` image — three of them fail for reasons that
|
||||
have nothing to do with any component.
|
||||
|
||||
| baseline | Arch | ubuntu:24.04 |
|
||||
|---|---|---|
|
||||
| `page-header` filtered-by-search | passes | ratio 0.03 differ, against a 0.02 allowance |
|
||||
| `track-info` | passes | ratio 0.03 differ |
|
||||
| `seek-bar` | 1152×18 | 1152×17 |
|
||||
|
||||
The two references that were genuinely stale did not even agree about
|
||||
their *new* size — `now-playing` renders 1152×65 on Arch and 1152×64 in
|
||||
the container. So moving CI's `check` job from `make ui-test` to
|
||||
`make ui-visual` is not a one-line change: it needs a second,
|
||||
container-recorded baseline set, which every local run would then fail
|
||||
against. That is the same trap the other way round, and a pre-push hook
|
||||
is the same fault again — one machine's baselines against everybody
|
||||
else's renderer.
|
||||
|
||||
So the tier stays local and opt-in, and the rule that replaces the gate
|
||||
is:
|
||||
|
||||
- **A change that moves a component's geometry refreshes that
|
||||
component's reference in the same commit, having read the image.**
|
||||
Look at the PNG; the dimensions in the failure message are the cheap
|
||||
half of the answer.
|
||||
- **Never refresh a reference you did not cause.** #196 exists because
|
||||
four of them drifted across three unrelated merges, and every red run
|
||||
made the next person likelier to stop running the tier than to read
|
||||
it.
|
||||
- **State the world the shot is taken in.** The stores are singletons,
|
||||
so a visual case that sets nothing photographs whatever the previous
|
||||
case left behind — which is how the sidebar's baseline came to have
|
||||
Tracks lit and `now-playing`'s to be playing from a dynamic mix.
|
||||
- **Record one file with `make ui-visual-update UI_ARGS=<path>`**, and
|
||||
check `git status` before committing either way. That filter is only
|
||||
honoured since #204: the recipe was a bare `--update`, and vitest
|
||||
takes the following positional as the flag's value, so the path was
|
||||
swallowed and *every* baseline was re-recorded — blessing any stale
|
||||
one in silence.
|
||||
|
||||
What the tier is worth, for the record: it is a *layout* check, blind to
|
||||
colour (the component tier has no `:root`, so it renders the fallbacks —
|
||||
`make ui-visual` passed unchanged through a whole palette rewrite,
|
||||
twice), and it has caught one thing nothing else could — swapping
|
||||
`library-status-indicator`'s `<button>` for a `<span>` lost the UA
|
||||
stylesheet's `box-sizing` and grew the badge 36→38px.
|
||||
Visual baselines are font-hinting and compositing sensitive, which is
|
||||
why they are opt-in: they only mean anything on the machine that
|
||||
recorded them.
|
||||
|
||||
## Bindings
|
||||
|
||||
|
||||
@@ -1,306 +0,0 @@
|
||||
---
|
||||
name: yj-loop
|
||||
description: Operating the autonomous backlog loop — the crank that works the YellowJacket tracker one issue at a time (tick mechanics, the state machine in Gitea, which agent and model take each leg, the escalation ladder, merge authority and the rails that stop it doing damage). Use whenever a scheduled tick fires, and when piloting or debugging the loop.
|
||||
---
|
||||
|
||||
# The YellowJacket backlog loop
|
||||
|
||||
Design and arguments: `.planning/plans/active/020-autonomous-backlog-loop.md`.
|
||||
This skill is the **operating procedure**; the plan is the reasoning.
|
||||
`yellowjacket-dev` is the harness doctrine (tiers, seeds, traps); this
|
||||
skill is the loop doctrine (who acts, on what model, with what authority).
|
||||
Read the plan first, once. Then this file every tick.
|
||||
|
||||
## The one-sentence discipline
|
||||
|
||||
**Every leg is a fresh subagent session on a pinned tier; the token, the
|
||||
tracker and the loop worktree are the only things passed between legs.
|
||||
Never switch a model mid-session, never let two writers exist at once,
|
||||
never keep state in a conversation.**
|
||||
|
||||
## Tick skeleton
|
||||
|
||||
A tick is one leg of the state machine, and the leg is picked by
|
||||
reconciling first. Execute in this order:
|
||||
|
||||
1. **Lock.** `/tmp/yj-loop.lock` holds `pid + start-iso`. If a live
|
||||
process owns it and is younger than 2 h: exit immediately, report
|
||||
"tick skipped (lock held)". If the PID is dead, take the lock.
|
||||
Remove it before every exit.
|
||||
2. **Reconcile.** Fresh reads, never cached: open issues
|
||||
(`scripts/issue.sh list`), PRs and CI via the REST API, branches via
|
||||
`git ls-remote --heads origin`, `.pi/loop/state.json`. GITEA_TOKEN
|
||||
refusing = the tick reports and exits; the identity rails below are
|
||||
not optional.
|
||||
3. **Pick the leg.** See the state machine below; the leg follows the
|
||||
issue's lifecycle (claim→plan→…→merge→…→housekeep). Exactly one leg.
|
||||
4. **Execute** — the leg table below says who acts and what they must
|
||||
return.
|
||||
5. **Journal** — one line per tick in the state file (issue, leg, result,
|
||||
tick cost if leg reports it).
|
||||
6. **Report** — three lines: issue taken or continued, its state now,
|
||||
anomalies. Then stop. A tick that reports is a tick that can leave a
|
||||
conversation behind.
|
||||
|
||||
## The state machine
|
||||
|
||||
The tracker is the truth. The state file (`.pi/loop/state.json`,
|
||||
gitignored) is an index plus flags (`emulator`, `drain`); the tracker
|
||||
wins every disagreement.
|
||||
|
||||
| Stage | Where it lives | Leg → actor |
|
||||
|---|---|---|
|
||||
| selected | nothing written until claim is possible | select |
|
||||
| in flight | `Status/In Progress`, assignee, comment with branch+approach | claim (orchestrator, `scripts/issue.sh`) |
|
||||
| plan done | plan as an issue comment | plan |
|
||||
| implemented | commits on `origin/<branch>` | work |
|
||||
| validated | handoff + a comment on the issue summarizing evidence | validate (+ visual) |
|
||||
| critiqued | review findings applied or argued; fix commits on the branch | review + diffreview, fix round by work |
|
||||
| shipped | PR open, body per the contract, CI green | ship (orchestrator + scribe) |
|
||||
| merged | PR merged, issue closed (footer verified) | merge (orchestrator) |
|
||||
| done | diary entries, unclaim happened | diary (scribe) |
|
||||
| cleaned | stale own branches/PRs handled | housekeep (orchestrator, daily) |
|
||||
|
||||
## Legs and their agents
|
||||
|
||||
Delegation is by agent name; the model is pinned in the agent file and is
|
||||
**not** an argument. Every leg prompt names: the issue, the evidence so
|
||||
far (plan comment, handoffs), what the leg must produce, and its stop
|
||||
rules. Never "go fix it" — the leg contract is in this file.
|
||||
|
||||
| Leg | Agent | Model (tier) | Produces |
|
||||
|---|---|---|---|
|
||||
| gather/mechanical dump | `yj-loop.inspect` | go/mimo-v2.5 (T0) | tracker/PR/CI/branch digest, verbatim |
|
||||
| select next issue | `yj-loop.select` | glm/glm-5.3 (T2) | one issue + reasons, or "nothing qualifies" |
|
||||
| plan | `yj-loop.plan` | glm/glm-5.3 (T2) | a plan comment on the issue |
|
||||
| implement | `yj-loop.work` | qwen/deepseek-v4-pro-0813 (T1) | commits + a handoff (see contract below) |
|
||||
| validate | `yj-loop.validate` | glm/glm-5.3 (T2) | pass/fail with evidence per acceptance item |
|
||||
| visual evidence | `yj-loop.visual` | glm/glm-5.3-flash (T2) | what the screenshot actually shows |
|
||||
| consequences review | `yj-loop.review` | glm/glm-5.3 (T2) | blockers / fix-worthy / optional findings |
|
||||
| understood-diff review | `yj-loop.diffreview` | qwen/deepseek-v4-pro-0813 (T1) | same shape, scope-tight |
|
||||
| escalation | `yj-loop.escalate` | go/kimi-k3 (T3) | same leg re-run, seeded with failure summary |
|
||||
| prose (PR body, commit msgs, journal) | `yj-loop.scribe` | go/mimo-v2.5 (T0) | text only, from supplied facts |
|
||||
|
||||
**Model fallback on quota exhaustion.** The pinned models are the
|
||||
intent, not a guarantee. The qwen token plan is a weekly pool and has
|
||||
run dry mid-tick (`429 … 1-week quota exhausted`). When a leg's launch
|
||||
fails with a 429, re-run it with a per-run `model` override one rung
|
||||
down and journal the substitution — never spend the T3 escalation
|
||||
model on a quota substitution. The qwen-pinned legs (`work`,
|
||||
`diffreview`) fall back `qwen/deepseek-v4-pro-0813` → `go/deepseek-v4-pro`
|
||||
→ `go/glm-5.3-flash`. Do **not** use the `deepseek/...` provider: it has
|
||||
no models, only catalog overrides, and fails silently (empty artifact,
|
||||
no session) — the model lives on the `go` gateway.
|
||||
|
||||
**Launch legs in the foreground.** The async subagent runner has died
|
||||
without persisting a child session (nothing to resume) and emits
|
||||
spurious "needs attention" nudges on runs that are already complete.
|
||||
Foreground `subagent` calls are the reliable mode here. A worker that
|
||||
dies mid-leg leaves uncommitted work: inspect the tree, then relaunch
|
||||
to *complete* — never to re-implement.
|
||||
|
||||
Orchestrator-only legs: **claim** (`issue.sh claim --branch` — atomic,
|
||||
refuses if held), **ship's PR/CI polling** (REST API below — `gitea_ci`
|
||||
job_logs 404s on this Gitea; the REST endpoints are the way), **merge**
|
||||
(API below), **housekeep**.
|
||||
|
||||
## Selection rules (`select`)
|
||||
|
||||
The rules from `.pi/prompts/next-issue.md` stay — priority order, #73's
|
||||
sequence overriding labels where it speaks, skipping `Status/*` states
|
||||
that mean busy, branch-collision check, verifiability, flakes. The
|
||||
emulator flag **adds** emulator-verifiable Android issues; it never
|
||||
reaches device-only ones. A "nothing qualifies" answer is a correct
|
||||
tick, not a failure — report it and stop.
|
||||
|
||||
## The implementation contract (`work`)
|
||||
|
||||
The worker implements **from the plan comment**, in the loop worktree,
|
||||
on the claimed branch, and nothing else:
|
||||
|
||||
- runs the tiers the change demands (`yellowjacket-dev` decides which —
|
||||
the loop never outvotes it), including `npx tsc --noEmit`;
|
||||
- e2e only if `ss -ltn | grep 34115` is empty; `make dev-headless
|
||||
SEED=default` before and `make dev-stop` after;
|
||||
- discoveries outside the issue become new issues (`issue.sh new`), never
|
||||
bigger diffs; a materially-larger-than-implied issue stops the leg with
|
||||
a comment and a label removal, not a hail-mary;
|
||||
- handoff must state: changed files, what was left undone, commands run
|
||||
with exit codes, verification evidence, surprises, decisions needing
|
||||
approval. A handoff without that list is a failed leg.
|
||||
|
||||
## Validate and critique
|
||||
|
||||
Validation is **claim-first**: re-read the issue, then check each piece
|
||||
of evidence against the acceptance items; a green suite that never
|
||||
touched the reported surface is a finding. Screenshots go to `visual`,
|
||||
never to a text-only tier.
|
||||
|
||||
Critique is the standing fan-out (`subagent` parallel: `yj-loop.review`
|
||||
consequences + `yj-loop.diffreview` scope-tight, both fresh). The
|
||||
orchestrator synthesizes: blockers and fix-worthy findings go back to
|
||||
`work` as one bounded fix round (maximum three rounds total; then the
|
||||
issue gets a `⟦loop⟧` comment stating what will not be fixed and why,
|
||||
and the ship leg proceeds unless a finding is a blocker). Reviewers do
|
||||
not edit files.
|
||||
|
||||
## Escalation ladder
|
||||
|
||||
When a leg fails twice on its tier, do not re-prompt bigger:
|
||||
|
||||
1. The failing session writes its summary: what it tried, what failed,
|
||||
what it observed.
|
||||
2. A **new** session on the next tier up is seeded with that summary and
|
||||
the original leg contract.
|
||||
3. T3 is the ceiling: fresh session, never parallel, **once per day**.
|
||||
A day's escalation is spent — the issue waits until tomorrow.
|
||||
|
||||
Routing down is free; routing up is the budget.
|
||||
|
||||
## Ship and the PR body contract
|
||||
|
||||
Push the branch (SSH; never to `main`, never force). The PR body —
|
||||
written by `scribe` from the validator's and reviewers' output — states:
|
||||
what the issue was, what changed and why, **which verification tiers ran
|
||||
and their results**, what was deliberately not done, the commit-to-issue
|
||||
table, and `Closes #n`. `Closes` also sits one-per-line in a commit body
|
||||
**inside the branch** — both, regardless of merge strategy, because the
|
||||
pairing was measured.
|
||||
|
||||
Poll CI until `check` and `e2e` finish. On failure: read the log via
|
||||
`GET /api/v1/repos/yonlu/yellowjacket/actions/runs/<run>/jobs` (per-step)
|
||||
and `…/actions/jobs/<id>/logs` (full). Fix on the branch. **Two
|
||||
consecutive identical failures = stop**: comment what is known on the
|
||||
PR and the issue, leave both, report. Do not burn ticks on a red wall.
|
||||
|
||||
## Merge authority
|
||||
|
||||
Merge when, and only when, **all** hold:
|
||||
|
||||
- the PR was opened by this loop (it is in the state file's index);
|
||||
- the protection contexts `CI / check` and `CI / e2e` are green on the
|
||||
PR's head, read from the API, not from the PR page's badge;
|
||||
- the PR reports mergeable;
|
||||
- the critique leg ran and no open blocker stands;
|
||||
- the branch is **not behind `origin/main`** — the protection's
|
||||
`block_on_outdated_branch: true` refuses it anyway; never
|
||||
`force_manually_merged` around it.
|
||||
|
||||
**Refresh before every merge.** In the loop worktree: `git fetch origin`
|
||||
in the same breath, then `git merge origin/main` on the PR branch,
|
||||
push. The fetch must be immediate — a cached `origin/main` merges
|
||||
against the wrong base, CI goes green on it, and the merge comes back
|
||||
405 "behind base", one whole CI cycle wasted (measured on the adoption
|
||||
wave). A textual conflict
|
||||
stops the leg there — as diff text, not as a failed merge click: hunks
|
||||
the loop authored are resolved by the loop; anything else is left with
|
||||
`⟦loop⟧` comment for a human, never forced. After any refresh push,
|
||||
re-poll the PR's own required contexts on the **new head** before
|
||||
merging.
|
||||
|
||||
**Merges happen one at a time**, each re-reading state — the previous
|
||||
merge moved `main`, and the next PR's mergeability is recomputed at
|
||||
its own turn.
|
||||
|
||||
```
|
||||
curl -sS -X POST -H "Authorization: token $GITEA_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
https://git.ljones.me/api/v1/repos/yonlu/yellowjacket/pulls/<n>/merge \
|
||||
-d '{"Do":"merge","merge_message_field":"default","force_manually_merged":false}'
|
||||
```
|
||||
|
||||
**Afterwards watch the `push` run on `main`** — the CI the merge
|
||||
started. A red main after a loop merge is a **halt**: comment what is
|
||||
known on the offending PR, mark the state file, stop taking new issues.
|
||||
That run is the only thing between a clean textual merge of
|
||||
independently-written PRs and a self-contradicting main; no
|
||||
mergeability check sees it. Only a green main lets the tick proceed (to
|
||||
footer verification, below).
|
||||
|
||||
Footer verification: `scripts/issue.sh list --state open` and check
|
||||
the footer took. Close stragglers with `issue.sh close`, naming the
|
||||
merge commit. `unclaim.yml` handles the label; it is not instant;
|
||||
reopening does not restore it. Merging fans out to nothing (releases
|
||||
are the manual `release.yml`, which the loop never runs) — the
|
||||
criticism stands before the merge because nothing stands after it.
|
||||
|
||||
## Rails — the loop's absolute rules
|
||||
|
||||
1. **Touch only its own.** Issues it claimed, branches it made, PRs it
|
||||
opened. `issue.sh claim` enforces the front gate; never work around a
|
||||
refusal.
|
||||
2. **One writer, one issue.** The loop worktree is the only dirty tree.
|
||||
3. **Never merge a PR it did not open.** Any merge that violates this is
|
||||
a hard stop.
|
||||
4. **Human work is holy.** Human branches, PRs, assignees: leave exactly
|
||||
as found. Cleanup never names them.
|
||||
5. **The token is identity.** If GITEA_TOKEN misbehaves, the tick stops.
|
||||
6. **New findings are new issues**, never scope creep. The tracker
|
||||
vocabulary (`Kind/`, `Area/`, `Priority/`) stays intact in one
|
||||
taxonomy; use `scripts/issue.sh new` with correct labels.
|
||||
7. **Conventional Commits**, enforced by `scripts/commit-check.sh`; the
|
||||
type list and `.releaserc.yml`'s must agree — a loop commit is a
|
||||
release grammar token even after months of no manual releases.
|
||||
8. **Tiers over vibes.** `yellowjacket-dev`'s tier table decides what a
|
||||
change must pass; a skipped tier is stated, never silent.
|
||||
9. **Two strikes on CI, three rounds of critique, one kimi a day.** The
|
||||
loop's patience is finite on purpose.
|
||||
10. **Every leg writes its evidence.** A leg that leaves nothing behind
|
||||
is indistinguishable from a leg that did not run — which is how the
|
||||
next tick re-does it.
|
||||
11. **The loop may not re-schedule itself** (the scheduler refuses it
|
||||
anyway — treat as an invariant, not a limitation).
|
||||
12. **Drain means drain.** `drain: true` = finish in flight, take
|
||||
nothing new, then stop.
|
||||
|
||||
## Emulator mode
|
||||
|
||||
Flag `emulator: true` in the state file **and** an already-booted
|
||||
emulator (`adb devices` answers) opts in: `make android` (build), `make
|
||||
android-install`, `make android-smoke` (crash check — the same pid
|
||||
surviving is the only signal that means started), `make
|
||||
android-screenshot` and `make android-eval` as evidence for `visual`.
|
||||
The loop never boots or stops an emulator; that is the user's machine.
|
||||
Device-only issues stay open under either setting. One-time setup the
|
||||
user performs: `make android-setup` (~3.5 GB, creates the `yj-test`
|
||||
AVD), then `make android-emulator` per session.
|
||||
|
||||
## ON / OFF / drain
|
||||
|
||||
- **Worktree:** `git worktree add ~/.paseo/worktrees/loop/jumpy-hound
|
||||
origin/main` (from any clone; branch from origin/main in the loop
|
||||
tree, never `git checkout main`). **Provision it once before the
|
||||
first push:** `make build-frontend` and `make testdata` — the pre-push
|
||||
`go-test` hook needs `frontend/dist` (the `//go:embed` in `main.go`)
|
||||
and the fixture library, and refuses the push without them.
|
||||
- **Session:** pi in that worktree, `/name loop`. Add the job via
|
||||
`/schedule-prompt` (name `yj-loop`, cron
|
||||
`0 0 10-18 * * 1-5`, prompt: "Read `.pi/skills/yj-loop/SKILL.md` and
|
||||
run exactly one tick. Stop.") — session-bound by default.
|
||||
- **OFF:** toggle the job, or close the session. **ON:** `pi --resume
|
||||
loop` in the worktree, job enabled. Courses of the tick appear in
|
||||
that session's transcript.
|
||||
- **Tune in:** the same resume. Talk to it only between; a tick is
|
||||
atomic.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
- `issue.sh: GITEA_TOKEN is not set` or a 401 — the token is the whole
|
||||
identity (rails 5). Stop, do not fall back to anything.
|
||||
- `gitea_ci`'s job log 404s — the REST endpoints above answer; this is
|
||||
a Gitea build, not a fault.
|
||||
- A spec fails that the tier doc says can fail from stale backend state
|
||||
— restart the app tier before believing it (`yellowjacket-dev`).
|
||||
- A tick that "did nothing" — reconcile again; the tracker usually says
|
||||
which leg it really is.
|
||||
- The job did not fire — the scheduler fires only while a session is
|
||||
open in its directory (documented); "the loop is off" is the correct
|
||||
reading, not a bug.
|
||||
- `error: object file … is empty` / `unpack-objects failed` / `bad
|
||||
object refs/heads/…` during a fetch or checkout — the shared object
|
||||
store was corrupted (a killed fetch leaves 0-byte object files, and a
|
||||
local ref can end up pointing at the dead sha1). **Halt and report**;
|
||||
do not retry, the churn only deepens it. Human repair: delete the
|
||||
0-byte objects, `git fetch origin --prune`, delete any ref that
|
||||
still dangles (`git update-ref -d refs/heads/<b>`), re-checkout the
|
||||
worktree at `origin/main`, then `git fsck --full`.
|
||||
@@ -4912,221 +4912,3 @@ bridge leaves the wizard up with its "Get Started" button correctly
|
||||
disabled — it gates on a directory chosen *in the wizard*, and the
|
||||
existing-library check runs once, on mount. A reload clears it. Nothing
|
||||
is broken; it cost twenty minutes of believing a tap had been swallowed.
|
||||
|
||||
## The sheet's scroll fade, and where a scrim may not go (measured 2026-08-23, headless)
|
||||
|
||||
#207's answer. The affordance is two background layers on
|
||||
`wa-dialog::part(body)` and the conditionality is
|
||||
`background-attachment`, not a scroll listener: a cover of the sheet's
|
||||
own colour painted at the end of the *content* (`local`) over a shadow
|
||||
pinned to the box (`scroll`), so the cover scrolls up and hides the
|
||||
shadow exactly when there is nothing more to see.
|
||||
|
||||
Measured at 424x360 (which is where a menu overflows on `main`, since
|
||||
`main` does not yet carry #67's eighth item — at 424x439 the track
|
||||
list's seven items are `scrollHeight` 364 against `clientHeight` 364,
|
||||
fitting exactly). Pixel at x=300, dark ramp, `bgElevated` `#343a40`:
|
||||
|
||||
| y | before | more below | at the end of the list |
|
||||
|---|---|---|---|
|
||||
| 330 | 52,58,64 | 50,56,62 | 52,58,64 |
|
||||
| 340 | 52,58,64 | 43,48,53 | 52,58,64 |
|
||||
| 350 | 52,58,64 | 33,37,40 | 52,58,64 |
|
||||
| 359 | 52,58,64 | 22,24,27 | 52,58,64 |
|
||||
|
||||
Three things worth keeping.
|
||||
|
||||
**A menu that fits draws nothing**, which is the same measurement: at
|
||||
424x439 the sheet is flat 52,58,64 to its bottom edge, because with no
|
||||
overflow the `local` layer's positioning area *is* the padding box and
|
||||
the cover lands on top of the shadow.
|
||||
|
||||
**A scrim over a menu row is that row's text surface**, so the 4.5:1
|
||||
rule reaches it and this is why the curve is steep rather than linear.
|
||||
A row is 48px with its label centred; 32px of scrim already down to a
|
||||
quarter strength at 14px puts about 0.06 at the label. Checked on the
|
||||
light ramp (`bgElevated` `#e9ecef`, text `#212529`) by overriding the
|
||||
two custom properties on `:root`: background at the label 205,207,210,
|
||||
which is **9.9:1**. The first draft — a linear 48px at 0.8 — put ~0.375
|
||||
on that label, 5.0:1, passing but visibly greyed. The bottom few pixels
|
||||
go to ~2.4:1 in either draft and are deliberately below where any
|
||||
label of a *fully visible* row sits; a label that lands there belongs
|
||||
to the half-cut row, which is the thing being signalled.
|
||||
|
||||
**A dark scrim on a dark surface reads far worse in a shrunk screenshot
|
||||
than on screen.** The first two probes (24px/0.45, then 32px/0.75) were
|
||||
measurably present — 52,58,64 down to 30,33,37 — and invisible in the
|
||||
inline preview. Crop the bottom 70px and scale it up before judging;
|
||||
the pixel values are the honest answer either way.
|
||||
|
||||
## The phone's context sheet is now longer than the phone (measured 2026-08-23, headless at 424x439)
|
||||
|
||||
#67 moves two destinations into every row menu, and the track list's
|
||||
menu is where that runs out of screen. Measured against the running
|
||||
app at the reference viewport, one row selected:
|
||||
|
||||
| menu | items | first item top | last item bottom |
|
||||
|---|---|---|---|
|
||||
| queue panel | 7 | 95 | 431 |
|
||||
| track list | 8 | 86 | **470** |
|
||||
|
||||
The viewport is 439. So the track list's last item — "Remove from
|
||||
Library" — is below the fold. It is **not unreachable**: the sheet is a
|
||||
`wa-dialog` whose body is `overflow-y: auto`, measured `scrollHeight`
|
||||
412 against `clientHeight` 373, and scrolling it 39px brings that item
|
||||
fully into view (383–431). What it has is no *affordance*: nothing on
|
||||
screen says the list continues.
|
||||
|
||||
Two things worth knowing before adding a ninth item anywhere.
|
||||
|
||||
**The limit was already reached, and this is what crossed it.** Seven
|
||||
48px rows in a 373px body is 364px — the queue's menu fits with 8px to
|
||||
spare and the track list's fitted exactly. Any item added to any of the
|
||||
fourteen menus after #60 was going to be the one that overflowed; the
|
||||
first one simply happened to be this.
|
||||
|
||||
**The measurement has to be taken with a row selected**, since the
|
||||
`Go to` items are drawn for a single selection only, and on the *first*
|
||||
track of the fixture library — which has no album (`01 Tone A`,
|
||||
`02 Tone B`) — only "Go to Artist" appears. That is the 8 above; an
|
||||
ordinary track makes it 9.
|
||||
|
||||
Filed as its own issue rather than fixed in #67's diff: it is a
|
||||
property of the shared sheet (`components/menu-surface/`), not of the
|
||||
items.
|
||||
|
||||
## The tap highlight is one inherited declaration (measured 2026-08-24)
|
||||
|
||||
`-webkit-tap-highlight-color` is an **inherited** property, and an
|
||||
inherited property crosses a shadow boundary — so `html { … :
|
||||
transparent }` in `index.css` reaches every shadow root in the app and
|
||||
no component needs a rule of its own. Measured in the running app
|
||||
(Chromium, `app-sidebar`'s `li button`, which is three shadow roots
|
||||
from the document): `rgba(0, 0, 0, 0)` with the rule, and
|
||||
`rgba(0, 0, 0, 0.18)` with it removed. That 0.18 grey over the bounding
|
||||
rect of whatever was tapped is what #54 reported.
|
||||
|
||||
The same argument was already spent once and is worth not
|
||||
re-deriving: `index.css`'s first rule is `*, *::before, *::after {
|
||||
user-select: none }`, which for the same reason already covers the
|
||||
shadow roots — #54's Findings ask for `user-select` on interactive
|
||||
surfaces and it has been done since before the issue was filed.
|
||||
|
||||
**What the highlight was, on the surfaces that had nothing else, is the
|
||||
press feedback.** Measured on a track row with the press rule removed
|
||||
and the button held down: `rgba(255, 255, 255, 0.05)` — the *hover*
|
||||
tint, arriving because the pointer is over the row, which is a
|
||||
synthesised hover on a phone and outlives the press. With the rule:
|
||||
0.12 while held, and the neighbouring row unchanged. So the press state
|
||||
is part of removing the highlight rather than a separate polish item,
|
||||
and the hover tints on those same surfaces moved behind
|
||||
`(hover: hover) and (pointer: fine)`, which is #68's gate applied to a
|
||||
tint rather than to a revealed control.
|
||||
|
||||
**`touch-action: manipulation` was considered and not taken.** The
|
||||
Findings offer it for the 300ms tap delay; this app's viewport is
|
||||
`width=device-width`, which is what removes that delay in Chrome, so
|
||||
the stated benefit is not there to win. What it would change is the
|
||||
gesture stack #63 tuned by measurement on the device (`pan-y` plus a
|
||||
non-passive `preventDefault`), and that is not measurable from here.
|
||||
|
||||
## Art pop-in is measurable in a browser, if you count frames rather than milliseconds (measured 2026-08-24)
|
||||
|
||||
#65 is an Android report ("scrolling through albums, the art pops in")
|
||||
and the desktop harness can measure it, which was not obvious: the
|
||||
first attempt waited 220 ms after each scroll jump and found **zero**
|
||||
blank covers on either build. The metric only discriminates at one and
|
||||
two animation frames after the jump, which is where a pop-in actually
|
||||
lives.
|
||||
|
||||
Protocol, on `make dev-headless SEED=bulk` (4 988 albums), ten
|
||||
2 400px jumps of `.grid-scroll-container`, counting covers whose rect
|
||||
intersects the viewport with `naturalWidth === 0`:
|
||||
|
||||
| build | blank at frame 1 | at frame 2 | at 50 ms |
|
||||
|---|---|---|---|
|
||||
| `main` | 254 / 258 | 214 / 258 | 0 |
|
||||
| `main`, second run | 254 / 258 | 190 / 258 | 0 |
|
||||
| prefetch | 117 / 258 | 77 / 258 | 0 |
|
||||
| prefetch, second run | 118 / 258 | 96 / 258 | 0 |
|
||||
|
||||
Two things this protocol gets wrong if repeated carelessly. **A second
|
||||
run in the same browser session measures the HTTP cache**, not the
|
||||
build — the skill already warns about this for `make perf`, and it
|
||||
applies to any image measurement; every row above is a fresh
|
||||
`playwright-cli close` + `open`. And **the frontend is embedded**, so
|
||||
comparing builds is a `git stash` *and* a rebuild, not a stash.
|
||||
|
||||
**The bulk library's covers are 300x300 and ~3.7 kB**, which is why
|
||||
both builds are clean by 50 ms here and why the phone's number cannot
|
||||
be inferred from this one — same caveat the skill already records
|
||||
about full-size artwork.
|
||||
|
||||
**`rangeChanged` and `visibilityChanged` are different ranges**, and
|
||||
the difference is the whole of this fix's value.
|
||||
`@lit-labs/virtualizer` reports `_first`/`_last` (rendered, including
|
||||
the ~1000px overhang) on the former and `_firstVisible`/`_lastVisible`
|
||||
on the latter. Both grids listen to `visibilityChanged` for scroll
|
||||
persistence, which wants the visible range and is correct; a prefetch
|
||||
window measured from it lands mostly on cards that already exist.
|
||||
Anchored there, the component test could see only one row past the
|
||||
last rendered card.
|
||||
|
||||
**`_overhang` is not configurable.** It is a `protected` field set to
|
||||
1000 in `BaseLayout` and read by every layout; there is no option on
|
||||
`grid()`/`flow()` and no property on the element. The issue's Direction
|
||||
("ask the virtualizer for a larger overscan") is therefore not
|
||||
available without patching a private, which is why the request is
|
||||
issued ahead of the element instead.
|
||||
|
||||
## Declined, and recorded nowhere else (2026-09-25, #256)
|
||||
|
||||
When CLAUDE.md was cut down to rules, most of its "considered and
|
||||
declined" paragraphs already had a home in a code or config comment
|
||||
beside what they explain. These three did not:
|
||||
|
||||
- **`touch-action: manipulation` was declined** (#54). The 300ms tap
|
||||
delay it is offered for is already absent on a `width=device-width`
|
||||
viewport; what it would actually change is the gesture stack #63 tuned
|
||||
by measurement on the reference device.
|
||||
- **There is no "Go to Genre"** in the phone row menu (#67). That menu
|
||||
replaces name links a phone cannot use, and there has never been a
|
||||
genre link to replace — it would be new navigation, which wants its
|
||||
own issue.
|
||||
- **The overlaid queue has no tap-outside gutter on a phone** (#171).
|
||||
The drawer-style gutter would buy the affordance by taking width off a
|
||||
full-screen surface on a 424px viewport; back and a 44px close button
|
||||
answer it instead.
|
||||
|
||||
## slskd's API, read from its source rather than a live daemon (2026-09-26)
|
||||
|
||||
`backend/download/provider_slskd.go` had never run against a real slskd
|
||||
when #263–#272 shipped, so its assumptions were checked against slskd
|
||||
0.26.0's source. One was wrong, and one design was only safe by luck.
|
||||
These are properties of someone else's server; re-check on an upgrade.
|
||||
`TestSlskdLive` (env-gated, see its comment) is the way to confirm them
|
||||
against a running one.
|
||||
|
||||
- **`searchTimeout` is seconds, from the last response**, minimum 5
|
||||
(`SearchRequest.cs`). We sent milliseconds (#274). The other search
|
||||
options — `responseLimit`, `fileLimit`, `filterResponses`,
|
||||
`minimumResponseFileCount`, `maximumPeerQueueLength` — are named as we
|
||||
send them; slskd's defaults are 100 responses, 10 000 files, queue
|
||||
1 000 000.
|
||||
- **`GET /searches/{id}/responses` exists**, and `DELETE
|
||||
/transfers/downloads/{user}/{id}?remove=true` cancels and removes.
|
||||
- **A finished download is moved to `<downloads>/<Subdirectory>/`**,
|
||||
where `Destination.Subdirectory` defaults to `${SOURCE_DIRECTORY}`
|
||||
(the remote leaf folder) and is user-configurable. A taken name is
|
||||
written as `name_<ticks>.ext` (`Destination.Exists = rename`, the
|
||||
default). No transfer record says where the file went.
|
||||
- **Batch enqueue (`POST /transfers/downloads/batches`) is new in 0.26.0**
|
||||
and is the only way to choose where a file lands: `options.destination`
|
||||
overrides the subdirectory pattern. A batch's files land flat in it,
|
||||
so one batch per disc. On an older daemon that path is routed to the
|
||||
per-user enqueue as username "batches" and the object body is
|
||||
rejected with 400 — which is why 400 means "no batches" here.
|
||||
- **`GET .../downloads/{user}/{id}/position` asks the peer** and returns
|
||||
a bare integer. slskd's own comment on `PlaceInQueue` is "may be
|
||||
wildly innacurate to the point of uselessness", which is why #275 acts
|
||||
only on two readings in a row.
|
||||
|
||||
@@ -1,244 +0,0 @@
|
||||
# 020 — The autonomous backlog loop
|
||||
|
||||
**Issue:** #236 (`Kind/Enhancement`, `Priority/Low`)
|
||||
**Status:** active — phase 0, supervised pilot
|
||||
**Relates:** #73 (the roadmap the loop follows), plan 005 (the harness the
|
||||
loop drives). Cost and model-tier doctrine is the `pi-session-reference`
|
||||
card handed to the session that designed this; the loop's copies of it
|
||||
are deliberate one-paragraph summaries, not the authority.
|
||||
|
||||
A pi coding-agent configuration that, toggled on, works the Gitea tracker
|
||||
one issue at a time — triage, claim, plan, implement, validate, critique,
|
||||
PR, CI, merge, verify-close, diary — and then does it again. The tracker is
|
||||
the state machine: whoever reads Gitea sees exactly where the loop is,
|
||||
which is the property this document's rails exist to protect.
|
||||
|
||||
---
|
||||
|
||||
## The shape: a crank, not a resident brain
|
||||
|
||||
Half the design is that **nothing lives in a conversation**. Each tick is a
|
||||
fresh, bounded unit of work; every transition writes evidence to Gitea
|
||||
(label, comment, branch, PR) or to the loop's own state file; a tick that
|
||||
dies mid-leg loses nothing, because the next tick resumes from what Gitea
|
||||
says.
|
||||
|
||||
The other half is that **no leg trusts the one before it**. The worker
|
||||
implements from the plan, not from the issue alone; the validator checks
|
||||
the *claim*, not the green CI row; the merger merges only after reading the
|
||||
protection contexts itself; the diary leg is what makes the next issue's
|
||||
triage cheaper.
|
||||
|
||||
One issue in flight at a time. That is a pacing decision, not a
|
||||
concurrency limit of the tooling — CI has a capacity-1 runner and the e2e
|
||||
tier owns one headless port on this machine, so two writers would serialize
|
||||
on infrastructure they cannot see and appear to be doing fine.
|
||||
|
||||
## The state machine
|
||||
|
||||
| Leg | Writes | Actor / model |
|
||||
|---|---|---|
|
||||
| reconcile | — | orchestrator + `inspect` (mimo-v2.5) |
|
||||
| select | nothing on the tracker; decision logged in the tick transcript | `select` (glm-5.3) |
|
||||
| claim | assignee + `Status/In Progress` + comment naming branch & approach | `scripts/issue.sh claim` |
|
||||
| plan | plan as an issue comment | `plan` (glm-5.3) |
|
||||
| implement | commits on the issue branch, in the loop worktree | `work` (qwen/deepseek-v4-pro-0813) |
|
||||
| validate | verification evidence in the handoff | `validate` (glm-5.3), `visual` (glm-5.3-flash) for screenshots |
|
||||
| critique | review findings; fix commits | `review` (glm-5.3) + `diffreview` (qwen) + fix round by `work` |
|
||||
| ship | push, PR with body contract, CI read + fixes | orchestrator + `scribe` (mimo-v2.5) |
|
||||
| merge | the merge; post-merge issue verification | orchestrator |
|
||||
| diary | `.pi/journal.md`, `CLAUDE.md` if structural | `scribe` |
|
||||
| housekeep | stale-branch/PR cleanup, state-file prune | orchestrator |
|
||||
|
||||
### Legs that are the orchestrator's alone
|
||||
|
||||
The orchestrator (the loop session) delegates every deliberative leg and
|
||||
keeps three for itself because they are script-shaped and must not be
|
||||
re-implemented by a model: claim (`issue.sh claim`, which refuses when
|
||||
someone else holds the issue — the backstop), merge (API calls below), and
|
||||
housekeep (branch deletion). If a tick does nothing else, it reconciles.
|
||||
|
||||
## Model routing
|
||||
|
||||
The routing authority is the card's four tiers, reproduced here as the
|
||||
loop's assignment, not as an argument:
|
||||
|
||||
- **T0 `go/mimo-v2.5`** — mechanical gathering, commit/PR/journal prose,
|
||||
any fan-out. Effectively free; wrong only where wrongness costs a
|
||||
debugging session, so nothing above takes its word for a *fact*.
|
||||
- **T1 `qwen/deepseek-v4-pro-0813`** — implement-from-a-written-plan,
|
||||
understood-diff review, the orchestrator itself. The default session
|
||||
model; half price 10:00–20:00 EDT, which the cron is shaped around.
|
||||
- **T2 `glm/glm-5.3`** — repo-scale reasoning: selection, planning,
|
||||
consequences review, validation judgement. Weekly credits with no
|
||||
rollover: the loop draws them every week by construction, which is the
|
||||
correct posture. **`glm-5.3-flash`** for anything multimodal
|
||||
(screenshots, UI inspection).
|
||||
- **T3 `go/kimi-k3`** — escalation only: two lower tiers already failed,
|
||||
or the issue is a named gnarly one. A fresh session seeded with the
|
||||
failing tier's own summary, never a mid-session switch, never parallel,
|
||||
at most once per day.
|
||||
|
||||
The invariant behind all four, from the card: **routing down is cheap,
|
||||
routing up is expensive.** An implementation that stalls is escalated by
|
||||
having the T1 session write *what it tried, what failed, what it observed*
|
||||
and handing that to a new session one tier up. Escalating a session in
|
||||
place is forbidden in both directions.
|
||||
|
||||
Fan-out is allowed on T0 and T1 only (the Go plan's $12/5 h constraint
|
||||
makes T3 fan-out self-defeating). Critique is the one standing fan-out:
|
||||
two reviewers, two angles, one synthesis.
|
||||
|
||||
## Scheduling
|
||||
|
||||
`0 0 10-18 * * 1-5` (local = EDT): hourly on weekdays inside Qwen's
|
||||
half-price window, clear of the card's ⚠ 2–6am band (DeepSeek peaks, GLM
|
||||
loses its off-peak discount — the window the old `yj-backlog` cron sat in,
|
||||
which this replaces as the loop supersedes it).
|
||||
|
||||
- A tick takes a lock (`/tmp/yj-loop.lock`, PID + timestamp). An overrun
|
||||
tick makes the next fire exit immediately; serialization survives
|
||||
whatever the scheduler does with overlapping fires.
|
||||
- ~9 ticks/day; an issue is 2–5 ticks; **one to two issues per day** is
|
||||
the natural rate. That also paces the bills without a budget flag.
|
||||
- The port check is part of reconcile: if `34115` is occupied, the tick
|
||||
refuses any leg that needs the headless app and defers to the next
|
||||
tick, without complaint. A human's interactive tier always wins.
|
||||
|
||||
## Runtime and ON/OFF
|
||||
|
||||
The scheduler (`pi-schedule-prompt`) fires only while a pi session is open
|
||||
in the job's directory — that limitation is the switch:
|
||||
|
||||
- **Worktree:** `git worktree add` a dedicated clone at
|
||||
`~/.paseo/worktrees/loop/jumpy-hound`. Loop edits happen only there; a
|
||||
dirty tree there is the loop's business and nobody else's. **Provision
|
||||
it once before its first push:** `make build-frontend` + `make testdata`
|
||||
— the pre-push `go-test` hook needs both and refuses without them.
|
||||
- **Session:** pi in that worktree, `/name loop`. The job is bound to that
|
||||
session, so another pi elsewhere in the same directory does not
|
||||
double-fire it.
|
||||
- **ON:** resume the loop session (`pi --resume loop`) and enable the job.
|
||||
**OFF:** toggle the job off in `/schedule-prompt`, or close the session.
|
||||
**Drain** (stop taking new work, finish in flight): set `drain: true` in
|
||||
the state file.
|
||||
- **Tune in:** the same `pi --resume loop` — the chat transcript *is* the
|
||||
loop's log, each tick's reasoning inline, each leg reporting in.
|
||||
|
||||
## Identity, claims, and what the loop may touch
|
||||
|
||||
The loop operates **as the owner** via `GITEA_TOKEN` (scopes: `read:user`,
|
||||
`write:issue`, `write:pull`, `write:repository`); pushes ride SSH and need
|
||||
no token. Every tracker comment the loop writes is prefixed `⟦loop⟧`, so
|
||||
the collaborator reads it as the pump and not as a person.
|
||||
|
||||
It may only ever touch work it created: issues it claimed, branches it
|
||||
made, PRs it opened. Two mechanisms make that enforced rather than
|
||||
intentional: `issue.sh claim` refuses an issue somebody else holds, and
|
||||
reconcile checks `git ls-remote --heads origin` so a branch name collision
|
||||
from a concurrent session is caught before the first edit.
|
||||
|
||||
## Merge lifecycle
|
||||
|
||||
- **Only PRs the loop opened.** A collaborator's PR is never merged, never
|
||||
commented on for pressure, never touched.
|
||||
- **Every branch is refreshed against main before its merge**, in the
|
||||
loop worktree — the refresh is where a textual conflict surfaces, as
|
||||
diff text: hunks the loop authored are resolved there, anything else
|
||||
is left to a human with a `⟦loop⟧` comment. The protection's
|
||||
`block_on_outdated_branch` makes the refresh mandatory for adopted
|
||||
(pre-loop) branches: behind `main`, a PR cannot merge at all.
|
||||
Required contexts are re-polled on the refreshed head.
|
||||
- **Merges are one at a time**, each re-reading state — the previous
|
||||
merge moved `main`, and the next PR's mergeability is recomputed at
|
||||
its own turn.
|
||||
- **Post-merge, the `push` run on `main` is watched.** A red main after
|
||||
a loop merge halts the loop. That run is the only guard against the
|
||||
class no mergeability check sees: two PRs touching the same file,
|
||||
merging cleanly, contradicting each other.
|
||||
- The gate is the protection rule itself, read from the API: contexts
|
||||
`CI / check*` and `CI / e2e*` green, PR mergeable. (Required approvals
|
||||
is 0 today; if a second person changes protection rules, the merge
|
||||
endpoint refuses and the tick stops and reports — human business.)
|
||||
- `Closes #n` goes **in a commit body inside the branch, one line per
|
||||
issue, and in the PR body**. Both, because a squash route and a merge
|
||||
route parse different texts, and this pairing was measured: a comma
|
||||
list partially matched, five of ten issues.
|
||||
- After merging: verify against `issue.sh list --state open` that the
|
||||
issue actually closed; close any straggler naming the merge commit.
|
||||
`unclaim.yml` strips `Status/In Progress` automatically; it is not
|
||||
instant, and a re-open does not restore it — the verification is
|
||||
against the open list, not against the label.
|
||||
- Merging to `main` fans out to nothing: releases are the manual
|
||||
`release.yml`, which this loop never runs. The blast radius of a
|
||||
merge is the main branch's CI, and the critique leg is what stands
|
||||
before it.
|
||||
|
||||
## Verification contract
|
||||
|
||||
The tier table is `yellowjacket-dev`'s; the loop re-states nothing above
|
||||
it except the *division of duty*: the worker runs the tiers the change
|
||||
demands, and the validator re-reads the issue and checks that the tier
|
||||
evidence actually answers the claim — a green suite that never touched
|
||||
the reported surface is a finding, not a pass. Cosmetics are read by a
|
||||
model that can see (`visual`, the multimodal tier); a change that moves
|
||||
geometry refreshes its `ui-visual` baseline in the same commit.
|
||||
`tsc --noEmit` is part of the gate and nothing else runs it. The e2e app
|
||||
is seeded (`SEED=default`) and stopped after.
|
||||
|
||||
## Android / emulator mode
|
||||
|
||||
The loop is **device-free by default**: issues whose verification is
|
||||
physical-device behaviour stay open for humans (the repo's own tags say
|
||||
which those are). One step of the ladder exists for the rest:
|
||||
|
||||
- `{"emulator": true}` in `.pi/loop/state.json` **plus an already-booted
|
||||
emulator** (`adb devices` answers) opts the loop into building the APK
|
||||
and using `android-smoke` (crash verification), and `android-screenshot`
|
||||
/ `android-eval` as rendering evidence for `visual`.
|
||||
- The loop **never boots or stops an emulator** — that is the user's
|
||||
machine and their gesture. Boot it with `make android-emulator`
|
||||
(one-time `make android-setup`, ~3.5 GB, creates the AVD), and
|
||||
`make android-emulator-stop` when done.
|
||||
- Real-device-only issues are skipped under either setting.
|
||||
|
||||
## Budgets and pacing
|
||||
|
||||
Expected spend: dominated by the T1 implementation leg inside the
|
||||
half-price window (pennies to tens of cents) and T2 on weekly credits;
|
||||
T3 bounded at one fresh call per day. The card's numbers ($12 per rolling
|
||||
5 h, $30/week as burst headroom not allowance, GLM reset weekly) are the
|
||||
sanity cells; the loop's own weekly check compares against them rather
|
||||
than against the month.
|
||||
|
||||
## Cleanup (housekeep leg, once per day)
|
||||
|
||||
- Loop-owned branches whose commits are in `origin/main`: deleted, local
|
||||
and remote.
|
||||
- Loop-owned PRs open >7 days or red on a second identical CI cause:
|
||||
commented with what is known (`⟦loop⟧`), and left — never silently
|
||||
deleted.
|
||||
- Anything not the loop's (assignee, branch, PR): left exactly as found.
|
||||
|
||||
## Pilot phases
|
||||
|
||||
- **P0 — supervised.** One tick, user watching the transcript: reconcile,
|
||||
select, claim, plan. No merge.
|
||||
- **P1 — observed.** Two ticks ending in the loop's first merge, watched
|
||||
through CI → merge → verify-close.
|
||||
- **P2 — unattended.** The schedule left on. Weekly check against the
|
||||
card's two-minute ritual.
|
||||
- **Hard stops** (any of these halts the loop and leaves a comment, never
|
||||
a silent retry): a tick dies twice with no explanation; a merge happens
|
||||
for a PR the loop did not open; spend outside the cells above by 2×.
|
||||
|
||||
## Not now, on purpose
|
||||
|
||||
- **Parallel worktrees** — blocked on e2e's exclusive port; viable only
|
||||
with per-worktree headless ports or CI-only e2e. The shape (
|
||||
supervisor + per-issue worktrees) is the target, not the first cut.
|
||||
- **Weekend batch refactors** — DeepSeek off-peak is real but is a
|
||||
scheduling knob on top of a working pump.
|
||||
- **More chain files** — the critique fan-out is a chain; the rest stay
|
||||
orchestrator-legs until two weeks of unattended runs say which legs
|
||||
are actually fixed-shape.
|
||||
@@ -1,263 +0,0 @@
|
||||
# 021 — Listening accounting: smart plays, skips, and a real history
|
||||
|
||||
**Issue:** none yet — open one before the first edit (tracker is the
|
||||
source of truth; `./scripts/issue.sh search "skip play count"` comes
|
||||
back empty as of this writing).
|
||||
|
||||
**Status:** plan — not started.
|
||||
|
||||
**Relates:** play-count rendering (`frontend/src/components/track-list/columns.ts`),
|
||||
smart playlists (`backend/smartplaylist/`), the event contract
|
||||
(`TrackPlayCountChanged`), and any future Wrapped / "minutes listened"
|
||||
surface.
|
||||
|
||||
---
|
||||
|
||||
## What exists now
|
||||
|
||||
Three facts, all load-bearing.
|
||||
|
||||
**A "play" is recorded only on a natural finish.** `recordPlay`
|
||||
(`backend/queue/playhistory.go:9`) is called from exactly one place —
|
||||
`OnPlaybackFinished` (`backend/queue/handlers.go:14`), and only when
|
||||
`srcErr == nil`. A track the user skips past at 90% is *not* a play;
|
||||
neither is one they pause at 60% and abandon. `play_count` /
|
||||
`last_played` on `audio_files` reflect "finished to the end," nothing
|
||||
more.
|
||||
|
||||
**There is no skip concept at all.** Skipping is indistinguishable
|
||||
from a natural finish, a pause, or a shutdown. Nothing records "the
|
||||
user rejected this track," so no downstream feature (smart playlists,
|
||||
shuffle, the revisit shelf, a future skip-rate heuristic) can ask
|
||||
about it.
|
||||
|
||||
**`play_history` is a write-only log.** It holds
|
||||
`(audio_file_id, played_at)` and nothing reads it — no sqlc query
|
||||
touches it, no `PlayHistory` read path exists. Its only recorded
|
||||
purpose is the timestamps a future "minutes listened over time"
|
||||
feature would need. It is classified `Authored, Cascade` in
|
||||
`backend/datamap/datamap.go:272` ("Listening history").
|
||||
|
||||
So the gaps are: (1) skips are invisible, and (2) "played" is
|
||||
under-counted — the opposite of the usual over-counting fear. The
|
||||
scrobble intuition (count a play once `min(50%, 4:00)` has been
|
||||
*heard*, independent of how it ends) is the fix for both.
|
||||
|
||||
---
|
||||
|
||||
## What we're building
|
||||
|
||||
A single classification of every track *exit*, plus one row per exit in
|
||||
a listening log, plus the existing denormalized `play_count` /
|
||||
`last_played` updated to match the new meaning. Three exit kinds:
|
||||
|
||||
| kind | condition |
|
||||
|---|---|
|
||||
| `complete` | reached natural end, **or** abandoned with `remaining <= tail` |
|
||||
| `play` | heard `>= playThreshold`, abandoned before the tail |
|
||||
| `skip` | user moved to a *different* track before `playThreshold` |
|
||||
|
||||
Not counted, not any kind: decode failure, pause/stop/shutdown before
|
||||
the threshold, and tracks shorter than `minTrackLength`.
|
||||
|
||||
### The thresholds — named judgements, one file
|
||||
|
||||
Follow the `PreviousRestartThreshold` precedent (`backend/queue/queue.go:28`,
|
||||
a bare `const` with a comment). A new `backend/queue/listen.go` (or a
|
||||
tiny `backend/listencount` package) declares:
|
||||
|
||||
```go
|
||||
const (
|
||||
// A track this short is deliberated jingle / interstitial and is
|
||||
// never counted, either way.
|
||||
minTrackLength = 30 * time.Second
|
||||
// The scrobble rule: half the track, or four minutes, whichever
|
||||
// comes first (Last.fm / ListenBrainz).
|
||||
playThresholdMax = 4 * time.Minute
|
||||
// "Finished enough": within 15s of the end, or the last 10%,
|
||||
// whichever is larger. A 10:00 ambient track gets a 60s fade
|
||||
// window; a 2:00 pop song gets 15s.
|
||||
tailWindowFloor = 15 * time.Second
|
||||
tailWindowFraction = 0.10
|
||||
)
|
||||
|
||||
func playThreshold(d time.Duration) time.Duration {
|
||||
return min(d/2, playThresholdMax)
|
||||
}
|
||||
func tailWindow(d time.Duration) time.Duration {
|
||||
return max(d/10, tailWindowFloor)
|
||||
}
|
||||
```
|
||||
|
||||
Classification is a pure function of `(reason, position, duration)` and
|
||||
*therefore unit-testable without a player*:
|
||||
|
||||
```go
|
||||
func classify(reason ExitReason, pos, dur time.Duration) Kind
|
||||
```
|
||||
|
||||
`ExitReason` is `finished | skipped | failed | abandoned`. `skipped`
|
||||
means the queue moved to a different track by user action (Next,
|
||||
Previous past the restart threshold, PlayIndex, queue replacement,
|
||||
select-from-a-list). `failed` is the decode-error path. `abandoned` is
|
||||
pause/stop/unload/shutdown — and in v1 is a no-op (see open question 3).
|
||||
|
||||
**"Heard" is approximated by the position at exit.** We read
|
||||
`player.CurrentPositionSeconds()` at the moment of the transition, not
|
||||
an accumulated listen-time ledger. A user who seeks to 80% and listens
|
||||
5 seconds reads as "heard 80%." That is deliberately accepted for v1:
|
||||
it is how most players actually behave, it is drastically simpler, and
|
||||
the failure mode ("counted a track you skimmed as played") is mild and
|
||||
exactly what the scrobble threshold already forgives. Written down
|
||||
because "position is not listen time" is the one assumption that will
|
||||
look like a bug if it is not.
|
||||
|
||||
**Fires once per listen.** Leaving a track already leaves it; the
|
||||
`chainID` guard in `player.onPlaybackFinished` (`backend/player/player.go:633`)
|
||||
already swallows a stale finish callback, and a transition advances
|
||||
`currentIndex` past the finished track. The classifier needs the same
|
||||
guard so a Next-then-stale-finish cannot produce two rows. Key it on the
|
||||
`(audioFileID, chainID)` the transition was about.
|
||||
|
||||
---
|
||||
|
||||
## Schema — resolved: fresh design, no migration
|
||||
|
||||
The A/B migration agonizing is moot. This app has two users and both
|
||||
are devs, and play counts are explicitly not worth preserving yet — so
|
||||
the schema is written as if listening accounting had been designed in
|
||||
from the start, and the existing two databases rebuild what they need
|
||||
(see below). There is no migration step and none is re-introduced.
|
||||
|
||||
**`play_history` is renamed to `listening_events`** and grows the three
|
||||
kinds, plus the raw position/duration the classification was made from:
|
||||
|
||||
```sql
|
||||
CREATE TABLE IF NOT EXISTS listening_events (
|
||||
id INTEGER PRIMARY KEY,
|
||||
audio_file_id INTEGER NOT NULL,
|
||||
kind TEXT NOT NULL DEFAULT 'complete'
|
||||
CHECK (kind IN ('complete','play','skip')),
|
||||
position_seconds INTEGER NOT NULL DEFAULT 0,
|
||||
duration_seconds INTEGER NOT NULL DEFAULT 0,
|
||||
occurred_at DATETIME NOT NULL DEFAULT (datetime('now')),
|
||||
FOREIGN KEY(audio_file_id) REFERENCES audio_files(id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_listening_events_audio_file_id
|
||||
ON listening_events(audio_file_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_listening_events_occurred_at
|
||||
ON listening_events(occurred_at);
|
||||
```
|
||||
|
||||
`position_seconds`/`duration_seconds` are kept raw so a future re-tune
|
||||
of the threshold does not force the events to be re-recorded. `kind`
|
||||
stays the write-time classification; the raw reading is evidence, not
|
||||
a second copy of the rule.
|
||||
|
||||
**The counters are denormalized onto `audio_files`** — `skip_count` /
|
||||
`last_skipped` join the existing `play_count` / `last_played`, because
|
||||
that is where the hot read path already lives and a log join per track
|
||||
row is not acceptable. This does grow the MIXED-KIND wart (see the
|
||||
survey below for the structural answer), but it is the *continuation* of
|
||||
the existing design, not a new leak: play counts sat on `audio_files`
|
||||
from before this feature existed.
|
||||
|
||||
**What happens to the two real databases on next launch.**
|
||||
`listening_events` is a new table, created verbatim. `audio_files`
|
||||
gains two columns, which `retireStaleTables` treats as a stale Owned
|
||||
table and rebuilds by rescan — dropping `play_count` / `last_played` /
|
||||
`tag_status` with it, which is the accepted cost stated in the issue.
|
||||
`play_history` is gone from the schema and the datamap, so
|
||||
`obsoleteTables` drops it; its (natural-finish-only) timestamp rows go
|
||||
with it. Nothing here is wrong on a fresh install, and on the two dev
|
||||
machines the answer is the documented "delete and rescan."
|
||||
|
||||
---
|
||||
|
||||
## Wiring: where the classifier is called
|
||||
|
||||
The risk is not the classifier — it is that **every track-replacement
|
||||
path must classify the outgoing track**, and there are many: `Next`,
|
||||
`Previous` (past the 3s restart threshold), `PlayIndex`, `playFromStart`,
|
||||
`SetQueue` / clear-and-play, remove-current, and select-from-a-list.
|
||||
Miss one and that path silently never records a skip.
|
||||
|
||||
So the classification is centralized in one queue method —
|
||||
|
||||
```go
|
||||
// leaveCurrent(reason) classifies the track at currentIndex as it is
|
||||
// about to be replaced, and records exactly one listening event.
|
||||
// Must be called without q.mu held (it writes to SQLite).
|
||||
func (q *Queue) leaveCurrent(reason ExitReason)
|
||||
```
|
||||
|
||||
— which reads position/duration from the player, calls `classify`, and
|
||||
emits the play/skip row + `TrackPlayCountChanged` when `kind != skip`.
|
||||
`OnPlaybackFinished(nil)` routes through `leaveCurrent(finished)`, the
|
||||
navigation methods route through `leaveCurrent(skipped)` before they
|
||||
advance, and `recordPlay` becomes the "did a play happen" half of it.
|
||||
|
||||
Because "one path forgot to call it" is the failure mode, a **source
|
||||
sweep** pins it, on the pattern of `TestNoDirectRuntimeEmits`
|
||||
(`backend/events/noemit_test.go`) and `TestCatalogCoversSchema`: a test
|
||||
walks `backend/queue` for assignments to `currentIndex` (and the
|
||||
`SetQueue` / remove paths) and fails if a mutation site does not sit
|
||||
adjacent to a `leaveCurrent` call. The sweep is the enforcement; the
|
||||
central method is the convenience.
|
||||
|
||||
`recordPlay` keeps its existing contract *when a play happens* —
|
||||
`TrackPlayCountChanged` with `{audioFileId, filePath, playCount,
|
||||
lastPlayed}` — so the frontend patch path and
|
||||
`playhistory_test.go` keep passing. A skip emits no per-track event in
|
||||
v1 (open question 4).
|
||||
|
||||
---
|
||||
|
||||
## Phases
|
||||
|
||||
1. **The classifier.** `listen.go`: the constants, `playThreshold`,
|
||||
`tailWindow`, `classify`. Table-driven unit tests covering every
|
||||
cell of the tristate, the <30s exemption, the tail window on both a
|
||||
10:00 and a 2:00 track, and the clip at the 4:00 cap. No I/O.
|
||||
2. **Schema.** *Done in this session.* `listening_events` replaces
|
||||
`play_history`; `skip_count` / `last_skipped` added to
|
||||
`audio_files`; datamap entry and `TestAuthoredCascadesAreDeliberate`
|
||||
allow-list renamed; `recordPlay` writes `listening_events
|
||||
('complete')`. `make generate` run; database / datamap / queue
|
||||
tests green.
|
||||
3. **Wiring.** `leaveCurrent`, the navigation/finish/error call sites,
|
||||
the `fires once per listen` guard, and the source sweep. Extend
|
||||
`playhistory_test.go` for skip/complete classification through the
|
||||
queue rather than the pure function.
|
||||
4. **Smart-playlist field.** `skip_count` (and optionally
|
||||
`days_since_skipped`) in `smartplaylist.go` field/numeric maps and
|
||||
the editor's field list, via subquery. A frontend event for skip —
|
||||
if a UI wants a skip column — follows separately.
|
||||
|
||||
## Verification
|
||||
|
||||
- **Go:** the classifier is pure and exhaustively unit-tested; the
|
||||
queue wiring is tested in-process with `events.WithSink`
|
||||
(`backend/queue/emit_test.go` is the model), asserting a Next at 90%
|
||||
emits a *play*, a Next at 10% emits a *skip and no play*, a natural
|
||||
finish emits a *complete*.
|
||||
- **Database:** schema + datamap tests fail-loud on any new or
|
||||
reclassified table; `database_test.go`'s listening-events round-trip
|
||||
asserts the new table and the four denormalized counter columns.
|
||||
- **e2e:** `e2e/specs/play-count.spec.ts` already awaits
|
||||
`TrackPlayCountChanged`; add the skip case (advance early, assert no
|
||||
`TrackPlayCountChanged` and a `skip` row via the `__/test/sql`
|
||||
endpoint if convenient, or via the playlist effect).
|
||||
- No visual/component tier needed unless a skip column ships (phase 4).
|
||||
|
||||
## Open questions / decisions needed
|
||||
|
||||
1. **Migration mechanism.** Resolved — fresh design, no migration (see the schema section). Play counts are not worth preserving, both users are devs, and `audio_files` / `play_history` rebuild-or-drop on next launch.
|
||||
2. **"Position is not listen time."** Accept the approximation for v1,
|
||||
or track accumulated listen seconds (a real ledger on the player) now?
|
||||
3. **Abandon on shutdown.** A track paused at 70% and then app-killed:
|
||||
count a `play` (scrobble says heard) or leave it unrecorded? v1
|
||||
proposes *unrecorded* — same as today — to keep the write path off
|
||||
the shutdown critical path.
|
||||
4. **Skip event to the frontend.** Emit now (parallel to
|
||||
`TrackPlayCountChanged`) or only when a surface consumes it?
|
||||
-173
@@ -1,173 +0,0 @@
|
||||
# Contributing to YellowJacket
|
||||
|
||||
This is the contributor's half of the [README](README.md): how to build it, how
|
||||
to check a change, and how a change gets in. [`CLAUDE.md`](CLAUDE.md) is the
|
||||
deep reference — the architecture, and the reasons behind the shape of it —
|
||||
and is worth reading before a change of any size, because most of this
|
||||
codebase's traps are written down there and nowhere else.
|
||||
|
||||
## Building from source
|
||||
|
||||
YellowJacket is [Go](https://go.dev/) with a [Lit](https://lit.dev/)/TypeScript
|
||||
frontend, bridged by [Wails v3](https://wails.io/).
|
||||
|
||||
| Tool | Version |
|
||||
|------|---------|
|
||||
| Go | 1.26+ |
|
||||
| Node.js | 22+ |
|
||||
| pnpm | 10+ |
|
||||
| Wails CLI | v3 — vendored, no install needed (`go tool wails3`) |
|
||||
|
||||
The Wails v3 CLI resolves from the `tool` block in `go.mod`, so there is nothing
|
||||
to install globally; `make setup` fetches it with the rest of the tooling.
|
||||
|
||||
On Linux, install the system libraries Wails needs. v3 builds against GTK4 +
|
||||
WebKitGTK 6.0 by default:
|
||||
|
||||
```bash
|
||||
sudo apt-get install libasound2-dev libgtk-4-dev libwebkitgtk-6.0-dev # Debian/Ubuntu
|
||||
sudo pacman -S alsa-lib gtk4 webkitgtk-6.0 # Arch
|
||||
```
|
||||
|
||||
A machine without `webkitgtk-6.0` can still build with `-tags gtk3` against the
|
||||
older WebKit2GTK 4.1 stack, but that is an escape hatch, not what CI or a
|
||||
release builds. macOS and Windows need no extra system packages. Run
|
||||
`go tool wails3 doctor` to check your environment.
|
||||
|
||||
```bash
|
||||
make setup # install tooling, frontend packages and the git hooks
|
||||
make dev # run with hot-reload
|
||||
make build-dev # debug build with symbols
|
||||
make build-prod # production build (stripped and trimmed)
|
||||
make android # the arm64 APK, into bin/
|
||||
```
|
||||
|
||||
The `Makefile` is the front door and carries a one-line description against
|
||||
each target; `Taskfile.yml` and `build/<platform>/Taskfile.yml` are the build
|
||||
implementation behind it and are not called directly.
|
||||
|
||||
## Generated code
|
||||
|
||||
Two generators run from `go generate ./...`, which `make generate` wraps:
|
||||
**sqlc** turns `backend/database/sql/queries/` into Go in
|
||||
`backend/database/sql/sqlcgen/`, and **templ** turns `.templ` files into
|
||||
`*_templ.go` beside them. Never edit either output by hand — run
|
||||
`make generate` after touching a `.sql` or a `.templ` file.
|
||||
|
||||
The TypeScript bindings in `frontend/bindings/` are generated by `wails3`
|
||||
rather than by `go generate`, so they are a separate step: `make bindings`
|
||||
regenerates them and `make bindings-check` fails if they are stale.
|
||||
`frontend/src/events.ts` is generated too, from `backend/events/events.go`.
|
||||
|
||||
A pre-commit hook checks that all of this is fresh, so the usual way to meet it
|
||||
is a failing commit rather than a bug.
|
||||
|
||||
## Checking a change
|
||||
|
||||
Run the tier the change actually demands, not the cheapest one.
|
||||
|
||||
| Change | Command |
|
||||
|---|---|
|
||||
| Go | `make lint` and `make test` — both cover all three build configurations |
|
||||
| A frontend component or store | `make ui-test` (Vitest in a real Chromium, no backend) |
|
||||
| A user-visible flow | `make e2e`, against a running `make dev-headless` |
|
||||
| CSS | `make css-check` — see the Chrome 113 note below |
|
||||
| Anything cosmetic | look at a screenshot; several bugs here were invisible to every assertion and obvious in an image |
|
||||
|
||||
`make test` needs the fixture library, which is generated rather than
|
||||
committed — it runs `make testdata` itself (about a second).
|
||||
|
||||
The end-to-end tier drives the real app with no display at all: `make
|
||||
dev-headless` starts it in the background on `:34115` (add `SEED=<name>` for a
|
||||
seeded library, built by `make sandbox-seed NAME=<name>`), `make dev-logs` tails
|
||||
it and `make dev-stop` stops it. **Check the port before starting one** — if
|
||||
`:34115` is already answering, someone else's app is there, and a green result
|
||||
about their build is worse than no result.
|
||||
|
||||
Two smaller checks exist because the failure they catch is silent:
|
||||
`make bindings-check` (stale generated bindings) and `make css-check`, which is
|
||||
two passes — one fails on a `css` literal ended early by a backtick inside a
|
||||
comment, the other on a nested CSS rule that begins with a bare element
|
||||
selector. Chrome 113 is what the reference Android device renders with, and it
|
||||
drops such a rule without a word.
|
||||
|
||||
`make vulncheck` runs govulncheck over the module.
|
||||
|
||||
## The issue tracker is the source of truth
|
||||
|
||||
Work is described by issues before it is described by branches, and the tracker
|
||||
is shared with people who cannot see your terminal.
|
||||
|
||||
- **Search before starting**, closed issues included: `./scripts/issue.sh search
|
||||
<terms>`. "That was fixed three weeks ago" is the cheapest possible answer.
|
||||
- **Claim before the first edit**, not before the commit:
|
||||
`./scripts/issue.sh claim <n>` sets the assignee, applies `Status/In Progress`
|
||||
and comments with the branch, so the work is visibly taken *while it is being
|
||||
done*. It refuses if somebody else holds it — talk to them rather than working
|
||||
around it.
|
||||
- **If no issue covers the work, open one first** (`./scripts/issue.sh new`).
|
||||
- **Findings get filed.** A bug tripped over on the way to something else is an
|
||||
issue with a reproduction, not a wider diff and not a sentence in a chat log.
|
||||
- **#73 is the roadmap** and states the order the backlog should be worked in.
|
||||
|
||||
`scripts/issue.sh` is the whole interface (`list`, `mine`, `search`, `show`,
|
||||
`new`, `claim`, `unclaim`, `comment`, `close`, `label`, `depends`, `labels`) and
|
||||
wants a `GITEA_TOKEN` with `write:issue`. The labels are a taxonomy rather than
|
||||
tags: `Kind/*`, `Area/*`, `Priority/*`, `Platform/*`, plus `Reviewed/*` and
|
||||
`Status/*`, of which the last two are exclusive scopes.
|
||||
|
||||
## Commits and pull requests
|
||||
|
||||
`main` is protected, so a branch and a PR are the only way in. Branch from
|
||||
`origin/main`, and name the branch after the issue (`fix/140-…`, `feat/25-…`).
|
||||
|
||||
Commit subjects are [Conventional Commits](https://www.conventionalcommits.org/)
|
||||
— `type(scope): subject`, imperative, ≤72 characters — and are enforced by a
|
||||
`commit-msg` hook and by CI (`make commit-check`). This is load-bearing rather
|
||||
than decorative: semantic-release reads the **type** to decide the next version,
|
||||
so a CI-only change is `ci:` and never `fix(ci):`, which would ship a patch
|
||||
release. `make release-dry` prints what a release would cut right now.
|
||||
|
||||
**The closing keyword goes in the commit body**, one issue per line, because
|
||||
Gitea parses commit messages that reach `main` and does not parse the PR body:
|
||||
|
||||
```
|
||||
docs: rewrite the README as a landing page
|
||||
|
||||
<why>
|
||||
|
||||
Closes #50
|
||||
```
|
||||
|
||||
A PR body carries a commit-to-issue table, the verification you actually ran
|
||||
(with results), and a `Closes` list for whoever reads it.
|
||||
|
||||
## Style
|
||||
|
||||
- **Go** — golangci-lint v2, strict: `err113` (static errors), `nlreturn`,
|
||||
`wsl_v5`, `godot`, `sloglint`, `perfsprint`, and imports grouped stdlib →
|
||||
third-party → `yellowjacket/…` by gci.
|
||||
- **TypeScript** — strict mode, no implicit `any`, no unused locals or
|
||||
parameters.
|
||||
- Match the surrounding code. Where `CLAUDE.md` explains why something is shaped
|
||||
the way it is, that shape is load-bearing and there is usually a test pinning
|
||||
it.
|
||||
|
||||
Hooks do most of the enforcing (`lefthook.yml`, installed by `make setup`):
|
||||
pre-commit runs vet, lint, the codegen checks, the frontend typecheck and the
|
||||
two CSS checks in parallel; pre-push runs the Go suite and the UI tier,
|
||||
deliberately one after the other rather than together.
|
||||
|
||||
## Where the rest of the documentation is
|
||||
|
||||
- [`CLAUDE.md`](CLAUDE.md) — architecture and constraints, in depth.
|
||||
- [`docs/PROFILING.md`](docs/PROFILING.md) — Go pprof and frontend profiling.
|
||||
- [`docs/android-release.md`](docs/android-release.md) — the APK, its signing
|
||||
key, and what the release workflow checks.
|
||||
- [`docs/index-cache.md`](docs/index-cache.md) — the search-index build cache
|
||||
and why it has a snapshot.
|
||||
- [`packaging/arch/README.md`](packaging/arch/README.md),
|
||||
[`packaging/homebrew/README.md`](packaging/homebrew/README.md) — the two
|
||||
package channels.
|
||||
- `.planning/` — design documents and measured history, not a queue. The queue
|
||||
is the tracker.
|
||||
@@ -160,11 +160,8 @@ ui-watch: ## Same suite, in watch mode
|
||||
ui-visual: ## Run the suite including screenshot comparisons
|
||||
@cd frontend && YJ_VISUAL=1 npx vitest run $(UI_ARGS)
|
||||
|
||||
# `--update=true`, never a bare `--update`: vitest takes the following
|
||||
# positional as the flag's value, so `--update <path>` swallows the path
|
||||
# and re-records every baseline in the repo instead of the one named.
|
||||
ui-visual-update: ## Re-record the screenshot baselines (UI_ARGS=<path> to filter)
|
||||
@cd frontend && YJ_VISUAL=1 npx vitest run --update=true $(UI_ARGS)
|
||||
ui-visual-update: ## Re-record the screenshot baselines
|
||||
@cd frontend && YJ_VISUAL=1 npx vitest run --update $(UI_ARGS)
|
||||
|
||||
ui-setup: ## Install the Vitest browser provider's own Chromium (once)
|
||||
@cd frontend && pnpm install && npx playwright install chromium
|
||||
@@ -192,7 +189,7 @@ css-check: ## Fail on a css`` literal ended early by a backtick, or a nested rul
|
||||
# Every command in them is a make target on purpose, so this is
|
||||
# checkable. It also asserts AGENTS.md is a symlink to CLAUDE.md, so the
|
||||
# two harnesses cannot drift onto two descriptions of one project.
|
||||
skill-check: ## Fail if the docs name a missing make target, or AGENTS.md is not a symlink
|
||||
skill-check: ## Fail if the agent docs name a missing make target, or AGENTS.md is not a symlink
|
||||
@./scripts/skill-check.sh
|
||||
|
||||
# Conventional Commits, which CLAUDE.md claimed CI enforced for a long
|
||||
|
||||
@@ -2,139 +2,112 @@
|
||||
|
||||
*Music how it was meant to bee.*
|
||||
|
||||
YellowJacket plays the music you already own. Point it at your folders and it
|
||||
scans them, reads the tags and the cover art, and gives you a library you can
|
||||
browse, search, queue and tidy up — on your own machine, with no account, no
|
||||
streaming service and no telemetry.
|
||||
YellowJacket is a fast, cross-platform desktop music player for your local
|
||||
collection. It plays your files, keeps your library tidy, and helps you discover
|
||||
and organize your music — all in a clean, responsive interface. No accounts, no
|
||||
streaming, no telemetry: just your music on your machine.
|
||||
|
||||
It plays **MP3**, **FLAC**, **OGG Vorbis** and **WAV**, on **Linux** and
|
||||
**Android**, and builds from source on **macOS**.
|
||||
Runs on **Linux**, **macOS**, and **Windows**.
|
||||
|
||||
|
||||
## Features
|
||||
|
||||
## What it does
|
||||
### Play your music
|
||||
- Plays **MP3, FLAC, OGG Vorbis, and WAV**
|
||||
- Play, pause, seek, and volume control with a mute toggle
|
||||
- Gapless, glitch-free seeking backed by a read-ahead buffer
|
||||
- A queue you can add to, reorder, and shuffle, with play-next support
|
||||
- Shuffle and repeat (off / all / one)
|
||||
- Picks up right where you left off — remembers your track, position, and volume between sessions
|
||||
- Media-key and MPRIS support on Linux, so your desktop's playback controls just work
|
||||
|
||||
**Plays your files.** Play, pause, seek and volume with a mute toggle; a
|
||||
read-ahead buffer so seeking is instant rather than gappy; a queue you can add
|
||||
to, reorder and shuffle, with play-next; shuffle and repeat (off / all / one).
|
||||
It remembers the track, the position and the queue between sessions, and it
|
||||
answers your desktop's media keys — MPRIS on Linux, a media notification and
|
||||
lock-screen controls on Android.
|
||||
### Keep your library organized
|
||||
- Point it at your music folders and it scans them automatically
|
||||
- Reads tags and embedded cover art, and de-duplicates artwork so it isn't stored twice
|
||||
- Incremental sync — only new or changed files get reprocessed, and deleted files are cleaned up
|
||||
- Browse by **album**, **artist**, or **genre**, or search across everything
|
||||
- Mark favorites and see what you've been listening to with play history
|
||||
- Edit track tags directly when something's off
|
||||
|
||||
**Keeps the library tidy.** It scans the folders you give it and rescans only
|
||||
what changed, so a big library costs its full scan once. It de-duplicates
|
||||
embedded cover art rather than storing the same image a hundred times, notices
|
||||
files that have gone away, and spots duplicate tracks. Browse by album, artist
|
||||
or genre, search across everything, mark favourites, and see what you have been
|
||||
playing.
|
||||
### Playlists
|
||||
- Create playlists, drag tracks in, and reorder them
|
||||
- **Smart playlists** that build themselves from rules (by genre, rating, play count, and more)
|
||||
- Pin a default playlist and spot duplicate tracks at a glance
|
||||
|
||||
**Playlists, and playlists that write themselves.** Drag tracks in and reorder
|
||||
them, or describe what you want — genre, play count, how long since you played
|
||||
it — and let a smart playlist keep itself up to date.
|
||||
|
||||
**Explore and auto-tag, from the MusicBrainz catalog.** Explore browses artists,
|
||||
releases and genres from the catalog rather than only from what you own, so an
|
||||
album page can tell you that you have nine of its twelve tracks. Auto-tag
|
||||
matches your files against MusicBrainz and fills in the metadata that is
|
||||
missing, with a review step before anything is written to disk. Lyrics search
|
||||
finds a track from a line you remember.
|
||||
|
||||
Explore needs its catalog, which is a one-off ~0.6 GB download from
|
||||
**Settings → Search Index**. It asks first on a metered connection, and
|
||||
everything else in the app works without it.
|
||||
### Discover and clean up (powered by MusicBrainz)
|
||||
- **Explore** — browse artists, releases, and genres from the MusicBrainz catalog, not just what's already in your library
|
||||
- **Auto-tag** — match your files against MusicBrainz to fill in correct artist, album, and track metadata, with a review step before anything is written
|
||||
- **Lyrics search** — find a track by a line you remember
|
||||
|
||||
## Install
|
||||
|
||||
Every download comes from the
|
||||
Download the latest build for your platform from the
|
||||
[releases page](https://git.ljones.me/yonlu/yellowjacket/releases).
|
||||
|
||||
### Linux
|
||||
| Platform | Download |
|
||||
|----------|----------|
|
||||
| Linux | `yellowjacket-linux-amd64` |
|
||||
| macOS | `yellowjacket-darwin-universal.app.zip` (Apple Silicon + Intel) |
|
||||
| Windows | `yellowjacket-windows-amd64.exe` |
|
||||
|
||||
Download `yellowjacket-<version>-linux-amd64.tar.gz` from the latest release and
|
||||
unpack it. It holds the binary, a `.desktop` entry and an icon.
|
||||
Prefer to build it yourself? See [Building from source](#building-from-source).
|
||||
|
||||
On **Arch**, install it from the package registry instead and get updates with
|
||||
the rest of your system — the one-time key import and `pacman.conf` block are in
|
||||
[`packaging/arch/README.md`](packaging/arch/README.md):
|
||||
|
||||
```bash
|
||||
sudo pacman -Sy yellowjacket
|
||||
```
|
||||
|
||||
### Android
|
||||
|
||||
Install the APK from the release page, or from the URL below, which always
|
||||
points at the newest build:
|
||||
|
||||
```
|
||||
https://git.ljones.me/api/packages/yonlu/generic/yellowjacket-android/latest/yellowjacket.apk
|
||||
```
|
||||
|
||||
That URL needs no credentials, so [Obtainium](https://obtainium.imranr.dev/) can
|
||||
poll it directly and keep the app up to date. The build is `arm64-v8a` only, and
|
||||
[`docs/android-release.md`](docs/android-release.md) says why.
|
||||
|
||||
### macOS
|
||||
|
||||
Homebrew builds it from source on your own Mac — there is no prebuilt `.app`,
|
||||
because a signed macOS bundle needs a macOS machine to produce it and the
|
||||
release runner is a Linux container.
|
||||
|
||||
```bash
|
||||
brew install shadow-puppet/yellowjacket/yellowjacket
|
||||
```
|
||||
|
||||
See [`packaging/homebrew/README.md`](packaging/homebrew/README.md).
|
||||
|
||||
### Windows
|
||||
|
||||
Not published. It cross-compiles cleanly, but no Windows build of this app has
|
||||
ever been *run*, and nothing here can exercise one — so shipping it would be a
|
||||
promise that cannot be kept. You can still build it yourself: see
|
||||
[`CONTRIBUTING.md`](CONTRIBUTING.md).
|
||||
|
||||
### Coming from a 1.x install?
|
||||
|
||||
Versions restarted at **0.0.1** when releases became automatic, which every
|
||||
package manager reads as a downgrade. It costs one reinstall, once — the details
|
||||
are with each channel: [Homebrew](packaging/homebrew/README.md#upgrading-from-1x-needs-a-reinstall-once),
|
||||
[Android](docs/android-release.md#the-1x-installs-cannot-be-upgraded-to-00x).
|
||||
|
||||
## First run
|
||||
## Getting started
|
||||
|
||||
1. Launch YellowJacket.
|
||||
2. Add the folder your music lives in — the first-run wizard asks, and
|
||||
**Settings → Libraries** is where you add more later.
|
||||
3. Watch the scan finish. It reports progress, and you can browse while it runs.
|
||||
4. Queue something and press play.
|
||||
2. Open **Settings** and add the folder(s) where your music lives.
|
||||
3. Let the initial scan finish — you'll see progress as it works.
|
||||
4. Browse by album, artist, or genre, queue something up, and press play.
|
||||
|
||||
Your library and settings stay on your machine:
|
||||
Your library and settings are stored locally:
|
||||
|
||||
| | Linux / macOS | Windows |
|
||||
|---|---|---|
|
||||
| Config | `~/.config/yellowjacket/` | `%LOCALAPPDATA%\yellowjacket\config` |
|
||||
| Library data | `~/.local/share/yellowjacket/` | `%LOCALAPPDATA%\yellowjacket\data` |
|
||||
|
||||
Setting `YJ_HOME` moves both, which is how you keep a second library separate.
|
||||
## Building from source
|
||||
|
||||
## More screenshots
|
||||
YellowJacket is built with [Go](https://go.dev/) and a
|
||||
[Lit](https://lit.dev/)/TypeScript frontend, bridged by the
|
||||
[Wails](https://wails.io/) framework.
|
||||
|
||||
An album page knows what you own, and says so:
|
||||
**Prerequisites**
|
||||
|
||||
|
||||
| Tool | Version |
|
||||
|------|---------|
|
||||
| Go | 1.25+ |
|
||||
| Node.js | 22+ |
|
||||
| pnpm | 10+ |
|
||||
| Wails CLI | v3 — vendored, no install needed (`go tool wails3`) |
|
||||
|
||||
The home page suggests somewhere to start rather than opening on a wall of
|
||||
everything:
|
||||
The Wails v3 CLI resolves from the `tool` block in `go.mod`, so there is nothing
|
||||
to install globally; `make setup` fetches it with the rest of the tooling.
|
||||
|
||||
|
||||
On Linux, install the system libraries Wails needs. v3 builds against GTK4 +
|
||||
WebKitGTK 6.0 by default:
|
||||
|
||||
## Contributing, and the rest of the documentation
|
||||
```bash
|
||||
sudo apt-get install libasound2-dev libgtk-4-dev libwebkitgtk-6.0-dev # Debian/Ubuntu
|
||||
sudo pacman -S alsa-lib gtk4 webkitgtk-6.0 # Arch
|
||||
```
|
||||
|
||||
- [`CONTRIBUTING.md`](CONTRIBUTING.md) — build it from source, run the tests,
|
||||
and how a change gets in.
|
||||
- [`CLAUDE.md`](CLAUDE.md) — the deep reference: the architecture and the reasons
|
||||
behind the shape of it.
|
||||
- [The issue tracker](https://git.ljones.me/yonlu/yellowjacket/issues) is what
|
||||
is wanted and what is being worked on; **#73** is the roadmap.
|
||||
- [Releases](https://git.ljones.me/yonlu/yellowjacket/releases) double as the
|
||||
changelog — every one is generated from the commits it contains.
|
||||
A machine without `webkitgtk-6.0` can still build with `-tags gtk3` against the
|
||||
older WebKit2GTK 4.1 stack, but that is an escape hatch, not what CI or a
|
||||
release builds.
|
||||
|
||||
macOS and Windows need no extra system packages. Run `go tool wails3 doctor` to
|
||||
check your environment.
|
||||
|
||||
**Build**
|
||||
|
||||
```bash
|
||||
make setup # install tooling and git hooks
|
||||
make dev # run with hot-reload
|
||||
make build-prod # produce a release binary
|
||||
```
|
||||
|
||||
More detail for contributors lives in [`CLAUDE.md`](./CLAUDE.md) — the
|
||||
architecture, the conventions and the reasons behind them. What is
|
||||
being worked on is [the issue
|
||||
tracker](https://git.ljones.me/yonlu/yellowjacket/issues); #73 is the
|
||||
roadmap.
|
||||
|
||||
@@ -499,10 +499,6 @@ func (yj *YellowJacketApp) OnStartup(ctx context.Context) {
|
||||
PostRemove: yj.explore.InvalidateLibrarySync,
|
||||
})
|
||||
|
||||
// A deleted playlist must not leave the queue's "Playing from"
|
||||
// label pointing at it.
|
||||
yj.playlist.SetOnPlaylistDeleted(yj.queue.DropSourceForPlaylist)
|
||||
|
||||
// Register playback finished handler to drive queue auto-advance.
|
||||
yj.player.SetPlaybackFinishedHandler(yj.queue.OnPlaybackFinished)
|
||||
|
||||
@@ -779,8 +775,6 @@ func (yj *YellowJacketApp) startJanitor() {
|
||||
}
|
||||
|
||||
yj.janitor.Register(maintenance.ExpiredHTTPCacheJob(yj.database))
|
||||
yj.janitor.Register(maintenance.StaleArtistMetadataJob(yj.database))
|
||||
yj.janitor.Register(maintenance.StaleSearchClicksJob(yj.database))
|
||||
yj.janitor.Register(maintenance.OrphanedCoverFilesJob(
|
||||
yj.database, coversDir, library.CoverArtFileSet,
|
||||
))
|
||||
|
||||
@@ -303,24 +303,6 @@ func (c *Config) GetLibraryDirectory() string {
|
||||
return string(c.Library.DirectoryPath)
|
||||
}
|
||||
|
||||
// A rejected setter puts the old value back, and that is not tidiness
|
||||
// (#231). Save validates the *whole* config, so a value left behind by
|
||||
// a failed write does not merely fail its own call: it fails every
|
||||
// later save, of every unrelated setting, silently and for the rest of
|
||||
// the session. Nothing reaches disk, so a restart clears it -- which
|
||||
// is exactly what makes the fault hard to see and impossible to report.
|
||||
//
|
||||
// The setters below that assign and then validate therefore snapshot
|
||||
// the field first and restore it on the error path. SetLibraryDirectory
|
||||
// is the other safe shape and the better one where the value can be
|
||||
// built on its own: it validates a candidate *before* assigning
|
||||
// anything, so there is nothing to undo.
|
||||
//
|
||||
// Not every setter needs either. A bool, an int64 and the shortcut
|
||||
// bindings pass through no validation that can reject them, and
|
||||
// SetViewVisible refuses an unknown, non-hideable or launch-page view
|
||||
// up front, so GeneralConfig.Validate never sees one it would fail on.
|
||||
|
||||
// SetLibraryDirectory validates and saves a new library directory,
|
||||
// then emits the LibraryConfigChanged event so listeners (e.g. the
|
||||
// Library scanner) can react.
|
||||
@@ -378,14 +360,11 @@ func (c *Config) SetScanConcurrency(mode string) error {
|
||||
c.Library.ApplyDefaults()
|
||||
}
|
||||
|
||||
previous := c.Library.ScanConcurrency
|
||||
c.Library.ScanConcurrency = library.ScanConcurrency(
|
||||
mode,
|
||||
)
|
||||
|
||||
if err := c.Library.Validate(); err != nil {
|
||||
c.Library.ScanConcurrency = previous
|
||||
|
||||
return fmt.Errorf(
|
||||
"invalid scan concurrency mode: %w", err,
|
||||
)
|
||||
@@ -476,12 +455,9 @@ func (c *Config) SetThemeAccentColor(
|
||||
c.Theme.ApplyDefaults()
|
||||
}
|
||||
|
||||
previous := c.Theme.AccentColor
|
||||
c.Theme.AccentColor = color
|
||||
|
||||
if err := c.Theme.Validate(); err != nil {
|
||||
c.Theme.AccentColor = previous
|
||||
|
||||
return fmt.Errorf(
|
||||
"invalid theme accent color: %w", err,
|
||||
)
|
||||
@@ -512,12 +488,9 @@ func (c *Config) SetThemeBackgroundShade(
|
||||
c.Theme.ApplyDefaults()
|
||||
}
|
||||
|
||||
previous := c.Theme.BackgroundShade
|
||||
c.Theme.BackgroundShade = theme.BackgroundShade(shade)
|
||||
|
||||
if err := c.Theme.Validate(); err != nil {
|
||||
c.Theme.BackgroundShade = previous
|
||||
|
||||
return fmt.Errorf(
|
||||
"invalid theme background shade: %w", err,
|
||||
)
|
||||
@@ -571,12 +544,9 @@ func (c *Config) SetDefaultPage(page string) error {
|
||||
c.General.ApplyDefaults()
|
||||
}
|
||||
|
||||
previous := c.General.DefaultPage
|
||||
c.General.DefaultPage = View(page)
|
||||
|
||||
if err := c.General.Validate(); err != nil {
|
||||
c.General.DefaultPage = previous
|
||||
|
||||
return fmt.Errorf(
|
||||
"invalid default page: %w", err,
|
||||
)
|
||||
@@ -621,12 +591,9 @@ func (c *Config) SetQueueFallback(mode string) error {
|
||||
c.General.ApplyDefaults()
|
||||
}
|
||||
|
||||
previous := c.General.QueueFallback
|
||||
c.General.QueueFallback = QueueFallback(mode)
|
||||
|
||||
if err := c.General.Validate(); err != nil {
|
||||
c.General.QueueFallback = previous
|
||||
|
||||
return fmt.Errorf(
|
||||
"invalid queue fallback: %w", err,
|
||||
)
|
||||
@@ -834,12 +801,9 @@ func (c *Config) SetTrackListColumns(
|
||||
c.TrackList = &tracklist.Config{}
|
||||
}
|
||||
|
||||
previous := c.TrackList.Columns
|
||||
c.TrackList.Columns = columns
|
||||
|
||||
if err := c.TrackList.Validate(); err != nil {
|
||||
c.TrackList.Columns = previous
|
||||
|
||||
return fmt.Errorf(
|
||||
"invalid track-list columns: %w", err,
|
||||
)
|
||||
@@ -937,12 +901,9 @@ func (c *Config) SetFavoritesIconStyle(
|
||||
c.Favorites.ApplyDefaults()
|
||||
}
|
||||
|
||||
previous := c.Favorites.IconStyle
|
||||
c.Favorites.IconStyle = favorites.IconStyle(style)
|
||||
|
||||
if err := c.Favorites.Validate(); err != nil {
|
||||
c.Favorites.IconStyle = previous
|
||||
|
||||
return fmt.Errorf(
|
||||
"invalid favorites icon style: %w", err,
|
||||
)
|
||||
|
||||
@@ -1,262 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"yellowjacket/backend/library"
|
||||
"yellowjacket/backend/tracklist"
|
||||
)
|
||||
|
||||
// newSavableConfig builds a loaded, valid config in a temp directory,
|
||||
// so Save() writes rather than refusing with errSaveBeforeLoad.
|
||||
//
|
||||
// The library directory is real and set, because Config.Validate only
|
||||
// validates the Library section when DirectoryPath is non-empty -- an
|
||||
// empty one would hide a poisoned ScanConcurrency from the whole-config
|
||||
// save that is the symptom under test.
|
||||
func newSavableConfig(t *testing.T) *Config {
|
||||
t.Helper()
|
||||
|
||||
c := &Config{
|
||||
logger: slog.Default(),
|
||||
filePath: filepath.Join(t.TempDir(), "config.toml"),
|
||||
Library: &library.Config{
|
||||
DirectoryPath: library.Directory(t.TempDir()),
|
||||
},
|
||||
}
|
||||
|
||||
c.applyDefaults()
|
||||
|
||||
if err := c.Load(); err != nil {
|
||||
t.Fatalf("Load() error: %v", err)
|
||||
}
|
||||
|
||||
if err := c.Save(); err != nil {
|
||||
t.Fatalf("Save() on a fresh config error: %v", err)
|
||||
}
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
// TestSetterRejectionDoesNotPoisonTheConfig is the whole of #231.
|
||||
//
|
||||
// Every setter here assigns to the in-memory config and then validates.
|
||||
// When the validation rejects the argument, the rejected value has to go
|
||||
// back -- not because the caller sees it (it gets an error either way),
|
||||
// but because Config.Save() validates the *whole* config. A value left
|
||||
// behind by a failed setter therefore fails every later save, of every
|
||||
// unrelated setting, silently and for the rest of the session.
|
||||
//
|
||||
// So each case asserts three things in order: the setter reports the
|
||||
// error, the getter still reports the old value, and an unrelated save
|
||||
// still works. The third is the one the user feels.
|
||||
func TestSetterRejectionDoesNotPoisonTheConfig(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
// reject calls the setter with an argument its own Validate
|
||||
// refuses.
|
||||
reject func(*Config) error
|
||||
// read reports the value the setter writes, so the rollback is
|
||||
// asserted on the config rather than only on the save.
|
||||
read func(*Config) string
|
||||
}{
|
||||
{
|
||||
name: "scan concurrency",
|
||||
reject: func(c *Config) error {
|
||||
return c.SetScanConcurrency("telepathy")
|
||||
},
|
||||
read: (*Config).GetScanConcurrency,
|
||||
},
|
||||
{
|
||||
name: "theme accent colour",
|
||||
reject: func(c *Config) error {
|
||||
return c.SetThemeAccentColor("not-a-hex")
|
||||
},
|
||||
read: (*Config).GetThemeAccentColor,
|
||||
},
|
||||
{
|
||||
name: "theme background shade",
|
||||
reject: func(c *Config) error {
|
||||
return c.SetThemeBackgroundShade("chartreuse")
|
||||
},
|
||||
read: (*Config).GetThemeBackgroundShade,
|
||||
},
|
||||
{
|
||||
name: "default page",
|
||||
reject: func(c *Config) error {
|
||||
return c.SetDefaultPage("nowhere")
|
||||
},
|
||||
read: (*Config).GetDefaultPage,
|
||||
},
|
||||
{
|
||||
name: "queue fallback",
|
||||
reject: func(c *Config) error {
|
||||
return c.SetQueueFallback("improvise")
|
||||
},
|
||||
read: (*Config).GetQueueFallback,
|
||||
},
|
||||
{
|
||||
name: "favorites icon style",
|
||||
reject: func(c *Config) error {
|
||||
return c.SetFavoritesIconStyle("asterisk")
|
||||
},
|
||||
read: (*Config).GetFavoritesIconStyle,
|
||||
},
|
||||
{
|
||||
name: "track-list columns",
|
||||
reject: func(c *Config) error {
|
||||
// titleArtist is a drawing definition, not a
|
||||
// configurable column (#197), so it is exactly what
|
||||
// the frontend used to be able to send.
|
||||
return c.SetTrackListColumns([]tracklist.Column{
|
||||
{ID: "titleArtist"},
|
||||
})
|
||||
},
|
||||
read: func(c *Config) string {
|
||||
return columnIDs(c.GetTrackListColumns())
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "track-list columns, duplicated",
|
||||
reject: func(c *Config) error {
|
||||
// The route #197 closed was one invalid id; a
|
||||
// duplicate is the one still reachable from a client
|
||||
// that assembles the list itself.
|
||||
return c.SetTrackListColumns([]tracklist.Column{
|
||||
{ID: tracklist.ColTrackName},
|
||||
{ID: tracklist.ColTrackName},
|
||||
})
|
||||
},
|
||||
read: func(c *Config) string {
|
||||
return columnIDs(c.GetTrackListColumns())
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
c := newSavableConfig(t)
|
||||
before := tc.read(c)
|
||||
|
||||
if err := tc.reject(c); err == nil {
|
||||
t.Fatal("setter accepted an invalid value, want an error")
|
||||
}
|
||||
|
||||
if after := tc.read(c); after != before {
|
||||
t.Errorf(
|
||||
"value after a rejected write = %q, want the previous %q",
|
||||
after, before,
|
||||
)
|
||||
}
|
||||
|
||||
// The symptom: an unrelated setting can no longer be saved.
|
||||
if err := c.SetPopupVolume(true); err != nil {
|
||||
t.Errorf("an unrelated setter failed after a rejected write: %v", err)
|
||||
}
|
||||
|
||||
if err := c.Save(); err != nil {
|
||||
t.Errorf("Save() failed after a rejected write: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestRejectedSetterLeavesNothingOnDisk pairs with the sweep above: the
|
||||
// rollback must not be undone by what the file already holds, so a
|
||||
// config reloaded from disk after a rejected write agrees with memory.
|
||||
func TestRejectedSetterLeavesNothingOnDisk(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
c := newSavableConfig(t)
|
||||
|
||||
if err := c.SetThemeAccentColor("#123456"); err != nil {
|
||||
t.Fatalf("SetThemeAccentColor() error: %v", err)
|
||||
}
|
||||
|
||||
if err := c.SetThemeAccentColor("not-a-hex"); err == nil {
|
||||
t.Fatal("SetThemeAccentColor accepted a non-colour, want an error")
|
||||
}
|
||||
|
||||
reloaded := &Config{logger: slog.Default(), filePath: c.filePath}
|
||||
reloaded.applyDefaults()
|
||||
|
||||
if err := reloaded.Load(); err != nil {
|
||||
t.Fatalf("Load() error: %v", err)
|
||||
}
|
||||
|
||||
if got := reloaded.GetThemeAccentColor(); got != "#123456" {
|
||||
t.Errorf("accent colour on disk = %q, want %q", got, "#123456")
|
||||
}
|
||||
|
||||
if c.GetThemeAccentColor() != reloaded.GetThemeAccentColor() {
|
||||
t.Errorf(
|
||||
"in-memory accent %q disagrees with disk %q after a rejected write",
|
||||
c.GetThemeAccentColor(), reloaded.GetThemeAccentColor(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSetLibraryDirectoryValidatesBeforeAssigning pins the precedent the
|
||||
// seven rolled-back setters follow: this one has always built and
|
||||
// validated a candidate before assigning, so a bad path never reaches
|
||||
// the config at all.
|
||||
func TestSetLibraryDirectoryValidatesBeforeAssigning(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
c := newSavableConfig(t)
|
||||
before := c.GetLibraryDirectory()
|
||||
|
||||
if err := c.SetLibraryDirectory(filepath.Join(t.TempDir(), "no-such-dir")); err == nil {
|
||||
t.Fatal("SetLibraryDirectory accepted a missing directory, want an error")
|
||||
}
|
||||
|
||||
if after := c.GetLibraryDirectory(); after != before {
|
||||
t.Errorf("library directory = %q, want the previous %q", after, before)
|
||||
}
|
||||
|
||||
if err := c.Save(); err != nil {
|
||||
t.Errorf("Save() failed after a rejected library directory: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSetViewVisibleRefusesBeforeAssigning covers the other setter left
|
||||
// out of the rollback pass: it guards its own argument up front, so
|
||||
// GeneralConfig.Validate never sees a view it would reject.
|
||||
func TestSetViewVisibleRefusesBeforeAssigning(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
c := newSavableConfig(t)
|
||||
|
||||
if err := c.SetViewVisible("no-such-view", false); err == nil {
|
||||
t.Fatal("SetViewVisible accepted an unknown view, want an error")
|
||||
}
|
||||
|
||||
if err := c.SetViewVisible(c.GetDefaultPage(), false); err == nil {
|
||||
t.Fatal("SetViewVisible hid the launch page, want an error")
|
||||
}
|
||||
|
||||
if err := c.Save(); err != nil {
|
||||
t.Errorf("Save() failed after a refused view visibility change: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// columnIDs renders a column list for comparison in the table above.
|
||||
func columnIDs(cols []tracklist.Column) string {
|
||||
ids := make([]byte, 0, len(cols)*8)
|
||||
|
||||
for i, col := range cols {
|
||||
if i > 0 {
|
||||
ids = append(ids, ',')
|
||||
}
|
||||
|
||||
ids = append(ids, col.ID...)
|
||||
}
|
||||
|
||||
return string(ids)
|
||||
}
|
||||
@@ -662,19 +662,19 @@ func TestSmartPlaylistColumns(t *testing.T) {
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Listening events tracking
|
||||
// Migration 10 — play history tracking
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func TestListeningEventsTable(t *testing.T) {
|
||||
func TestPlayHistoryTable(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db := NewTestDB(t)
|
||||
|
||||
// Verify listening_events table exists.
|
||||
// Verify play_history table exists.
|
||||
var tableCount int64
|
||||
|
||||
tblRows, err := db.QueryContext(
|
||||
"SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='listening_events'",
|
||||
"SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='play_history'",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("query sqlite_master: %v", err)
|
||||
@@ -695,14 +695,12 @@ func TestListeningEventsTable(t *testing.T) {
|
||||
_ = tblRows.Close()
|
||||
|
||||
if tableCount != 1 {
|
||||
t.Errorf("listening_events table count = %d, want 1", tableCount)
|
||||
t.Errorf("play_history table count = %d, want 1", tableCount)
|
||||
}
|
||||
|
||||
// Verify audio_files has the denormalized listening counters.
|
||||
// Verify audio_files has play_count and last_played columns.
|
||||
hasPlayCount := false
|
||||
hasLastPlayed := false
|
||||
hasSkipCount := false
|
||||
hasLastSkipped := false
|
||||
|
||||
colRows, err := db.QueryContext("PRAGMA table_info(audio_files)")
|
||||
if err != nil {
|
||||
@@ -734,14 +732,6 @@ func TestListeningEventsTable(t *testing.T) {
|
||||
if name == "last_played" {
|
||||
hasLastPlayed = true
|
||||
}
|
||||
|
||||
if name == "skip_count" {
|
||||
hasSkipCount = true
|
||||
}
|
||||
|
||||
if name == "last_skipped" {
|
||||
hasLastSkipped = true
|
||||
}
|
||||
}
|
||||
|
||||
_ = colRows.Close()
|
||||
@@ -754,14 +744,6 @@ func TestListeningEventsTable(t *testing.T) {
|
||||
t.Error("audio_files missing last_played column")
|
||||
}
|
||||
|
||||
if !hasSkipCount {
|
||||
t.Error("audio_files missing skip_count column")
|
||||
}
|
||||
|
||||
if !hasLastSkipped {
|
||||
t.Error("audio_files missing last_skipped column")
|
||||
}
|
||||
|
||||
// Verify track_metadata VIEW includes play_count and last_played.
|
||||
viewCols := map[string]bool{}
|
||||
|
||||
@@ -801,7 +783,7 @@ func TestListeningEventsTable(t *testing.T) {
|
||||
t.Error("track_metadata VIEW missing last_played column")
|
||||
}
|
||||
|
||||
// Round-trip: insert a listening_events row and verify play_count update.
|
||||
// Round-trip: insert a play_history row and verify play_count update.
|
||||
// First, set up test data. The test DB already has library id=0.
|
||||
InsertTestTrack(t, db, TestTrack{
|
||||
FilePath: "/test/play_history.mp3",
|
||||
@@ -839,12 +821,12 @@ func TestListeningEventsTable(t *testing.T) {
|
||||
t.Errorf("initial play_count = %d, want 0", playCount)
|
||||
}
|
||||
|
||||
// Insert a listening_events row (kind defaults to 'complete').
|
||||
// Insert a play_history row and update play_count.
|
||||
_, err = db.ExecContext(
|
||||
"INSERT INTO listening_events (audio_file_id, kind) VALUES (1, 'complete')",
|
||||
"INSERT INTO play_history (audio_file_id) VALUES (1)",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("insert listening_events: %v", err)
|
||||
t.Fatalf("insert play_history: %v", err)
|
||||
}
|
||||
|
||||
_, err = db.ExecContext(
|
||||
|
||||
@@ -151,26 +151,14 @@ func (d *DB) SetLyrics(audioFileID int64, lyrics, source, recordingMBID string)
|
||||
return d.upsertLyricsIndex(audioFileID, lyrics)
|
||||
}
|
||||
|
||||
// DeleteLyricsIndex removes one file's entry from the contentless
|
||||
// lyrics_index. It is called wherever a file row is deleted — the
|
||||
// `lyrics` table cascades with its file, but the FTS entry does not and
|
||||
// would otherwise accumulate for the life of the install (#249).
|
||||
func (d *DB) DeleteLyricsIndex(audioFileID int64) error {
|
||||
if _, err := d.db.ExecContext(d.Ctx,
|
||||
"DELETE FROM lyrics_index WHERE rowid = ?", audioFileID,
|
||||
); err != nil {
|
||||
return fmt.Errorf("could not delete lyrics_index row: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// upsertLyricsIndex refreshes a single file's entry in the contentless
|
||||
// lyrics_index. contentless_delete=1 makes the DELETE valid; an empty
|
||||
// lyrics string leaves the row deleted.
|
||||
func (d *DB) upsertLyricsIndex(audioFileID int64, lyrics string) error {
|
||||
if err := d.DeleteLyricsIndex(audioFileID); err != nil {
|
||||
return err
|
||||
if _, err := d.db.ExecContext(d.Ctx,
|
||||
"DELETE FROM lyrics_index WHERE rowid = ?", audioFileID,
|
||||
); err != nil {
|
||||
return fmt.Errorf("could not delete lyrics_index row: %w", err)
|
||||
}
|
||||
|
||||
if strings.TrimSpace(lyrics) == "" {
|
||||
|
||||
@@ -1,177 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Preserving a playlist entry across the loss of its track is two
|
||||
// statements, not one, and the split is not tidiness -- it is what
|
||||
// makes the important half work in the situation that needs it most.
|
||||
//
|
||||
// `playlist_tracks.audio_file_id` is ON DELETE SET NULL, so an entry
|
||||
// outlives its file as an id-less row that says nothing about what the
|
||||
// user put in the playlist. The phantom_* columns carry the answer
|
||||
// across and ResolvePhantomTracksAfterScan re-links them afterwards --
|
||||
// but only if something fills them *before* the rows go.
|
||||
//
|
||||
// The two halves are not equally important and are not equally
|
||||
// available:
|
||||
//
|
||||
// - **phantom_file_path is the one that matters.**
|
||||
// ResolvePhantomTracksAfterScan matches it against
|
||||
// `audio_files.file_path`, so without it an entry can never be
|
||||
// re-linked and the playlist is empty for good. It comes straight
|
||||
// off `audio_files`, whose `file_path` is the table's natural key
|
||||
// and has been present in every shape it has ever had -- including
|
||||
// the pre-013 stub of `(id, file_path, recording_id)`.
|
||||
// - The rest is *display* for a phantom entry before a rescan
|
||||
// re-links it, and it comes from the `track_metadata` view, which
|
||||
// is the one definition of a track row and not worth restating.
|
||||
//
|
||||
// Reading the view is what cannot be relied on here, and that is the
|
||||
// whole reason for the split. This runs *before* applySchema, which is
|
||||
// precisely the moment the schema is inconsistent: the view is whatever
|
||||
// the last launch's schema declared, while `audio_files` is whatever
|
||||
// the launch before that left behind. A view over columns the table no
|
||||
// longer has is not merely empty -- `pragma_table_info` on it *errors*,
|
||||
// and so does selecting from it. `cmd/indexbuild`'s fixture is exactly
|
||||
// that shape and is what caught this.
|
||||
//
|
||||
// COALESCE keeps an existing phantom value in both halves: an entry
|
||||
// already phantom is one whose file went missing in an earlier pass,
|
||||
// and its recorded metadata is the only copy left. Overwriting that
|
||||
// from a NULL join erases the rows this exists to protect.
|
||||
const (
|
||||
preservePhantomPathSQL = `
|
||||
UPDATE playlist_tracks
|
||||
SET phantom_file_path = COALESCE(phantom_file_path, (
|
||||
SELECT af.file_path FROM audio_files af
|
||||
WHERE af.id = playlist_tracks.audio_file_id
|
||||
))
|
||||
WHERE audio_file_id IS NOT NULL
|
||||
`
|
||||
|
||||
preservePhantomDisplaySQL = `
|
||||
UPDATE playlist_tracks
|
||||
SET
|
||||
phantom_title = COALESCE(phantom_title, (
|
||||
SELECT tm.title FROM track_metadata tm
|
||||
WHERE tm.id = playlist_tracks.audio_file_id
|
||||
)),
|
||||
phantom_artist = COALESCE(phantom_artist, (
|
||||
SELECT tm.artist_name FROM track_metadata tm
|
||||
WHERE tm.id = playlist_tracks.audio_file_id
|
||||
)),
|
||||
phantom_album = COALESCE(phantom_album, (
|
||||
SELECT tm.album FROM track_metadata tm
|
||||
WHERE tm.id = playlist_tracks.audio_file_id
|
||||
)),
|
||||
phantom_duration_ms = COALESCE(phantom_duration_ms, (
|
||||
SELECT af.length_milliseconds FROM audio_files af
|
||||
WHERE af.id = playlist_tracks.audio_file_id
|
||||
)),
|
||||
phantom_genre = COALESCE(phantom_genre, (
|
||||
SELECT tm.genre FROM track_metadata tm
|
||||
WHERE tm.id = playlist_tracks.audio_file_id
|
||||
)),
|
||||
phantom_cover_art_path = COALESCE(phantom_cover_art_path, (
|
||||
SELECT tm.cover_art_path FROM track_metadata tm
|
||||
WHERE tm.id = playlist_tracks.audio_file_id
|
||||
))
|
||||
WHERE audio_file_id IS NOT NULL
|
||||
`
|
||||
)
|
||||
|
||||
// PreservePlaylistPhantoms records every linked playlist entry's track
|
||||
// metadata on the entry itself, so the entry survives the rows being
|
||||
// deleted underneath it.
|
||||
//
|
||||
// Every path that empties `audio_files` must call this (or the scoped
|
||||
// variant below) first, inside the same transaction as the delete.
|
||||
// These paths have drifted before: the full rescan in backend/library
|
||||
// did this and the stale-shape retire in this package did not, so the
|
||||
// *documented* repair ("delete and rescan") preserved playlists while
|
||||
// the automatic one that exists to spare the user that work silently
|
||||
// emptied them (#183). The incremental scan's orphan cleanup and
|
||||
// RemoveFromLibrary drifted the same way and are #246.
|
||||
//
|
||||
// The display half is skipped, with a warning, when `track_metadata`
|
||||
// cannot answer -- see the note above. Skipping it costs a phantom
|
||||
// entry its title until a rescan re-links it; skipping the path half
|
||||
// would cost the entry outright, so that one is an error.
|
||||
func PreservePlaylistPhantoms(
|
||||
ctx context.Context, tx *sql.Tx, logger *slog.Logger,
|
||||
) error {
|
||||
return preservePlaylistPhantoms(ctx, tx, nil, logger)
|
||||
}
|
||||
|
||||
// PreservePlaylistPhantomsForFiles is PreservePlaylistPhantoms scoped to
|
||||
// the given audio file ids, for the two removal paths that delete a
|
||||
// known subset of the table rather than all of it: the incremental
|
||||
// scan's orphan cleanup and RemoveFromLibrary. A bulk pass there would
|
||||
// rewrite every linked playlist row on every scan for nothing.
|
||||
func PreservePlaylistPhantomsForFiles(
|
||||
ctx context.Context, tx *sql.Tx, ids []int64, logger *slog.Logger,
|
||||
) error {
|
||||
return preservePlaylistPhantoms(ctx, tx, ids, logger)
|
||||
}
|
||||
|
||||
func preservePlaylistPhantoms(
|
||||
ctx context.Context, tx *sql.Tx, ids []int64, logger *slog.Logger,
|
||||
) error {
|
||||
clause, args, skip := phantomIDFilter(ids)
|
||||
if skip {
|
||||
return nil
|
||||
}
|
||||
|
||||
if _, err := tx.ExecContext(
|
||||
ctx, preservePhantomPathSQL+clause, args...,
|
||||
); err != nil {
|
||||
return fmt.Errorf(
|
||||
"could not preserve playlist track file paths: %w", err,
|
||||
)
|
||||
}
|
||||
|
||||
if _, err := tx.ExecContext(
|
||||
ctx, preservePhantomDisplaySQL+clause, args...,
|
||||
); err != nil {
|
||||
// A failed statement does not roll back a SQLite transaction,
|
||||
// so the path half above stands and the entries remain
|
||||
// re-linkable.
|
||||
logger.Warn(
|
||||
"could not record display metadata for playlist entries; "+
|
||||
"they will be re-linked by the next scan but read as "+
|
||||
"unknown until then",
|
||||
"err", err,
|
||||
)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// phantomIDFilter builds the extra WHERE terms and arguments that scope
|
||||
// a preservation pass to a set of audio file ids. A nil ids returns the
|
||||
// empty clause (a bulk run over every linked entry); an empty slice
|
||||
// reports skip, since there is nothing to preserve.
|
||||
func phantomIDFilter(ids []int64) (clause string, args []any, skip bool) {
|
||||
switch {
|
||||
case ids == nil:
|
||||
return "", nil, false
|
||||
case len(ids) == 0:
|
||||
return "", nil, true
|
||||
}
|
||||
|
||||
clause = " AND audio_file_id IN (" +
|
||||
strings.Repeat("?,", len(ids)-1) + "?)"
|
||||
|
||||
args = make([]any, len(ids))
|
||||
for i, id := range ids {
|
||||
args[i] = id
|
||||
}
|
||||
|
||||
return clause, args, false
|
||||
}
|
||||
@@ -1,94 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestPreservePlaylistPhantomsForFilesScopesToTheRequestedIDs is the
|
||||
// scoping half of the scoped variant: a run over one file's id must
|
||||
// fill that file's playlist entries and leave every other entry alone,
|
||||
// because the incremental scan calls this once per orphan batch and a
|
||||
// pass that rewrote the whole table would touch every playlist row on
|
||||
// every scan.
|
||||
func TestPreservePlaylistPhantomsForFilesScopesToTheRequestedIDs(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
db := openRaw(t, t.TempDir())
|
||||
|
||||
if _, err := db.ExecContext(ctx, "PRAGMA foreign_keys = ON"); err != nil {
|
||||
t.Fatalf("pragma: %v", err)
|
||||
}
|
||||
|
||||
if err := applySchema(ctx, db); err != nil {
|
||||
t.Fatalf("applySchema: %v", err)
|
||||
}
|
||||
|
||||
if _, err := db.ExecContext(ctx, `
|
||||
INSERT INTO playlists (id, name) VALUES (1, 'keepme');
|
||||
INSERT INTO libraries (id, name, path) VALUES (0, 'test', '/music');
|
||||
INSERT INTO artists (id, name) VALUES (3, 'Aurora Fields');
|
||||
INSERT INTO cover_art (id, file_path, mime_type)
|
||||
VALUES (9, 'covers/7.jpg', 'image/jpeg');
|
||||
INSERT INTO albums (id, name, artist_id, cover_art_id)
|
||||
VALUES (4, 'Tideline', 3, 9);
|
||||
INSERT INTO audio_files
|
||||
(id, file_path, file_type_id, length_milliseconds,
|
||||
title, artist_credit, artist_id, album_id)
|
||||
VALUES
|
||||
(7, '/music/a.flac', 1, 1000,
|
||||
'Slack Water', 'Aurora Fields', 3, 4),
|
||||
(8, '/music/b.flac', 1, 2000,
|
||||
'Second Tide', 'Aurora Fields', 3, 4);
|
||||
INSERT INTO playlist_tracks (playlist_id, audio_file_id, position)
|
||||
VALUES (1, 7, 0), (1, 8, 1);
|
||||
`); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
|
||||
tx, err := db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("begin: %v", err)
|
||||
}
|
||||
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
if err := PreservePlaylistPhantomsForFiles(
|
||||
ctx, tx, []int64{7}, testLogger(),
|
||||
); err != nil {
|
||||
t.Fatalf("preserve: %v", err)
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
t.Fatalf("commit: %v", err)
|
||||
}
|
||||
|
||||
var filled, untouched sql.NullString
|
||||
|
||||
if err := db.QueryRowContext(ctx,
|
||||
"SELECT phantom_file_path FROM playlist_tracks WHERE audio_file_id = 7",
|
||||
).Scan(&filled); err != nil {
|
||||
t.Fatalf("read the requested entry: %v", err)
|
||||
}
|
||||
|
||||
if filled.String != "/music/a.flac" {
|
||||
t.Errorf(
|
||||
"requested entry phantom_file_path = %q, want %q",
|
||||
filled.String, "/music/a.flac",
|
||||
)
|
||||
}
|
||||
|
||||
if err := db.QueryRowContext(ctx,
|
||||
"SELECT phantom_file_path FROM playlist_tracks WHERE audio_file_id = 8",
|
||||
).Scan(&untouched); err != nil {
|
||||
t.Fatalf("read the untouched entry: %v", err)
|
||||
}
|
||||
|
||||
if untouched.Valid {
|
||||
t.Errorf(
|
||||
"untouched entry got phantom_file_path = %q, want NULL "+
|
||||
"(a scoped run must not rewrite the whole table)",
|
||||
untouched.String,
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -68,14 +68,8 @@ CREATE TABLE IF NOT EXISTS audio_files (
|
||||
-- compared against the on-disk mtime during a scan to detect files
|
||||
-- another application retagged in place.
|
||||
modified_at INTEGER NOT NULL DEFAULT 0,
|
||||
-- Listening counts, denormalized from listening_events so the hot
|
||||
-- read path (track list sort, shelves, smart playlists) never joins
|
||||
-- a log table. Authored: a rescan cannot rebuild them. This is the
|
||||
-- "MIXED KIND" half of audio_files the datamap notes.
|
||||
play_count INTEGER NOT NULL DEFAULT 0,
|
||||
last_played DATETIME,
|
||||
skip_count INTEGER NOT NULL DEFAULT 0,
|
||||
last_skipped DATETIME,
|
||||
tag_status TEXT NOT NULL DEFAULT 'untagged'
|
||||
CHECK(tag_status IN (
|
||||
'untagged', 'auto_matched', 'user_confirmed', 'user_skipped_permanent'
|
||||
|
||||
@@ -45,6 +45,8 @@ CREATE INDEX IF NOT EXISTS idx_download_items_live
|
||||
CREATE INDEX IF NOT EXISTS idx_download_items_state
|
||||
ON download_items(state);
|
||||
|
||||
-- ListDownloadItemsForDownload filters on the parent download.
|
||||
CREATE INDEX IF NOT EXISTS idx_download_items_download
|
||||
ON download_items(download_id);
|
||||
-- idx_download_items_download is deliberately NOT declared here: on an
|
||||
-- existing database this table already exists at schema-pass time with
|
||||
-- its old column still named request_id, so an inline CREATE INDEX on
|
||||
-- download_id would fail outright. See ensureDownloadIndexes in
|
||||
-- backend/database/download_rename_migration.go.
|
||||
|
||||
@@ -66,11 +66,14 @@ CREATE TABLE IF NOT EXISTS download_requests (
|
||||
FOREIGN KEY(parent_id) REFERENCES download_requests(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_download_requests_due
|
||||
ON download_requests(state, next_try_at);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_download_requests_entity
|
||||
ON download_requests(entity, state);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_download_requests_parent
|
||||
ON download_requests(parent_id) WHERE parent_id IS NOT NULL;
|
||||
-- idx_download_requests_{due,entity,parent} are deliberately NOT
|
||||
-- declared here. This table name is reused from the old one-shot
|
||||
-- attempt table (also called download_requests before the Want/Request
|
||||
-- rename), so on an existing database this CREATE TABLE is a no-op
|
||||
-- against a table that, at schema-pass time, is still shaped like the
|
||||
-- OLD attempts table and lacks these columns entirely — an inline
|
||||
-- CREATE INDEX here would fail outright rather than just no-op. See
|
||||
-- migrateDownloadRename/ensureDownloadIndexes in
|
||||
-- backend/database/download_rename_migration.go, which create these
|
||||
-- once the rename has actually happened (or immediately, on a fresh
|
||||
-- database where the columns exist from the start).
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
-- One row per track *exit*, three ways a listen can end: it reached
|
||||
-- the end, it was heard enough to count and then skipped past, or it
|
||||
-- was abandoned for another track before anyone had really listened.
|
||||
--
|
||||
-- This is the source of truth for listening behaviour. The
|
||||
-- denormalized `play_count` / `last_played` / `skip_count` /
|
||||
-- `last_skipped` on audio_files are materialized from it, because the
|
||||
-- hot read path (track-list sort, the shelves, smart playlists) must
|
||||
-- not join a log that grows by one row per song forever.
|
||||
--
|
||||
-- `kind` is the classification, applied at write time:
|
||||
--
|
||||
-- complete the track reached its natural end, or was skipped in
|
||||
-- its tail window (the last few seconds of a long fade).
|
||||
-- play the scrobble threshold was heard — half the track or
|
||||
-- four minutes, whichever is less — and the user moved on
|
||||
-- before the end.
|
||||
-- skip the user moved to a different track before that.
|
||||
--
|
||||
-- `position_seconds` / `duration_seconds` are the raw reading the
|
||||
-- classification was made from, kept so a future re-tune of the
|
||||
-- threshold does not need the events re-recorded. 0/0 on a row means
|
||||
-- "not captured for this event" (e.g. a natural finish recorded before
|
||||
-- these columns existed), not "a zero-second track".
|
||||
CREATE TABLE IF NOT EXISTS listening_events (
|
||||
id INTEGER PRIMARY KEY,
|
||||
audio_file_id INTEGER NOT NULL,
|
||||
kind TEXT NOT NULL DEFAULT 'complete'
|
||||
CHECK (kind IN ('complete', 'play', 'skip')),
|
||||
position_seconds INTEGER NOT NULL DEFAULT 0,
|
||||
duration_seconds INTEGER NOT NULL DEFAULT 0,
|
||||
occurred_at DATETIME NOT NULL DEFAULT (datetime('now')),
|
||||
FOREIGN KEY(audio_file_id) REFERENCES audio_files(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_listening_events_audio_file_id
|
||||
ON listening_events(audio_file_id);
|
||||
|
||||
-- "What did I listen to this month" walks this, rather than the
|
||||
-- per-track index above.
|
||||
CREATE INDEX IF NOT EXISTS idx_listening_events_occurred_at
|
||||
ON listening_events(occurred_at);
|
||||
@@ -0,0 +1,9 @@
|
||||
CREATE TABLE IF NOT EXISTS play_history (
|
||||
id INTEGER PRIMARY KEY,
|
||||
audio_file_id INTEGER NOT NULL,
|
||||
played_at DATETIME NOT NULL DEFAULT (datetime('now')),
|
||||
FOREIGN KEY(audio_file_id) REFERENCES audio_files(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_play_history_audio_file_id
|
||||
ON play_history(audio_file_id);
|
||||
@@ -1,15 +1,18 @@
|
||||
CREATE TABLE IF NOT EXISTS queue (
|
||||
id INTEGER PRIMARY KEY CHECK(id = 1),
|
||||
source_playlist_id INTEGER,
|
||||
current_position INTEGER NOT NULL DEFAULT 0,
|
||||
shuffle_mode BOOLEAN NOT NULL DEFAULT false,
|
||||
repeat_mode TEXT NOT NULL DEFAULT 'off',
|
||||
shuffle_order TEXT,
|
||||
-- What the queue was built from ("Playing from: X"): an album,
|
||||
-- playlist, smart playlist, genre or artist, identified by the id
|
||||
-- that source_type's namespace gives it.
|
||||
-- source_playlist_id above is unused dead weight (nothing has ever
|
||||
-- written it a nonzero value); source_type/source_id/source_label
|
||||
-- below are its generalized replacement, covering albums, playlists,
|
||||
-- smart playlists, genres and artists rather than playlists alone.
|
||||
source_type TEXT NOT NULL DEFAULT '',
|
||||
source_id INTEGER NOT NULL DEFAULT 0,
|
||||
source_label TEXT NOT NULL DEFAULT ''
|
||||
source_label TEXT NOT NULL DEFAULT '',
|
||||
FOREIGN KEY(source_playlist_id) REFERENCES playlists(id) ON DELETE SET NULL
|
||||
);
|
||||
|
||||
-- Singleton row: there is exactly one playback queue.
|
||||
|
||||
@@ -26,10 +26,17 @@ CREATE TABLE IF NOT EXISTS tagging_items (
|
||||
-- complete rip of their own directory. parent_group_key is the
|
||||
-- original folder group they were split from.
|
||||
--
|
||||
-- These columns are appended after created_at rather than grouped
|
||||
-- with the rest of the row: sqlc's `SELECT *` scans (GetTaggingItem)
|
||||
-- bind column order positionally, so new columns always go at the
|
||||
-- end.
|
||||
-- These two columns are declared LAST, after created_at, even
|
||||
-- though that reads oddly next to the rest of the table: sql/
|
||||
-- migrations/0001 brings a pre-existing tagging_items up to date
|
||||
-- with `ALTER TABLE ADD COLUMN`, which SQLite always appends at
|
||||
-- the end of the column list. A fresh install (this file) and an
|
||||
-- upgraded database (this file + the migration) must end up with
|
||||
-- IDENTICAL column order, because sqlc-generated `SELECT *` scans
|
||||
-- (e.g. GetTaggingItem) bind columns positionally — see the
|
||||
-- schema/migration column-order test in database_test.go. Put
|
||||
-- new columns wherever reads best when adding a table for the
|
||||
-- first time; append-only from the second migration on.
|
||||
synthetic INTEGER NOT NULL DEFAULT 0,
|
||||
parent_group_key TEXT NOT NULL DEFAULT '',
|
||||
-- album_artist_conflict latches to 1 the first time two tracks
|
||||
@@ -51,3 +58,10 @@ CREATE INDEX IF NOT EXISTS idx_tagging_items_library_status
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_tagging_items_status_pending
|
||||
ON tagging_items(library_id) WHERE status = 'pending';
|
||||
|
||||
-- idx_tagging_items_parent_group_key is NOT declared here on
|
||||
-- purpose: this file runs unconditionally, before migrations, even
|
||||
-- against a database that hasn't run 0001 yet — an index predicate
|
||||
-- referencing parent_group_key would fail on that table. It lives
|
||||
-- solely in sql/migrations/0001_tagging_items_synthetic.sql, which
|
||||
-- runs after the column exists either way (see database.go).
|
||||
|
||||
@@ -39,7 +39,7 @@ INSERT INTO audio_files (
|
||||
?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?
|
||||
)
|
||||
RETURNING id, file_path, library_id, file_type_id, length_milliseconds, sample_rate, bit_depth, channels, bitrate, file_size, title, artist_credit, artist_id, album_id, track_number, disc_number, total_tracks, year, composer, comment, recording_mbid, basename, group_key, modified_at, play_count, last_played, skip_count, last_skipped, tag_status
|
||||
RETURNING id, file_path, library_id, file_type_id, length_milliseconds, sample_rate, bit_depth, channels, bitrate, file_size, title, artist_credit, artist_id, album_id, track_number, disc_number, total_tracks, year, composer, comment, recording_mbid, basename, group_key, modified_at, play_count, last_played, tag_status
|
||||
`
|
||||
|
||||
type CreateAudioFileParams struct {
|
||||
@@ -135,8 +135,6 @@ func (q *Queries) CreateAudioFile(ctx context.Context, arg CreateAudioFileParams
|
||||
&i.ModifiedAt,
|
||||
&i.PlayCount,
|
||||
&i.LastPlayed,
|
||||
&i.SkipCount,
|
||||
&i.LastSkipped,
|
||||
&i.TagStatus,
|
||||
)
|
||||
return i, err
|
||||
@@ -194,7 +192,7 @@ func (q *Queries) GetAllAudioFilePaths(ctx context.Context) ([]GetAllAudioFilePa
|
||||
|
||||
const getAudioFile = `-- name: GetAudioFile :one
|
||||
|
||||
SELECT id, file_path, library_id, file_type_id, length_milliseconds, sample_rate, bit_depth, channels, bitrate, file_size, title, artist_credit, artist_id, album_id, track_number, disc_number, total_tracks, year, composer, comment, recording_mbid, basename, group_key, modified_at, play_count, last_played, skip_count, last_skipped, tag_status FROM audio_files WHERE id = ? LIMIT 1
|
||||
SELECT id, file_path, library_id, file_type_id, length_milliseconds, sample_rate, bit_depth, channels, bitrate, file_size, title, artist_credit, artist_id, album_id, track_number, disc_number, total_tracks, year, composer, comment, recording_mbid, basename, group_key, modified_at, play_count, last_played, tag_status FROM audio_files WHERE id = ? LIMIT 1
|
||||
`
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
@@ -230,15 +228,13 @@ func (q *Queries) GetAudioFile(ctx context.Context, id int64) (AudioFile, error)
|
||||
&i.ModifiedAt,
|
||||
&i.PlayCount,
|
||||
&i.LastPlayed,
|
||||
&i.SkipCount,
|
||||
&i.LastSkipped,
|
||||
&i.TagStatus,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getAudioFileByPath = `-- name: GetAudioFileByPath :one
|
||||
SELECT id, file_path, library_id, file_type_id, length_milliseconds, sample_rate, bit_depth, channels, bitrate, file_size, title, artist_credit, artist_id, album_id, track_number, disc_number, total_tracks, year, composer, comment, recording_mbid, basename, group_key, modified_at, play_count, last_played, skip_count, last_skipped, tag_status FROM audio_files WHERE file_path = ? LIMIT 1
|
||||
SELECT id, file_path, library_id, file_type_id, length_milliseconds, sample_rate, bit_depth, channels, bitrate, file_size, title, artist_credit, artist_id, album_id, track_number, disc_number, total_tracks, year, composer, comment, recording_mbid, basename, group_key, modified_at, play_count, last_played, tag_status FROM audio_files WHERE file_path = ? LIMIT 1
|
||||
`
|
||||
|
||||
func (q *Queries) GetAudioFileByPath(ctx context.Context, filePath string) (AudioFile, error) {
|
||||
@@ -271,8 +267,6 @@ func (q *Queries) GetAudioFileByPath(ctx context.Context, filePath string) (Audi
|
||||
&i.ModifiedAt,
|
||||
&i.PlayCount,
|
||||
&i.LastPlayed,
|
||||
&i.SkipCount,
|
||||
&i.LastSkipped,
|
||||
&i.TagStatus,
|
||||
)
|
||||
return i, err
|
||||
@@ -340,7 +334,7 @@ func (q *Queries) GetAudioFilesByPaths(ctx context.Context, paths []string) ([]G
|
||||
}
|
||||
|
||||
const getAudioFilesInLibrary = `-- name: GetAudioFilesInLibrary :many
|
||||
SELECT id, file_path, library_id, file_type_id, length_milliseconds, sample_rate, bit_depth, channels, bitrate, file_size, title, artist_credit, artist_id, album_id, track_number, disc_number, total_tracks, year, composer, comment, recording_mbid, basename, group_key, modified_at, play_count, last_played, skip_count, last_skipped, tag_status FROM audio_files WHERE library_id = ?
|
||||
SELECT id, file_path, library_id, file_type_id, length_milliseconds, sample_rate, bit_depth, channels, bitrate, file_size, title, artist_credit, artist_id, album_id, track_number, disc_number, total_tracks, year, composer, comment, recording_mbid, basename, group_key, modified_at, play_count, last_played, tag_status FROM audio_files WHERE library_id = ?
|
||||
`
|
||||
|
||||
func (q *Queries) GetAudioFilesInLibrary(ctx context.Context, libraryID int64) ([]AudioFile, error) {
|
||||
@@ -379,8 +373,6 @@ func (q *Queries) GetAudioFilesInLibrary(ctx context.Context, libraryID int64) (
|
||||
&i.ModifiedAt,
|
||||
&i.PlayCount,
|
||||
&i.LastPlayed,
|
||||
&i.SkipCount,
|
||||
&i.LastSkipped,
|
||||
&i.TagStatus,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -94,8 +94,6 @@ type AudioFile struct {
|
||||
ModifiedAt int64
|
||||
PlayCount int64
|
||||
LastPlayed sql.NullTime
|
||||
SkipCount int64
|
||||
LastSkipped sql.NullTime
|
||||
TagStatus string
|
||||
}
|
||||
|
||||
@@ -263,15 +261,6 @@ type Library struct {
|
||||
AutotagWarningAcked int64
|
||||
}
|
||||
|
||||
type ListeningEvent struct {
|
||||
ID int64
|
||||
AudioFileID int64
|
||||
Kind string
|
||||
PositionSeconds int64
|
||||
DurationSeconds int64
|
||||
OccurredAt time.Time
|
||||
}
|
||||
|
||||
type Lyric struct {
|
||||
AudioFileID int64
|
||||
Text string
|
||||
@@ -284,6 +273,12 @@ type LyricsIndex struct {
|
||||
Lyrics string
|
||||
}
|
||||
|
||||
type PlayHistory struct {
|
||||
ID int64
|
||||
AudioFileID int64
|
||||
PlayedAt time.Time
|
||||
}
|
||||
|
||||
type PlayerState struct {
|
||||
ID int64
|
||||
Volume int64
|
||||
@@ -317,14 +312,15 @@ type PlaylistTrack struct {
|
||||
}
|
||||
|
||||
type Queue struct {
|
||||
ID int64
|
||||
CurrentPosition int64
|
||||
ShuffleMode bool
|
||||
RepeatMode string
|
||||
ShuffleOrder sql.NullString
|
||||
SourceType string
|
||||
SourceID int64
|
||||
SourceLabel string
|
||||
ID int64
|
||||
SourcePlaylistID sql.NullInt64
|
||||
CurrentPosition int64
|
||||
ShuffleMode bool
|
||||
RepeatMode string
|
||||
ShuffleOrder sql.NullString
|
||||
SourceType string
|
||||
SourceID int64
|
||||
SourceLabel string
|
||||
}
|
||||
|
||||
type QueueTrack struct {
|
||||
|
||||
@@ -245,13 +245,6 @@ func dropDeferred(
|
||||
ctx context.Context, db *sql.DB, logger *slog.Logger,
|
||||
drop map[string]string,
|
||||
) error {
|
||||
// Asked before the transaction opens, because the answer is about
|
||||
// which tables are live and that cannot change underneath us here.
|
||||
preserve, err := shouldPreservePhantoms(ctx, db, drop)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tx, err := db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not begin the retire transaction: %w", err)
|
||||
@@ -263,22 +256,6 @@ func dropDeferred(
|
||||
return fmt.Errorf("could not defer foreign keys: %w", err)
|
||||
}
|
||||
|
||||
// Before any drop, so every entry still has a track to read. It is
|
||||
// in this transaction rather than beside it because the preservation
|
||||
// and the delete have to succeed or fail together: a commit that
|
||||
// dropped the files without the phantoms is the bug, and a commit
|
||||
// that wrote phantoms without dropping anything is a lie about rows
|
||||
// that are still there.
|
||||
if preserve {
|
||||
logger.Info(
|
||||
"preserving playlist entries across the retire of audio_files",
|
||||
)
|
||||
|
||||
if err := PreservePlaylistPhantoms(ctx, tx, logger); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Sorted, so a failure is reproducible. Map order is random, and a
|
||||
// bug that depends on which table happens to go first reproduces on
|
||||
// one run in three and passes review on the other two -- which is
|
||||
@@ -306,38 +283,6 @@ func dropDeferred(
|
||||
return nil
|
||||
}
|
||||
|
||||
// shouldPreservePhantoms reports whether this retire is about to take
|
||||
// `audio_files` out from under the playlists.
|
||||
//
|
||||
// The `playlist_tracks` check is not defensive padding. This runs
|
||||
// *before* applySchema, which is the moment the schema is by definition
|
||||
// mid-repair, and the preservation reads a table it does not drop. A
|
||||
// database old enough not to have it would otherwise fail here, and
|
||||
// failing here means the app does not open at all -- while nothing is
|
||||
// lost by skipping, since an absent `playlist_tracks` holds no
|
||||
// playlists to save.
|
||||
//
|
||||
// It deliberately does *not* ask after `track_metadata`. Whether that
|
||||
// view can answer is PreservePlaylistPhantoms's own business, because a
|
||||
// view broken against an older `audio_files` is a state this function
|
||||
// cannot detect without hitting the same error it is trying to avoid:
|
||||
// pragma_table_info on such a view errors rather than reporting no
|
||||
// columns.
|
||||
func shouldPreservePhantoms(
|
||||
ctx context.Context, db *sql.DB, drop map[string]string,
|
||||
) (bool, error) {
|
||||
if _, going := drop["audio_files"]; !going {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
cols, err := liveColumns(ctx, db, "playlist_tracks")
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
return len(cols) > 0, nil
|
||||
}
|
||||
|
||||
// staleReason reports why a live table disagrees with its declaration,
|
||||
// or "" when it agrees. A column the live table does not have is the
|
||||
// additive case; a column whose declared type changed is the one an
|
||||
|
||||
@@ -497,180 +497,3 @@ func TestParseCreateTablesReadsTheRealSchema(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestRetiringAudioFilesKeepsPlaylistContents is the symptom this
|
||||
// repair exists for: a playlist survived the retire as a row count and
|
||||
// nothing else.
|
||||
//
|
||||
// TestRetiringOwnedTablesDoesNotDangle already asserts the entry does
|
||||
// not keep a stale id, which is the *dangerous* half. It is satisfied
|
||||
// just as well by an entry that says nothing at all, which is the
|
||||
// half that quietly emptied every playlist -- so this asserts what the
|
||||
// entry still knows, and specifically phantom_file_path, because that
|
||||
// is the column ResolvePhantomTracksAfterScan matches back against
|
||||
// audio_files.file_path.
|
||||
//
|
||||
// Note the seed drops `comment`, not `artist_credit`: the mutation has
|
||||
// to leave `track_metadata` standing, since a real launch reaches the
|
||||
// retire with the view the previous launch created. A test that drops
|
||||
// the view first is testing the skip path, not this one.
|
||||
func TestRetiringAudioFilesKeepsPlaylistContents(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
db := openRaw(t, t.TempDir())
|
||||
|
||||
if _, err := db.ExecContext(ctx, "PRAGMA foreign_keys = ON"); err != nil {
|
||||
t.Fatalf("pragma: %v", err)
|
||||
}
|
||||
|
||||
if err := applySchema(ctx, db); err != nil {
|
||||
t.Fatalf("applySchema: %v", err)
|
||||
}
|
||||
|
||||
if _, err := db.ExecContext(ctx, `
|
||||
INSERT INTO playlists (id, name) VALUES (1, 'keepme');
|
||||
INSERT INTO libraries (id, name, path) VALUES (0, 'test', '/music');
|
||||
INSERT INTO artists (id, name) VALUES (3, 'Aurora Fields');
|
||||
INSERT INTO cover_art (id, file_path, mime_type)
|
||||
VALUES (9, 'covers/7.jpg', 'image/jpeg');
|
||||
INSERT INTO genres (id, name) VALUES (5, 'Ambient');
|
||||
INSERT INTO albums (id, name, artist_id, cover_art_id)
|
||||
VALUES (4, 'Tideline', 3, 9);
|
||||
INSERT INTO audio_files
|
||||
(id, file_path, file_type_id, length_milliseconds,
|
||||
title, artist_credit, artist_id, album_id)
|
||||
VALUES (7, '/music/a.flac', 1, 1000,
|
||||
'Slack Water', 'Aurora Fields', 3, 4);
|
||||
INSERT INTO file_genres (audio_file_id, genre_id) VALUES (7, 5);
|
||||
INSERT INTO playlist_tracks (playlist_id, audio_file_id, position)
|
||||
VALUES (1, 7, 0);
|
||||
ALTER TABLE audio_files DROP COLUMN comment;
|
||||
`); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
|
||||
if err := retireStaleTables(ctx, db, testLogger()); err != nil {
|
||||
t.Fatalf("retire: %v", err)
|
||||
}
|
||||
|
||||
if err := applySchema(ctx, db); err != nil {
|
||||
t.Fatalf("applySchema: %v", err)
|
||||
}
|
||||
|
||||
var (
|
||||
path, title, artist, album, genre, cover sql.NullString
|
||||
duration sql.NullInt64
|
||||
)
|
||||
|
||||
if err := db.QueryRowContext(ctx, `
|
||||
SELECT phantom_file_path, phantom_title, phantom_artist,
|
||||
phantom_album, phantom_duration_ms, phantom_genre,
|
||||
phantom_cover_art_path
|
||||
FROM playlist_tracks WHERE playlist_id = 1
|
||||
`).Scan(&path, &title, &artist, &album, &duration, &genre, &cover); err != nil {
|
||||
t.Fatalf("read the surviving entry: %v", err)
|
||||
}
|
||||
|
||||
// The one that matters: without it the entry can never be re-linked
|
||||
// by the rescan the retire itself provokes.
|
||||
if path.String != "/music/a.flac" {
|
||||
t.Fatalf(
|
||||
"phantom_file_path is %q, want %q -- the playlist entry "+
|
||||
"cannot be re-linked and the playlist is empty for good",
|
||||
path.String, "/music/a.flac",
|
||||
)
|
||||
}
|
||||
|
||||
if title.String != "Slack Water" {
|
||||
t.Errorf("phantom_title is %q, want %q", title.String, "Slack Water")
|
||||
}
|
||||
|
||||
if artist.String != "Aurora Fields" {
|
||||
t.Errorf("phantom_artist is %q, want %q", artist.String, "Aurora Fields")
|
||||
}
|
||||
|
||||
if album.String != "Tideline" {
|
||||
t.Errorf("phantom_album is %q, want %q", album.String, "Tideline")
|
||||
}
|
||||
|
||||
if duration.Int64 != 1000 {
|
||||
t.Errorf("phantom_duration_ms is %d, want 1000", duration.Int64)
|
||||
}
|
||||
|
||||
if genre.String != "Ambient" {
|
||||
t.Errorf("phantom_genre is %q, want %q", genre.String, "Ambient")
|
||||
}
|
||||
|
||||
if cover.String != "covers/7.jpg" {
|
||||
t.Errorf("phantom_cover_art_path is %q, want %q", cover.String, "covers/7.jpg")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRetiringAudioFilesKeepsPathsWhenTheViewCannotAnswer is the case
|
||||
// that broke cmd/indexbuild: this repair runs *before* applySchema, so
|
||||
// `track_metadata` is whatever the last launch declared while
|
||||
// `audio_files` is whatever the launch before that left behind, and a
|
||||
// view over columns the table no longer has does not read as empty --
|
||||
// it errors.
|
||||
//
|
||||
// The pre-013 stub shape below is the real one that fixture carries.
|
||||
// What must survive is phantom_file_path, because `file_path` is the
|
||||
// table's natural key and has been in every shape it ever had; the
|
||||
// display columns are allowed to be absent, and the open must not fail.
|
||||
func TestRetiringAudioFilesKeepsPathsWhenTheViewCannotAnswer(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
db := openRaw(t, t.TempDir())
|
||||
|
||||
if _, err := db.ExecContext(ctx, "PRAGMA foreign_keys = ON"); err != nil {
|
||||
t.Fatalf("pragma: %v", err)
|
||||
}
|
||||
|
||||
if err := applySchema(ctx, db); err != nil {
|
||||
t.Fatalf("applySchema: %v", err)
|
||||
}
|
||||
|
||||
// The rows go in *after* the reshape: dropping audio_files with
|
||||
// foreign keys on would fire the ON DELETE SET NULL and null the
|
||||
// entry this test is about, which would pass for the wrong reason.
|
||||
if _, err := db.ExecContext(ctx, `
|
||||
DROP TABLE audio_files;
|
||||
CREATE TABLE audio_files (
|
||||
id INTEGER PRIMARY KEY,
|
||||
file_path TEXT NOT NULL UNIQUE,
|
||||
recording_id INTEGER
|
||||
);
|
||||
INSERT INTO playlists (id, name) VALUES (1, 'keepme');
|
||||
INSERT INTO audio_files (id, file_path) VALUES (7, '/music/a.flac');
|
||||
INSERT INTO playlist_tracks (playlist_id, audio_file_id, position)
|
||||
VALUES (1, 7, 0);
|
||||
`); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
|
||||
// The symptom this guards: the repair must not turn a recoverable
|
||||
// database into one the app refuses to open.
|
||||
if err := retireStaleTables(ctx, db, testLogger()); err != nil {
|
||||
t.Fatalf(
|
||||
"the retire failed on a view it could not read, so the app "+
|
||||
"would not open at all: %v", err,
|
||||
)
|
||||
}
|
||||
|
||||
if err := applySchema(ctx, db); err != nil {
|
||||
t.Fatalf("applySchema: %v", err)
|
||||
}
|
||||
|
||||
var path sql.NullString
|
||||
if err := db.QueryRowContext(ctx,
|
||||
"SELECT phantom_file_path FROM playlist_tracks WHERE playlist_id = 1",
|
||||
).Scan(&path); err != nil {
|
||||
t.Fatalf("read the surviving entry: %v", err)
|
||||
}
|
||||
|
||||
if path.String != "/music/a.flac" {
|
||||
t.Fatalf(
|
||||
"phantom_file_path is %q, want %q -- the display half being "+
|
||||
"unavailable must not cost the entry its one re-link key",
|
||||
path.String, "/music/a.flac",
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -269,11 +269,10 @@ var tables = []Table{
|
||||
"from owned files plus the LRCLIB backfill.",
|
||||
},
|
||||
{
|
||||
Name: "listening_events", Kind: Authored, Lifetime: Cascade,
|
||||
Note: "Listening history, one row per track exit (complete, play " +
|
||||
"or skip). Authored, but intentionally cascades with its " +
|
||||
"track — history for a file no longer in the library has " +
|
||||
"nothing to point at.",
|
||||
Name: "play_history", Kind: Authored, Lifetime: Cascade,
|
||||
Note: "Listening history. Authored, but intentionally cascades " +
|
||||
"with its track — history for a file no longer in the library " +
|
||||
"has nothing to point at.",
|
||||
},
|
||||
{
|
||||
Name: "player_state", Kind: Authored, Lifetime: Retained,
|
||||
|
||||
@@ -212,14 +212,14 @@ func TestLifetimesMatchSchema(t *testing.T) {
|
||||
|
||||
// Authored data is unrecoverable, so it must never be removed as a side
|
||||
// effect of deleting owned data. Cascade is allowed only where the
|
||||
// catalog explains why (listening_events, queue_tracks); this test pins the
|
||||
// catalog explains why (play_history, queue_tracks); this test pins the
|
||||
// set so a new cascade onto authored data is a deliberate decision.
|
||||
func TestAuthoredCascadesAreDeliberate(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
allowed := map[string]bool{
|
||||
"listening_events": true,
|
||||
"queue_tracks": true,
|
||||
"play_history": true,
|
||||
"queue_tracks": true,
|
||||
|
||||
// Download history is scoped to the library it imported into.
|
||||
// When that library is removed the files it acquired go with
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -145,36 +144,17 @@ func (c *apiClient) checkStatus(resp *http.Response) error {
|
||||
case resp.StatusCode >= 400:
|
||||
snippet, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
|
||||
|
||||
return fmt.Errorf("%w: %w", c.errUnreachable, &httpStatusError{
|
||||
code: resp.StatusCode,
|
||||
body: strings.TrimSpace(string(snippet)),
|
||||
})
|
||||
return fmt.Errorf(
|
||||
"%w: HTTP %d: %s",
|
||||
c.errUnreachable,
|
||||
resp.StatusCode,
|
||||
strings.TrimSpace(string(snippet)),
|
||||
)
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// httpStatusError is a non-2xx answer, kept typed so a caller can tell
|
||||
// a missing endpoint from a daemon that is down.
|
||||
type httpStatusError struct {
|
||||
code int
|
||||
body string
|
||||
}
|
||||
|
||||
func (e *httpStatusError) Error() string {
|
||||
return fmt.Sprintf("HTTP %d: %s", e.code, e.body)
|
||||
}
|
||||
|
||||
// statusCode returns the HTTP status an error carries, or 0.
|
||||
func statusCode(err error) int {
|
||||
var se *httpStatusError
|
||||
if errors.As(err, &se) {
|
||||
return se.code
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
// decodeJSON decodes a JSON string into out. Providers whose auth or
|
||||
// response handling does not fit apiClient still parse bodies the same
|
||||
// way, so the helper lives here rather than being repeated.
|
||||
|
||||
@@ -1,248 +0,0 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Multi-disc rips, single-track results and coverage counted in tracks
|
||||
// rather than files (#270).
|
||||
|
||||
func TestParsePathReadsTheDiscFromItsFolder(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := []struct {
|
||||
path string
|
||||
disc int
|
||||
track int
|
||||
folder string
|
||||
}{
|
||||
{`\share\Pink Floyd - The Wall (1979)\CD2\03 Hey You.flac`, 2, 3, "The Wall"},
|
||||
{`\share\The Wall\Disc 1\01 In The Flesh.flac`, 1, 1, "The Wall"},
|
||||
{`\share\The Wall\[Disk-2]\01 Hey You.flac`, 2, 1, "The Wall"},
|
||||
{`\share\The Wall\CD1 - Live\04 Mother.flac`, 1, 4, "The Wall"},
|
||||
// The filename's own disc number is more specific than the folder.
|
||||
{`\share\The Wall\CD1\2-05 Comfortably Numb.flac`, 2, 5, "The Wall"},
|
||||
// Not a disc folder: a number is required.
|
||||
{`\share\CDs\The Wall\01 In The Flesh.flac`, 0, 1, "The Wall"},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.path, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
got := ParsePath(tc.path)
|
||||
if got.Disc != tc.disc || got.Track != tc.track || got.Folder != tc.folder {
|
||||
t.Errorf(
|
||||
"ParsePath = disc %d track %d folder %q, want %d %d %q",
|
||||
got.Disc, got.Track, got.Folder, tc.disc, tc.track, tc.folder,
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlbumDir(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := map[string]string{
|
||||
`\share\Album\CD1\01 A.flac`: "/share/Album",
|
||||
`\share\Album\01 A.flac`: "/share/Album",
|
||||
`CD1\01 A.flac`: "CD1",
|
||||
`\share\CD Collection\01.mp3`: "/share/CD Collection",
|
||||
}
|
||||
|
||||
for in, want := range cases {
|
||||
if got := AlbumDir(in); got != want {
|
||||
t.Errorf("AlbumDir(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// One album shared as CD1/CD2 is one candidate, named after the album.
|
||||
func TestSlskdGroupsDiscFoldersIntoOneCandidate(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newSlskdStub(t)
|
||||
stub.responses = []slskdResponse{{
|
||||
Username: "peer",
|
||||
Files: []slskdFile{
|
||||
{Filename: `\m\The Wall\CD1\01 In The Flesh.flac`, Size: 1},
|
||||
{Filename: `\m\The Wall\CD1\02 The Thin Ice.flac`, Size: 1},
|
||||
{Filename: `\m\The Wall\CD2\01 Hey You.flac`, Size: 1},
|
||||
{Filename: `\m\The Wall\CD2\02 Is There Anybody Out There.flac`, Size: 1},
|
||||
},
|
||||
}}
|
||||
|
||||
s, _ := newStubSlskd(t, stub)
|
||||
|
||||
got, err := s.Search(context.Background(), Download{Query: "the wall"})
|
||||
if err != nil {
|
||||
t.Fatalf("Search: %v", err)
|
||||
}
|
||||
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("got %d candidates, want the two discs as one", len(got))
|
||||
}
|
||||
|
||||
if got[0].Title != "The Wall" || len(got[0].Files) != 4 {
|
||||
t.Errorf(
|
||||
"candidate = %q with %d files, want \"The Wall\" with 4",
|
||||
got[0].Title, len(got[0].Files),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// A track search matches one file per folder, so a single-track request
|
||||
// must accept a one-file folder that an album request rightly drops.
|
||||
func TestSlskdKeepsASingleFileForATrackRequest(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newSlskdStub(t)
|
||||
stub.responses = []slskdResponse{{
|
||||
Username: "peer",
|
||||
Files: []slskdFile{
|
||||
{Filename: `\m\OK Computer\02 Paranoid Android.flac`, Size: 1},
|
||||
},
|
||||
}}
|
||||
|
||||
s, _ := newStubSlskd(t, stub)
|
||||
|
||||
track, err := s.Search(context.Background(), Download{
|
||||
RecordingMBID: "rec-1", Artist: "Radiohead", Album: "Paranoid Android",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Search: %v", err)
|
||||
}
|
||||
|
||||
if len(track) != 1 {
|
||||
t.Errorf("track request: got %d candidates, want 1", len(track))
|
||||
}
|
||||
|
||||
album, err := s.Search(context.Background(), Download{
|
||||
ReleaseMBID: "rel-1", Artist: "Radiohead", Album: "OK Computer",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Search: %v", err)
|
||||
}
|
||||
|
||||
if len(album) != 0 {
|
||||
t.Errorf("album request: got %d candidates, want the one-file folder dropped", len(album))
|
||||
}
|
||||
}
|
||||
|
||||
// Two discs with a file of the same name both reach staging, each under
|
||||
// its disc folder, where the importer reads the disc number from.
|
||||
func TestSlskdCollectKeepsDiscFolders(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newSlskdStub(t)
|
||||
s, downloads := newStubSlskd(t, stub)
|
||||
|
||||
for _, disc := range []string{"CD1", "CD2"} {
|
||||
dir := filepath.Join(downloads, disc)
|
||||
if err := os.MkdirAll(dir, 0o750); err != nil {
|
||||
t.Fatalf("mkdir: %v", err)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(
|
||||
filepath.Join(dir, "01 Intro.flac"), []byte(disc), 0o600,
|
||||
); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
dst := t.TempDir()
|
||||
|
||||
got, err := s.collect(Candidate{Files: []CandidateFile{
|
||||
{Path: `\m\Album\CD1\01 Intro.flac`, IsAudio: true},
|
||||
{Path: `\m\Album\CD2\01 Intro.flac`, IsAudio: true},
|
||||
}}, dst, "", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("collect: %v", err)
|
||||
}
|
||||
|
||||
if len(got.Files) != 2 {
|
||||
t.Fatalf("collected %d files, want 2", len(got.Files))
|
||||
}
|
||||
|
||||
for _, disc := range []string{"CD1", "CD2"} {
|
||||
data, err := os.ReadFile(filepath.Join(dst, disc, "01 Intro.flac"))
|
||||
if err != nil || string(data) != disc {
|
||||
t.Errorf("%s's file missing or overwritten: %q, %v", disc, data, err)
|
||||
}
|
||||
|
||||
if hint := ParsePath(filepath.Join(dst, disc, "01 Intro.flac")); hint.Disc == 0 {
|
||||
t.Errorf("staged %s file lost its disc number", disc)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A two-disc release whose discs both number from 01 aligns completely
|
||||
// once the disc comes from the folder; before, disc 2's 01 collided with
|
||||
// disc 1's.
|
||||
func TestMultiDiscCandidateAlignsEveryTrack(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dl := Download{
|
||||
ReleaseMBID: "the-wall",
|
||||
Artist: "Pink Floyd",
|
||||
Album: "The Wall",
|
||||
Expected: []ExpectedTrack{
|
||||
{DiscNumber: 1, Position: 1, Title: "In the Flesh?"},
|
||||
{DiscNumber: 1, Position: 2, Title: "The Thin Ice"},
|
||||
{DiscNumber: 2, Position: 1, Title: "Hey You"},
|
||||
{DiscNumber: 2, Position: 2, Title: "Is There Anybody Out There?"},
|
||||
},
|
||||
}
|
||||
|
||||
c := Candidate{
|
||||
Title: "The Wall",
|
||||
Files: []CandidateFile{
|
||||
{Path: `\m\Pink Floyd - The Wall\CD1\01 In the Flesh.flac`, Size: 1},
|
||||
{Path: `\m\Pink Floyd - The Wall\CD1\02 The Thin Ice.flac`, Size: 1},
|
||||
{Path: `\m\Pink Floyd - The Wall\CD2\01 Hey You.flac`, Size: 1},
|
||||
{Path: `\m\Pink Floyd - The Wall\CD2\02 Is There Anybody Out There.flac`, Size: 1},
|
||||
},
|
||||
}
|
||||
|
||||
got := Score(dl, c, 50, AutoDownloadPrefs{})
|
||||
|
||||
if got.Match.Completeness != 1 {
|
||||
t.Errorf("completeness = %f, want 1", got.Match.Completeness)
|
||||
}
|
||||
|
||||
if got.Match.AlbumFit < 0.99 {
|
||||
t.Errorf("album fit = %f, want the album's own name to match", got.Match.AlbumFit)
|
||||
}
|
||||
|
||||
if got.Match.Overall < minMatch {
|
||||
t.Errorf("match = %f, want it to clear the auto-pick bar %f", got.Match.Overall, minMatch)
|
||||
}
|
||||
}
|
||||
|
||||
// Ten files against a ten-track album is not a complete album when only
|
||||
// three of them are its tracks.
|
||||
func TestCompletenessCountsTracksNotFiles(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dl := okComputer()
|
||||
|
||||
c := Candidate{Title: "OK Computer", Files: []CandidateFile{
|
||||
{Path: `\m\Radiohead - OK Computer\Airbag.flac`, Size: 1},
|
||||
{Path: `\m\Radiohead - OK Computer\Paranoid Android.flac`, Size: 1},
|
||||
{Path: `\m\Radiohead - OK Computer\Exit Music (For a Film).flac`, Size: 1},
|
||||
{Path: `\m\Radiohead - OK Computer\Creep.flac`, Size: 1},
|
||||
}}
|
||||
|
||||
got := Score(dl, c, 50, AutoDownloadPrefs{})
|
||||
|
||||
if got.Match.Completeness > 0.76 {
|
||||
t.Errorf(
|
||||
"completeness = %f with 3 of 4 tracks present, want at most 0.75",
|
||||
got.Match.Completeness,
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -47,7 +47,7 @@ func grabAll(
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
f.manager.grab(ctx, dl, candidate, nil, false)
|
||||
f.manager.grab(ctx, dl, candidate, nil)
|
||||
}()
|
||||
}
|
||||
|
||||
@@ -79,9 +79,9 @@ func TestConcurrencyForPrefersOverrideThenKind(t *testing.T) {
|
||||
want int
|
||||
}{
|
||||
{
|
||||
name: "slskd defaults to a few peers",
|
||||
name: "slskd defaults to one",
|
||||
cfg: Config{Kind: KindSlskd},
|
||||
want: 3,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "usenet defaults higher",
|
||||
@@ -92,9 +92,9 @@ func TestConcurrencyForPrefersOverrideThenKind(t *testing.T) {
|
||||
name: "explicit override wins",
|
||||
cfg: Config{
|
||||
Kind: KindSlskd,
|
||||
Settings: map[string]string{concurrencyKey: "1"},
|
||||
Settings: map[string]string{concurrencyKey: "3"},
|
||||
},
|
||||
want: 1,
|
||||
want: 3,
|
||||
},
|
||||
{
|
||||
name: "nonsense override falls back",
|
||||
@@ -102,7 +102,7 @@ func TestConcurrencyForPrefersOverrideThenKind(t *testing.T) {
|
||||
Kind: KindSlskd,
|
||||
Settings: map[string]string{concurrencyKey: "not a number"},
|
||||
},
|
||||
want: 3,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "zero override falls back",
|
||||
@@ -110,7 +110,7 @@ func TestConcurrencyForPrefersOverrideThenKind(t *testing.T) {
|
||||
Kind: KindSlskd,
|
||||
Settings: map[string]string{concurrencyKey: "0"},
|
||||
},
|
||||
want: 3,
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "unknown kind falls back to the global default",
|
||||
@@ -126,9 +126,9 @@ func TestConcurrencyForPrefersOverrideThenKind(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The reason the per-provider cap exists: a daemon capped at one
|
||||
// transfer must serialize, even when the global cap would allow more and
|
||||
// the user has queued several albums at once.
|
||||
// The reason the per-provider cap exists: a Soulseek daemon capped at
|
||||
// one transfer must serialize, even when the global cap would allow
|
||||
// more and the user has queued several albums at once.
|
||||
func TestPerProviderCapSerializesTransfers(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -142,7 +142,6 @@ func TestPerProviderCapSerializesTransfers(t *testing.T) {
|
||||
ID: 1,
|
||||
Kind: KindSlskd,
|
||||
Priority: 50,
|
||||
Settings: map[string]string{concurrencyKey: "1"},
|
||||
}, slow)
|
||||
|
||||
// Three requests against the same one-at-a-time provider.
|
||||
@@ -211,8 +210,8 @@ func TestSyncSemaphoresReplacesChangedLimits(t *testing.T) {
|
||||
f.manager.installProvider(Config{ID: 1, Kind: KindSlskd}, nil)
|
||||
|
||||
first := f.manager.semaphoreFor(1)
|
||||
if want := kindConcurrency[KindSlskd]; cap(first) != want {
|
||||
t.Fatalf("slskd semaphore cap = %d, want %d", cap(first), want)
|
||||
if cap(first) != 1 {
|
||||
t.Fatalf("slskd semaphore cap = %d, want 1", cap(first))
|
||||
}
|
||||
|
||||
// Same limit: the semaphore is kept, so in-flight accounting is not
|
||||
|
||||
@@ -1,261 +0,0 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A transfer that fails on one copy of an album is not a failed
|
||||
// download while another acceptable copy exists. On Soulseek the usual
|
||||
// failure is one peer being offline, with several others offering the
|
||||
// same folder.
|
||||
|
||||
var errPeerOffline = errors.New("peer went offline")
|
||||
|
||||
func TestManagerFallsBackToTheNextCandidate(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newManagerFixture(t)
|
||||
|
||||
// The failing source ranks first on priority, so the fallback is
|
||||
// what reaches the one that works.
|
||||
bad := fakeWithAlbum(1, "offline-peer", ".flac")
|
||||
bad.GrabErr = errPeerOffline
|
||||
good := fakeWithAlbum(2, "online-peer", ".flac")
|
||||
|
||||
f.manager.installProvider(Config{ID: 1, Priority: 90}, bad)
|
||||
f.manager.installProvider(Config{ID: 2, Priority: 10}, good)
|
||||
|
||||
dl := fourTrackDownload()
|
||||
|
||||
if _, err := f.manager.Start(context.Background(), dl); err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
|
||||
waitForDownloadState(t, f.store, dl.ID, StateComplete)
|
||||
|
||||
if bad.GrabCalls != 1 || good.GrabCalls != 1 {
|
||||
t.Errorf(
|
||||
"grabs: failing=%d working=%d, want 1 and 1",
|
||||
bad.GrabCalls, good.GrabCalls,
|
||||
)
|
||||
}
|
||||
|
||||
// The abandoned attempt's staging goes with it; only a request that
|
||||
// fails outright keeps its staging for inspection.
|
||||
waitFor(t, func() bool {
|
||||
entries, err := os.ReadDir(f.staging.Root())
|
||||
|
||||
return err == nil && len(entries) == 0
|
||||
}, "the failed attempt's staging was never released")
|
||||
}
|
||||
|
||||
// Falling back must not lower the bar. A second choice outside the
|
||||
// user's guardrails is not a choice auto-pick may make, first or second.
|
||||
func TestManagerFallbackRespectsTheGuardrails(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newManagerFixture(t)
|
||||
f.manager.SetPreferences(AutoDownloadPrefs{MaxSizeMB: 50})
|
||||
|
||||
bad := fakeWithAlbum(1, "offline-peer", ".flac")
|
||||
bad.GrabErr = errPeerOffline
|
||||
bad.Candidates[0].TotalSize = 40 << 20
|
||||
|
||||
huge := fakeWithAlbum(2, "oversized", ".flac")
|
||||
huge.Candidates[0].TotalSize = 900 << 20
|
||||
|
||||
f.manager.installProvider(Config{ID: 1, Priority: 90}, bad)
|
||||
f.manager.installProvider(Config{ID: 2, Priority: 10}, huge)
|
||||
|
||||
dl := fourTrackDownload()
|
||||
|
||||
if _, err := f.manager.Start(context.Background(), dl); err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
|
||||
waitForDownloadState(t, f.store, dl.ID, StateFailed)
|
||||
|
||||
if huge.GrabCalls != 0 {
|
||||
t.Errorf("fell back to a candidate over the size ceiling")
|
||||
}
|
||||
}
|
||||
|
||||
// A copy the user picked by hand is the copy they asked for. Quietly
|
||||
// substituting another is a decision they did not make.
|
||||
func TestManagerPickDoesNotFallBack(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newManagerFixture(t)
|
||||
|
||||
bad := fakeWithAlbum(1, "offline-peer", ".flac")
|
||||
bad.GrabErr = errPeerOffline
|
||||
good := fakeWithAlbum(2, "online-peer", ".flac")
|
||||
|
||||
f.manager.installProvider(Config{ID: 1, Priority: 90}, bad)
|
||||
f.manager.installProvider(Config{ID: 2, Priority: 10}, good)
|
||||
|
||||
// A ceiling below both copies parks the result set for the user.
|
||||
f.manager.SetPreferences(AutoDownloadPrefs{MaxSizeMB: 1})
|
||||
|
||||
bad.Candidates[0].TotalSize = 30 << 20
|
||||
good.Candidates[0].TotalSize = 30 << 20
|
||||
|
||||
dl := fourTrackDownload()
|
||||
|
||||
if _, err := f.manager.Start(context.Background(), dl); err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
|
||||
if err := f.manager.Pick(
|
||||
context.Background(), dl.ID, "offline-peer-cand",
|
||||
); err != nil {
|
||||
t.Fatalf("Pick: %v", err)
|
||||
}
|
||||
|
||||
waitForDownloadState(t, f.store, dl.ID, StateFailed)
|
||||
|
||||
if good.GrabCalls != 0 {
|
||||
t.Errorf("a hand-picked grab fell back to another candidate")
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback is for surviving an offline peer or two, not for walking a
|
||||
// forty-peer list for six hours.
|
||||
func TestManagerFallbackIsBounded(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newManagerFixture(t)
|
||||
|
||||
var providers []*FakeProvider
|
||||
|
||||
for i := int64(1); i <= maxGrabAttempts+2; i++ {
|
||||
p := fakeWithAlbum(i, "peer-"+itoa(int(i)), ".flac")
|
||||
p.GrabErr = errPeerOffline
|
||||
|
||||
f.manager.installProvider(Config{ID: i, Priority: 50}, p)
|
||||
providers = append(providers, p)
|
||||
}
|
||||
|
||||
dl := fourTrackDownload()
|
||||
|
||||
if _, err := f.manager.Start(context.Background(), dl); err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
|
||||
waitForDownloadState(t, f.store, dl.ID, StateFailed)
|
||||
|
||||
grabs := 0
|
||||
for _, p := range providers {
|
||||
grabs += p.GrabCalls
|
||||
}
|
||||
|
||||
if grabs != maxGrabAttempts {
|
||||
t.Errorf("grabs = %d, want %d", grabs, maxGrabAttempts)
|
||||
}
|
||||
}
|
||||
|
||||
// The veto judges the best candidate *inside* the guardrails, so the
|
||||
// grab has to take that one — not the overall best, which may be the
|
||||
// very copy the user said not to take unattended.
|
||||
func TestManagerAutoPickTakesTheBestEligibleCandidate(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newManagerFixture(t)
|
||||
f.manager.SetPreferences(AutoDownloadPrefs{MaxSizeMB: 50})
|
||||
|
||||
huge := fakeWithAlbum(1, "oversized", ".flac")
|
||||
huge.Candidates[0].TotalSize = 900 << 20
|
||||
|
||||
fits := fakeWithAlbum(2, "fits", ".flac")
|
||||
fits.Candidates[0].TotalSize = 40 << 20
|
||||
|
||||
f.manager.installProvider(Config{ID: 1, Priority: 90}, huge)
|
||||
f.manager.installProvider(Config{ID: 2, Priority: 10}, fits)
|
||||
|
||||
dl := fourTrackDownload()
|
||||
|
||||
ranked, err := f.manager.Start(context.Background(), dl)
|
||||
if err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
|
||||
if ranked[0].ID != "oversized-cand" {
|
||||
t.Fatalf("fixture: best overall is %s, want the oversized copy", ranked[0].ID)
|
||||
}
|
||||
|
||||
waitForDownloadState(t, f.store, dl.ID, StateComplete)
|
||||
|
||||
if huge.GrabCalls != 0 || fits.GrabCalls != 1 {
|
||||
t.Errorf(
|
||||
"grabs: oversized=%d fits=%d, want 0 and 1",
|
||||
huge.GrabCalls, fits.GrabCalls,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// On Soulseek a failure is the peer's, so every folder that peer offered
|
||||
// goes with it. Elsewhere a failure is the release's, and one indexer's
|
||||
// other releases are still worth trying.
|
||||
func TestRuledOutBy(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
failed := []Candidate{
|
||||
{ID: "slskd:alice:Album", Kind: KindSlskd, ProviderID: 1, Origin: "alice"},
|
||||
{ID: "tracker-1", Kind: KindProwlarr, ProviderID: 2, Origin: "indexer"},
|
||||
}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
c Candidate
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "the same candidate",
|
||||
c: Candidate{ID: "tracker-1", Kind: KindProwlarr, ProviderID: 2, Origin: "indexer"},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "another folder from a failed peer",
|
||||
c: Candidate{
|
||||
ID: "slskd:alice:Album (2)",
|
||||
Kind: KindSlskd,
|
||||
ProviderID: 1,
|
||||
Origin: "alice",
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "another peer",
|
||||
c: Candidate{ID: "slskd:bob:Album", Kind: KindSlskd, ProviderID: 1, Origin: "bob"},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "another release from the same indexer",
|
||||
c: Candidate{ID: "tracker-2", Kind: KindProwlarr, ProviderID: 2, Origin: "indexer"},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "a peer of the same name on a different daemon",
|
||||
c: Candidate{
|
||||
ID: "slskd:alice:Album",
|
||||
Kind: KindSlskd,
|
||||
ProviderID: 3,
|
||||
Origin: "alice",
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if got := ruledOutBy(tc.c, failed); got != tc.want {
|
||||
t.Errorf("ruledOutBy = %v, want %v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,190 +0,0 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"yellowjacket/backend/jobs"
|
||||
)
|
||||
|
||||
// Filling in an almost-complete album (#276).
|
||||
//
|
||||
// A grab that delivers nine of twelve tracks clears the completeness
|
||||
// floor and is imported, and before this the other three were never
|
||||
// looked for. On Soulseek that is the commonest way an album ends up
|
||||
// almost right: one peer's folder is missing a track, or one file
|
||||
// failed. So after an import, each missing track is searched for on
|
||||
// its own and fetched from somewhere else, into the same album.
|
||||
|
||||
// maxFillInTracks bounds how many tracks are fetched one by one. An
|
||||
// album missing more than a few is a different candidate's job, not a
|
||||
// dozen single-track grabs.
|
||||
const maxFillInTracks = 3
|
||||
|
||||
// fillIn fetches the tracks a successful import did not deliver. It
|
||||
// never fails the download: the album is already imported, and a track
|
||||
// it cannot find is logged and left.
|
||||
func (m *Manager) fillIn(
|
||||
ctx context.Context,
|
||||
dl Download,
|
||||
main Candidate,
|
||||
imported ImportResult,
|
||||
job *jobs.Handle,
|
||||
) {
|
||||
missing := missingTracks(dl, imported.Matched)
|
||||
if len(missing) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
for _, t := range missing {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
|
||||
paths, err := m.fillInTrack(ctx, dl, main, t, job)
|
||||
if err != nil {
|
||||
m.logger.Info(
|
||||
"could not fill in a missing track",
|
||||
"download", dl.ID,
|
||||
"track", t.Title,
|
||||
"error", err,
|
||||
)
|
||||
|
||||
if job != nil {
|
||||
job.Logf(jobs.LevelWarn, fmt.Sprintf(
|
||||
"Could not find %q elsewhere: %v", t.Title, err,
|
||||
))
|
||||
}
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
if job != nil {
|
||||
job.Logf(jobs.LevelInfo, fmt.Sprintf(
|
||||
"Filled in %q from another source (%d file)", t.Title, len(paths),
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// missingTracks is what an import left out, when filling it in is
|
||||
// worth trying: an album with a tracklist, a few tracks short.
|
||||
func missingTracks(dl Download, matched []ExpectedTrack) []ExpectedTrack {
|
||||
// A recording request is one track; there is no album to complete.
|
||||
if dl.RecordingMBID != "" || len(dl.Expected) < 2 {
|
||||
return nil
|
||||
}
|
||||
|
||||
have := make(map[trackKey]bool, len(matched))
|
||||
for _, t := range matched {
|
||||
have[keyOf(t)] = true
|
||||
}
|
||||
|
||||
var missing []ExpectedTrack
|
||||
|
||||
for _, t := range dl.Expected {
|
||||
if !have[keyOf(t)] {
|
||||
missing = append(missing, t)
|
||||
}
|
||||
}
|
||||
|
||||
// A half-empty album was a poor copy, not a nearly complete one.
|
||||
if len(missing) > maxFillInTracks || 2*len(missing) >= len(dl.Expected) {
|
||||
return nil
|
||||
}
|
||||
|
||||
return missing
|
||||
}
|
||||
|
||||
// fillInTrack searches for one track and grabs the first acceptable
|
||||
// copy that is not from the source that already failed to supply it.
|
||||
// One attempt: a fill-in that walks a candidate list per track would
|
||||
// multiply a download's grabs by the number of gaps.
|
||||
func (m *Manager) fillInTrack(
|
||||
ctx context.Context,
|
||||
dl Download,
|
||||
main Candidate,
|
||||
t ExpectedTrack,
|
||||
job *jobs.Handle,
|
||||
) ([]string, error) {
|
||||
want := trackRequest(dl, t)
|
||||
|
||||
ranked, err := m.Search(ctx, want)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
prefs := m.preferences()
|
||||
|
||||
for _, c := range ranked {
|
||||
if ruledOutBy(c, []Candidate{main}) || !autoAcceptable(want, c, prefs) {
|
||||
continue
|
||||
}
|
||||
|
||||
// The track is being put into an album this app placed; a
|
||||
// delegate would put it in its own library instead.
|
||||
if plan, err := m.planTransfer(want, c); err != nil || plan.delegated() {
|
||||
continue
|
||||
}
|
||||
|
||||
narrowed, ok := narrowTo(c, t)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
out := m.attemptGrab(ctx, dl, narrowed, job, []ExpectedTrack{t})
|
||||
|
||||
if out.item.StagingDir != "" {
|
||||
if err := m.staging.Release(out.item.StagingDir); err != nil {
|
||||
m.logger.Warn("could not release staging dir", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
if out.err != nil {
|
||||
return nil, out.err
|
||||
}
|
||||
|
||||
if err := m.store.SetItemImported(
|
||||
ctx, out.item.ID, out.imported.Paths,
|
||||
); err != nil {
|
||||
m.logger.Warn("could not record imported paths", "error", err)
|
||||
}
|
||||
|
||||
return out.imported.Paths, nil
|
||||
}
|
||||
|
||||
return nil, ErrNoCandidates
|
||||
}
|
||||
|
||||
// trackRequest is the search for one track of an album: the track's
|
||||
// artist and title as the query, the album kept so a copy from that
|
||||
// album outranks the same song off a compilation, and one expected
|
||||
// track so a single file is a complete answer.
|
||||
func trackRequest(dl Download, t ExpectedTrack) Download {
|
||||
artist := cmp.Or(t.Artist, dl.Artist)
|
||||
|
||||
want := dl
|
||||
want.Query = strings.TrimSpace(artist + " " + t.Title)
|
||||
want.Expected = []ExpectedTrack{t}
|
||||
|
||||
return want
|
||||
}
|
||||
|
||||
// narrowTo trims a candidate to the one file that aligns to t, so the
|
||||
// grab fetches a track rather than whatever else the folder offered.
|
||||
func narrowTo(c Candidate, t ExpectedTrack) (Candidate, bool) {
|
||||
aligned, _ := matchFiles(c.Files, []ExpectedTrack{t})
|
||||
|
||||
for _, f := range aligned {
|
||||
if f.IsAudio && f.MatchedTo == t.Position {
|
||||
c.Files = []CandidateFile{f}
|
||||
c.TotalSize = f.Size
|
||||
|
||||
return c, true
|
||||
}
|
||||
}
|
||||
|
||||
return Candidate{}, false
|
||||
}
|
||||
@@ -1,168 +0,0 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Filling in the tracks an almost-complete album is missing (#276).
|
||||
|
||||
func fiveTrackTitles() []string {
|
||||
return append(allTitles(), "Let Down")
|
||||
}
|
||||
|
||||
func fiveTrackDownload() Download {
|
||||
dl := fourTrackDownload()
|
||||
dl.Expected = append(dl.Expected, ExpectedTrack{Position: 5, Title: "Let Down"})
|
||||
|
||||
return dl
|
||||
}
|
||||
|
||||
func TestMissingTracks(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dl := fiveTrackDownload()
|
||||
got := func(positions ...int) []ExpectedTrack {
|
||||
out := make([]ExpectedTrack, 0, len(positions))
|
||||
for _, p := range positions {
|
||||
out = append(out, dl.Expected[p-1])
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
dl Download
|
||||
matched []ExpectedTrack
|
||||
want int
|
||||
}{
|
||||
{"complete", dl, got(1, 2, 3, 4, 5), 0},
|
||||
{"one short", dl, got(1, 2, 3, 4), 1},
|
||||
{"two short", dl, got(1, 2, 3), 2},
|
||||
{"half gone is a poor copy", dl, got(1, 2), 0},
|
||||
{"a recording is not an album", func() Download {
|
||||
d := dl
|
||||
d.RecordingMBID = "rec"
|
||||
|
||||
return d
|
||||
}(), got(1, 2, 3, 4), 0},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
if n := len(missingTracks(tc.dl, tc.matched)); n != tc.want {
|
||||
t.Errorf("%s: %d missing, want %d", tc.name, n, tc.want)
|
||||
}
|
||||
}
|
||||
|
||||
big := fiveTrackDownload()
|
||||
for i := 6; i <= 20; i++ {
|
||||
big.Expected = append(big.Expected, ExpectedTrack{Position: i, Title: "T" + itoa(i)})
|
||||
}
|
||||
|
||||
if n := len(missingTracks(big, big.Expected[:16])); n != 0 {
|
||||
t.Errorf("four of twenty missing: %d filled in, want none past %d", n, maxFillInTracks)
|
||||
}
|
||||
}
|
||||
|
||||
// A fill-in import takes only the file that is the missing track, and
|
||||
// places it in the album with the album's tags; anything else the grab
|
||||
// brought is left out.
|
||||
func TestImportOnlyTakesTheMissingTrack(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newImportFixture(t,
|
||||
"05 - Let Down.flac",
|
||||
"02 - Paranoid Android.flac",
|
||||
)
|
||||
|
||||
dl := fiveTrackDownload()
|
||||
|
||||
got, err := f.importer.Import(
|
||||
context.Background(), dl,
|
||||
Result{Dir: f.dir, Files: f.files},
|
||||
ImportOptions{LibraryRoot: f.root, WriteTags: true, Only: dl.Expected[4:]},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("Import: %v", err)
|
||||
}
|
||||
|
||||
want := filepath.Join(f.root, "Radiohead", "OK Computer", "05 Let Down.flac")
|
||||
if len(got.Paths) != 1 || got.Paths[0] != want {
|
||||
t.Errorf("imported %q, want only %s", got.Paths, want)
|
||||
}
|
||||
|
||||
// A file that is not the missing track is not imported at all.
|
||||
g := newImportFixture(t, "02 - Paranoid Android.flac")
|
||||
|
||||
if _, err := g.importer.Import(
|
||||
context.Background(), dl,
|
||||
Result{Dir: g.dir, Files: g.files},
|
||||
ImportOptions{LibraryRoot: g.root, WriteTags: true, Only: dl.Expected[4:]},
|
||||
); !errors.Is(err, ErrTooIncomplete) {
|
||||
t.Errorf("Import = %v, want nothing matched", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The album comes from one source missing its fifth track; the fifth is
|
||||
// then found on its own at another and lands in the same album.
|
||||
func TestManagerFillsInAMissingTrack(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newManagerFixture(t)
|
||||
|
||||
titles := fiveTrackTitles()
|
||||
|
||||
album := NewFakeProvider(1, "album", Caps{CanSearch: true, CanTransport: true})
|
||||
ac := candidateFor("album-cand", titles, ".flac", 30_000_000)
|
||||
ac.ProviderID = 1
|
||||
album.Candidates = []Candidate{ac}
|
||||
|
||||
for i, tt := range titles[:4] {
|
||||
album.Written[trackToken(i+1)+" - "+tt+".flac"] = []byte("audio-data")
|
||||
}
|
||||
|
||||
single := NewFakeProvider(2, "single", Caps{CanSearch: true, CanTransport: true})
|
||||
sc := Candidate{
|
||||
ID: "single-cand",
|
||||
Protocol: ProtocolDirect,
|
||||
Title: "Radiohead - OK Computer",
|
||||
Artist: "Radiohead",
|
||||
Files: []CandidateFile{{
|
||||
Path: "Radiohead - OK Computer/05 - Let Down.flac",
|
||||
Size: 30_000_000,
|
||||
}},
|
||||
Health: 0.5,
|
||||
ProviderID: 2,
|
||||
}
|
||||
single.Candidates = []Candidate{sc}
|
||||
single.Written["05 - Let Down.flac"] = []byte("audio-data")
|
||||
|
||||
f.manager.installProvider(Config{ID: 1, Priority: 90}, album)
|
||||
f.manager.installProvider(Config{ID: 2, Priority: 10}, single)
|
||||
|
||||
dl := fiveTrackDownload()
|
||||
|
||||
if _, err := f.manager.Start(context.Background(), dl); err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
|
||||
waitForDownloadState(t, f.store, dl.ID, StateComplete)
|
||||
|
||||
if album.GrabCallCount() != 1 || single.GrabCallCount() != 1 {
|
||||
t.Errorf(
|
||||
"grabs: album=%d single=%d, want 1 and 1",
|
||||
album.GrabCallCount(), single.GrabCallCount(),
|
||||
)
|
||||
}
|
||||
|
||||
for i, tt := range titles {
|
||||
p := filepath.Join(f.root, "Radiohead", "OK Computer", trackToken(i+1)+" "+tt+".flac")
|
||||
if _, err := os.Stat(p); err != nil {
|
||||
t.Errorf("track %d not in the library: %v", i+1, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -79,12 +79,6 @@ type ImportOptions struct {
|
||||
// them. Off for delegate providers, which have already imported
|
||||
// and tagged the files themselves.
|
||||
WriteTags bool
|
||||
|
||||
// Only, when set, imports just the files that align to these
|
||||
// tracks of the download and skips the completeness check: it is a
|
||||
// fill-in for tracks an earlier grab of the same album did not
|
||||
// deliver (#276), tagged and placed as part of that album.
|
||||
Only []ExpectedTrack
|
||||
}
|
||||
|
||||
// DefaultPathTemplate is the layout used when none is configured.
|
||||
@@ -124,10 +118,6 @@ type ImportResult struct {
|
||||
// Skipped counts non-audio files left in staging (logs, cue sheets,
|
||||
// scene .nfo files) — deliberately not imported.
|
||||
Skipped int
|
||||
|
||||
// Matched are the expected tracks an imported file was aligned to,
|
||||
// which is how a caller learns what the grab did not deliver.
|
||||
Matched []ExpectedTrack
|
||||
}
|
||||
|
||||
// Import verifies, tags and moves a completed grab into the library.
|
||||
@@ -150,25 +140,14 @@ func (i *Importer) Import(
|
||||
return ImportResult{}, ErrNoAudio
|
||||
}
|
||||
|
||||
if len(opts.Only) == 0 {
|
||||
if err := checkCompleteness(len(audio), dl); err != nil {
|
||||
return ImportResult{}, err
|
||||
}
|
||||
if err := checkCompleteness(len(audio), dl); err != nil {
|
||||
return ImportResult{}, err
|
||||
}
|
||||
|
||||
// Align staged files to the expected tracklist so tags and
|
||||
// filenames reflect the release, not the uploader's naming.
|
||||
plan := i.planFiles(audio, dl)
|
||||
|
||||
if len(opts.Only) > 0 {
|
||||
plan = onlyTracks(plan, opts.Only)
|
||||
if len(plan) == 0 {
|
||||
return ImportResult{}, fmt.Errorf(
|
||||
"%w: no file matched the missing track", ErrTooIncomplete,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
out := ImportResult{
|
||||
Paths: make([]string, 0, len(plan)),
|
||||
Skipped: skipped,
|
||||
@@ -205,49 +184,11 @@ func (i *Importer) Import(
|
||||
}
|
||||
|
||||
out.Paths = append(out.Paths, dest)
|
||||
|
||||
if p.Matched {
|
||||
out.Matched = append(out.Matched, p.Track)
|
||||
}
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// trackKey identifies an expected track within a release.
|
||||
type trackKey struct{ disc, position int }
|
||||
|
||||
func keyOf(t ExpectedTrack) trackKey {
|
||||
return trackKey{disc: t.DiscNumber, position: t.Position}
|
||||
}
|
||||
|
||||
// onlyTracks keeps the planned files aligned to one of want. A fill-in
|
||||
// grab can bring more than the one file it was after — a folder where
|
||||
// the title also matched a live take — and anything else would land in
|
||||
// the album as a duplicate or a stranger.
|
||||
func onlyTracks(plan []plannedFile, want []ExpectedTrack) []plannedFile {
|
||||
keys := make(map[trackKey]bool, len(want))
|
||||
for _, t := range want {
|
||||
keys[keyOf(t)] = true
|
||||
}
|
||||
|
||||
out := make([]plannedFile, 0, len(want))
|
||||
seen := map[trackKey]bool{}
|
||||
|
||||
for _, p := range plan {
|
||||
k := keyOf(p.Track)
|
||||
if !p.Matched || !keys[k] || seen[k] {
|
||||
continue
|
||||
}
|
||||
|
||||
seen[k] = true
|
||||
|
||||
out = append(out, p)
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// plannedFile pairs a staged file with the expected track it matched.
|
||||
type plannedFile struct {
|
||||
Source string
|
||||
|
||||
@@ -1,77 +0,0 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// keyedLock is a set of mutexes created on demand, one per key, that
|
||||
// honour a context while waiting. An entry lives only while someone
|
||||
// holds or waits on it, so a key per Soulseek peer or per folder name
|
||||
// does not accumulate for the life of the process.
|
||||
type keyedLock[K comparable] struct {
|
||||
mu sync.Mutex
|
||||
held map[K]*keyedEntry
|
||||
}
|
||||
|
||||
type keyedEntry struct {
|
||||
ch chan struct{}
|
||||
|
||||
// refs counts holders and waiters; the entry is dropped at zero.
|
||||
refs int
|
||||
}
|
||||
|
||||
// acquire blocks until k is free or ctx ends, and returns the function
|
||||
// that frees it.
|
||||
func (l *keyedLock[K]) acquire(ctx context.Context, k K) (func(), error) {
|
||||
l.mu.Lock()
|
||||
|
||||
if l.held == nil {
|
||||
l.held = map[K]*keyedEntry{}
|
||||
}
|
||||
|
||||
e, ok := l.held[k]
|
||||
if !ok {
|
||||
e = &keyedEntry{ch: make(chan struct{}, 1)}
|
||||
l.held[k] = e
|
||||
}
|
||||
|
||||
e.refs++
|
||||
|
||||
l.mu.Unlock()
|
||||
|
||||
select {
|
||||
case e.ch <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
l.drop(k, e)
|
||||
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
|
||||
var once sync.Once
|
||||
|
||||
return func() {
|
||||
once.Do(func() {
|
||||
<-e.ch
|
||||
l.drop(k, e)
|
||||
})
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (l *keyedLock[K]) drop(k K, e *keyedEntry) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
|
||||
e.refs--
|
||||
if e.refs == 0 {
|
||||
delete(l.held, k)
|
||||
}
|
||||
}
|
||||
|
||||
// size reports how many keys are held or awaited, for tests.
|
||||
func (l *keyedLock[K]) size() int {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
|
||||
return len(l.held)
|
||||
}
|
||||
+65
-248
@@ -59,15 +59,13 @@ const concurrencyKey = "maxConcurrent"
|
||||
// A single global cap is the wrong shape here: usenet and torrent
|
||||
// clients are built to run many transfers at once and are throttled by
|
||||
// bandwidth, while Soulseek transfers come from one person's home
|
||||
// upload slot. Politeness there is per *peer* — asking one user for two
|
||||
// folders at once gets you queued behind everyone else at best and
|
||||
// banned at worst — and the manager holds that line separately, one
|
||||
// grab per peer (peerLocks). Two different users do not compete for
|
||||
// anyone's slot, so the daemon-wide number only bounds how many peers
|
||||
// are asked at once, and one slow peer no longer serialises every other
|
||||
// Soulseek download behind it.
|
||||
// upload slot. Hitting the same peer with parallel requests gets you
|
||||
// queued behind everyone else at best and banned at worst, so slskd is
|
||||
// capped at one — the polite number, and the one that actually
|
||||
// completes fastest, because a Soulseek peer serves one file at a time
|
||||
// regardless of how many you ask for.
|
||||
var kindConcurrency = map[Kind]int{
|
||||
KindSlskd: 3,
|
||||
KindSlskd: 1,
|
||||
KindYtDlp: 2,
|
||||
KindQBittorrent: 4,
|
||||
KindSABnzbd: 4,
|
||||
@@ -157,11 +155,6 @@ type Manager struct {
|
||||
semMu sync.Mutex
|
||||
provSem map[int64]chan struct{}
|
||||
|
||||
// peerLocks holds one grab per Soulseek peer, taken before any
|
||||
// slot: a grab waiting for a busy peer must not sit on a provider
|
||||
// slot another peer could be using.
|
||||
peerLocks keyedLock[peerKey]
|
||||
|
||||
// delegatePoll is how often delegating managers are asked for
|
||||
// status. A field rather than the constant so tests can drive the
|
||||
// full delegate flow without sleeping through it.
|
||||
@@ -584,12 +577,12 @@ func (m *Manager) Start(
|
||||
))
|
||||
}
|
||||
|
||||
if pick, ok := autoPick(dl, ranked, m.preferences()); ok {
|
||||
if m.AutoPickable(dl, ranked) {
|
||||
if job != nil {
|
||||
job.Logf(jobs.LevelInfo, "Auto-selected best candidate")
|
||||
}
|
||||
|
||||
go m.grab(context.WithoutCancel(ctx), dl, pick, job, true)
|
||||
go m.grab(context.WithoutCancel(ctx), dl, ranked[0], job)
|
||||
|
||||
return ranked, nil
|
||||
}
|
||||
@@ -629,13 +622,6 @@ func (m *Manager) Attempt(
|
||||
return false, veto, nil
|
||||
}
|
||||
|
||||
pick, ok := autoPick(dl, ranked, m.preferences())
|
||||
if !ok {
|
||||
// Unreachable while autoPick and AutoPickVeto agree; kept so a
|
||||
// future divergence refuses rather than grabbing blind.
|
||||
return false, "no candidate clears the auto-download bar", nil
|
||||
}
|
||||
|
||||
if err := m.store.CreateDownload(ctx, dl); err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
@@ -657,7 +643,7 @@ func (m *Manager) Attempt(
|
||||
))
|
||||
}
|
||||
|
||||
go m.grab(context.WithoutCancel(ctx), dl, pick, job, true)
|
||||
go m.grab(context.WithoutCancel(ctx), dl, ranked[0], job)
|
||||
|
||||
return true, "", nil
|
||||
}
|
||||
@@ -692,7 +678,7 @@ func (m *Manager) Pick(
|
||||
|
||||
job := m.startJob(dl)
|
||||
|
||||
go m.grab(context.WithoutCancel(ctx), dl, *chosen, job, false)
|
||||
go m.grab(context.WithoutCancel(ctx), dl, *chosen, job)
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -716,20 +702,13 @@ func (m *Manager) Cancel(ctx context.Context, downloadID string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// grab drives one request all the way to the library. It runs on its
|
||||
// grab drives one candidate all the way to the library. It runs on its
|
||||
// own goroutine and owns the job from here on.
|
||||
//
|
||||
// When fallback is set and a candidate's transfer fails, the next
|
||||
// candidate that auto-pick would itself have accepted is tried in its
|
||||
// place (see nextCandidate). It is set for the two unattended routes
|
||||
// and not for a candidate the user picked by hand: they chose that copy,
|
||||
// and quietly substituting another is a decision they did not make.
|
||||
func (m *Manager) grab(
|
||||
ctx context.Context,
|
||||
dl Download,
|
||||
c Candidate,
|
||||
job *jobs.Handle,
|
||||
fallback bool,
|
||||
) {
|
||||
ctx, cancel := context.WithTimeout(ctx, grabTimeout)
|
||||
defer cancel()
|
||||
@@ -744,101 +723,6 @@ func (m *Manager) grab(
|
||||
m.actMu.Unlock()
|
||||
}()
|
||||
|
||||
var failed []Candidate
|
||||
|
||||
for {
|
||||
out := m.attemptGrab(ctx, dl, c, job, nil)
|
||||
if out.err == nil {
|
||||
m.fillIn(ctx, dl, c, out.imported, job)
|
||||
m.finishGrab(ctx, dl, out.item, out.imported, job)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
failed = append(failed, c)
|
||||
|
||||
next, ok := m.nextCandidate(ctx, dl, failed, out, fallback)
|
||||
if !ok {
|
||||
m.failDownload(ctx, job, dl.ID, out.err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
m.logger.Info(
|
||||
"download candidate failed; trying the next",
|
||||
"download", dl.ID,
|
||||
"failed", c.ID,
|
||||
"next", next.ID,
|
||||
"error", out.err,
|
||||
)
|
||||
|
||||
if job != nil {
|
||||
job.Logf(jobs.LevelWarn, fmt.Sprintf(
|
||||
"%s failed (%v); trying %s instead",
|
||||
describeCandidate(c), out.err, describeCandidate(next),
|
||||
))
|
||||
}
|
||||
|
||||
// The failed attempt's staging holds at most a partial folder
|
||||
// nobody is going to import, and the next attempt reserves its
|
||||
// own. Only the final failure keeps its staging for inspection.
|
||||
if out.item.StagingDir != "" {
|
||||
if err := m.staging.Release(out.item.StagingDir); err != nil {
|
||||
m.logger.Warn("could not release staging dir", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
c = next
|
||||
}
|
||||
}
|
||||
|
||||
// maxGrabAttempts bounds how many candidates one request will try. A
|
||||
// popular album can have dozens of peers; the point of falling back is
|
||||
// to survive the ordinary one or two that are offline, not to walk the
|
||||
// whole list for six hours.
|
||||
const maxGrabAttempts = 3
|
||||
|
||||
// peerKey names one Soulseek user on one daemon. The same username on
|
||||
// two daemons is two logins and two queues.
|
||||
type peerKey struct {
|
||||
provider int64
|
||||
peer string
|
||||
}
|
||||
|
||||
// peerKeyFor returns the peer a candidate is fetched from, when the
|
||||
// source is one where asking a peer for two things at once is rude.
|
||||
func peerKeyFor(c Candidate) (peerKey, bool) {
|
||||
if c.Kind != KindSlskd || c.Origin == "" {
|
||||
return peerKey{}, false
|
||||
}
|
||||
|
||||
return peerKey{provider: c.ProviderID, peer: c.Origin}, true
|
||||
}
|
||||
|
||||
// grabOutcome is how one candidate's attempt ended.
|
||||
type grabOutcome struct {
|
||||
item DownloadItem
|
||||
imported ImportResult
|
||||
err error
|
||||
|
||||
// retryable reports whether another candidate might succeed where
|
||||
// this one failed: the transfer failed, or delivered too little of
|
||||
// the album. Anything else — no staging space, no library root, a
|
||||
// tag write failing — would fail the next candidate identically.
|
||||
retryable bool
|
||||
}
|
||||
|
||||
// attemptGrab takes one candidate through transfer and import. It
|
||||
// records the item's own failure, but not the download's: whether the
|
||||
// download has failed is the caller's decision, since another candidate
|
||||
// may yet succeed.
|
||||
func (m *Manager) attemptGrab(
|
||||
ctx context.Context,
|
||||
dl Download,
|
||||
c Candidate,
|
||||
job *jobs.Handle,
|
||||
only []ExpectedTrack,
|
||||
) grabOutcome {
|
||||
// Who will move the bytes is decided before any slot is taken, so
|
||||
// the transfer waits in its own provider's queue rather than in a
|
||||
// global one. A delegate takes no slot at all: the transfer is
|
||||
@@ -847,26 +731,21 @@ func (m *Manager) attemptGrab(
|
||||
// work against our budget.
|
||||
plan, err := m.planTransfer(dl, c)
|
||||
if err != nil {
|
||||
return grabOutcome{err: err}
|
||||
m.failDownload(ctx, job, dl.ID, err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if !plan.delegated() {
|
||||
if key, ok := peerKeyFor(c); ok {
|
||||
release, err := m.peerLocks.acquire(ctx, key)
|
||||
if err != nil {
|
||||
return grabOutcome{err: err}
|
||||
}
|
||||
|
||||
defer release()
|
||||
}
|
||||
|
||||
provSem := m.semaphoreFor(plan.transportID)
|
||||
|
||||
select {
|
||||
case provSem <- struct{}{}:
|
||||
defer func() { <-provSem }()
|
||||
case <-ctx.Done():
|
||||
return grabOutcome{err: ctx.Err()}
|
||||
m.failDownload(ctx, job, dl.ID, ctx.Err())
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
globalSem := m.globalSem()
|
||||
@@ -875,7 +754,9 @@ func (m *Manager) attemptGrab(
|
||||
case globalSem <- struct{}{}:
|
||||
defer func() { <-globalSem }()
|
||||
case <-ctx.Done():
|
||||
return grabOutcome{err: ctx.Err()}
|
||||
m.failDownload(ctx, job, dl.ID, ctx.Err())
|
||||
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
@@ -890,30 +771,24 @@ func (m *Manager) attemptGrab(
|
||||
|
||||
dir, err := m.staging.Reserve(item.ID)
|
||||
if err != nil {
|
||||
return grabOutcome{err: err}
|
||||
m.failDownload(ctx, job, dl.ID, err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
item.StagingDir = dir
|
||||
|
||||
if err := m.store.CreateItem(ctx, item); err != nil {
|
||||
return grabOutcome{item: item, err: err}
|
||||
}
|
||||
m.failDownload(ctx, job, dl.ID, err)
|
||||
|
||||
fail := func(err error, retryable bool) grabOutcome {
|
||||
if serr := m.store.SetItemState(
|
||||
ctx, item.ID, StateFailed, err.Error(),
|
||||
); serr != nil {
|
||||
m.logger.Warn("could not record item failure", "error", serr)
|
||||
}
|
||||
|
||||
return grabOutcome{item: item, err: err, retryable: retryable}
|
||||
return
|
||||
}
|
||||
|
||||
result, err := m.transfer(ctx, dl, item, plan, job)
|
||||
if err != nil {
|
||||
// A delegate's failure is the external manager's verdict on the
|
||||
// whole request, not on one copy of it.
|
||||
return fail(err, !plan.delegated())
|
||||
m.failItem(ctx, job, item, dl.ID, err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
m.setStates(ctx, dl.ID, item.ID, StateImporting)
|
||||
@@ -923,117 +798,42 @@ func (m *Manager) attemptGrab(
|
||||
job.SetStages(importStages(2))
|
||||
}
|
||||
|
||||
var imported ImportResult
|
||||
|
||||
if result.Delegated {
|
||||
// The external manager already placed and tagged these files in
|
||||
// its own library. Moving them out from under a system that is
|
||||
// still managing them would be worse than useless, so the files
|
||||
// are recorded where they are and the library scan picks them
|
||||
// up in place.
|
||||
imported = ImportResult{Paths: result.Files}
|
||||
|
||||
if job != nil {
|
||||
job.Logf(jobs.LevelInfo, fmt.Sprintf(
|
||||
"External manager imported %d files; recording them in place",
|
||||
len(result.Files),
|
||||
))
|
||||
}
|
||||
} else {
|
||||
opts := m.importOptions()
|
||||
opts.WriteTags = true
|
||||
|
||||
return grabOutcome{
|
||||
item: item,
|
||||
imported: ImportResult{Paths: result.Files},
|
||||
opts.LibraryRoot, err = m.library.LibraryPath(dl.LibraryID)
|
||||
if err != nil {
|
||||
m.failItem(ctx, job, item, dl.ID,
|
||||
fmt.Errorf("resolve library root: %w", err))
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
imported, err = m.importer.Import(ctx, dl, result, opts)
|
||||
if err != nil {
|
||||
m.failItem(ctx, job, item, dl.ID, err)
|
||||
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
opts := m.importOptions()
|
||||
opts.WriteTags = true
|
||||
opts.Only = only
|
||||
|
||||
opts.LibraryRoot, err = m.library.LibraryPath(dl.LibraryID)
|
||||
if err != nil {
|
||||
return fail(fmt.Errorf("resolve library root: %w", err), false)
|
||||
}
|
||||
|
||||
imported, err := m.importer.Import(ctx, dl, result, opts)
|
||||
if err != nil {
|
||||
return fail(err, errors.Is(err, ErrTooIncomplete))
|
||||
}
|
||||
|
||||
return grabOutcome{item: item, imported: imported}
|
||||
}
|
||||
|
||||
// nextCandidate picks the candidate to try after the ones in failed.
|
||||
//
|
||||
// It only ever offers a candidate auto-pick would have taken on its own
|
||||
// (autoAcceptable), so falling back cannot lower the bar an unattended
|
||||
// download is held to: the second choice has to clear the same gates
|
||||
// the first did.
|
||||
//
|
||||
// On Soulseek a failure belongs to the *peer* — offline, refusing, or
|
||||
// holding us in a queue — so every folder that peer offered is skipped
|
||||
// with it. Elsewhere a failure belongs to the release, and only that
|
||||
// candidate is.
|
||||
func (m *Manager) nextCandidate(
|
||||
ctx context.Context,
|
||||
dl Download,
|
||||
failed []Candidate,
|
||||
out grabOutcome,
|
||||
fallback bool,
|
||||
) (Candidate, bool) {
|
||||
if !fallback || !out.retryable || ctx.Err() != nil ||
|
||||
len(failed) >= maxGrabAttempts {
|
||||
return Candidate{}, false
|
||||
}
|
||||
|
||||
m.resMu.RLock()
|
||||
ranked := m.results[dl.ID]
|
||||
m.resMu.RUnlock()
|
||||
|
||||
prefs := m.preferences()
|
||||
|
||||
for _, c := range ranked {
|
||||
if ruledOutBy(c, failed) || !autoAcceptable(dl, c, prefs) {
|
||||
continue
|
||||
}
|
||||
|
||||
return c, true
|
||||
}
|
||||
|
||||
return Candidate{}, false
|
||||
}
|
||||
|
||||
// ruledOutBy reports whether a failure among failed also rules out c.
|
||||
func ruledOutBy(c Candidate, failed []Candidate) bool {
|
||||
for _, f := range failed {
|
||||
if c.ID == f.ID && c.ProviderID == f.ProviderID {
|
||||
return true
|
||||
}
|
||||
|
||||
if c.Kind == KindSlskd && f.Kind == KindSlskd &&
|
||||
c.ProviderID == f.ProviderID && c.Origin != "" &&
|
||||
c.Origin == f.Origin {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// describeCandidate names a candidate for the job log.
|
||||
func describeCandidate(c Candidate) string {
|
||||
if c.Origin != "" {
|
||||
return fmt.Sprintf("%q from %s", c.Title, c.Origin)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%q", c.Title)
|
||||
}
|
||||
|
||||
// finishGrab records a successful import and retires what the request
|
||||
// was holding.
|
||||
func (m *Manager) finishGrab(
|
||||
ctx context.Context,
|
||||
dl Download,
|
||||
item DownloadItem,
|
||||
imported ImportResult,
|
||||
job *jobs.Handle,
|
||||
) {
|
||||
if err := m.store.SetItemImported(
|
||||
ctx, item.ID, imported.Paths,
|
||||
); err != nil {
|
||||
@@ -1377,6 +1177,23 @@ func (m *Manager) failDownload(
|
||||
}
|
||||
}
|
||||
|
||||
// failItem records an item-level failure and fails its download.
|
||||
func (m *Manager) failItem(
|
||||
ctx context.Context,
|
||||
job *jobs.Handle,
|
||||
item DownloadItem,
|
||||
downloadID string,
|
||||
err error,
|
||||
) {
|
||||
if serr := m.store.SetItemState(
|
||||
ctx, item.ID, StateFailed, err.Error(),
|
||||
); serr != nil {
|
||||
m.logger.Warn("could not record item failure", "error", serr)
|
||||
}
|
||||
|
||||
m.failDownload(ctx, job, downloadID, err)
|
||||
}
|
||||
|
||||
// startJob registers the request in the background jobs panel.
|
||||
func (m *Manager) startJob(dl Download) *jobs.Handle {
|
||||
if m.jobsReg == nil {
|
||||
|
||||
@@ -66,14 +66,6 @@ var (
|
||||
|
||||
// separatorPattern splits "Artist - Album" style folder names.
|
||||
separatorPattern = regexp.MustCompile(`\s+[-–—]\s+`)
|
||||
|
||||
// discFolderPattern matches a directory that holds one disc of an
|
||||
// album rather than the album: "CD1", "CD 2", "Disc 3", "Disk-1",
|
||||
// "[Disc 2]", "CD1 - The Early Years". A number is required, so a
|
||||
// folder merely called "CDs" is not one.
|
||||
discFolderPattern = regexp.MustCompile(
|
||||
`(?i)^\s*[\[(]?\s*(?:cd|disc|disk)\s*[-_.#]?\s*(\d{1,2})\b`,
|
||||
)
|
||||
)
|
||||
|
||||
// FormatForPath returns the audio format implied by a path's extension,
|
||||
@@ -102,63 +94,16 @@ type TrackHint struct {
|
||||
Folder string
|
||||
}
|
||||
|
||||
// discFolder reports whether a directory name is one disc of an album,
|
||||
// and which.
|
||||
func discFolder(name string) (int, bool) {
|
||||
m := discFolderPattern.FindStringSubmatch(name)
|
||||
if m == nil {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
n, err := strconv.Atoi(m[1])
|
||||
if err != nil || n == 0 {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
return n, true
|
||||
}
|
||||
|
||||
// AlbumDir is the directory that holds a file's *album*: its parent,
|
||||
// or its grandparent when the parent is a disc folder.
|
||||
//
|
||||
// Multi-disc rips are shared as `Album/CD1/…` and `Album/CD2/…`, and
|
||||
// grouping candidates by the immediate parent split one album into two
|
||||
// half-albums, each titled "CD1". Neither could clear the completeness
|
||||
// or album-title bars, so a multi-disc release could not be auto-picked
|
||||
// at all. A disc folder at the root has no album above it and is
|
||||
// returned as it is.
|
||||
func AlbumDir(p string) string {
|
||||
dir := path.Dir(strings.ReplaceAll(p, `\`, "/"))
|
||||
|
||||
if _, ok := discFolder(path.Base(dir)); !ok {
|
||||
return dir
|
||||
}
|
||||
|
||||
parent := path.Dir(dir)
|
||||
if parent == "." || parent == "/" || parent == "" {
|
||||
return dir
|
||||
}
|
||||
|
||||
return parent
|
||||
}
|
||||
|
||||
// ParsePath extracts what it can from one candidate file path.
|
||||
func ParsePath(p string) TrackHint {
|
||||
// Soulseek paths are Windows-style; normalize before splitting.
|
||||
norm := strings.ReplaceAll(p, `\`, "/")
|
||||
base := path.Base(norm)
|
||||
folder := path.Base(path.Dir(norm))
|
||||
|
||||
name := strings.TrimSuffix(base, path.Ext(base))
|
||||
|
||||
// The album's name is the album directory's, not a disc folder's,
|
||||
// and the disc folder is where a multi-disc rip says which disc a
|
||||
// file is on. A disc number in the filename ("2-01 …") is more
|
||||
// specific and overrides it below.
|
||||
hint := TrackHint{Folder: cleanAlbumName(path.Base(AlbumDir(norm)))}
|
||||
|
||||
if disc, ok := discFolder(path.Base(path.Dir(norm))); ok {
|
||||
hint.Disc = disc
|
||||
}
|
||||
hint := TrackHint{Folder: cleanAlbumName(folder)}
|
||||
|
||||
if m := trackNumPattern.FindStringSubmatch(name); m != nil {
|
||||
if m[1] != "" {
|
||||
@@ -258,72 +203,21 @@ func AnnotateFiles(files []CandidateFile) []CandidateFile {
|
||||
|
||||
// matchFiles aligns a candidate's audio files to the expected tracklist
|
||||
// and returns the per-file assignment plus the mean title similarity of
|
||||
// the aligned pairs. alignFiles is the same alignment with the
|
||||
// duration evidence as well.
|
||||
func matchFiles(
|
||||
files []CandidateFile,
|
||||
expected []ExpectedTrack,
|
||||
) ([]CandidateFile, float64) {
|
||||
a := alignFiles(files, expected)
|
||||
|
||||
return a.files, a.titleFit
|
||||
}
|
||||
|
||||
// alignment is what aligning a candidate to a tracklist found.
|
||||
type alignment struct {
|
||||
files []CandidateFile
|
||||
|
||||
// titleFit is the mean title similarity over aligned pairs.
|
||||
titleFit float64
|
||||
|
||||
// durationFit is the mean duration agreement over aligned pairs
|
||||
// where both sides state a length, and timedPairs is how many such
|
||||
// pairs there were.
|
||||
durationFit float64
|
||||
timedPairs int
|
||||
aligned int
|
||||
}
|
||||
|
||||
// durationAgreement scores how well a file's length matches the
|
||||
// expected track's, in 0..1. Rips of the same master differ by a
|
||||
// second or two of silence; a different edit, a live take or a
|
||||
// truncated file differs by tens of seconds.
|
||||
func durationAgreement(got, want int64) float64 {
|
||||
const (
|
||||
exactMillis = 3_000
|
||||
wrongMillis = 30_000
|
||||
)
|
||||
|
||||
d := got - want
|
||||
if d < 0 {
|
||||
d = -d
|
||||
}
|
||||
|
||||
switch {
|
||||
case d <= exactMillis:
|
||||
return 1
|
||||
case d >= wrongMillis:
|
||||
return 0
|
||||
default:
|
||||
return 1 - float64(d-exactMillis)/float64(wrongMillis-exactMillis)
|
||||
}
|
||||
}
|
||||
|
||||
// alignFiles aligns a candidate's audio files to the expected tracklist.
|
||||
// the aligned pairs.
|
||||
//
|
||||
// Alignment is greedy by score rather than optimal: candidate folders
|
||||
// are small (a few dozen files at most) and the common cases — correct
|
||||
// track numbers, or clean "NN Title" names — are unambiguous, so the
|
||||
// extra machinery of Hungarian assignment buys nothing here.
|
||||
func alignFiles(
|
||||
func matchFiles(
|
||||
files []CandidateFile,
|
||||
expected []ExpectedTrack,
|
||||
) alignment {
|
||||
) ([]CandidateFile, float64) {
|
||||
annotated := make([]CandidateFile, len(files))
|
||||
copy(annotated, files)
|
||||
|
||||
if len(expected) == 0 {
|
||||
return alignment{files: annotated}
|
||||
return annotated, 0
|
||||
}
|
||||
|
||||
hints := make([]TrackHint, len(annotated))
|
||||
@@ -336,19 +230,8 @@ func alignFiles(
|
||||
var (
|
||||
total float64
|
||||
matched int
|
||||
|
||||
durTotal float64
|
||||
timed int
|
||||
)
|
||||
|
||||
// timing adds a pair's duration evidence when both sides state one.
|
||||
timing := func(f CandidateFile, e ExpectedTrack) {
|
||||
if f.LengthMillis > 0 && e.LengthMillis > 0 {
|
||||
durTotal += durationAgreement(f.LengthMillis, e.LengthMillis)
|
||||
timed++
|
||||
}
|
||||
}
|
||||
|
||||
// Pass 1: trust explicit track numbers when they are unique and in
|
||||
// range. A folder that numbers its files correctly is the strong
|
||||
// case, and title comparison only adds noise there.
|
||||
@@ -367,8 +250,6 @@ func alignFiles(
|
||||
|
||||
total += autotag.TitleSimilarity(hints[i].Title, expected[idx].Title)
|
||||
matched++
|
||||
|
||||
timing(annotated[i], expected[idx])
|
||||
}
|
||||
|
||||
// Pass 2: title similarity for whatever is left.
|
||||
@@ -403,26 +284,13 @@ func alignFiles(
|
||||
|
||||
total += bestSim
|
||||
matched++
|
||||
|
||||
timing(annotated[i], expected[bestIdx])
|
||||
}
|
||||
|
||||
if matched == 0 {
|
||||
return alignment{files: annotated}
|
||||
return annotated, 0
|
||||
}
|
||||
|
||||
a := alignment{
|
||||
files: annotated,
|
||||
titleFit: total / float64(matched),
|
||||
timedPairs: timed,
|
||||
aligned: matched,
|
||||
}
|
||||
|
||||
if timed > 0 {
|
||||
a.durationFit = durTotal / float64(timed)
|
||||
}
|
||||
|
||||
return a
|
||||
return annotated, total / float64(matched)
|
||||
}
|
||||
|
||||
// indexForPosition finds the expected track at a disc/track position.
|
||||
|
||||
@@ -1,224 +0,0 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Soulseek politeness is per peer, not per daemon (#272).
|
||||
|
||||
func TestKeyedLockSerialisesOneKeyOnly(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var l keyedLock[string]
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
releaseA, err := l.acquire(ctx, "a")
|
||||
if err != nil {
|
||||
t.Fatalf("acquire a: %v", err)
|
||||
}
|
||||
|
||||
// Another key is free while "a" is held.
|
||||
releaseB, err := l.acquire(ctx, "b")
|
||||
if err != nil {
|
||||
t.Fatalf("acquire b: %v", err)
|
||||
}
|
||||
|
||||
releaseB()
|
||||
|
||||
// The same key waits, and gives up with its context.
|
||||
short, cancel := context.WithTimeout(ctx, 20*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
if _, err := l.acquire(short, "a"); !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("second acquire of a held key = %v, want the deadline", err)
|
||||
}
|
||||
|
||||
releaseA()
|
||||
releaseA() // Idempotent: a second call must not free someone else's hold.
|
||||
|
||||
if n := l.size(); n != 0 {
|
||||
t.Errorf("%d keys left behind, want none once nobody holds or waits", n)
|
||||
}
|
||||
}
|
||||
|
||||
// grabEach runs one grab per candidate and returns a function that waits
|
||||
// for all of them; grabAll's reasons for waiting apply.
|
||||
func grabEach(t *testing.T, f managerFixture, cands []Candidate) func() {
|
||||
t.Helper()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for i, c := range cands {
|
||||
dl := fourTrackDownload()
|
||||
dl.ID = "dl-" + string(rune('a'+i))
|
||||
|
||||
if err := f.store.CreateDownload(ctx, dl); err != nil {
|
||||
t.Fatalf("CreateDownload: %v", err)
|
||||
}
|
||||
|
||||
wg.Add(1)
|
||||
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
f.manager.grab(ctx, dl, c, nil, false)
|
||||
}()
|
||||
}
|
||||
|
||||
return func() {
|
||||
done := make(chan struct{})
|
||||
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(done)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Error("transfers did not finish")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func slskdCandidates(p *FakeProvider, peers ...string) []Candidate {
|
||||
out := make([]Candidate, 0, len(peers))
|
||||
|
||||
for i, peer := range peers {
|
||||
c := p.Candidates[0]
|
||||
c.ID = c.ID + "-" + itoa(i)
|
||||
c.Kind = KindSlskd
|
||||
c.ProviderID = 1
|
||||
c.Origin = peer
|
||||
out = append(out, c)
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// Three albums from one user are asked for one at a time, even though
|
||||
// the daemon would allow three transfers.
|
||||
func TestOnePeerIsAskedForOneThingAtATime(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newManagerFixture(t)
|
||||
f.manager.SetMaxConcurrent(4)
|
||||
|
||||
p := fakeWithAlbum(1, "slskd", ".flac")
|
||||
p.GrabGate = make(chan struct{})
|
||||
|
||||
f.manager.installProvider(Config{ID: 1, Kind: KindSlskd, Priority: 50}, p)
|
||||
|
||||
wait := grabEach(t, f, slskdCandidates(p, "alice", "alice", "alice"))
|
||||
|
||||
waitFor(t, func() bool { return p.GrabCallCount() >= 1 }, "no grab started")
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
|
||||
if got := p.MaxParallelGrabs(); got != 1 {
|
||||
t.Errorf("%d simultaneous grabs from one peer, want 1", got)
|
||||
}
|
||||
|
||||
close(p.GrabGate)
|
||||
|
||||
waitFor(t, func() bool { return p.GrabCallCount() == 3 }, "queued grabs never ran")
|
||||
wait()
|
||||
|
||||
if n := f.manager.peerLocks.size(); n != 0 {
|
||||
t.Errorf("%d peer locks left behind", n)
|
||||
}
|
||||
}
|
||||
|
||||
// Different users run at once, up to the daemon's cap — the point of
|
||||
// the change: one slow peer no longer holds up every other.
|
||||
func TestDifferentPeersRunTogether(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := newManagerFixture(t)
|
||||
f.manager.SetMaxConcurrent(8)
|
||||
|
||||
p := fakeWithAlbum(1, "slskd", ".flac")
|
||||
p.GrabGate = make(chan struct{})
|
||||
|
||||
f.manager.installProvider(Config{ID: 1, Kind: KindSlskd, Priority: 50}, p)
|
||||
|
||||
wait := grabEach(t, f, slskdCandidates(p, "alice", "bob", "carol", "dave"))
|
||||
|
||||
waitFor(
|
||||
t,
|
||||
func() bool { return p.MaxParallelGrabs() >= kindConcurrency[KindSlskd] },
|
||||
"different peers were serialised",
|
||||
)
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
if got := p.MaxParallelGrabs(); got != kindConcurrency[KindSlskd] {
|
||||
t.Errorf("%d simultaneous grabs, want the daemon cap %d", got, kindConcurrency[KindSlskd])
|
||||
}
|
||||
|
||||
close(p.GrabGate)
|
||||
wait()
|
||||
}
|
||||
|
||||
func TestSlskdLocalFolders(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s := &slskd{downloadsPath: "/dl"}
|
||||
|
||||
got := s.localFolders(Candidate{Files: []CandidateFile{
|
||||
{Path: `\m\The Wall\CD2\01 Hey You.flac`},
|
||||
{Path: `\m\The Wall\CD1\01 In The Flesh.flac`},
|
||||
{Path: `\m\The Wall\CD1\02 The Thin Ice.flac`},
|
||||
}})
|
||||
|
||||
want := []string{filepath.Join("/dl", "CD1"), filepath.Join("/dl", "CD2")}
|
||||
if len(got) != len(want) || got[0] != want[0] || got[1] != want[1] {
|
||||
t.Errorf("localFolders = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// Two peers' "Greatest Hits" land in one slskd directory, so the second
|
||||
// grab does not enqueue until the first has collected its files.
|
||||
func TestSlskdSameFolderNameWaits(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newSlskdStub(t)
|
||||
s, downloads := newStubSlskd(t, stub)
|
||||
|
||||
c := Candidate{
|
||||
Payload: map[string]string{"username": "bob"},
|
||||
Files: []CandidateFile{
|
||||
{Path: `\music\Greatest Hits\01 Intro.flac`, Size: 1, IsAudio: true},
|
||||
},
|
||||
}
|
||||
|
||||
release, err := lockSlskdFolders(
|
||||
context.Background(), []string{filepath.Join(downloads, "Greatest Hits")},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("lock: %v", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
if _, err := s.Grab(ctx, c, t.TempDir(), nil); !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("Grab = %v, want it to wait on the held folder", err)
|
||||
}
|
||||
|
||||
release()
|
||||
|
||||
stub.mu.Lock()
|
||||
posted := stub.posted
|
||||
stub.mu.Unlock()
|
||||
|
||||
if posted {
|
||||
t.Error("enqueued transfers into a folder another grab held")
|
||||
}
|
||||
}
|
||||
@@ -236,18 +236,17 @@ func Register(d Descriptor, c Constructor) {
|
||||
}
|
||||
|
||||
// concurrencyField describes the per-provider transfer limit, with help
|
||||
// text explaining what the number means where it means something
|
||||
// unusual: on slskd it counts peers, since each peer is only ever asked
|
||||
// for one folder at a time whatever it is set to.
|
||||
// text explaining why the default is what it is — a user who raises
|
||||
// slskd from 1 to 8 and gets themselves queued behind every other
|
||||
// Soulseek user deserves to have been warned.
|
||||
func concurrencyField(k Kind) Field {
|
||||
help := "Maximum simultaneous transfers from this client."
|
||||
|
||||
if k == KindSlskd {
|
||||
help = "How many Soulseek users to download from at once. " +
|
||||
"Each user is only ever asked for one album at a time, " +
|
||||
"since peers queue or ban clients that ask for more; " +
|
||||
"this bounds how many different users are asked in " +
|
||||
"parallel."
|
||||
help = "Maximum simultaneous transfers. Soulseek peers serve " +
|
||||
"one file at a time and queue or ban clients that ask for " +
|
||||
"more, so 1 is both the polite setting and usually the " +
|
||||
"fastest."
|
||||
}
|
||||
|
||||
return Field{
|
||||
|
||||
+92
-1033
File diff suppressed because it is too large
Load Diff
@@ -1,130 +0,0 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestSlskdLive runs the provider against a real slskd daemon. It is
|
||||
// skipped unless YJ_SLSKD_URL, YJ_SLSKD_API_KEY and YJ_SLSKD_DOWNLOADS
|
||||
// are set, and it downloads something only when YJ_SLSKD_GRAB=1 — then
|
||||
// the smallest candidate the search returns, from whichever stranger
|
||||
// is sharing it.
|
||||
//
|
||||
// Everything else here tests the provider against a stub written from
|
||||
// reading slskd's source. This is where those readings are checked:
|
||||
// the search options, the responses endpoint, the batch destination,
|
||||
// the cancel.
|
||||
//
|
||||
// YJ_SLSKD_URL=http://localhost:5030 YJ_SLSKD_API_KEY=… \
|
||||
// YJ_SLSKD_DOWNLOADS=/path/to/slskd/downloads YJ_SLSKD_GRAB=1 \
|
||||
// go test -run TestSlskdLive -v ./backend/download/
|
||||
func TestSlskdLive(t *testing.T) {
|
||||
base, key, downloads := os.Getenv("YJ_SLSKD_URL"),
|
||||
os.Getenv("YJ_SLSKD_API_KEY"), os.Getenv("YJ_SLSKD_DOWNLOADS")
|
||||
if base == "" || key == "" || downloads == "" {
|
||||
t.Skip(
|
||||
"set YJ_SLSKD_URL, YJ_SLSKD_API_KEY and YJ_SLSKD_DOWNLOADS to run against a real slskd",
|
||||
)
|
||||
}
|
||||
|
||||
query := os.Getenv("YJ_SLSKD_QUERY")
|
||||
if query == "" {
|
||||
query = "Radiohead OK Computer"
|
||||
}
|
||||
|
||||
p, err := newSlskd(
|
||||
Config{
|
||||
ID: 1, Kind: KindSlskd, Name: "live", Enabled: true,
|
||||
Settings: map[string]string{"url": base, "downloadsPath": downloads},
|
||||
},
|
||||
func(string) (string, error) { return key, nil },
|
||||
slogDiscard(),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("newSlskd: %v", err)
|
||||
}
|
||||
|
||||
s, ok := p.(*slskd)
|
||||
if !ok {
|
||||
t.Fatalf("provider is %T", p)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
if err := s.Check(ctx); err != nil {
|
||||
t.Fatalf("Check: %v", err)
|
||||
}
|
||||
|
||||
started := time.Now()
|
||||
|
||||
got, err := s.Search(ctx, Download{Query: query})
|
||||
if err != nil {
|
||||
t.Fatalf("Search: %v", err)
|
||||
}
|
||||
|
||||
t.Logf(
|
||||
"search %q: %d candidates in %s",
|
||||
query,
|
||||
len(got),
|
||||
time.Since(started).Round(time.Millisecond),
|
||||
)
|
||||
|
||||
if len(got) == 0 {
|
||||
t.Fatal("no candidates; try a more common YJ_SLSKD_QUERY")
|
||||
}
|
||||
|
||||
timed := 0
|
||||
|
||||
for _, c := range got {
|
||||
for _, f := range c.Files {
|
||||
if f.LengthMillis > 0 {
|
||||
timed++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
t.Logf("%d files carry a length", timed)
|
||||
|
||||
if os.Getenv("YJ_SLSKD_GRAB") != "1" {
|
||||
return
|
||||
}
|
||||
|
||||
smallest := slices.MinFunc(got, func(a, b Candidate) int {
|
||||
return int(a.TotalSize - b.TotalSize)
|
||||
})
|
||||
|
||||
t.Logf("grabbing %q from %s (%d files, %d bytes)",
|
||||
smallest.Title, smallest.Origin, len(smallest.Files), smallest.TotalSize)
|
||||
|
||||
s.stallAfter = 3 * time.Minute
|
||||
|
||||
gctx, cancel := context.WithTimeout(ctx, 15*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
res, err := s.Grab(gctx, smallest, t.TempDir(), func(p Progress) {
|
||||
t.Logf("%s: %d/%d bytes", p.Phase, p.Current, p.Total)
|
||||
})
|
||||
if err != nil {
|
||||
// A stranger going offline is not a defect; what matters is
|
||||
// that the transfers were cancelled, which slskd's UI shows.
|
||||
t.Fatalf("Grab: %v", err)
|
||||
}
|
||||
|
||||
t.Logf("batches: %v", s.batches.Load() == batchesSupported)
|
||||
|
||||
for _, f := range res.Files {
|
||||
info, err := os.Stat(f)
|
||||
if err != nil || info.Size() == 0 {
|
||||
t.Errorf("collected %s is missing or empty: %v", f, err)
|
||||
}
|
||||
}
|
||||
|
||||
if entries, _ := os.ReadDir(filepath.Join(downloads, slskdDestRoot)); len(entries) != 0 {
|
||||
t.Errorf("%d batch folders left in slskd's downloads", len(entries))
|
||||
}
|
||||
}
|
||||
@@ -7,9 +7,7 @@ import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -33,45 +31,11 @@ type slskdStub struct {
|
||||
transfers [][]slskdTransfer
|
||||
pollCount int
|
||||
|
||||
// before is what the downloads endpoint reports until something is
|
||||
// enqueued: records slskd already held from earlier attempts.
|
||||
before []slskdTransfer
|
||||
|
||||
// enqueued records what was requested for download.
|
||||
enqueued []map[string]any
|
||||
posted bool
|
||||
|
||||
// paths records the escaped path of every transfers call, and
|
||||
// cancelled the escaped request URI of every DELETE.
|
||||
paths []string
|
||||
cancelled []string
|
||||
|
||||
// unauthorized makes every call return 401.
|
||||
unauthorized bool
|
||||
|
||||
// searches records every search request body, and searchGets the
|
||||
// request URI of every search GET.
|
||||
searches []map[string]any
|
||||
searchGets []string
|
||||
|
||||
// noResponsesEndpoint makes /searches/{id}/responses 404, as an
|
||||
// older daemon would.
|
||||
noResponsesEndpoint bool
|
||||
|
||||
// batches makes the daemon take batch downloads, as 0.26 does.
|
||||
// Without it the batch endpoint answers 400, which is what an older
|
||||
// daemon's per-user route does with a batch body. batchBodies
|
||||
// records each batch, and delivered is written into its destination
|
||||
// under downloads when it is enqueued, keyed by file base name.
|
||||
batches bool
|
||||
batchBodies []map[string]any
|
||||
|
||||
// positions is what the queue-position endpoint answers, in order;
|
||||
// the last repeats. positionAsks counts the calls.
|
||||
positions []int
|
||||
positionAsks int
|
||||
delivered map[string]string
|
||||
downloads string
|
||||
}
|
||||
|
||||
func newSlskdStub(t *testing.T) *slskdStub {
|
||||
@@ -93,16 +57,6 @@ func newSlskdStub(t *testing.T) *slskdStub {
|
||||
return
|
||||
}
|
||||
|
||||
var body map[string]any
|
||||
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Errorf("decode search body: %v", err)
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
s.searches = append(s.searches, body)
|
||||
s.mu.Unlock()
|
||||
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
})
|
||||
|
||||
@@ -119,26 +73,8 @@ func newSlskdStub(t *testing.T) *slskdStub {
|
||||
|
||||
s.mu.Lock()
|
||||
responses := s.responses
|
||||
noEndpoint := s.noResponsesEndpoint
|
||||
s.searchGets = append(s.searchGets, r.URL.RequestURI())
|
||||
s.mu.Unlock()
|
||||
|
||||
if strings.HasSuffix(r.URL.Path, "/responses") {
|
||||
if noEndpoint {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(t, w, responses)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if r.URL.Query().Get("includeResponses") != "true" {
|
||||
responses = nil
|
||||
}
|
||||
|
||||
writeJSON(t, w, slskdSearch{
|
||||
ID: "search-1",
|
||||
IsComplete: true,
|
||||
@@ -151,35 +87,7 @@ func newSlskdStub(t *testing.T) *slskdStub {
|
||||
return
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
s.paths = append(s.paths, r.URL.EscapedPath())
|
||||
s.mu.Unlock()
|
||||
|
||||
if r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/position") {
|
||||
s.mu.Lock()
|
||||
idx := min(s.positionAsks, len(s.positions)-1)
|
||||
s.positionAsks++
|
||||
place := 0
|
||||
|
||||
if idx >= 0 {
|
||||
place = s.positions[idx]
|
||||
}
|
||||
|
||||
s.mu.Unlock()
|
||||
|
||||
writeJSON(t, w, place)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/batches") {
|
||||
s.enqueueBatch(t, w, r)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
switch r.Method {
|
||||
case http.MethodPost:
|
||||
if r.Method == http.MethodPost {
|
||||
var body []map[string]any
|
||||
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
@@ -188,42 +96,15 @@ func newSlskdStub(t *testing.T) *slskdStub {
|
||||
|
||||
s.mu.Lock()
|
||||
s.enqueued = body
|
||||
s.posted = true
|
||||
|
||||
for _, file := range body {
|
||||
name, _ := file["filename"].(string)
|
||||
norm := strings.ReplaceAll(name, `\`, "/")
|
||||
s.write(t, path.Base(path.Dir(norm)), path.Base(norm))
|
||||
}
|
||||
|
||||
s.mu.Unlock()
|
||||
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
|
||||
return
|
||||
case http.MethodDelete:
|
||||
s.mu.Lock()
|
||||
s.cancelled = append(s.cancelled, r.URL.RequestURI())
|
||||
s.mu.Unlock()
|
||||
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
|
||||
if !s.posted {
|
||||
before := s.before
|
||||
s.mu.Unlock()
|
||||
|
||||
writeJSON(t, w, map[string]any{
|
||||
"directories": []map[string]any{{"files": before}},
|
||||
})
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
idx := s.pollCount
|
||||
if idx >= len(s.transfers) {
|
||||
idx = len(s.transfers) - 1
|
||||
@@ -249,89 +130,6 @@ func newSlskdStub(t *testing.T) *slskdStub {
|
||||
return s
|
||||
}
|
||||
|
||||
// enqueueBatch answers the batch endpoint.
|
||||
func (s *slskdStub) enqueueBatch(t *testing.T, w http.ResponseWriter, r *http.Request) {
|
||||
t.Helper()
|
||||
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
if !s.batches {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
var body map[string]any
|
||||
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Errorf("decode batch body: %v", err)
|
||||
}
|
||||
|
||||
s.batchBodies = append(s.batchBodies, body)
|
||||
s.posted = true
|
||||
|
||||
files, _ := body["files"].([]any)
|
||||
options, _ := body["options"].(map[string]any)
|
||||
dest, _ := options["destination"].(string)
|
||||
|
||||
for _, f := range files {
|
||||
file, _ := f.(map[string]any)
|
||||
s.enqueued = append(s.enqueued, file)
|
||||
|
||||
name, _ := file["filename"].(string)
|
||||
base := path.Base(strings.ReplaceAll(name, `\`, "/"))
|
||||
|
||||
s.write(t, filepath.FromSlash(dest), base)
|
||||
}
|
||||
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
}
|
||||
|
||||
// deliver names the files that arrive once enqueued.
|
||||
func (s *slskdStub) deliver(names ...string) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
if s.delivered == nil {
|
||||
s.delivered = map[string]string{}
|
||||
}
|
||||
|
||||
for _, n := range names {
|
||||
s.delivered[n] = "audio"
|
||||
}
|
||||
}
|
||||
|
||||
// write puts a delivered file where slskd would: under dir in the
|
||||
// downloads folder, renamed name_<ticks>.ext when the name is taken, as
|
||||
// slskd's default Destination.Exists does. Callers hold s.mu.
|
||||
func (s *slskdStub) write(t *testing.T, dir, base string) {
|
||||
t.Helper()
|
||||
|
||||
content, ok := s.delivered[base]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
full := filepath.Join(s.downloads, dir)
|
||||
if err := os.MkdirAll(full, 0o750); err != nil {
|
||||
t.Errorf("mkdir: %v", err)
|
||||
}
|
||||
|
||||
target := filepath.Join(full, base)
|
||||
if _, err := os.Stat(target); err == nil {
|
||||
ext := filepath.Ext(base)
|
||||
target = filepath.Join(
|
||||
full,
|
||||
strings.TrimSuffix(base, ext)+"_"+strconv.FormatInt(time.Now().UnixNano(), 10)+ext,
|
||||
)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(target, []byte(content), 0o600); err != nil {
|
||||
t.Errorf("write: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// reject enforces API-key auth like the real daemon.
|
||||
func (s *slskdStub) reject(w http.ResponseWriter, r *http.Request) bool {
|
||||
s.mu.Lock()
|
||||
@@ -364,10 +162,6 @@ func newStubSlskd(t *testing.T, stub *slskdStub) (*slskd, string) {
|
||||
|
||||
downloads := t.TempDir()
|
||||
|
||||
stub.mu.Lock()
|
||||
stub.downloads = downloads
|
||||
stub.mu.Unlock()
|
||||
|
||||
p, err := newSlskd(
|
||||
Config{
|
||||
ID: 1,
|
||||
@@ -397,13 +191,6 @@ func newStubSlskd(t *testing.T, stub *slskdStub) (*slskd, string) {
|
||||
s.searchWait = 200 * time.Millisecond
|
||||
s.transferPoll = time.Millisecond
|
||||
|
||||
// Long enough that no existing test trips them by accident; the
|
||||
// tests about stalls and absences set their own.
|
||||
s.stallAfter = time.Minute
|
||||
s.absentGrace = time.Minute
|
||||
s.positionPoll = time.Millisecond
|
||||
s.queueCeiling = time.Hour
|
||||
|
||||
return s, downloads
|
||||
}
|
||||
|
||||
@@ -607,9 +394,24 @@ func TestSlskdGrabCollectsFromDownloadsFolder(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
s, _ := newStubSlskd(t, stub)
|
||||
s, downloads := newStubSlskd(t, stub)
|
||||
|
||||
stub.deliver("01 Airbag.flac", "02 Paranoid Android.flac")
|
||||
// slskd writes into <downloads>/<folder>/<file>.
|
||||
folder := filepath.Join(downloads, "OK Computer")
|
||||
if err := os.MkdirAll(folder, 0o750); err != nil {
|
||||
t.Fatalf("mkdir: %v", err)
|
||||
}
|
||||
|
||||
for _, name := range []string{
|
||||
"01 Airbag.flac",
|
||||
"02 Paranoid Android.flac",
|
||||
} {
|
||||
if err := os.WriteFile(
|
||||
filepath.Join(folder, name), []byte("audio"), 0o600,
|
||||
); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
c := Candidate{
|
||||
ID: "slskd:peer:OK Computer",
|
||||
@@ -668,9 +470,18 @@ func TestSlskdGrabToleratesPartialFailure(t *testing.T) {
|
||||
{Filename: `\s\Album\02 B.flac`, State: "Completed, Errored"},
|
||||
}}
|
||||
|
||||
s, _ := newStubSlskd(t, stub)
|
||||
s, downloads := newStubSlskd(t, stub)
|
||||
|
||||
stub.deliver("01 A.flac")
|
||||
folder := filepath.Join(downloads, "Album")
|
||||
if err := os.MkdirAll(folder, 0o750); err != nil {
|
||||
t.Fatalf("mkdir: %v", err)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(
|
||||
filepath.Join(folder, "01 A.flac"), []byte("audio"), 0o600,
|
||||
); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
|
||||
c := Candidate{
|
||||
Files: []CandidateFile{
|
||||
@@ -754,284 +565,3 @@ func TestSlskdRequiresConfiguration(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// slskdAlbum is a two-file candidate from peer, whose arrived files
|
||||
// the stub writes where slskd would once they are enqueued.
|
||||
func slskdAlbum(t *testing.T, stub *slskdStub, peer string, arrived ...string) Candidate {
|
||||
t.Helper()
|
||||
|
||||
stub.deliver(arrived...)
|
||||
|
||||
return Candidate{
|
||||
Files: []CandidateFile{
|
||||
{Path: `\s\Album\01 A.flac`, Size: 500, IsAudio: true},
|
||||
{Path: `\s\Album\02 B.flac`, Size: 500, IsAudio: true},
|
||||
},
|
||||
TotalSize: 1000,
|
||||
Payload: map[string]string{"username": peer},
|
||||
}
|
||||
}
|
||||
|
||||
// cancelledURIs returns what the stub was asked to cancel.
|
||||
func (s *slskdStub) cancelledURIs() []string {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
return append([]string(nil), s.cancelled...)
|
||||
}
|
||||
|
||||
// A peer that queues us and never sends a byte is given up on, and the
|
||||
// queued transfers are cancelled in slskd rather than left to start
|
||||
// hours later for a request nobody is waiting on.
|
||||
func TestSlskdGrabGivesUpOnAStalledPeer(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newSlskdStub(t)
|
||||
stub.transfers = [][]slskdTransfer{{
|
||||
{ID: "t1", Filename: `\s\Album\01 A.flac`, State: "Queued, Remotely"},
|
||||
{ID: "t2", Filename: `\s\Album\02 B.flac`, State: "Queued, Remotely"},
|
||||
}}
|
||||
|
||||
s, _ := newStubSlskd(t, stub)
|
||||
s.stallAfter = 30 * time.Millisecond
|
||||
|
||||
_, err := s.Grab(
|
||||
context.Background(), slskdAlbum(t, stub, "peer"), t.TempDir(), nil,
|
||||
)
|
||||
if !errors.Is(err, ErrSlskdTimeout) {
|
||||
t.Fatalf("error = %v, want ErrSlskdTimeout", err)
|
||||
}
|
||||
|
||||
got := stub.cancelledURIs()
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("cancelled %v, want both queued transfers", got)
|
||||
}
|
||||
|
||||
for _, uri := range got {
|
||||
if !strings.HasSuffix(uri, "?remove=true") {
|
||||
t.Errorf("cancel %s does not remove the record", uri)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A folder that stalls on its last track goes forward with what
|
||||
// arrived, the same as one whose last track failed; the importer's
|
||||
// completeness check decides whether that is enough.
|
||||
func TestSlskdGrabKeepsWhatArrivedBeforeAStall(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newSlskdStub(t)
|
||||
stub.transfers = [][]slskdTransfer{{
|
||||
{
|
||||
ID: "t1", Filename: `\s\Album\01 A.flac`,
|
||||
State: "Completed, Succeeded", BytesTransferred: 500,
|
||||
},
|
||||
{ID: "t2", Filename: `\s\Album\02 B.flac`, State: "Queued, Remotely"},
|
||||
}}
|
||||
|
||||
s, _ := newStubSlskd(t, stub)
|
||||
s.stallAfter = 30 * time.Millisecond
|
||||
|
||||
got, err := s.Grab(
|
||||
context.Background(),
|
||||
slskdAlbum(t, stub, "peer", "01 A.flac"),
|
||||
t.TempDir(), nil,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("Grab: %v", err)
|
||||
}
|
||||
|
||||
if len(got.Files) != 1 {
|
||||
t.Errorf("collected %d files, want the 1 that arrived", len(got.Files))
|
||||
}
|
||||
|
||||
if cancelled := stub.cancelledURIs(); len(cancelled) != 1 ||
|
||||
!strings.Contains(cancelled[0], "/t2") {
|
||||
t.Errorf("cancelled %v, want only the stalled t2", cancelled)
|
||||
}
|
||||
}
|
||||
|
||||
// Progress is what holds the stall timer off. A transfer that keeps
|
||||
// moving bytes is never abandoned, however long it takes.
|
||||
func TestSlskdGrabWaitsOnATransferThatIsMoving(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newSlskdStub(t)
|
||||
|
||||
for b := int64(1); b <= 100; b++ {
|
||||
stub.transfers = append(stub.transfers, []slskdTransfer{
|
||||
{ID: "t1", Filename: `\s\Album\01 A.flac`, State: "InProgress", BytesTransferred: b},
|
||||
{ID: "t2", Filename: `\s\Album\02 B.flac`, State: "Queued, Remotely"},
|
||||
})
|
||||
}
|
||||
|
||||
stub.transfers = append(stub.transfers, []slskdTransfer{
|
||||
{
|
||||
ID: "t1",
|
||||
Filename: `\s\Album\01 A.flac`,
|
||||
State: "Completed, Succeeded",
|
||||
BytesTransferred: 500,
|
||||
},
|
||||
{
|
||||
ID: "t2",
|
||||
Filename: `\s\Album\02 B.flac`,
|
||||
State: "Completed, Succeeded",
|
||||
BytesTransferred: 500,
|
||||
},
|
||||
})
|
||||
|
||||
s, _ := newStubSlskd(t, stub)
|
||||
// A hundred polls take several times the stall window; each one
|
||||
// moves a byte. The window is kept well above one poll so a
|
||||
// descheduled test runner does not read as a stall.
|
||||
s.transferPoll = 5 * time.Millisecond
|
||||
s.stallAfter = 150 * time.Millisecond
|
||||
|
||||
got, err := s.Grab(
|
||||
context.Background(),
|
||||
slskdAlbum(t, stub, "peer", "01 A.flac", "02 B.flac"),
|
||||
t.TempDir(), nil,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("Grab: %v", err)
|
||||
}
|
||||
|
||||
if len(got.Files) != 2 {
|
||||
t.Errorf("collected %d files, want 2", len(got.Files))
|
||||
}
|
||||
}
|
||||
|
||||
// A file slskd never lists was refused at enqueue and will never reach
|
||||
// a terminal state. It counts as failed once the grace period is up,
|
||||
// rather than being waited on until the six-hour ceiling.
|
||||
func TestSlskdGrabCountsAnUnlistedFileAsFailed(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newSlskdStub(t)
|
||||
stub.transfers = [][]slskdTransfer{{
|
||||
{
|
||||
ID: "t1", Filename: `\s\Album\01 A.flac`,
|
||||
State: "Completed, Succeeded", BytesTransferred: 500,
|
||||
},
|
||||
}}
|
||||
|
||||
s, _ := newStubSlskd(t, stub)
|
||||
s.absentGrace = 20 * time.Millisecond
|
||||
|
||||
got, err := s.Grab(
|
||||
context.Background(),
|
||||
slskdAlbum(t, stub, "peer", "01 A.flac"),
|
||||
t.TempDir(), nil,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("Grab: %v", err)
|
||||
}
|
||||
|
||||
if len(got.Files) != 1 {
|
||||
t.Errorf("collected %d files, want 1", len(got.Files))
|
||||
}
|
||||
}
|
||||
|
||||
// A finished record left by an earlier attempt at the same file is not
|
||||
// this attempt's answer. Without the snapshot it would fail the grab on
|
||||
// the first poll, before the new transfer had started.
|
||||
func TestSlskdGrabIgnoresAnEarlierAttemptsRecord(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stale := slskdTransfer{
|
||||
ID: "old", Filename: `\s\Album\01 A.flac`, State: "Completed, Errored",
|
||||
}
|
||||
|
||||
stub := newSlskdStub(t)
|
||||
stub.before = []slskdTransfer{stale}
|
||||
stub.transfers = [][]slskdTransfer{
|
||||
{stale},
|
||||
{
|
||||
stale,
|
||||
{
|
||||
ID: "new", Filename: `\s\Album\01 A.flac`,
|
||||
State: "Completed, Succeeded", BytesTransferred: 500,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
s, _ := newStubSlskd(t, stub)
|
||||
|
||||
c := slskdAlbum(t, stub, "peer", "01 A.flac")
|
||||
c.Files = c.Files[:1]
|
||||
|
||||
got, err := s.Grab(context.Background(), c, t.TempDir(), nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Grab: %v", err)
|
||||
}
|
||||
|
||||
if len(got.Files) != 1 {
|
||||
t.Errorf("collected %d files, want 1", len(got.Files))
|
||||
}
|
||||
}
|
||||
|
||||
// Cancelling the download cancels the transfer in slskd too. The
|
||||
// cleanup must not inherit the cancelled context, or it is never sent.
|
||||
func TestSlskdGrabCancelsTransfersWhenTheCallerGivesUp(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newSlskdStub(t)
|
||||
stub.transfers = [][]slskdTransfer{{
|
||||
{ID: "t1", Filename: `\s\Album\01 A.flac`, State: "InProgress", BytesTransferred: 10},
|
||||
{ID: "t2", Filename: `\s\Album\02 B.flac`, State: "Queued, Remotely"},
|
||||
}}
|
||||
|
||||
s, _ := newStubSlskd(t, stub)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
_, err := s.Grab(ctx, slskdAlbum(t, stub, "peer"), t.TempDir(), nil)
|
||||
if !errors.Is(err, ErrSlskdTimeout) {
|
||||
t.Fatalf("error = %v, want ErrSlskdTimeout", err)
|
||||
}
|
||||
|
||||
if got := stub.cancelledURIs(); len(got) != 2 {
|
||||
t.Errorf("cancelled %v, want both live transfers", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Soulseek usernames carry spaces and punctuation; spliced raw into the
|
||||
// path, a name with a slash addresses a different endpoint entirely.
|
||||
func TestSlskdEscapesTheUsername(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newSlskdStub(t)
|
||||
stub.transfers = [][]slskdTransfer{{
|
||||
{
|
||||
ID: "t1", Filename: `\s\Album\01 A.flac`,
|
||||
State: "Completed, Succeeded", BytesTransferred: 500,
|
||||
},
|
||||
{
|
||||
ID: "t2", Filename: `\s\Album\02 B.flac`,
|
||||
State: "Completed, Succeeded", BytesTransferred: 500,
|
||||
},
|
||||
}}
|
||||
|
||||
s, _ := newStubSlskd(t, stub)
|
||||
|
||||
if _, err := s.Grab(
|
||||
context.Background(),
|
||||
slskdAlbum(t, stub, "dj a/b", "01 A.flac", "02 B.flac"),
|
||||
t.TempDir(), nil,
|
||||
); err != nil {
|
||||
t.Fatalf("Grab: %v", err)
|
||||
}
|
||||
|
||||
stub.mu.Lock()
|
||||
paths := append([]string(nil), stub.paths...)
|
||||
stub.mu.Unlock()
|
||||
|
||||
for _, p := range paths {
|
||||
// The batch endpoint carries the name in its body.
|
||||
if p != "/api/v0/transfers/downloads/dj%20a%2Fb" &&
|
||||
p != "/api/v0/transfers/downloads/batches" {
|
||||
t.Errorf("transfers call went to %s", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@@ -18,21 +17,6 @@ import (
|
||||
|
||||
// stubYtDlp writes an executable script that echoes the given stdout
|
||||
// and returns it as a provider config binary path.
|
||||
//
|
||||
// The write is held under syscall.ForkLock, and that is not tidiness:
|
||||
// the kernel refuses to exec a file that is open for writing anywhere
|
||||
// in the process, and these tests are parallel, so a *sibling* test's
|
||||
// fork can duplicate this descriptor in the moment it is open and
|
||||
// carry it past our close — the exec a moment later then fails with
|
||||
// ETXTBSY, "text file busy". That is #146, seen once in CI and once
|
||||
// locally, on trees containing no Go at all. Closing sooner is not
|
||||
// available (os.WriteFile has already closed the file before anything
|
||||
// execs it) and O_CLOEXEC does not help, because the window is between
|
||||
// another goroutine's fork and its own exec. ForkLock is the lock
|
||||
// syscall.forkExec takes across that fork, so holding it here means no
|
||||
// child can exist while the descriptor does. Measured on this helper
|
||||
// under 12 concurrent writers: 176-189 of 2400 execs refused without
|
||||
// it, 0 of 2400 with it.
|
||||
func stubYtDlp(t *testing.T, script string) string {
|
||||
t.Helper()
|
||||
|
||||
@@ -42,11 +26,9 @@ func stubYtDlp(t *testing.T, script string) string {
|
||||
|
||||
path := filepath.Join(t.TempDir(), "yt-dlp")
|
||||
|
||||
syscall.ForkLock.Lock()
|
||||
err := os.WriteFile(path, []byte("#!/bin/sh\n"+script), 0o700)
|
||||
syscall.ForkLock.Unlock()
|
||||
|
||||
if err != nil {
|
||||
if err := os.WriteFile(
|
||||
path, []byte("#!/bin/sh\n"+script), 0o700,
|
||||
); err != nil {
|
||||
t.Fatalf("write stub: %v", err)
|
||||
}
|
||||
|
||||
|
||||
+18
-104
@@ -35,15 +35,6 @@ const (
|
||||
weightArtistFit = 0.12
|
||||
)
|
||||
|
||||
// Match sub-weights when the candidate's durations are known. Duration
|
||||
// takes its weight from title fit, the signal it corroborates: a title
|
||||
// says which song a file claims to be, a length says whether it is that
|
||||
// recording — the right edit, the whole file, not the live take.
|
||||
const (
|
||||
timedWeightTitleFit = 0.25
|
||||
timedWeightDurationFit = 0.15
|
||||
)
|
||||
|
||||
// Quality sub-weights. Each set sums to 1.0.
|
||||
//
|
||||
// There are two of them because a stated preference changes what the
|
||||
@@ -328,13 +319,13 @@ func Score(dl Download, c Candidate, priority int, prefs AutoDownloadPrefs) Cand
|
||||
|
||||
audio := c.AudioFiles()
|
||||
|
||||
a := alignFiles(audio, dl.Expected)
|
||||
matched, titleFit := matchFiles(audio, dl.Expected)
|
||||
|
||||
// Write the alignment back so the picker can show which file maps
|
||||
// to which track.
|
||||
c.Files = mergeMatched(c.Files, a.files)
|
||||
c.Files = mergeMatched(c.Files, matched)
|
||||
|
||||
c.Match = scoreMatch(dl, c, audio, a)
|
||||
c.Match = scoreMatch(dl, c, audio, titleFit)
|
||||
c.Quality = scoreQuality(
|
||||
c, audio, priority, prefs, dl.runtimeMillis(),
|
||||
)
|
||||
@@ -349,21 +340,14 @@ func scoreMatch(
|
||||
dl Download,
|
||||
c Candidate,
|
||||
audio []CandidateFile,
|
||||
a alignment,
|
||||
titleFit float64,
|
||||
) MatchScore {
|
||||
m := MatchScore{
|
||||
Anchored: dl.Anchored(),
|
||||
TitleFit: a.titleFit,
|
||||
DurationFit: a.durationFit,
|
||||
|
||||
// Durations count once at least half the aligned pairs state
|
||||
// one; a single timed pair would be a coin toss carrying 15%.
|
||||
DurationKnown: a.timedPairs > 0 && a.timedPairs*2 >= a.aligned,
|
||||
Anchored: dl.Anchored(),
|
||||
TitleFit: titleFit,
|
||||
}
|
||||
|
||||
m.Completeness = completeness(
|
||||
alignedCount(c.Files), len(audio), len(dl.Expected),
|
||||
)
|
||||
m.Completeness = completeness(len(audio), len(dl.Expected))
|
||||
|
||||
// The candidate's own title, and the folder its files sit in, are
|
||||
// two independent guesses at the album name. Take the better one:
|
||||
@@ -383,16 +367,9 @@ func scoreMatch(
|
||||
// With no expected tracklist there is no title signal at all, so
|
||||
// redistribute its weight onto the album/artist evidence rather
|
||||
// than scoring every free-text result as half-wrong.
|
||||
switch {
|
||||
case len(dl.Expected) == 0:
|
||||
if len(dl.Expected) == 0 {
|
||||
m.Overall = 0.55*m.AlbumFit + 0.45*m.ArtistFit
|
||||
case m.DurationKnown:
|
||||
m.Overall = timedWeightTitleFit*m.TitleFit +
|
||||
timedWeightDurationFit*m.DurationFit +
|
||||
weightCompleteness*m.Completeness +
|
||||
weightAlbumFit*m.AlbumFit +
|
||||
weightArtistFit*m.ArtistFit
|
||||
default:
|
||||
} else {
|
||||
m.Overall = weightTitleFit*m.TitleFit +
|
||||
weightCompleteness*m.Completeness +
|
||||
weightAlbumFit*m.AlbumFit +
|
||||
@@ -438,54 +415,30 @@ func artistFit(want string, c Candidate) float64 {
|
||||
return best
|
||||
}
|
||||
|
||||
// completeness scores how much of the expected tracklist a candidate
|
||||
// covers. Extra files are penalized far more gently than missing ones:
|
||||
// completeness scores audio file count against the expected track
|
||||
// count. Extra files are penalized far more gently than missing ones:
|
||||
// a folder with bonus tracks or a stray intro is still the album, while
|
||||
// a folder missing half the tracks is not.
|
||||
//
|
||||
// **Coverage is counted in aligned tracks, not in files.** It used to
|
||||
// be the audio file count, so any ten files scored full marks against
|
||||
// a ten-track album whether or not they were its tracks — and since
|
||||
// title fit is the mean over the files that *did* align, a folder where
|
||||
// three titles matched read as a near-perfect candidate on both counts.
|
||||
// `aligned` is how many files matchFiles assigned to an expected track;
|
||||
// `audio` still sets the penalty for extras, because a folder of thirty
|
||||
// files holding the ten wanted is a worse copy than one holding ten.
|
||||
func completeness(aligned, audio, want int) float64 {
|
||||
func completeness(got, want int) float64 {
|
||||
if want == 0 {
|
||||
if audio > 0 {
|
||||
if got > 0 {
|
||||
return 0.5
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
if aligned == 0 {
|
||||
if got == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
cover := float64(min(aligned, want)) / float64(want)
|
||||
if got >= want {
|
||||
extra := float64(got-want) / float64(want)
|
||||
|
||||
if audio > want {
|
||||
extra := float64(audio-want) / float64(want)
|
||||
cover *= math.Max(0.75, 1.0-0.25*extra)
|
||||
return math.Max(0.75, 1.0-0.25*extra)
|
||||
}
|
||||
|
||||
return cover
|
||||
}
|
||||
|
||||
// alignedCount is how many audio files were assigned to an expected
|
||||
// track.
|
||||
func alignedCount(files []CandidateFile) int {
|
||||
n := 0
|
||||
|
||||
for _, f := range files {
|
||||
if f.IsAudio && f.MatchedTo != 0 {
|
||||
n++
|
||||
}
|
||||
}
|
||||
|
||||
return n
|
||||
return float64(got) / float64(want)
|
||||
}
|
||||
|
||||
// scoreQuality answers whether this is a good copy.
|
||||
@@ -782,45 +735,6 @@ func AutoPickVeto(
|
||||
return ""
|
||||
}
|
||||
|
||||
// autoAcceptable reports whether auto-pick may take this one candidate
|
||||
// without asking: the request is anchored to a tracklist, and the
|
||||
// candidate is inside the user's guardrails and clears the match and
|
||||
// quality bars. It is AutoPickVeto's test applied to a single
|
||||
// candidate, which is what falling back to a second choice needs.
|
||||
func autoAcceptable(dl Download, c Candidate, prefs AutoDownloadPrefs) bool {
|
||||
return dl.Anchored() &&
|
||||
len(dl.Expected) > 0 &&
|
||||
prefs.eligible(c, dl.runtimeMillis()) &&
|
||||
c.Match.Overall >= minMatch &&
|
||||
c.Quality.Overall >= minQuality
|
||||
}
|
||||
|
||||
// autoPick returns the candidate auto-pick takes: the best-ranked one
|
||||
// it may take at all.
|
||||
//
|
||||
// That is not `ranked[0]`. AutoPickVeto judges the best candidate
|
||||
// *inside* the guardrails, so when the overall best is outside them —
|
||||
// over the size ceiling, say — the veto passes on the strength of the
|
||||
// second, and grabbing the first would download exactly the copy the
|
||||
// user said not to take unattended.
|
||||
func autoPick(
|
||||
dl Download,
|
||||
ranked []Candidate,
|
||||
prefs AutoDownloadPrefs,
|
||||
) (Candidate, bool) {
|
||||
if AutoPickVeto(dl, ranked, prefs) != "" {
|
||||
return Candidate{}, false
|
||||
}
|
||||
|
||||
for _, c := range ranked {
|
||||
if autoAcceptable(dl, c, prefs) {
|
||||
return c, true
|
||||
}
|
||||
}
|
||||
|
||||
return Candidate{}, false
|
||||
}
|
||||
|
||||
// mergeMatched copies MatchedTo assignments from the audio-only slice
|
||||
// back onto the full file list.
|
||||
func mergeMatched(all, matched []CandidateFile) []CandidateFile {
|
||||
|
||||
@@ -282,32 +282,28 @@ func TestCompleteness(t *testing.T) {
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
aligned int
|
||||
audio int
|
||||
got int
|
||||
want int
|
||||
minScore float64
|
||||
maxScore float64
|
||||
}{
|
||||
{"exact", 10, 10, 10, 1.0, 1.0},
|
||||
{"half missing", 5, 5, 10, 0.49, 0.51},
|
||||
{"one bonus track", 10, 11, 10, 0.95, 1.0},
|
||||
{"double", 10, 20, 10, 0.74, 0.76},
|
||||
{"nothing", 0, 0, 10, 0, 0},
|
||||
{"no expectation", 0, 5, 0, 0.5, 0.5},
|
||||
// Ten files are not ten tracks: three that align are three.
|
||||
{"right count, wrong tracks", 3, 10, 10, 0.29, 0.31},
|
||||
{"files that align to nothing", 0, 10, 10, 0, 0},
|
||||
{"exact", 10, 10, 1.0, 1.0},
|
||||
{"half missing", 5, 10, 0.49, 0.51},
|
||||
{"one bonus track", 11, 10, 0.95, 1.0},
|
||||
{"double", 20, 10, 0.74, 0.76},
|
||||
{"nothing", 0, 10, 0, 0},
|
||||
{"no expectation", 5, 0, 0.5, 0.5},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
got := completeness(tt.aligned, tt.audio, tt.want)
|
||||
got := completeness(tt.got, tt.want)
|
||||
if got < tt.minScore || got > tt.maxScore {
|
||||
t.Errorf(
|
||||
"completeness(%d, %d, %d) = %f, want in [%f, %f]",
|
||||
tt.aligned, tt.audio, tt.want, got, tt.minScore, tt.maxScore,
|
||||
"completeness(%d, %d) = %f, want in [%f, %f]",
|
||||
tt.got, tt.want, got, tt.minScore, tt.maxScore,
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -1,238 +0,0 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// What Soulseek is asked, how, and what is kept from the answer (#271).
|
||||
|
||||
func TestSlskdQueries(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
dl Download
|
||||
want []string
|
||||
}{
|
||||
{
|
||||
name: "a plain request is searched once",
|
||||
dl: Download{Artist: "Radiohead", Album: "OK Computer"},
|
||||
want: []string{"Radiohead OK Computer"},
|
||||
},
|
||||
{
|
||||
name: "an edition qualifier gets a second query without it",
|
||||
dl: Download{Artist: "Radiohead", Album: "OK Computer (Collector's Edition)"},
|
||||
want: []string{
|
||||
"Radiohead OK Computer (Collector's Edition)",
|
||||
"Radiohead OK Computer",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "a trailing remaster note",
|
||||
dl: Download{Artist: "Pink Floyd", Album: "Animals - 2018 Remaster"},
|
||||
want: []string{
|
||||
"Pink Floyd Animals - 2018 Remaster",
|
||||
"Pink Floyd Animals",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "a leading dash would be an exclusion",
|
||||
dl: Download{Artist: "Mocky", Album: "-ism"},
|
||||
want: []string{"Mocky -ism", "Mocky ism"},
|
||||
},
|
||||
{
|
||||
name: "a compilation is not searched by its placeholder artist",
|
||||
dl: Download{Artist: "Various Artists", Album: "Pulp Fiction"},
|
||||
want: []string{"Various Artists Pulp Fiction", "Pulp Fiction"},
|
||||
},
|
||||
{
|
||||
name: "what the user typed is searched as written",
|
||||
dl: Download{Query: "ok computer (deluxe)", Album: "OK Computer (Deluxe)"},
|
||||
want: []string{"ok computer (deluxe)"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
got := slskdQueries(tc.dl)
|
||||
if strings.Join(got, "|") != strings.Join(tc.want, "|") {
|
||||
t.Errorf("slskdQueries = %q, want %q", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Both queries run, the options are stated rather than left to the
|
||||
// daemon's defaults, and a folder both queries found is one candidate.
|
||||
func TestSlskdSearchRunsBothQueriesAndMerges(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newSlskdStub(t)
|
||||
stub.responses = []slskdResponse{{
|
||||
Username: "peer",
|
||||
Files: []slskdFile{
|
||||
{Filename: `\m\Radiohead - OK Computer\01 Airbag.flac`, Size: 1, Length: 284},
|
||||
{Filename: `\m\Radiohead - OK Computer\02 Paranoid Android.flac`, Size: 1, Length: 383},
|
||||
},
|
||||
}}
|
||||
|
||||
s, _ := newStubSlskd(t, stub)
|
||||
|
||||
got, err := s.Search(context.Background(), Download{
|
||||
ReleaseMBID: "rel", Artist: "Radiohead", Album: "OK Computer (Deluxe Edition)",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Search: %v", err)
|
||||
}
|
||||
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("got %d candidates, want the one folder once", len(got))
|
||||
}
|
||||
|
||||
if got[0].Files[0].LengthMillis != 284_000 {
|
||||
t.Errorf("length = %d ms, want 284000 from slskd's seconds", got[0].Files[0].LengthMillis)
|
||||
}
|
||||
|
||||
stub.mu.Lock()
|
||||
searches := append([]map[string]any(nil), stub.searches...)
|
||||
gets := append([]string(nil), stub.searchGets...)
|
||||
stub.mu.Unlock()
|
||||
|
||||
if len(searches) != 2 {
|
||||
t.Fatalf("ran %d searches, want 2", len(searches))
|
||||
}
|
||||
|
||||
for _, body := range searches {
|
||||
for _, key := range []string{
|
||||
"searchTimeout", "responseLimit", "fileLimit",
|
||||
"minimumResponseFileCount", "maximumPeerQueueLength",
|
||||
} {
|
||||
if _, ok := body[key]; !ok {
|
||||
t.Errorf("search %q does not state %s", body["searchText"], key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The responses are fetched once at the end, not with every poll.
|
||||
for _, uri := range gets {
|
||||
if strings.Contains(uri, "includeResponses") {
|
||||
t.Errorf("poll %s asked for every response", uri)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A daemon without the responses endpoint still returns results.
|
||||
func TestSlskdSearchFallsBackForAnOlderDaemon(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newSlskdStub(t)
|
||||
stub.noResponsesEndpoint = true
|
||||
stub.responses = []slskdResponse{{
|
||||
Username: "peer",
|
||||
Files: []slskdFile{
|
||||
{Filename: `\m\Album\01 A.flac`, Size: 1},
|
||||
{Filename: `\m\Album\02 B.flac`, Size: 1},
|
||||
},
|
||||
}}
|
||||
|
||||
s, _ := newStubSlskd(t, stub)
|
||||
|
||||
got, err := s.Search(context.Background(), Download{Query: "album"})
|
||||
if err != nil {
|
||||
t.Fatalf("Search: %v", err)
|
||||
}
|
||||
|
||||
if len(got) != 1 {
|
||||
t.Errorf("got %d candidates, want 1 through the fallback", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDurationAgreement(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := []struct {
|
||||
got, want int64
|
||||
score float64
|
||||
}{
|
||||
{300_000, 300_000, 1},
|
||||
{301_500, 300_000, 1}, // a second of silence
|
||||
{300_000, 316_500, 0.5},
|
||||
{300_000, 345_000, 0}, // a different edit
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
if got := durationAgreement(tc.got, tc.want); got < tc.score-0.01 || got > tc.score+0.01 {
|
||||
t.Errorf("durationAgreement(%d, %d) = %f, want %f", tc.got, tc.want, got, tc.score)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Two folders with the same track names are told apart by their
|
||||
// lengths: one is the album, the other a live record of the same songs.
|
||||
func TestDurationsSeparateTheRightRecording(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dl := okComputer()
|
||||
|
||||
timed := func(id string, lengths ...int64) Candidate {
|
||||
c := candidateFor(id, allTitles(), ".flac", 30_000_000)
|
||||
for i := range c.Files {
|
||||
c.Files[i].LengthMillis = lengths[i]
|
||||
}
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
studio := timed("studio", trackMillis, trackMillis+1_000, trackMillis, trackMillis-500)
|
||||
live := timed(
|
||||
"live",
|
||||
trackMillis+60_000,
|
||||
trackMillis+75_000,
|
||||
trackMillis+50_000,
|
||||
trackMillis+90_000,
|
||||
)
|
||||
|
||||
ranked := Rank(dl, []Candidate{live, studio}, nil, AutoDownloadPrefs{})
|
||||
|
||||
if ranked[0].ID != "studio" {
|
||||
t.Fatalf("winner = %s, want the recording whose lengths match", ranked[0].ID)
|
||||
}
|
||||
|
||||
if !ranked[0].Match.DurationKnown || ranked[0].Match.DurationFit < 0.99 {
|
||||
t.Errorf(
|
||||
"studio duration fit = %f known=%v",
|
||||
ranked[0].Match.DurationFit,
|
||||
ranked[0].Match.DurationKnown,
|
||||
)
|
||||
}
|
||||
|
||||
if ranked[1].Match.DurationFit != 0 {
|
||||
t.Errorf("live duration fit = %f, want 0", ranked[1].Match.DurationFit)
|
||||
}
|
||||
}
|
||||
|
||||
// Without lengths the score is exactly what it was before durations
|
||||
// were read, so a provider that reports none is not penalised.
|
||||
func TestUnknownDurationsLeaveTheScoreAlone(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dl := okComputer()
|
||||
c := Score(dl, candidateFor("c", allTitles(), ".flac", 30_000_000), 50, AutoDownloadPrefs{})
|
||||
|
||||
if c.Match.DurationKnown {
|
||||
t.Fatal("no file states a length, yet durations are known")
|
||||
}
|
||||
|
||||
want := weightTitleFit*c.Match.TitleFit +
|
||||
weightCompleteness*c.Match.Completeness +
|
||||
weightAlbumFit*c.Match.AlbumFit +
|
||||
weightArtistFit*c.Match.ArtistFit
|
||||
|
||||
if c.Match.Overall != want {
|
||||
t.Errorf("match = %f, want the untimed formula's %f", c.Match.Overall, want)
|
||||
}
|
||||
}
|
||||
@@ -1,328 +0,0 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Where slskd writes a grab's files, and how collect finds them (#274).
|
||||
|
||||
// slskd reads searchTimeout in whole seconds, from the last response.
|
||||
func TestSlskdSearchTimeoutIsInSeconds(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := []struct {
|
||||
wait time.Duration
|
||||
want int
|
||||
}{
|
||||
{20 * time.Second, 18},
|
||||
{200 * time.Millisecond, slskdMinSearchTimeout},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
s := &slskd{searchWait: tc.wait}
|
||||
|
||||
got, ok := s.searchRequest("id", "text", 2)["searchTimeout"].(int)
|
||||
if !ok || got != tc.want {
|
||||
t.Errorf("wait %s: searchTimeout = %v, want %d seconds", tc.wait, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsRenamedCopy(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := map[string]bool{
|
||||
"01 A_638912345678901234.flac": true,
|
||||
"01 A.flac": false,
|
||||
"01 A_.flac": false,
|
||||
"01 A_v2.flac": false,
|
||||
"01 A_123.mp3": false,
|
||||
"01 AB_123.flac": false,
|
||||
}
|
||||
|
||||
for name, want := range cases {
|
||||
if got := isRenamedCopy(name, "01 A", ".flac"); got != want {
|
||||
t.Errorf("isRenamedCopy(%q) = %v, want %v", name, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func succeeded(names ...string) [][]slskdTransfer {
|
||||
out := make([]slskdTransfer, 0, len(names))
|
||||
for i, n := range names {
|
||||
out = append(out, slskdTransfer{
|
||||
ID: "t" + itoa(i),
|
||||
Filename: n,
|
||||
State: "Completed, Succeeded",
|
||||
BytesTransferred: 500,
|
||||
})
|
||||
}
|
||||
|
||||
return [][]slskdTransfer{out}
|
||||
}
|
||||
|
||||
// On a daemon with batches, each grab writes into a folder of its own,
|
||||
// collect reads from exactly there, and the folder is gone afterwards.
|
||||
func TestSlskdBatchGrabUsesItsOwnFolder(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newSlskdStub(t)
|
||||
stub.batches = true
|
||||
stub.transfers = succeeded(`\s\Album\01 A.flac`, `\s\Album\02 B.flac`)
|
||||
|
||||
s, downloads := newStubSlskd(t, stub)
|
||||
|
||||
// A same-named file in the folder a per-user enqueue would use is
|
||||
// someone else's, and must not be touched.
|
||||
other := filepath.Join(downloads, "Album", "01 A.flac")
|
||||
if err := os.MkdirAll(filepath.Dir(other), 0o750); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(other, []byte("the user's"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
dst := t.TempDir()
|
||||
|
||||
got, err := s.Grab(
|
||||
context.Background(), slskdAlbum(t, stub, "peer", "01 A.flac", "02 B.flac"), dst, nil,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("Grab: %v", err)
|
||||
}
|
||||
|
||||
if len(got.Files) != 2 {
|
||||
t.Fatalf("collected %d files, want 2", len(got.Files))
|
||||
}
|
||||
|
||||
stub.mu.Lock()
|
||||
bodies := append([]map[string]any(nil), stub.batchBodies...)
|
||||
stub.mu.Unlock()
|
||||
|
||||
if len(bodies) != 1 {
|
||||
t.Fatalf("%d batches, want 1", len(bodies))
|
||||
}
|
||||
|
||||
if bodies[0]["username"] != "peer" {
|
||||
t.Errorf("batch username = %v", bodies[0]["username"])
|
||||
}
|
||||
|
||||
dest, _ := bodies[0]["options"].(map[string]any)["destination"].(string)
|
||||
if !strings.HasPrefix(dest, slskdDestRoot+"/") {
|
||||
t.Errorf("destination %q is not under %s", dest, slskdDestRoot)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(filepath.Join(downloads, filepath.FromSlash(dest))); !os.IsNotExist(err) {
|
||||
t.Errorf("batch folder left behind: %v", err)
|
||||
}
|
||||
|
||||
if data, _ := os.ReadFile(other); string(data) != "the user's" {
|
||||
t.Errorf("the user's own file was taken or changed: %q", data)
|
||||
}
|
||||
}
|
||||
|
||||
// A batch's files land flat in its destination, so a two-disc rip is
|
||||
// two batches, one per disc, or disc 2's "01" is renamed out of the way
|
||||
// of disc 1's.
|
||||
func TestSlskdBatchSplitsDiscs(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newSlskdStub(t)
|
||||
stub.batches = true
|
||||
stub.transfers = succeeded(`\s\Wall\CD1\01 In.flac`, `\s\Wall\CD2\01 Hey You.flac`)
|
||||
stub.deliver("01 In.flac", "01 Hey You.flac")
|
||||
|
||||
s, _ := newStubSlskd(t, stub)
|
||||
|
||||
dst := t.TempDir()
|
||||
|
||||
got, err := s.Grab(context.Background(), Candidate{
|
||||
Files: []CandidateFile{
|
||||
{Path: `\s\Wall\CD1\01 In.flac`, Size: 500, IsAudio: true},
|
||||
{Path: `\s\Wall\CD2\01 Hey You.flac`, Size: 500, IsAudio: true},
|
||||
},
|
||||
Payload: map[string]string{"username": "peer"},
|
||||
}, dst, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Grab: %v", err)
|
||||
}
|
||||
|
||||
stub.mu.Lock()
|
||||
n := len(stub.batchBodies)
|
||||
stub.mu.Unlock()
|
||||
|
||||
if n != 2 {
|
||||
t.Errorf("%d batches, want one per disc", n)
|
||||
}
|
||||
|
||||
for _, want := range []string{
|
||||
filepath.Join(dst, "CD1", "01 In.flac"),
|
||||
filepath.Join(dst, "CD2", "01 Hey You.flac"),
|
||||
} {
|
||||
found := false
|
||||
|
||||
for _, f := range got.Files {
|
||||
found = found || f == want
|
||||
}
|
||||
|
||||
if !found {
|
||||
t.Errorf("%s not collected; got %q", want, got.Files)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// An abandoned batch grab leaves nothing in slskd's folder either.
|
||||
func TestSlskdBatchFailureDiscardsItsFolder(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newSlskdStub(t)
|
||||
stub.batches = true
|
||||
stub.transfers = [][]slskdTransfer{{
|
||||
{ID: "t0", Filename: `\s\Album\01 A.flac`, State: "Completed, Errored"},
|
||||
{ID: "t1", Filename: `\s\Album\02 B.flac`, State: "Completed, Errored"},
|
||||
}}
|
||||
|
||||
s, downloads := newStubSlskd(t, stub)
|
||||
|
||||
// The stub delivers the file, as a partial slskd left behind would.
|
||||
if _, err := s.Grab(
|
||||
context.Background(), slskdAlbum(t, stub, "peer", "01 A.flac"), t.TempDir(), nil,
|
||||
); err == nil {
|
||||
t.Fatal("Grab succeeded with every transfer failed")
|
||||
}
|
||||
|
||||
entries, _ := os.ReadDir(filepath.Join(downloads, slskdDestRoot))
|
||||
if len(entries) != 0 {
|
||||
t.Errorf("%d batch folders left behind", len(entries))
|
||||
}
|
||||
}
|
||||
|
||||
// An older daemon answers the batch endpoint with 400; the grab falls
|
||||
// back to the per-user enqueue, and later grabs do not ask again.
|
||||
func TestSlskdFallsBackWithoutBatches(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newSlskdStub(t)
|
||||
stub.transfers = succeeded(`\s\Album\01 A.flac`, `\s\Album\02 B.flac`)
|
||||
|
||||
s, _ := newStubSlskd(t, stub)
|
||||
|
||||
for range 2 {
|
||||
stub.mu.Lock()
|
||||
stub.posted = false
|
||||
stub.pollCount = 0
|
||||
stub.mu.Unlock()
|
||||
|
||||
if _, err := s.Grab(
|
||||
context.Background(), slskdAlbum(t, stub, "peer", "01 A.flac"), t.TempDir(), nil,
|
||||
); err != nil {
|
||||
t.Fatalf("Grab: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
stub.mu.Lock()
|
||||
paths := append([]string(nil), stub.paths...)
|
||||
stub.mu.Unlock()
|
||||
|
||||
batchCalls := 0
|
||||
|
||||
for _, p := range paths {
|
||||
if strings.HasSuffix(p, "/batches") {
|
||||
batchCalls++
|
||||
}
|
||||
}
|
||||
|
||||
if batchCalls != 1 {
|
||||
t.Errorf("asked for a batch %d times, want once", batchCalls)
|
||||
}
|
||||
}
|
||||
|
||||
// Without batches, a file of the same name already in slskd's folder is
|
||||
// not this grab's: slskd wrote ours beside it as name_<ticks>.ext, and
|
||||
// that is the one collected.
|
||||
func TestSlskdCollectsTheRenamedCopyNotTheOldFile(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newSlskdStub(t)
|
||||
stub.transfers = succeeded(`\s\Album\01 A.flac`, `\s\Album\02 B.flac`)
|
||||
|
||||
s, downloads := newStubSlskd(t, stub)
|
||||
|
||||
old := filepath.Join(downloads, "Album", "01 A.flac")
|
||||
if err := os.MkdirAll(filepath.Dir(old), 0o750); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(old, []byte("left by an earlier attempt"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
dst := t.TempDir()
|
||||
|
||||
got, err := s.Grab(
|
||||
context.Background(), slskdAlbum(t, stub, "peer", "01 A.flac", "02 B.flac"), dst, nil,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("Grab: %v", err)
|
||||
}
|
||||
|
||||
if len(got.Files) != 2 {
|
||||
t.Fatalf("collected %d files, want 2", len(got.Files))
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(filepath.Join(dst, "01 A.flac"))
|
||||
if err != nil || string(data) != "audio" {
|
||||
t.Errorf("collected %q, want this grab's file", data)
|
||||
}
|
||||
|
||||
if data, _ := os.ReadFile(old); string(data) != "left by an earlier attempt" {
|
||||
t.Error("the file that was already there was moved")
|
||||
}
|
||||
}
|
||||
|
||||
// And when this grab's copy never arrived, the old one is not taken in
|
||||
// its place.
|
||||
func TestSlskdDoesNotCollectAFileThatWasAlreadyThere(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newSlskdStub(t)
|
||||
stub.transfers = [][]slskdTransfer{
|
||||
{
|
||||
{ID: "t0", Filename: `\s\Album\01 A.flac`, State: "Completed, Errored"},
|
||||
{
|
||||
ID: "t1",
|
||||
Filename: `\s\Album\02 B.flac`,
|
||||
State: "Completed, Succeeded",
|
||||
BytesTransferred: 500,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
s, downloads := newStubSlskd(t, stub)
|
||||
|
||||
old := filepath.Join(downloads, "Album", "01 A.flac")
|
||||
if err := os.MkdirAll(filepath.Dir(old), 0o750); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(old, []byte("stale"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got, err := s.Grab(
|
||||
context.Background(), slskdAlbum(t, stub, "peer", "02 B.flac"), t.TempDir(), nil,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("Grab: %v", err)
|
||||
}
|
||||
|
||||
if len(got.Files) != 1 || filepath.Base(got.Files[0]) != "02 B.flac" {
|
||||
t.Errorf("collected %q, want only 02 B.flac", got.Files)
|
||||
}
|
||||
}
|
||||
@@ -1,132 +0,0 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A peer that has queued us is judged by where we are in its queue, not
|
||||
// only by a timer (#275).
|
||||
|
||||
func queuedThen(polls int, final string) [][]slskdTransfer {
|
||||
queued := []slskdTransfer{
|
||||
{ID: "t0", Filename: `\s\Album\01 A.flac`, State: "Queued, Remotely"},
|
||||
{ID: "t1", Filename: `\s\Album\02 B.flac`, State: "Queued, Remotely"},
|
||||
}
|
||||
|
||||
out := make([][]slskdTransfer, 0, polls+1)
|
||||
for range polls {
|
||||
out = append(out, queued)
|
||||
}
|
||||
|
||||
return append(out, []slskdTransfer{
|
||||
{ID: "t0", Filename: `\s\Album\01 A.flac`, State: final, BytesTransferred: 500},
|
||||
{ID: "t1", Filename: `\s\Album\02 B.flac`, State: final, BytesTransferred: 500},
|
||||
})
|
||||
}
|
||||
|
||||
func descending(from int) []int {
|
||||
out := make([]int, 0, from)
|
||||
for p := from; p >= 1; p-- {
|
||||
out = append(out, p)
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// Two readings far back in the queue give the peer up at once, rather
|
||||
// than after the stall timer.
|
||||
func TestSlskdGivesUpOnALongQueue(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newSlskdStub(t)
|
||||
stub.transfers = queuedThen(100_000, "Completed, Succeeded")
|
||||
stub.positions = []int{400}
|
||||
|
||||
s, _ := newStubSlskd(t, stub)
|
||||
s.stallAfter = time.Hour
|
||||
|
||||
started := time.Now()
|
||||
|
||||
_, err := s.Grab(context.Background(), slskdAlbum(t, stub, "peer"), t.TempDir(), nil)
|
||||
if !errors.Is(err, ErrSlskdTimeout) || !strings.Contains(err.Error(), "position 400") {
|
||||
t.Fatalf("Grab = %v, want a queue-position give-up", err)
|
||||
}
|
||||
|
||||
if time.Since(started) > 5*time.Second {
|
||||
t.Error("the give-up waited on something other than the position")
|
||||
}
|
||||
|
||||
if len(stub.cancelledURIs()) == 0 {
|
||||
t.Error("the queued transfers were not cancelled")
|
||||
}
|
||||
}
|
||||
|
||||
// One far reading is not enough: slskd says the figure can be wildly
|
||||
// wrong.
|
||||
func TestSlskdOneBadPositionIsNotEnough(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newSlskdStub(t)
|
||||
stub.transfers = queuedThen(20, "Completed, Succeeded")
|
||||
stub.positions = []int{400, 3}
|
||||
|
||||
s, _ := newStubSlskd(t, stub)
|
||||
s.positionPoll = 0
|
||||
|
||||
if _, err := s.Grab(
|
||||
context.Background(),
|
||||
slskdAlbum(t, stub, "peer", "01 A.flac", "02 B.flac"),
|
||||
t.TempDir(), nil,
|
||||
); err != nil {
|
||||
t.Fatalf("Grab: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A queue that is moving is progress: the grab outlives the stall timer
|
||||
// while its position improves.
|
||||
func TestSlskdAMovingQueueIsProgress(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newSlskdStub(t)
|
||||
// Near enough to wait for, with more improving readings (45) than
|
||||
// there are queued polls (30), so the queue outlives the stall timer
|
||||
// (30 polls of at least 2 ms against 40 ms) while still improving,
|
||||
// however slowly the machine runs the loop.
|
||||
stub.transfers = queuedThen(30, "Completed, Succeeded")
|
||||
stub.positions = descending(slskdMaxQueuePosition - 5)
|
||||
|
||||
s, _ := newStubSlskd(t, stub)
|
||||
s.transferPoll = 2 * time.Millisecond
|
||||
s.positionPoll = 0
|
||||
s.stallAfter = 40 * time.Millisecond
|
||||
|
||||
if _, err := s.Grab(
|
||||
context.Background(),
|
||||
slskdAlbum(t, stub, "peer", "01 A.flac", "02 B.flac"),
|
||||
t.TempDir(), nil,
|
||||
); err != nil {
|
||||
t.Fatalf("Grab: %v; a moving queue was treated as a stall", err)
|
||||
}
|
||||
}
|
||||
|
||||
// However steadily the queue moves, waiting in it has a ceiling.
|
||||
func TestSlskdQueueHasACeiling(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stub := newSlskdStub(t)
|
||||
stub.transfers = queuedThen(100_000, "Completed, Succeeded")
|
||||
stub.positions = descending(100_000)
|
||||
|
||||
s, _ := newStubSlskd(t, stub)
|
||||
s.stallAfter = time.Hour
|
||||
s.queueCeiling = 50 * time.Millisecond
|
||||
|
||||
_, err := s.Grab(context.Background(), slskdAlbum(t, stub, "peer"), t.TempDir(), nil)
|
||||
if !errors.Is(err, ErrSlskdTimeout) {
|
||||
t.Fatalf("Grab = %v, want the queue ceiling", err)
|
||||
}
|
||||
}
|
||||
@@ -232,17 +232,12 @@ type Candidate struct {
|
||||
// results give paths and sizes but no tags, so Format and duration are
|
||||
// inferred from the path and size where possible.
|
||||
type CandidateFile struct {
|
||||
Path string `json:"path"`
|
||||
Size int64 `json:"size"`
|
||||
Format Format `json:"format"`
|
||||
Bitrate int `json:"bitrate,omitempty"` // kbps, 0 when unknown
|
||||
IsAudio bool `json:"isAudio"`
|
||||
|
||||
// LengthMillis is the file's duration as the source reports it, or
|
||||
// 0 when it does not. Soulseek reports it for most audio files.
|
||||
LengthMillis int64 `json:"lengthMillis,omitempty"`
|
||||
|
||||
MatchedTo int `json:"matchedTo,omitempty"` // expected track position
|
||||
Path string `json:"path"`
|
||||
Size int64 `json:"size"`
|
||||
Format Format `json:"format"`
|
||||
Bitrate int `json:"bitrate,omitempty"` // kbps, 0 when unknown
|
||||
IsAudio bool `json:"isAudio"`
|
||||
MatchedTo int `json:"matchedTo,omitempty"` // expected track position
|
||||
}
|
||||
|
||||
// Format is a normalized audio container/codec name.
|
||||
@@ -291,14 +286,7 @@ type MatchScore struct {
|
||||
TitleFit float64 `json:"titleFit"` // filenames vs expected titles
|
||||
ArtistFit float64 `json:"artistFit"` // path/origin vs expected artist
|
||||
AlbumFit float64 `json:"albumFit"` // folder name vs album title
|
||||
Completeness float64 `json:"completeness"` // aligned tracks vs expected count
|
||||
|
||||
// DurationFit is how well the aligned files' lengths agree with the
|
||||
// expected tracks', and DurationKnown whether enough of them stated
|
||||
// a length for that to count. When it does not, the score is the
|
||||
// four text signals alone, exactly as before durations were read.
|
||||
DurationFit float64 `json:"durationFit"`
|
||||
DurationKnown bool `json:"durationKnown"`
|
||||
Completeness float64 `json:"completeness"` // audio files vs expected count
|
||||
|
||||
// Anchored records whether an MBID drove this score. Unanchored
|
||||
// matches are capped, because there is nothing to be right about.
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
package explore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"database/sql/driver"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
@@ -285,25 +283,6 @@ func (si *SearchIndex) importCoreArtifact(ctx context.Context, path string) erro
|
||||
}
|
||||
|
||||
merged, mergeErr := si.mergeArtifactRows(ctx, info.rows)
|
||||
|
||||
// Every row the artifact declares has to land. The walk partitions
|
||||
// the artifact's key space, so a total short of info.rows does not
|
||||
// mean the artifact was smaller than it said -- it means a predicate
|
||||
// filtered rows out, and the catalog is quietly partial. Equality
|
||||
// rather than a lower bound because RowsAffected counts an upsert
|
||||
// that changes nothing, and a row already merged locally is counted
|
||||
// again here.
|
||||
//
|
||||
// One reachable case, so this is not merely a tripwire: a row whose
|
||||
// mbid is empty is excluded by `mbid > ?` in both encodings, and an
|
||||
// artifact carrying one would otherwise import as complete.
|
||||
if mergeErr == nil && merged != info.rows {
|
||||
mergeErr = fmt.Errorf(
|
||||
"%w: merged %d of %d rows — a row the artifact holds was not selected",
|
||||
ErrArtifactUnusable, merged, info.rows,
|
||||
)
|
||||
}
|
||||
|
||||
if mergeErr == nil {
|
||||
si.mergeArtifactCredits(ctx)
|
||||
}
|
||||
@@ -369,12 +348,8 @@ func (si *SearchIndex) analyzeIndex() {
|
||||
// is an index range scan and a cancelled import leaves committed work
|
||||
// behind rather than rolling it all back.
|
||||
func (si *SearchIndex) mergeArtifactRows(ctx context.Context, total int) (int, error) {
|
||||
// Asked once, because it is a property of the file and it decides
|
||||
// how the walk's own comparisons are typed. See artifactKey.
|
||||
storesText := si.artifactStoresText()
|
||||
|
||||
selectColumns := artifactSelectColumns(
|
||||
storesText, si.artifactHasTotals(),
|
||||
si.artifactStoresText(), si.artifactHasTotals(),
|
||||
)
|
||||
|
||||
insertSQL := `
|
||||
@@ -392,7 +367,7 @@ func (si *SearchIndex) mergeArtifactRows(ctx context.Context, total int) (int, e
|
||||
WHERE mbid > ? AND mbid <= ?` + upsertIndexConflictSQL
|
||||
|
||||
var (
|
||||
cursor artifactKey
|
||||
cursor string
|
||||
merged int
|
||||
)
|
||||
|
||||
@@ -401,30 +376,17 @@ func (si *SearchIndex) mergeArtifactRows(ctx context.Context, total int) (int, e
|
||||
return merged, err
|
||||
}
|
||||
|
||||
upper, hasUpper, err := si.artifactBatchBound(storesText, cursor)
|
||||
upper, hasUpper, err := si.artifactBatchBound(cursor)
|
||||
if err != nil {
|
||||
return merged, err
|
||||
}
|
||||
|
||||
if hasUpper && bytes.Compare(upper, cursor) <= 0 {
|
||||
// The predicate matched the cursor itself, so the walk can
|
||||
// never advance. SQLite says nothing when a comparison is
|
||||
// made between types it will not coerce - the query simply
|
||||
// answers wrongly - so a mismatch here would otherwise spin
|
||||
// forever behind an unmoving progress bar. Fail instead.
|
||||
return merged, fmt.Errorf(
|
||||
"%w: artifact walk did not advance past %x",
|
||||
ErrArtifactUnusable, []byte(cursor),
|
||||
)
|
||||
}
|
||||
|
||||
var res sql.Result
|
||||
|
||||
if hasUpper {
|
||||
res, err = si.db.ExecContext(insertRangeSQL,
|
||||
cursor.bind(storesText), upper.bind(storesText))
|
||||
res, err = si.db.ExecContext(insertRangeSQL, cursor, upper)
|
||||
} else {
|
||||
res, err = si.db.ExecContext(insertSQL, cursor.bind(storesText))
|
||||
res, err = si.db.ExecContext(insertSQL, cursor)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
@@ -451,65 +413,26 @@ func (si *SearchIndex) mergeArtifactRows(ctx context.Context, total int) (int, e
|
||||
}
|
||||
}
|
||||
|
||||
// artifactKey is one MBID as the attached artifact stores it: 16 raw
|
||||
// bytes in a compact artifact, the dashed 36-character form in one
|
||||
// published before that storage change.
|
||||
//
|
||||
// It is a type with a bind method rather than a string because the
|
||||
// comparison it feeds is typed, and the wrong type is silent. SQLite
|
||||
// does not coerce between TEXT and BLOB and orders every blob after
|
||||
// every text value, so a cursor bound as text against a byte column
|
||||
// makes `mbid > ?` true of the whole table - the walk rediscovers the
|
||||
// same batch bound forever, and `mbid <= ?` false of the whole table,
|
||||
// so no batch merges at all. Nothing errors; the import simply never
|
||||
// finishes. bind is the one place that knows which form the column is
|
||||
// in, decided by artifactStoresText, which asks the artifact rather than
|
||||
// trusting a version number.
|
||||
type artifactKey []byte
|
||||
|
||||
// bind renders the key as a statement argument in the artifact's own
|
||||
// encoding.
|
||||
func (k artifactKey) bind(storesText bool) driver.Value {
|
||||
if storesText {
|
||||
return string(k)
|
||||
}
|
||||
|
||||
// Never nil. database/sql converts a nil []byte to SQL NULL, and
|
||||
// `mbid > NULL` is NULL for every row - so an unset cursor would
|
||||
// agree with nothing and import nothing, which is the same silently
|
||||
// empty merge this type exists to prevent, one type over.
|
||||
if k == nil {
|
||||
return []byte{}
|
||||
}
|
||||
|
||||
return []byte(k)
|
||||
}
|
||||
|
||||
// artifactBatchBound returns the MBID that ends the next batch, and
|
||||
// whether one exists — no bound means the remainder is the last batch.
|
||||
//
|
||||
// The bound is read out of the artifact and handed back as an
|
||||
// artifactKey, because it becomes the next comparison the walk makes.
|
||||
func (si *SearchIndex) artifactBatchBound(
|
||||
storesText bool, cursor artifactKey,
|
||||
) (artifactKey, bool, error) {
|
||||
var bound []byte
|
||||
func (si *SearchIndex) artifactBatchBound(cursor string) (string, bool, error) {
|
||||
var bound string
|
||||
|
||||
err := si.db.QueryRowWriter(
|
||||
`SELECT mbid FROM core.explore_index
|
||||
WHERE mbid > ? ORDER BY mbid LIMIT 1 OFFSET ?`,
|
||||
cursor.bind(storesText), artifactMergeBatch-1,
|
||||
cursor, artifactMergeBatch-1,
|
||||
).Scan(&bound)
|
||||
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, false, nil
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("%w: batch bound: %w", ErrArtifactUnusable, err)
|
||||
return "", false, fmt.Errorf("%w: batch bound: %w", ErrArtifactUnusable, err)
|
||||
}
|
||||
|
||||
return artifactKey(bound), true, nil
|
||||
return bound, true, nil
|
||||
}
|
||||
|
||||
// stampArtifactMeta records what the merge established: the catalog half
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package explore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
@@ -72,7 +71,13 @@ func writeTestArtifact(
|
||||
}
|
||||
}
|
||||
|
||||
stampArtifactMeta(t, db, meta)
|
||||
for k, v := range meta {
|
||||
if _, err := db.Exec(
|
||||
`INSERT INTO artifact_meta (key, value) VALUES (?, ?)`, k, v,
|
||||
); err != nil {
|
||||
t.Fatalf("stamp artifact meta: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, r := range rows {
|
||||
if _, err := db.Exec(`
|
||||
@@ -88,101 +93,6 @@ func writeTestArtifact(
|
||||
return path
|
||||
}
|
||||
|
||||
// compactArtifactSchema is the artifact cmd/indexexport publishes: the
|
||||
// catalog's ids as 16 raw bytes, its entity types as codes, and the
|
||||
// per-release-group total_tracks the exporter added after the first
|
||||
// artifact was shipped.
|
||||
//
|
||||
// It matters that a fixture carries this encoding and not the older
|
||||
// text one, because SQLite does not coerce between TEXT and BLOB and
|
||||
// every comparison the importer makes against an mbid is therefore
|
||||
// encoding-sensitive. writeTestArtifact above is the *other* fixture:
|
||||
// it still writes the text form, which is what the first published
|
||||
// artifact carries and what the importer must keep reading.
|
||||
var compactArtifactSchema = []string{
|
||||
`CREATE TABLE explore_index (
|
||||
entity_type INTEGER NOT NULL,
|
||||
mbid BLOB NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
artist_name TEXT NOT NULL,
|
||||
artist_mbid BLOB NOT NULL,
|
||||
aliases TEXT NOT NULL DEFAULT '',
|
||||
popularity INTEGER NOT NULL DEFAULT 0,
|
||||
listener_count INTEGER NOT NULL DEFAULT 0,
|
||||
duration INTEGER NOT NULL DEFAULT 0,
|
||||
caa_release_mbid BLOB NOT NULL DEFAULT x'',
|
||||
release_name TEXT NOT NULL DEFAULT '',
|
||||
primary_type TEXT NOT NULL DEFAULT '',
|
||||
secondary_types TEXT NOT NULL DEFAULT '',
|
||||
release_date TEXT NOT NULL DEFAULT '',
|
||||
total_tracks INTEGER NOT NULL DEFAULT 0,
|
||||
artist_type TEXT NOT NULL DEFAULT '',
|
||||
country TEXT NOT NULL DEFAULT '',
|
||||
disambiguation TEXT NOT NULL DEFAULT '',
|
||||
sort_name TEXT NOT NULL DEFAULT '',
|
||||
discog_fetched INTEGER NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (mbid)
|
||||
) WITHOUT ROWID`,
|
||||
`CREATE TABLE artifact_meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
)`,
|
||||
}
|
||||
|
||||
// writeCompactTestArtifact builds the artifact the exporter publishes
|
||||
// today, in its own encoding, so the importer is exercised against what
|
||||
// a client actually downloads rather than against what it was written
|
||||
// for.
|
||||
func writeCompactTestArtifact(
|
||||
t *testing.T, meta map[string]string, rows []artifactRow,
|
||||
) string {
|
||||
t.Helper()
|
||||
|
||||
path := filepath.Join(t.TempDir(), "core-index.db")
|
||||
|
||||
db, err := sql.Open("sqlite", "file:"+path)
|
||||
if err != nil {
|
||||
t.Fatalf("open artifact: %v", err)
|
||||
}
|
||||
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
for _, stmt := range compactArtifactSchema {
|
||||
if _, err := db.Exec(stmt); err != nil {
|
||||
t.Fatalf("create artifact schema: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
stampArtifactMeta(t, db, meta)
|
||||
|
||||
for _, r := range rows {
|
||||
if _, err := db.Exec(`
|
||||
INSERT INTO explore_index
|
||||
(entity_type, mbid, title, artist_name, artist_mbid, popularity)
|
||||
VALUES (?, ?, ?, ?, ?, ?)`,
|
||||
entityCode(r.entityType), mbidBytes(r.mbid), r.title,
|
||||
r.artistName, mbidBytes(r.artistMBID), r.popularity,
|
||||
); err != nil {
|
||||
t.Fatalf("insert artifact row: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
return path
|
||||
}
|
||||
|
||||
// stampArtifactMeta writes the artifact_meta rows a fixture declares.
|
||||
func stampArtifactMeta(t *testing.T, db *sql.DB, meta map[string]string) {
|
||||
t.Helper()
|
||||
|
||||
for k, v := range meta {
|
||||
if _, err := db.Exec(
|
||||
`INSERT INTO artifact_meta (key, value) VALUES (?, ?)`, k, v,
|
||||
); err != nil {
|
||||
t.Fatalf("stamp artifact meta: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// validMeta is the artifact_meta a well-formed artifact carries.
|
||||
func validMeta() map[string]string {
|
||||
return map[string]string{
|
||||
@@ -360,71 +270,6 @@ func TestImportCoreArtifactBatchWalkCoversAllRows(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestImportCoreArtifactBatchWalkCoversAllRowsCompact is the batch walk
|
||||
// on the encoding the exporter actually publishes.
|
||||
//
|
||||
// The walk positions itself by comparing the artifact's own mbid column
|
||||
// against the last id it reached, and that column holds 16 raw bytes.
|
||||
// SQLite does not coerce between TEXT and BLOB, and a blob sorts after
|
||||
// every text value, so a cursor bound as text is a predicate that either
|
||||
// matches every row or none: `mbid > ?` with an empty text key is true
|
||||
// of the whole table, so
|
||||
// the 100th row is always the 100th row and the bound never advances,
|
||||
// while `mbid <= <text>` is false of the whole table, so no batch ever
|
||||
// merges. The result is not a wrong import but an unbounded loop that
|
||||
// merges nothing and never fails.
|
||||
//
|
||||
// Both encodings are covered on purpose. The walk was only ever tested
|
||||
// against the text fixture above, which is why it shipped broken on the
|
||||
// one the clients download.
|
||||
func TestImportCoreArtifactBatchWalkCoversAllRowsCompact(t *testing.T) {
|
||||
db := database.NewTestDB(t)
|
||||
si := NewSearchIndex(db, nil, nil, testLogger())
|
||||
|
||||
original := artifactMergeBatch
|
||||
artifactMergeBatch = 100
|
||||
|
||||
t.Cleanup(func() { artifactMergeBatch = original })
|
||||
|
||||
const total = 337
|
||||
|
||||
rows := make([]artifactRow, 0, total)
|
||||
for i := range total {
|
||||
rows = append(rows, artifactRow{
|
||||
entityType: EntityRecording,
|
||||
mbid: syntheticMBID(i),
|
||||
title: "Song",
|
||||
artistName: "Artist",
|
||||
artistMBID: artA,
|
||||
popularity: i,
|
||||
})
|
||||
}
|
||||
|
||||
path := writeCompactTestArtifact(t, validMeta(), rows)
|
||||
|
||||
if err := si.importCoreArtifact(context.Background(), path); err != nil {
|
||||
t.Fatalf("importCoreArtifact: %v", err)
|
||||
}
|
||||
|
||||
var got, top int
|
||||
|
||||
if err := db.QueryRowWriter(
|
||||
"SELECT COUNT(*), MAX(popularity) FROM explore_index",
|
||||
).Scan(&got, &top); err != nil {
|
||||
t.Fatalf("count rows: %v", err)
|
||||
}
|
||||
|
||||
if got != total {
|
||||
t.Errorf("merged %d rows, want %d", got, total)
|
||||
}
|
||||
|
||||
// A count alone would pass if the walk re-merged the same first
|
||||
// batch forever, so the far end of the artifact is checked too.
|
||||
if top != total-1 {
|
||||
t.Errorf("highest popularity = %d, want %d", top, total-1)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportCoreArtifactRejectsBadArtifacts(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
@@ -609,9 +454,61 @@ func TestArtifactColumnsMatchExporter(t *testing.T) {
|
||||
// the importer decides by asking the artifact, not by trusting a
|
||||
// version number, and both must land identically.
|
||||
func TestImportCoreArtifactAcceptsBothEncodings(t *testing.T) {
|
||||
compact := writeCompactTestArtifact(t, validMeta(), []artifactRow{
|
||||
{EntityArtist, artA, "Artist A", "Artist A", artA, 5000},
|
||||
})
|
||||
compact := filepath.Join(t.TempDir(), "core-index.db")
|
||||
|
||||
db, err := sql.Open("sqlite", "file:"+compact)
|
||||
if err != nil {
|
||||
t.Fatalf("open artifact: %v", err)
|
||||
}
|
||||
|
||||
if _, err := db.Exec(`CREATE TABLE explore_index (
|
||||
entity_type INTEGER NOT NULL,
|
||||
mbid BLOB NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
artist_name TEXT NOT NULL,
|
||||
artist_mbid BLOB NOT NULL,
|
||||
aliases TEXT NOT NULL DEFAULT '',
|
||||
popularity INTEGER NOT NULL DEFAULT 0,
|
||||
listener_count INTEGER NOT NULL DEFAULT 0,
|
||||
duration INTEGER NOT NULL DEFAULT 0,
|
||||
caa_release_mbid BLOB NOT NULL DEFAULT x'',
|
||||
release_name TEXT NOT NULL DEFAULT '',
|
||||
primary_type TEXT NOT NULL DEFAULT '',
|
||||
secondary_types TEXT NOT NULL DEFAULT '',
|
||||
release_date TEXT NOT NULL DEFAULT '',
|
||||
artist_type TEXT NOT NULL DEFAULT '',
|
||||
country TEXT NOT NULL DEFAULT '',
|
||||
disambiguation TEXT NOT NULL DEFAULT '',
|
||||
sort_name TEXT NOT NULL DEFAULT '',
|
||||
discog_fetched INTEGER NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (mbid)
|
||||
)`); err != nil {
|
||||
t.Fatalf("create artifact table: %v", err)
|
||||
}
|
||||
|
||||
if _, err := db.Exec(
|
||||
`CREATE TABLE artifact_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)`,
|
||||
); err != nil {
|
||||
t.Fatalf("create artifact meta: %v", err)
|
||||
}
|
||||
|
||||
for k, v := range validMeta() {
|
||||
if _, err := db.Exec(
|
||||
"INSERT INTO artifact_meta (key, value) VALUES (?, ?)", k, v,
|
||||
); err != nil {
|
||||
t.Fatalf("write artifact meta: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := db.Exec(`
|
||||
INSERT INTO explore_index (entity_type, mbid, title, artist_name, artist_mbid, popularity)
|
||||
VALUES (1, ?, 'Artist A', 'Artist A', ?, 5000)`,
|
||||
mbidBytes(artA), mbidBytes(artA),
|
||||
); err != nil {
|
||||
t.Fatalf("write artifact row: %v", err)
|
||||
}
|
||||
|
||||
_ = db.Close()
|
||||
|
||||
live := database.NewTestDB(t)
|
||||
si := NewSearchIndex(live, nil, nil, testLogger())
|
||||
@@ -851,88 +748,3 @@ func TestImportCoreArtifactWithoutCredits(t *testing.T) {
|
||||
t.Errorf("credit refs = %d, want 0", refs)
|
||||
}
|
||||
}
|
||||
|
||||
// TestImportCoreArtifactRefusesAMergeThatLosesRows is the count guard's
|
||||
// positive case.
|
||||
//
|
||||
// The walk's predicates partition the artifact's key space, so a merge
|
||||
// that lands fewer rows than the artifact declares means a predicate
|
||||
// dropped some — and the failure is a catalog that looks populated and
|
||||
// is missing things nobody can name. An empty mbid is the reachable
|
||||
// way to get there: `mbid > ?` is false of it in both encodings, so it
|
||||
// is never selected, and nothing else in the import would notice.
|
||||
func TestImportCoreArtifactRefusesAMergeThatLosesRows(t *testing.T) {
|
||||
db := database.NewTestDB(t)
|
||||
si := NewSearchIndex(db, nil, nil, testLogger())
|
||||
|
||||
path := writeCompactTestArtifact(t, validMeta(), []artifactRow{
|
||||
{EntityArtist, artA, "Artist A", "Artist A", artA, 5000},
|
||||
{EntityArtist, "", "Nameless", "Artist A", artA, 4000},
|
||||
})
|
||||
|
||||
err := si.importCoreArtifact(context.Background(), path)
|
||||
if err == nil {
|
||||
t.Fatal("a merge that lost a row was reported as a complete import")
|
||||
}
|
||||
|
||||
if !strings.Contains(err.Error(), "merged 1 of 2 rows") {
|
||||
t.Errorf("error = %v, want it to name the shortfall", err)
|
||||
}
|
||||
|
||||
// And the same rule as every other rejection: a failed merge must not
|
||||
// leave the index claiming it has a catalog, or the real build would
|
||||
// never run again.
|
||||
if si.hasMeta(dumpImportDoneKey) {
|
||||
t.Error("a failed import still stamped dump_import_done")
|
||||
}
|
||||
}
|
||||
|
||||
// TestArtifactKeyBindsInTheArtifactsOwnEncoding pins the one place the
|
||||
// batch walk's comparison type is decided.
|
||||
//
|
||||
// Every wrong answer is silent, which is why it is worth pinning all
|
||||
// four. SQLite does not coerce TEXT to BLOB and orders every blob after
|
||||
// every text value, so a text key against a byte column makes
|
||||
// `mbid > ?` true of the whole artifact - the cursor never advances and
|
||||
// the walk spins forever without merging a row - while a byte key
|
||||
// against a text column makes it false of the whole artifact, so every
|
||||
// batch merges nothing and the import "succeeds" empty. An unset cursor
|
||||
// is the same fault once more: database/sql converts a nil []byte to
|
||||
// SQL NULL, and `mbid > NULL` matches no row at all.
|
||||
func TestArtifactKeyBindsInTheArtifactsOwnEncoding(t *testing.T) {
|
||||
raw := mbidBytes(artA)
|
||||
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
key artifactKey
|
||||
want []byte
|
||||
}{
|
||||
{"unset", nil, []byte{}},
|
||||
{"set", artifactKey(raw), raw},
|
||||
} {
|
||||
t.Run("bytes/"+tt.name, func(t *testing.T) {
|
||||
got, ok := tt.key.bind(false).([]byte)
|
||||
if !ok {
|
||||
t.Fatalf("bind(false) = %T, want []byte", tt.key.bind(false))
|
||||
}
|
||||
|
||||
if got == nil {
|
||||
t.Fatal("bound to SQL NULL, which matches no row")
|
||||
}
|
||||
|
||||
if !bytes.Equal(got, tt.want) {
|
||||
t.Errorf("bind(false) = %x, want %x", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// The dashed form is what an artifact published before the storage
|
||||
// change carries, and it has to compare as text against text.
|
||||
if got := artifactKey(nil).bind(true); got != "" {
|
||||
t.Errorf("bind(true) on an unset cursor = %#v, want an empty string", got)
|
||||
}
|
||||
|
||||
if got := artifactKey(artA).bind(true); got != artA {
|
||||
t.Errorf("bind(true) = %#v, want %q", got, artA)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,266 +0,0 @@
|
||||
package explore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"yellowjacket/backend/database"
|
||||
)
|
||||
|
||||
// Import of the artifact we actually publish, as a client imports it.
|
||||
//
|
||||
// Every other test here builds a fixture, and a fixture is a second
|
||||
// description of the storage format that can be wrong in the same
|
||||
// direction as the code reading it. That is how #258 shipped: the
|
||||
// importer positioned its batch walk with a Go `string` cursor against
|
||||
// the artifact's 16-byte `mbid` column, and SQLite neither coerces
|
||||
// between TEXT and BLOB nor complains about the comparison — so the walk
|
||||
// merged nothing and never advanced, and no install could finish its
|
||||
// first index build. The fixture that guards the walk writes the old
|
||||
// text encoding; the only compact fixture is one row, below the batch
|
||||
// size, so the bound query never ran. Both passed throughout.
|
||||
//
|
||||
// So this one takes the published file and runs the client's own path
|
||||
// over it — checksum, decompress, merge — and asserts that what the
|
||||
// artifact holds is what the client ends up with.
|
||||
//
|
||||
// It skips without the path, so an ordinary test run pays nothing for
|
||||
// it, and the publish job is where it is meant to run:
|
||||
//
|
||||
// YJ_CORE_INDEX_ARTIFACT=/tmp/core-index.db.zst \
|
||||
// go test -tags indexbuild -run TestImportPublishedArtifact \
|
||||
// ./backend/explore/
|
||||
//
|
||||
// The indexbuild tag is not incidental: that job's container has no GTK,
|
||||
// and the default tag set links the app through Wails.
|
||||
|
||||
// publishedArtifactEnv points at the published artifact: the compressed
|
||||
// core-index.db.zst, or the unpacked core-index.db.
|
||||
const publishedArtifactEnv = "YJ_CORE_INDEX_ARTIFACT"
|
||||
|
||||
// artifactTotals is the pair this test compares across the boundary.
|
||||
//
|
||||
// Rows is the whole point — a merge that lands fewer of them than the
|
||||
// artifact declares is a catalog that looks populated and is missing
|
||||
// things nobody can name — and popularity is the half whose absence was
|
||||
// reported when it happened, because it arrives only through the merge.
|
||||
type artifactTotals struct {
|
||||
rows int
|
||||
withListen int
|
||||
}
|
||||
|
||||
func TestImportPublishedArtifact(t *testing.T) {
|
||||
published := strings.TrimSpace(os.Getenv(publishedArtifactEnv))
|
||||
if published == "" {
|
||||
t.Skipf("set %s=<core-index.db.zst> to import the published artifact",
|
||||
publishedArtifactEnv)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(published); err != nil {
|
||||
t.Fatalf("%s: %v", publishedArtifactEnv, err)
|
||||
}
|
||||
|
||||
// A file-backed database rather than NewTestDB's in-memory one: the
|
||||
// artifact is ~135MB and a million rows, which is not a thing to hold
|
||||
// in RAM inside a test. YJ_HOME is how NewDB is pointed somewhere
|
||||
// disposable, and going through NewDB means this is the constructor,
|
||||
// the schema and the read pool the app itself opens.
|
||||
//
|
||||
// Nothing closes it, because nothing can: `DB` has no Close and the
|
||||
// app's handles are process-lifetime by design. The directory is
|
||||
// unlinked at cleanup and the file goes with it.
|
||||
t.Setenv("YJ_HOME", t.TempDir())
|
||||
|
||||
db, err := database.NewDB(testLogger())
|
||||
if err != nil {
|
||||
t.Fatalf("open database: %v", err)
|
||||
}
|
||||
|
||||
si := NewSearchIndex(db, nil, nil, testLogger())
|
||||
|
||||
// The checksum the publisher shipped, if it shipped one. Every
|
||||
// client verifies it and refuses the artifact when it does not
|
||||
// match, so a wrong one breaks Explore for everyone who has not
|
||||
// already imported — and nothing else would see it, because the
|
||||
// comparison is between two files only the publisher has.
|
||||
if want, ok := publishedChecksum(published); ok {
|
||||
got, err := fileSHA256(published)
|
||||
if err != nil {
|
||||
t.Fatalf("checksum the artifact: %v", err)
|
||||
}
|
||||
|
||||
if got != want {
|
||||
t.Errorf("published artifact hashes to %s, but its .sha256 says %s",
|
||||
got, want)
|
||||
}
|
||||
}
|
||||
|
||||
unpacked := unpackPublishedArtifact(t, si, published)
|
||||
|
||||
want, err := artifactTotalsOf(unpacked)
|
||||
if err != nil {
|
||||
t.Fatalf("count the artifact's rows: %v", err)
|
||||
}
|
||||
|
||||
if err := si.importCoreArtifact(context.Background(), unpacked); err != nil {
|
||||
t.Fatalf("importCoreArtifact: %v", err)
|
||||
}
|
||||
|
||||
got, err := indexTotalsOf(db)
|
||||
if err != nil {
|
||||
t.Fatalf("count the index's rows: %v", err)
|
||||
}
|
||||
|
||||
if got.rows != want.rows {
|
||||
t.Errorf("merged %d rows, but the artifact holds %d",
|
||||
got.rows, want.rows)
|
||||
}
|
||||
|
||||
if got.withListen != want.withListen {
|
||||
t.Errorf("%d rows carry a listen count, but the artifact holds %d of them",
|
||||
got.withListen, want.withListen)
|
||||
}
|
||||
|
||||
// The FTS index is rebuilt from the table once the merge is done, and
|
||||
// it is what search actually reads: a merge that lands without it
|
||||
// leaves Explore silently matching nothing, which is the state #258
|
||||
// produced by a different route.
|
||||
var indexed int
|
||||
if err := db.QueryRowWriter(
|
||||
"SELECT COUNT(*) FROM explore_index_fts",
|
||||
).Scan(&indexed); err != nil {
|
||||
t.Fatalf("count the FTS index: %v", err)
|
||||
}
|
||||
|
||||
if indexed != got.rows {
|
||||
t.Errorf("FTS index holds %d rows against the table's %d",
|
||||
indexed, got.rows)
|
||||
}
|
||||
|
||||
// And one row read back through the app's own path, which is the
|
||||
// other direction of every conversion the merge makes: a byte MBID
|
||||
// out of the table, the app's dashed form, and back in as a lookup.
|
||||
var raw []byte
|
||||
if err := db.QueryRowWriter(`
|
||||
SELECT mbid FROM explore_index
|
||||
WHERE entity_type = 1 /* artist */ AND popularity > 0
|
||||
ORDER BY popularity DESC LIMIT 1`).Scan(&raw); err != nil {
|
||||
t.Fatalf("read a stored mbid: %v", err)
|
||||
}
|
||||
|
||||
dashed, err := mbidFromBytes(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("the stored mbid is not one: %v", err)
|
||||
}
|
||||
|
||||
artist := si.LookupArtistByMBID(dashed)
|
||||
if artist == nil {
|
||||
t.Fatalf("the artifact's most popular artist %s does not look up", dashed)
|
||||
}
|
||||
|
||||
if artist.Popularity == 0 {
|
||||
t.Errorf("artist %s came back with no popularity", dashed)
|
||||
}
|
||||
}
|
||||
|
||||
// publishedChecksum reads the sha256 the publisher wrote beside the
|
||||
// artifact, in `sha256sum` output form. A missing file is not a
|
||||
// failure: it is only there when the artifact came from the publish job.
|
||||
func publishedChecksum(path string) (string, bool) {
|
||||
body, err := os.ReadFile(path + ".sha256")
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
|
||||
sum := strings.TrimSpace(string(body))
|
||||
if i := strings.IndexAny(sum, " \t"); i > 0 {
|
||||
sum = sum[:i]
|
||||
}
|
||||
|
||||
if len(sum) != 64 {
|
||||
return "", false
|
||||
}
|
||||
|
||||
return strings.ToLower(sum), true
|
||||
}
|
||||
|
||||
// unpackPublishedArtifact returns a path to the unpacked database,
|
||||
// going through the client's own decompression when it is handed the
|
||||
// compressed file that is actually published.
|
||||
func unpackPublishedArtifact(t *testing.T, si *SearchIndex, path string) string {
|
||||
t.Helper()
|
||||
|
||||
if strings.HasSuffix(path, ".db") {
|
||||
return path
|
||||
}
|
||||
|
||||
// Copied into the test's own directory first: decompress writes
|
||||
// beside the compressed file, and the publisher's directory is not
|
||||
// this test's to write in.
|
||||
staging := t.TempDir()
|
||||
dst := filepath.Join(staging, coreArtifactFile)
|
||||
|
||||
src, err := os.Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("open the published artifact: %v", err)
|
||||
}
|
||||
|
||||
defer func() { _ = src.Close() }()
|
||||
|
||||
out, err := os.Create(dst)
|
||||
if err != nil {
|
||||
t.Fatalf("create a staging copy: %v", err)
|
||||
}
|
||||
|
||||
if _, err := io.Copy(out, src); err != nil {
|
||||
t.Fatalf("copy the published artifact: %v", err)
|
||||
}
|
||||
|
||||
if err := out.Close(); err != nil {
|
||||
t.Fatalf("close the staging copy: %v", err)
|
||||
}
|
||||
|
||||
fetcher := &artifactFetcher{si: si, stagingDir: staging}
|
||||
|
||||
if err := fetcher.decompress(context.Background()); err != nil {
|
||||
t.Fatalf("decompress the published artifact: %v", err)
|
||||
}
|
||||
|
||||
return fetcher.unpackedPath()
|
||||
}
|
||||
|
||||
// artifactTotalsOf counts what an artifact file holds, read directly so
|
||||
// the numbers do not depend on anything the client does.
|
||||
func artifactTotalsOf(path string) (artifactTotals, error) {
|
||||
db, err := sql.Open("sqlite", "file:"+path+"?mode=ro")
|
||||
if err != nil {
|
||||
return artifactTotals{}, err
|
||||
}
|
||||
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
var totals artifactTotals
|
||||
|
||||
err = db.QueryRow(`SELECT COUNT(*), COALESCE(SUM(popularity > 0), 0)
|
||||
FROM explore_index`).Scan(&totals.rows, &totals.withListen)
|
||||
if err != nil {
|
||||
return artifactTotals{}, err
|
||||
}
|
||||
|
||||
return totals, nil
|
||||
}
|
||||
|
||||
// indexTotalsOf counts what the client ended up with.
|
||||
func indexTotalsOf(db *database.DB) (artifactTotals, error) {
|
||||
var totals artifactTotals
|
||||
|
||||
err := db.QueryRowWriter(`SELECT COUNT(*), COALESCE(SUM(popularity > 0), 0)
|
||||
FROM explore_index`).Scan(&totals.rows, &totals.withListen)
|
||||
|
||||
return totals, err
|
||||
}
|
||||
+11
-17
@@ -383,13 +383,9 @@ func (si *SearchIndex) topByPopularity(
|
||||
// MusicBrainz IDs, so this never touches the library tables and asks
|
||||
// one query rather than one per artist.
|
||||
//
|
||||
// **The row is drawn at random, and that is the whole point of it.**
|
||||
// Ordered by popularity it was a second leaderboard: the same handful of
|
||||
// big names appeared every time the page opened, which is not what "you
|
||||
// own one album by these artists" is saying. The pool is still bounded
|
||||
// to `pool` artists — a 4 000-artist library does not need all of them
|
||||
// ranked — but which of them, and which of their albums, is `RANDOM()`,
|
||||
// so the shelf is a different sample each visit.
|
||||
// The artists are drawn most-popular-owned-album first, so a large
|
||||
// library's pool is the part of it the user is likeliest to recognise
|
||||
// rather than whichever artists sort first.
|
||||
func (si *SearchIndex) unownedAlbumsBySinglyOwnedArtists(
|
||||
ctx context.Context,
|
||||
pool, limit int,
|
||||
@@ -400,17 +396,15 @@ func (si *SearchIndex) unownedAlbumsBySinglyOwnedArtists(
|
||||
WHERE entity_type = 2 /* release_group */
|
||||
AND in_library = 0
|
||||
AND artist_mbid IN (
|
||||
SELECT artist_mbid FROM (
|
||||
SELECT artist_mbid FROM explore_index
|
||||
WHERE entity_type = 2 /* release_group */
|
||||
AND in_library = 1
|
||||
AND artist_mbid != x''
|
||||
GROUP BY artist_mbid
|
||||
HAVING COUNT(*) = 1
|
||||
)
|
||||
ORDER BY RANDOM()
|
||||
SELECT artist_mbid FROM explore_index
|
||||
WHERE entity_type = 2 /* release_group */
|
||||
AND in_library = 1
|
||||
AND artist_mbid != x''
|
||||
GROUP BY artist_mbid
|
||||
HAVING COUNT(*) = 1
|
||||
ORDER BY MAX(popularity) DESC
|
||||
LIMIT ?)
|
||||
ORDER BY RANDOM()
|
||||
ORDER BY popularity DESC
|
||||
LIMIT ?`,
|
||||
pool, limit,
|
||||
))
|
||||
|
||||
@@ -3,7 +3,6 @@ package explore
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"yellowjacket/backend/database"
|
||||
@@ -203,11 +202,6 @@ func TestShelves_MoreFromOwnedNeedsExactlyOneOwnedAlbum(t *testing.T) {
|
||||
titles = append(titles, album.Title)
|
||||
}
|
||||
|
||||
// The row is a random sample, so the *set* is what is asserted and
|
||||
// not the order — see `unownedAlbumsBySinglyOwnedArtists` for why
|
||||
// the ordering was given up.
|
||||
sort.Strings(titles)
|
||||
|
||||
if len(titles) != 2 || titles[0] != "Second" || titles[1] != "Third" {
|
||||
t.Fatalf("albums = %v, want [Second Third]", titles)
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ package library
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"image"
|
||||
@@ -70,77 +69,6 @@ func CoverArtFileSet(coverPath string) []string {
|
||||
return paths
|
||||
}
|
||||
|
||||
// sweepOrphanedCoverArt deletes the cover_art rows no album references
|
||||
// and returns their file paths, for the caller to remove from disk
|
||||
// after the transaction commits. Cover art is referenced only by
|
||||
// albums.cover_art_id, so an orphan is a cover whose album is gone —
|
||||
// which is every album the caller just swept.
|
||||
//
|
||||
// One implementation because the scan path, RemoveFromLibrary and
|
||||
// RemoveLibrary all reach this state, and the scan side used to skip it
|
||||
// entirely while RemoveLibrary did it inline (#247).
|
||||
func (l *Library) sweepOrphanedCoverArt(tx *sql.Tx) ([]string, error) {
|
||||
const orphanSQL = `
|
||||
SELECT file_path FROM cover_art WHERE id NOT IN (
|
||||
SELECT DISTINCT cover_art_id FROM albums
|
||||
WHERE cover_art_id IS NOT NULL
|
||||
)`
|
||||
|
||||
rows, err := tx.QueryContext(l.ctx, orphanSQL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not query orphaned cover art: %w", err)
|
||||
}
|
||||
|
||||
var paths []string
|
||||
|
||||
for rows.Next() {
|
||||
var filePath string
|
||||
|
||||
if err := rows.Scan(&filePath); err != nil {
|
||||
l.logger.Warn("could not scan cover art path", "err", err)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
paths = append(paths, filePath)
|
||||
}
|
||||
|
||||
// Close before the DELETE: the two run on the one writer connection.
|
||||
if err := rows.Close(); err != nil {
|
||||
l.logger.Warn("could not close cover art rows", "err", err)
|
||||
}
|
||||
|
||||
if len(paths) > 0 {
|
||||
if _, err := tx.ExecContext(l.ctx, `
|
||||
DELETE FROM cover_art WHERE id NOT IN (
|
||||
SELECT DISTINCT cover_art_id FROM albums
|
||||
WHERE cover_art_id IS NOT NULL
|
||||
)`); err != nil {
|
||||
return nil, fmt.Errorf("could not delete orphaned cover_art: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return paths, nil
|
||||
}
|
||||
|
||||
// removeCoverArtFiles removes a cover original and its derived size
|
||||
// variants. Only the original is stored in cover_art.file_path; the
|
||||
// _sm/_md/_lg tiers are derived filenames beside it, so they have to be
|
||||
// removed by name or they accumulate forever.
|
||||
func (l *Library) removeCoverArtFiles(coverPaths []string) {
|
||||
for _, coverPath := range coverPaths {
|
||||
for _, path := range CoverArtFileSet(coverPath) {
|
||||
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
|
||||
l.logger.Warn(
|
||||
"could not remove orphaned cover art file",
|
||||
"path", path,
|
||||
"err", err,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// saveCoverArt saves embedded cover art to the cache directory.
|
||||
// Returns the file path where the art was saved, or empty string
|
||||
// if no picture data. Timing is recorded in the provided metrics.
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
|
||||
"yellowjacket/backend/coverart"
|
||||
"yellowjacket/backend/database/sql/sqlcgen"
|
||||
"yellowjacket/internal/testfixtures"
|
||||
)
|
||||
|
||||
// TestScan_StoresOnlyCoverTiers pins the size decision: a scan writes
|
||||
@@ -27,9 +26,10 @@ func TestScan_StoresOnlyCoverTiers(t *testing.T) {
|
||||
|
||||
lib, db := setupTestLibrary(t)
|
||||
|
||||
// Load skips when the fixture library has not been generated, as
|
||||
// every other fixture test does.
|
||||
root := testfixtures.Load(t).Root()
|
||||
root, err := filepath.Abs("../../test_data/music_library_test")
|
||||
if err != nil {
|
||||
t.Fatalf("resolve fixture path: %v", err)
|
||||
}
|
||||
|
||||
library, err := db.Queries.CreateLibrary(lib.ctx, sqlcgen.CreateLibraryParams{
|
||||
Name: "Fixtures",
|
||||
|
||||
+50
-8
@@ -320,12 +320,43 @@ func (l *Library) RemoveLibrary(id int64) (*RemovalSummary, error) {
|
||||
|
||||
genresRemoved, _ := result.RowsAffected()
|
||||
|
||||
// Collect and delete orphaned cover_art rows before the commit. The
|
||||
// shared helper is the one place this sweep lives, so the scan path,
|
||||
// RemoveFromLibrary and this removal cannot drift (#247).
|
||||
orphanedCoverArtPaths, err := l.sweepOrphanedCoverArt(tx)
|
||||
// 15. Collect orphaned cover_art file paths for post-commit cleanup.
|
||||
// SAFETY: Hand-crafted SELECT for orphaned cover art identification.
|
||||
// Parameterless.
|
||||
rows, err := tx.QueryContext(l.ctx,
|
||||
`SELECT file_path FROM cover_art WHERE id NOT IN (
|
||||
SELECT DISTINCT cover_art_id FROM albums
|
||||
WHERE cover_art_id IS NOT NULL
|
||||
)`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("could not query orphaned cover art: %w", err)
|
||||
}
|
||||
|
||||
var orphanedCoverArtPaths []string
|
||||
|
||||
for rows.Next() {
|
||||
var filePath string
|
||||
if err := rows.Scan(&filePath); err != nil {
|
||||
l.logger.Warn("could not scan cover art path", "err", err)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
orphanedCoverArtPaths = append(orphanedCoverArtPaths, filePath)
|
||||
}
|
||||
|
||||
if err := rows.Close(); err != nil {
|
||||
l.logger.Warn("could not close cover art rows", "err", err)
|
||||
}
|
||||
|
||||
// 16. Delete orphaned cover_art rows.
|
||||
// SAFETY: Hand-crafted orphan cleanup SQL. Parameterless.
|
||||
if _, err := tx.ExecContext(l.ctx,
|
||||
`DELETE FROM cover_art WHERE id NOT IN (
|
||||
SELECT DISTINCT cover_art_id FROM albums
|
||||
WHERE cover_art_id IS NOT NULL
|
||||
)`); err != nil {
|
||||
return nil, fmt.Errorf("could not delete orphaned cover_art: %w", err)
|
||||
}
|
||||
|
||||
// 17. Delete the library's tagging queue. tagging_items holds a
|
||||
@@ -361,9 +392,20 @@ func (l *Library) RemoveLibrary(id int64) (*RemovalSummary, error) {
|
||||
// avoids a costly full re-index of all remaining tracks (~10s for
|
||||
// 25K tracks).
|
||||
|
||||
// Post-commit: remove the orphaned cover art files and their sized
|
||||
// variants.
|
||||
l.removeCoverArtFiles(orphanedCoverArtPaths)
|
||||
// 21. Post-commit: Delete orphaned cover art files and their sized
|
||||
// variants. Only the original is stored in cover_art.file_path; the
|
||||
// _sm/_md/_lg thumbnails are derived filenames beside it, so they
|
||||
// have to be removed by name or they accumulate forever.
|
||||
for _, coverPath := range orphanedCoverArtPaths {
|
||||
for _, path := range CoverArtFileSet(coverPath) {
|
||||
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
|
||||
l.logger.Warn("could not remove orphaned cover art file",
|
||||
"path", path,
|
||||
"err", err,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 22. Post-commit: Compact queue.
|
||||
if l.removalHooks.CompactQueue != nil {
|
||||
|
||||
+32
-119
@@ -911,96 +911,30 @@ func (l *Library) scanInternal(
|
||||
|
||||
orphanStart := time.Now()
|
||||
|
||||
// Snapshot the orphan set first. The playlist-phantom
|
||||
// preservation and the deletes are one transaction (the
|
||||
// preservation has to land before the ON DELETE SET NULL, and
|
||||
// both have to succeed or neither does), and the ids are what
|
||||
// scope that preservation to just these files instead of
|
||||
// rewriting every playlist row on a routine scan.
|
||||
var (
|
||||
orphans []sqlcgen.AudioFile
|
||||
orphanPaths []string
|
||||
)
|
||||
|
||||
existingPaths.Range(func(key, value any) bool {
|
||||
orphanPaths = append(orphanPaths, key.(string))
|
||||
orphans = append(orphans, value.(sqlcgen.AudioFile))
|
||||
path := key.(string)
|
||||
audioFile := value.(sqlcgen.AudioFile)
|
||||
|
||||
return true
|
||||
})
|
||||
l.logger.Debug(
|
||||
"removing orphaned database entry",
|
||||
"path", path, "id", audioFile.ID,
|
||||
)
|
||||
|
||||
deleted := make([]bool, len(orphans))
|
||||
if err := l.db.Queries.DeleteAudioFile(
|
||||
l.ctx, audioFile.ID,
|
||||
); err != nil {
|
||||
l.logger.Warn(
|
||||
"failed to delete orphaned audio file",
|
||||
"path", path,
|
||||
"id", audioFile.ID,
|
||||
"err", err,
|
||||
)
|
||||
|
||||
if len(orphans) > 0 {
|
||||
orphanIDs := make([]int64, len(orphans))
|
||||
for i, f := range orphans {
|
||||
orphanIDs[i] = f.ID
|
||||
metrics.addWarning(path, "orphan", err)
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
tx, beginErr := l.db.BeginTx()
|
||||
if beginErr != nil {
|
||||
metrics.addWarning("", "orphan", beginErr)
|
||||
} else {
|
||||
defer func() { _ = tx.Rollback() }() // no-op after commit
|
||||
|
||||
if err := database.PreservePlaylistPhantomsForFiles(
|
||||
l.ctx, tx, orphanIDs, l.logger,
|
||||
); err != nil {
|
||||
// Deleting without the phantoms is exactly the
|
||||
// playlist-emptying bug the preservation exists to
|
||||
// prevent, so leave the rows for the next scan
|
||||
// rather than empty the playlists now.
|
||||
l.logger.Error(
|
||||
"skipping orphan deletion: could not preserve "+
|
||||
"playlist entries",
|
||||
"err", err,
|
||||
)
|
||||
|
||||
metrics.addWarning("", "orphan", err)
|
||||
|
||||
_ = tx.Rollback()
|
||||
} else {
|
||||
txq := l.db.Queries.WithTx(tx)
|
||||
|
||||
for i, f := range orphans {
|
||||
if err := txq.DeleteAudioFile(
|
||||
l.ctx, f.ID,
|
||||
); err != nil {
|
||||
l.logger.Warn(
|
||||
"failed to delete orphaned audio file",
|
||||
"path", orphanPaths[i],
|
||||
"id", f.ID,
|
||||
"err", err,
|
||||
)
|
||||
|
||||
metrics.addWarning(orphanPaths[i], "orphan", err)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
deleted[i] = true
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
l.logger.Error(
|
||||
"could not commit orphan deletion",
|
||||
"err", err,
|
||||
)
|
||||
|
||||
metrics.addWarning("", "orphan", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Post-commit bookkeeping for the files that actually went.
|
||||
for i, f := range orphans {
|
||||
if !deleted[i] {
|
||||
continue
|
||||
}
|
||||
|
||||
path := orphanPaths[i]
|
||||
|
||||
// Keep the file's tagging group in sync: drop the group's
|
||||
// track count and clear it out once empty, mirroring the
|
||||
// bookkeeping maybeRebindTaggingGroup does for a group_key
|
||||
@@ -1008,25 +942,25 @@ func (l *Library) scanInternal(
|
||||
// and replaced leaves a stale tagging_items row behind —
|
||||
// its track_count still counts the deleted files, and it
|
||||
// never clears from the autotag queue.
|
||||
if f.GroupKey != "" {
|
||||
if audioFile.GroupKey != "" {
|
||||
if err := l.db.Queries.DecrementTaggingItemTrackCount(
|
||||
l.ctx, f.GroupKey,
|
||||
l.ctx, audioFile.GroupKey,
|
||||
); err != nil {
|
||||
l.logger.Warn(
|
||||
"failed to decrement tagging group for orphan",
|
||||
"path", path,
|
||||
"group_key", f.GroupKey,
|
||||
"group_key", audioFile.GroupKey,
|
||||
"err", err,
|
||||
)
|
||||
|
||||
metrics.addWarning(path, "orphan", err)
|
||||
} else if err := l.db.Queries.DeleteTaggingItemIfEmpty(
|
||||
l.ctx, f.GroupKey,
|
||||
l.ctx, audioFile.GroupKey,
|
||||
); err != nil {
|
||||
l.logger.Warn(
|
||||
"failed to clean up emptied tagging group for orphan",
|
||||
"path", path,
|
||||
"group_key", f.GroupKey,
|
||||
"group_key", audioFile.GroupKey,
|
||||
"err", err,
|
||||
)
|
||||
|
||||
@@ -1034,21 +968,13 @@ func (l *Library) scanInternal(
|
||||
}
|
||||
}
|
||||
|
||||
// Remove from FTS5 search index and the lyrics index.
|
||||
if err := l.db.DeleteSearchIndex(f.ID); err != nil {
|
||||
// Remove from FTS5 search index.
|
||||
if err := l.db.DeleteSearchIndex(
|
||||
audioFile.ID,
|
||||
); err != nil {
|
||||
l.logger.Warn(
|
||||
"failed to delete FTS entry for orphan",
|
||||
"id", f.ID,
|
||||
"err", err,
|
||||
)
|
||||
|
||||
metrics.addWarning(path, "orphan", err)
|
||||
}
|
||||
|
||||
if err := l.db.DeleteLyricsIndex(f.ID); err != nil {
|
||||
l.logger.Warn(
|
||||
"failed to delete lyrics index entry for orphan",
|
||||
"id", f.ID,
|
||||
"id", audioFile.ID,
|
||||
"err", err,
|
||||
)
|
||||
|
||||
@@ -1056,7 +982,9 @@ func (l *Library) scanInternal(
|
||||
}
|
||||
|
||||
removed.Add(1)
|
||||
}
|
||||
|
||||
return true
|
||||
})
|
||||
|
||||
metrics.OrphanCleanup = time.Since(orphanStart)
|
||||
|
||||
@@ -1265,32 +1193,17 @@ func (l *Library) pruneEmptyEntities() {
|
||||
}
|
||||
}
|
||||
|
||||
// Cover art after albums: a cover whose album just went is
|
||||
// unreferenced, and leaving the row behind keeps its files exempt
|
||||
// from the janitor's covers sweep forever (#247).
|
||||
orphanedCovers, err := l.sweepOrphanedCoverArt(tx)
|
||||
if err != nil {
|
||||
l.logger.Warn("could not sweep orphaned cover art", "err", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
l.logger.Warn("could not commit entity cleanup", "err", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Post-commit: the rows are gone, so their files can go too.
|
||||
l.removeCoverArtFiles(orphanedCovers)
|
||||
|
||||
if len(albumIDs) > 0 || len(artistIDs) > 0 || len(genreIDs) > 0 ||
|
||||
len(orphanedCovers) > 0 {
|
||||
if len(albumIDs) > 0 || len(artistIDs) > 0 || len(genreIDs) > 0 {
|
||||
l.logger.Info("pruned empty library entities",
|
||||
"albums", len(albumIDs),
|
||||
"artists", len(artistIDs),
|
||||
"genres", len(genreIDs),
|
||||
"covers", len(orphanedCovers),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -94,57 +94,6 @@ func countRows(
|
||||
return n
|
||||
}
|
||||
|
||||
// RemoveFromLibrary is the one path that empties a track *deliberately*:
|
||||
// the file stays on disk but is excluded, so nothing re-imports it. The
|
||||
// playlist entry must still survive as a re-linkable phantom rather than
|
||||
// an empty row, because a later full rescan clears the exclusion and is
|
||||
// what re-links the entry then (#246).
|
||||
func TestRemoveFromLibrary_PreservesPlaylistPhantoms(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
lib, db := setupTestLibrary(t)
|
||||
|
||||
seedRemovableLibrary(t, lib, "/nonexistent/cover.jpg")
|
||||
|
||||
if _, err := lib.db.ExecContext(
|
||||
`INSERT INTO playlists (name) VALUES ('keepme')`,
|
||||
); err != nil {
|
||||
t.Fatalf("seed playlist: %v", err)
|
||||
}
|
||||
|
||||
playlistID := queryInt(
|
||||
t, db, `SELECT id FROM playlists WHERE name = 'keepme'`,
|
||||
)
|
||||
|
||||
if _, err := lib.db.ExecContext(
|
||||
`INSERT INTO playlist_tracks (playlist_id, audio_file_id, position)
|
||||
SELECT ?, id, 0 FROM audio_files
|
||||
WHERE file_path = '/music/song.mp3'`,
|
||||
playlistID,
|
||||
); err != nil {
|
||||
t.Fatalf("seed playlist_tracks: %v", err)
|
||||
}
|
||||
|
||||
if _, err := lib.RemoveFromLibrary([]string{"/music/song.mp3"}); err != nil {
|
||||
t.Fatalf("RemoveFromLibrary: %v", err)
|
||||
}
|
||||
|
||||
phantomPath := queryString(
|
||||
t, db,
|
||||
`SELECT phantom_file_path FROM playlist_tracks
|
||||
WHERE playlist_id = ?`,
|
||||
playlistID,
|
||||
)
|
||||
|
||||
if phantomPath != "/music/song.mp3" {
|
||||
t.Fatalf(
|
||||
"phantom_file_path is %q, want %q -- the entry cannot be "+
|
||||
"re-linked after a later full rescan",
|
||||
phantomPath, "/music/song.mp3",
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// A library with tagging_items must still be removable. tagging_items
|
||||
// FK-references libraries with no ON DELETE clause, so leaving those
|
||||
// rows behind fails the DELETE and rolls back the entire removal.
|
||||
@@ -249,53 +198,3 @@ func TestCoverArtFileSet(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Removing the last track of an album must take the album's cover art
|
||||
// with it — both the row and every derived file — or the row keeps its
|
||||
// files exempt from the janitor's covers sweep forever (#247).
|
||||
func TestRemoveFromLibrary_DeletesOrphanedCoverArt(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
lib, _ := setupTestLibrary(t)
|
||||
|
||||
dir := t.TempDir()
|
||||
|
||||
// The largest tier is what cover_art.file_path names; write every
|
||||
// variant so the sweep has a real set to remove.
|
||||
for _, tier := range thumbnailTiers {
|
||||
p := filepath.Join(dir, coverart.SizedFilename("abc123.jpg", tier.Suffix))
|
||||
if err := os.WriteFile(p, []byte("img"), 0o600); err != nil {
|
||||
t.Fatalf("write %s: %v", p, err)
|
||||
}
|
||||
}
|
||||
|
||||
cover := filepath.Join(dir, coverart.SizedFilename("abc123.jpg", "_lg"))
|
||||
|
||||
seedRemovableLibrary(t, lib, cover)
|
||||
|
||||
// Link the album to the cover so it is not orphaned until the track
|
||||
// (and with it the album) goes.
|
||||
if _, err := lib.db.ExecContext(
|
||||
`UPDATE albums SET cover_art_id =
|
||||
(SELECT id FROM cover_art WHERE file_path = ?)
|
||||
WHERE name = 'Test Album'`,
|
||||
cover,
|
||||
); err != nil {
|
||||
t.Fatalf("link cover art: %v", err)
|
||||
}
|
||||
|
||||
if _, err := lib.RemoveFromLibrary([]string{"/music/song.mp3"}); err != nil {
|
||||
t.Fatalf("RemoveFromLibrary: %v", err)
|
||||
}
|
||||
|
||||
if n := countRows(t, lib, "cover_art"); n != 0 {
|
||||
t.Errorf("cover_art has %d rows after removal, want 0", n)
|
||||
}
|
||||
|
||||
for _, tier := range thumbnailTiers {
|
||||
p := filepath.Join(dir, coverart.SizedFilename("abc123.jpg", tier.Suffix))
|
||||
if _, err := os.Stat(p); !os.IsNotExist(err) {
|
||||
t.Errorf("cover art file still present: %s", filepath.Base(p))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"yellowjacket/backend/database"
|
||||
"yellowjacket/backend/events"
|
||||
)
|
||||
|
||||
@@ -58,25 +57,6 @@ func (l *Library) RemoveFromLibrary(filePaths []string) (*RemovalResult, error)
|
||||
|
||||
var result RemovalResult
|
||||
|
||||
// Preserve the playlist entries before the rows go, so they survive
|
||||
// as re-linkable phantoms rather than empty rows. The track is
|
||||
// excluded and will not be re-imported on its own, but a later full
|
||||
// rescan clears the exclusion and this is what lets the entry
|
||||
// re-link then — the same preservation every other path that empties
|
||||
// audio_files performs (#246).
|
||||
rowIDs := make([]int64, len(rows))
|
||||
for i, row := range rows {
|
||||
rowIDs[i] = row.ID
|
||||
}
|
||||
|
||||
if err := database.PreservePlaylistPhantomsForFiles(
|
||||
l.ctx, tx, rowIDs, l.logger,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf(
|
||||
"could not preserve playlist entries for removal: %w", err,
|
||||
)
|
||||
}
|
||||
|
||||
// Exclude every path the caller named, including one whose row has
|
||||
// already gone: the user asked for that file to stay out, and a row
|
||||
// that disappeared between the click and the commit is not a reason
|
||||
@@ -147,11 +127,6 @@ func (l *Library) RemoveFromLibrary(filePaths []string) (*RemovalResult, error)
|
||||
l.logger.Warn("could not delete FTS entry for removed track",
|
||||
"path", row.FilePath, "id", row.ID, "err", err)
|
||||
}
|
||||
|
||||
if err := l.db.DeleteLyricsIndex(row.ID); err != nil {
|
||||
l.logger.Warn("could not delete lyrics index entry for removed track",
|
||||
"path", row.FilePath, "id", row.ID, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Deleting an audio_files row cascades to queue_tracks, so the
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"time"
|
||||
|
||||
"yellowjacket/backend/coverart"
|
||||
"yellowjacket/backend/database"
|
||||
)
|
||||
|
||||
var errNoLibrariesConfigured = errors.New(
|
||||
@@ -138,12 +137,34 @@ func (l *Library) clearLibraryTables() error {
|
||||
// metadata for all linked tracks before audio_files are deleted.
|
||||
// ON DELETE SET NULL will null out audio_file_id, converting them
|
||||
// to phantoms that ResolvePhantomTracksAfterScan can re-link.
|
||||
//
|
||||
// Shared with the stale-shape retire in backend/database, which is
|
||||
// the other path that empties this table and which did not do this
|
||||
// (#183): the statement lives there so the two cannot drift again.
|
||||
if err := database.PreservePlaylistPhantoms(l.ctx, tx, l.logger); err != nil {
|
||||
return err
|
||||
if _, err := tx.ExecContext(l.ctx, `
|
||||
UPDATE playlist_tracks
|
||||
SET
|
||||
phantom_title = COALESCE(phantom_title, (
|
||||
SELECT tm.title FROM track_metadata tm
|
||||
WHERE tm.id = playlist_tracks.audio_file_id
|
||||
)),
|
||||
phantom_artist = COALESCE(phantom_artist, (
|
||||
SELECT tm.artist_name FROM track_metadata tm
|
||||
WHERE tm.id = playlist_tracks.audio_file_id
|
||||
)),
|
||||
phantom_album = COALESCE(phantom_album, (
|
||||
SELECT tm.album FROM track_metadata tm
|
||||
WHERE tm.id = playlist_tracks.audio_file_id
|
||||
)),
|
||||
phantom_duration_ms = COALESCE(phantom_duration_ms, (
|
||||
SELECT af.length_milliseconds FROM audio_files af
|
||||
WHERE af.id = playlist_tracks.audio_file_id
|
||||
)),
|
||||
phantom_file_path = COALESCE(phantom_file_path, (
|
||||
SELECT af.file_path FROM audio_files af
|
||||
WHERE af.id = playlist_tracks.audio_file_id
|
||||
))
|
||||
WHERE audio_file_id IS NOT NULL
|
||||
`); err != nil {
|
||||
return fmt.Errorf(
|
||||
"could not preserve playlist track metadata: %w", err,
|
||||
)
|
||||
}
|
||||
|
||||
// Phase 2: the files. file_genres cascades with them.
|
||||
|
||||
@@ -231,98 +231,3 @@ func TestScan_MultipleDirectoriesDoNotCrossContaminate(t *testing.T) {
|
||||
t.Errorf("Album A and Album B must not share a group_key: %+v", keys)
|
||||
}
|
||||
}
|
||||
|
||||
// TestScan_OrphanCleanupPreservesPlaylistPhantoms guards #246: a file
|
||||
// deleted from the library folder *outside* YellowJacket is discovered
|
||||
// as an orphan by the next scan, and its playlist entry must survive as
|
||||
// a re-linkable phantom — the same preservation the full rescan and
|
||||
// stale-retire paths already perform, scoped here to just the orphaned
|
||||
// file. Before the fix the entry became an empty row (audio_file_id
|
||||
// NULL and no phantom_file_path), which nothing can ever re-link.
|
||||
func TestScan_OrphanCleanupPreservesPlaylistPhantoms(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
lib, db := setupTestLibrary(t)
|
||||
|
||||
root := t.TempDir()
|
||||
track := filepath.Join(root, "gone.mp3")
|
||||
|
||||
writeTestTrack(t, track, 0)
|
||||
|
||||
library, err := db.Queries.CreateLibrary(lib.ctx, sqlcgen.CreateLibraryParams{
|
||||
Name: "orphans",
|
||||
Path: root,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create library: %v", err)
|
||||
}
|
||||
|
||||
if metrics := lib.scanInternal(library.ID, library.Name, library.Path); metrics == nil {
|
||||
t.Fatal("first scan returned nil metrics")
|
||||
}
|
||||
|
||||
trackID := queryInt(
|
||||
t, db, "SELECT id FROM audio_files WHERE file_path = ?", track,
|
||||
)
|
||||
if trackID == 0 {
|
||||
t.Fatal("first scan did not import the track")
|
||||
}
|
||||
|
||||
if _, err := db.ExecContext(
|
||||
`INSERT INTO playlists (name) VALUES ('keepme')`,
|
||||
); err != nil {
|
||||
t.Fatalf("seed playlist: %v", err)
|
||||
}
|
||||
|
||||
playlistID := queryInt(
|
||||
t, db, "SELECT id FROM playlists WHERE name = 'keepme'",
|
||||
)
|
||||
|
||||
if _, err := db.ExecContext(
|
||||
`INSERT INTO playlist_tracks (playlist_id, audio_file_id, position)
|
||||
VALUES (?, ?, 0)`,
|
||||
playlistID, trackID,
|
||||
); err != nil {
|
||||
t.Fatalf("seed playlist_tracks: %v", err)
|
||||
}
|
||||
|
||||
// The file goes away outside the app.
|
||||
if err := os.Remove(track); err != nil {
|
||||
t.Fatalf("remove track: %v", err)
|
||||
}
|
||||
|
||||
if metrics := lib.scanInternal(library.ID, library.Name, library.Path); metrics == nil {
|
||||
t.Fatal("second scan returned nil metrics")
|
||||
}
|
||||
|
||||
if n := queryInt(
|
||||
t, db, "SELECT COUNT(*) FROM audio_files WHERE file_path = ?", track,
|
||||
); n != 0 {
|
||||
t.Fatalf("audio_files still holds the removed path: %d rows", n)
|
||||
}
|
||||
|
||||
if n := queryInt(
|
||||
t, db,
|
||||
"SELECT COUNT(*) FROM playlist_tracks WHERE playlist_id = ? "+
|
||||
"AND audio_file_id IS NULL",
|
||||
playlistID,
|
||||
); n != 1 {
|
||||
t.Fatalf(
|
||||
"playlist entry did not become a phantom: %d null-id rows, want 1",
|
||||
n,
|
||||
)
|
||||
}
|
||||
|
||||
phantomPath := queryString(
|
||||
t, db,
|
||||
"SELECT phantom_file_path FROM playlist_tracks WHERE playlist_id = ?",
|
||||
playlistID,
|
||||
)
|
||||
if phantomPath != track {
|
||||
t.Fatalf(
|
||||
"phantom_file_path = %q, want %q -- the entry cannot be "+
|
||||
"re-linked if the file comes back",
|
||||
phantomPath, track,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -666,120 +666,3 @@ func TestExpiredHTTPCacheJob_TrimsToBudget(t *testing.T) {
|
||||
t.Errorf("kept %q, want the longest-lived row", kept)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStaleArtistMetadataJob pins the sweep's two keep rules: an owned
|
||||
// artist's metadata survives, a browsed artist's survives while it still
|
||||
// holds cached artwork, and everything else goes (#248).
|
||||
func TestStaleArtistMetadataJob(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db := database.NewTestDB(t)
|
||||
|
||||
const (
|
||||
ownedMBID = "11111111-1111-1111-1111-111111111111"
|
||||
browsedMBID = "22222222-2222-2222-2222-222222222222"
|
||||
staleMBID = "33333333-3333-3333-3333-333333333333"
|
||||
)
|
||||
|
||||
// The owned artist is in the library - which means a *file* says
|
||||
// so. An artists row on its own is not ownership.
|
||||
database.InsertTestTrack(t, db, database.TestTrack{
|
||||
FilePath: "/music/owned.mp3",
|
||||
Artist: "Owned",
|
||||
ArtistMBID: ownedMBID,
|
||||
})
|
||||
|
||||
for _, mbid := range []string{ownedMBID, browsedMBID, staleMBID} {
|
||||
if _, err := db.ExecContext(
|
||||
`INSERT INTO artist_metadata (mbid, source, data, fetched_at)
|
||||
VALUES (?, 'wikidata-p18', x'00', CURRENT_TIMESTAMP)`,
|
||||
mbid,
|
||||
); err != nil {
|
||||
t.Fatalf("seed artist_metadata for %s: %v", mbid, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The browsed artist holds cached artwork, so its metadata is still
|
||||
// referenced and must survive.
|
||||
if _, err := db.ExecContext(
|
||||
`INSERT INTO artist_images
|
||||
(artist_mbid, source, source_url, file_path)
|
||||
VALUES (?, 'test', 'http://x', '/art/primary.jpg')`,
|
||||
browsedMBID,
|
||||
); err != nil {
|
||||
t.Fatalf("seed artist_images: %v", err)
|
||||
}
|
||||
|
||||
if _, err := StaleArtistMetadataJob(db).Run(context.Background()); err != nil {
|
||||
t.Fatalf("run job: %v", err)
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
mbid string
|
||||
want int
|
||||
}{
|
||||
{ownedMBID, 1},
|
||||
{browsedMBID, 1},
|
||||
{staleMBID, 0},
|
||||
} {
|
||||
var n int
|
||||
if err := db.QueryRowWriter(
|
||||
"SELECT COUNT(*) FROM artist_metadata WHERE mbid = ?", tc.mbid,
|
||||
).Scan(&n); err != nil {
|
||||
t.Fatalf("count %s: %v", tc.mbid, err)
|
||||
}
|
||||
|
||||
if n != tc.want {
|
||||
t.Errorf("artist_metadata rows for %s = %d, want %d", tc.mbid, n, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestStaleSearchClicksJob deletes only the clicks old enough to have
|
||||
// left the retention window (#249).
|
||||
func TestStaleSearchClicksJob(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db := database.NewTestDB(t)
|
||||
|
||||
count := func(mbid string) int {
|
||||
t.Helper()
|
||||
|
||||
var n int
|
||||
if err := db.QueryRowWriter(
|
||||
"SELECT COUNT(*) FROM search_clicks WHERE entity_mbid = ?", mbid,
|
||||
).Scan(&n); err != nil {
|
||||
t.Fatalf("count %s: %v", mbid, err)
|
||||
}
|
||||
|
||||
return n
|
||||
}
|
||||
|
||||
seed := func(query, mbid, lastClicked string) {
|
||||
t.Helper()
|
||||
|
||||
if _, err := db.ExecContext(
|
||||
`INSERT INTO search_clicks
|
||||
(query, entity_mbid, entity_type, click_count, last_clicked)
|
||||
VALUES (?, ?, 'recording', 1, ?)`,
|
||||
query, mbid, lastClicked,
|
||||
); err != nil {
|
||||
t.Fatalf("seed search_clicks: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
seed("tide", "aaaa", "2024-01-01 00:00:00") // stale
|
||||
seed("tide", "bbbb", "2999-01-01 00:00:00") // recent
|
||||
|
||||
if _, err := StaleSearchClicksJob(db).Run(context.Background()); err != nil {
|
||||
t.Fatalf("run job: %v", err)
|
||||
}
|
||||
|
||||
if n := count("bbbb"); n != 1 {
|
||||
t.Errorf("recent click was deleted: %d rows, want 1", n)
|
||||
}
|
||||
|
||||
if n := count("aaaa"); n != 0 {
|
||||
t.Errorf("stale click survived: %d rows, want 0", n)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -628,69 +628,3 @@ func dirSize(dir string) (bytes, files int64) {
|
||||
|
||||
return bytes, files
|
||||
}
|
||||
|
||||
// StaleArtistMetadataJob evicts long-lived artist metadata (bios, wiki
|
||||
// leads, relationships) for artists the user no longer has any reason
|
||||
// to keep around: not owned and holding no cached artwork.
|
||||
//
|
||||
// artist_metadata has no TTL by design — entity data changes rarely and
|
||||
// re-fetching spends someone else's rate limit — so without a sweep it
|
||||
// grows for the life of the install. This is the "swept when the
|
||||
// artist is no longer referenced" contract the datamap always declared
|
||||
// for it and nothing ever performed (#248).
|
||||
func StaleArtistMetadataJob(db *database.DB) Job {
|
||||
return Job{
|
||||
Name: "artist-metadata-sweep",
|
||||
MinInterval: dailyInterval,
|
||||
Run: func(_ context.Context) (Result, error) {
|
||||
res, err := db.ExecContext(
|
||||
`DELETE FROM artist_metadata
|
||||
WHERE mbid NOT IN (` + ownedArtistMBIDs + `)
|
||||
AND mbid NOT IN (
|
||||
SELECT artist_mbid FROM artist_images
|
||||
)`,
|
||||
)
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf(
|
||||
"delete stale artist_metadata rows: %w", err,
|
||||
)
|
||||
}
|
||||
|
||||
rows, _ := res.RowsAffected()
|
||||
|
||||
return Result{RowsDeleted: rows}, nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// searchClicksRetention is how long a search-click ranking signal stays
|
||||
// useful. search_clicks is authored behavioural data — nothing that
|
||||
// owns a row ever drops it — so age is the ceiling that keeps the table
|
||||
// from growing without bound for the life of the install (#249).
|
||||
const searchClicksRetention = "-180 days"
|
||||
|
||||
// StaleSearchClicksJob deletes search-click ranking rows older than the
|
||||
// retention window. Rows are small and the table grows slowly, so this
|
||||
// runs daily and does almost nothing most runs.
|
||||
func StaleSearchClicksJob(db *database.DB) Job {
|
||||
return Job{
|
||||
Name: "search-clicks-sweep",
|
||||
MinInterval: dailyInterval,
|
||||
Run: func(_ context.Context) (Result, error) {
|
||||
res, err := db.ExecContext(
|
||||
`DELETE FROM search_clicks
|
||||
WHERE last_clicked < datetime('now', ?)`,
|
||||
searchClicksRetention,
|
||||
)
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf(
|
||||
"delete stale search_clicks rows: %w", err,
|
||||
)
|
||||
}
|
||||
|
||||
rows, _ := res.RowsAffected()
|
||||
|
||||
return Result{RowsDeleted: rows}, nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -74,14 +74,6 @@ func ExtractTags(path string) (*TrackMetadata, error) {
|
||||
|
||||
// ExtractTagsFromReader reads metadata from an io.ReadSeeker.
|
||||
func ExtractTagsFromReader(r io.ReadSeeker) (*TrackMetadata, error) {
|
||||
// The container decides, so this is asked before tag.ReadFrom and
|
||||
// not after its failure: a WAV's tags live in a RIFF chunk that
|
||||
// dhowden/tag cannot see, and its fallback -- an ID3v1 trailer --
|
||||
// would otherwise outrank them.
|
||||
if meta, ok := wavTags(r); ok {
|
||||
return meta, nil
|
||||
}
|
||||
|
||||
m, err := tag.ReadFrom(r)
|
||||
if err != nil {
|
||||
// No tags found is not necessarily an error - return empty metadata
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
package metadata
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"yellowjacket/backend/riff"
|
||||
)
|
||||
|
||||
// wavTags reads the ID3v2 tag a WAV carries in its RIFF "id3 " chunk,
|
||||
// which is where backend/tagwriter puts it and where dhowden/tag --
|
||||
// having no RIFF reader at all -- cannot look. Without this a WAV
|
||||
// scans as an untagged file however carefully it was tagged.
|
||||
//
|
||||
// ok is false when r is not a RIFF/WAVE container, and the read
|
||||
// position is restored either way so the caller can carry on.
|
||||
func wavTags(r io.ReadSeeker) (*TrackMetadata, bool) {
|
||||
start, err := r.Seek(0, io.SeekCurrent)
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
id3Data, chunkErr := riff.ID3Chunk(r)
|
||||
|
||||
if _, err := r.Seek(start, io.SeekStart); err != nil {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
switch {
|
||||
case chunkErr == nil:
|
||||
return wavTagsFrom(id3Data), true
|
||||
|
||||
// Not ours to read: let the ordinary dispatch have the file.
|
||||
case errors.Is(chunkErr, riff.ErrNotRIFF), errors.Is(chunkErr, riff.ErrNotWAVE):
|
||||
return nil, false
|
||||
|
||||
// A RIFF container we cannot get a tag out of -- no chunk, an RF64
|
||||
// file, a truncated header. That is a file with no readable tags,
|
||||
// which is what the scanner's filename fallback is for.
|
||||
default:
|
||||
return &TrackMetadata{}, true
|
||||
}
|
||||
}
|
||||
|
||||
// wavTagsFrom parses the bytes of a WAV's ID3v2 chunk.
|
||||
func wavTagsFrom(id3Data []byte) *TrackMetadata {
|
||||
meta, err := extractID3v2Lenient(bytes.NewReader(id3Data))
|
||||
if err != nil {
|
||||
// A tag holding no frames is not a damaged tag: writing every
|
||||
// field back out empty leaves one, and warning about it would
|
||||
// put a fault on a file that has none.
|
||||
if errors.Is(err, ErrTagsUnreadable) {
|
||||
return &TrackMetadata{}
|
||||
}
|
||||
|
||||
return &TrackMetadata{
|
||||
TagReadWarning: fmt.Errorf("%w: %w", ErrTagsUnreadable, err),
|
||||
}
|
||||
}
|
||||
|
||||
// extractID3v2Lenient names MP3, being the recovery path for one.
|
||||
meta.FileFormat = strings.ToUpper(strings.TrimPrefix(string(WAV), "."))
|
||||
|
||||
return meta
|
||||
}
|
||||
@@ -134,12 +134,6 @@ type Service struct {
|
||||
libraryDir LibraryDirProvider
|
||||
favoritesConf FavoritesConfigProvider
|
||||
|
||||
// onDeleted, when set, is called after a playlist is deleted so
|
||||
// cross-cutting state that points at it (the queue's "Playing
|
||||
// from" label) can stop pointing at a playlist that no longer
|
||||
// exists. Wired from app.go, like Library.SetRemovalHooks.
|
||||
onDeleted func(playlistID int64)
|
||||
|
||||
// dataDirOverride, when non-empty, replaces the OS user data
|
||||
// directory as the base for the playlists folder. Set by tests to
|
||||
// keep M3U writes out of the real user data directory.
|
||||
@@ -172,17 +166,6 @@ func (s *Service) SetFavoritesConfig(
|
||||
s.favoritesConf = provider
|
||||
}
|
||||
|
||||
// SetOnPlaylistDeleted registers a callback invoked after a playlist is
|
||||
// deleted, for cross-cutting invalidation.
|
||||
//
|
||||
//wails:ignore // internal wiring, not part of the app's IPC surface.
|
||||
func (s *Service) SetOnPlaylistDeleted(onDeleted func(playlistID int64)) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
s.onDeleted = onDeleted
|
||||
}
|
||||
|
||||
// ServiceStartup is v3's service lifecycle hook: it runs once the
|
||||
// runtime exists, and ctx is cancelled when the app shuts down. It
|
||||
// replaces v2's SetContext, which had to be called by hand from
|
||||
@@ -783,17 +766,6 @@ func (s *Service) DeletePlaylist(playlistID int64) error {
|
||||
|
||||
s.emitEvent(events.PlaylistDeleted, playlistID)
|
||||
|
||||
// Cross-cutting invalidation: the queue's "Playing from" label may
|
||||
// point at this playlist, and a link to a playlist that no longer
|
||||
// exists is worse than none.
|
||||
s.mu.Lock()
|
||||
onDeleted := s.onDeleted
|
||||
s.mu.Unlock()
|
||||
|
||||
if onDeleted != nil {
|
||||
onDeleted(playlistID)
|
||||
}
|
||||
|
||||
// Recreate the default playlist if we just deleted it.
|
||||
if s.defaultPlaylistID() == playlistID {
|
||||
s.EnsureDefaultPlaylist()
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"yellowjacket/backend/events"
|
||||
)
|
||||
|
||||
// recordPlay inserts a listening_events row and updates the denormalized
|
||||
// recordPlay inserts a play_history row and updates the denormalized
|
||||
// play_count / last_played columns on audio_files. Called from
|
||||
// OnPlaybackFinished for the track that just finished.
|
||||
//
|
||||
@@ -20,12 +20,10 @@ func (q *Queue) recordPlay(audioFileID int64) {
|
||||
|
||||
now := time.Now().UTC().Format(time.DateTime)
|
||||
|
||||
// Insert the listening event. A natural finish is a 'complete' by
|
||||
// construction; position/duration are the classifier's to fill once
|
||||
// skips are recorded (see .planning/plans/active/021).
|
||||
// Insert play_history row.
|
||||
_, err := q.db.ExecContext(
|
||||
`INSERT INTO listening_events (audio_file_id, kind, occurred_at)
|
||||
VALUES (?, 'complete', ?)`,
|
||||
`INSERT INTO play_history (audio_file_id, played_at)
|
||||
VALUES (?, ?)`,
|
||||
audioFileID, now,
|
||||
)
|
||||
if err != nil {
|
||||
|
||||
@@ -1581,21 +1581,6 @@ func (q *Queue) dropSource() {
|
||||
q.source = Source{}
|
||||
}
|
||||
|
||||
// DropSourceForPlaylist clears the queue's "Playing from" label when
|
||||
// its source playlist is deleted. A link back to a playlist that no
|
||||
// longer exists is worse than none, and the label otherwise survives
|
||||
// the deletion until the next SetQueue (#249).
|
||||
func (q *Queue) DropSourceForPlaylist(playlistID int64) {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
|
||||
if (q.source.Type == "playlist" || q.source.Type == "smartPlaylist") &&
|
||||
q.source.ID == playlistID {
|
||||
q.dropSource()
|
||||
q.persistState()
|
||||
}
|
||||
}
|
||||
|
||||
// commitMutation persists the current queue state after a mutation.
|
||||
// When reindex is true, track positions are renumbered first.
|
||||
// The caller must hold q.mu.
|
||||
|
||||
@@ -535,35 +535,3 @@ func TestCycleRepeat_CyclesThroughModes(t *testing.T) {
|
||||
t.Errorf("after third cycle: got %q, want %q", state.RepeatMode, RepeatOff)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDropSourceForPlaylist clears the "Playing from" label when the
|
||||
// queue's source playlist is deleted, and leaves it alone otherwise
|
||||
// (#249).
|
||||
func TestDropSourceForPlaylist(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
q, db := setupTestQueue(t)
|
||||
paths := seedAudioFiles(t, db, 2)
|
||||
|
||||
q.SetQueue(paths, 0, false, Source{Type: "playlist", ID: 42, Label: "Road Trip"})
|
||||
q.DropSourceForPlaylist(42)
|
||||
|
||||
if got := q.GetState().Source; got != (Source{}) {
|
||||
t.Errorf("source = %+v, want empty after playlist 42 deleted", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDropSourceForPlaylistIgnoresOtherPlaylists(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
q, db := setupTestQueue(t)
|
||||
paths := seedAudioFiles(t, db, 2)
|
||||
|
||||
source := Source{Type: "smartPlaylist", ID: 42, Label: "Road Trip"}
|
||||
q.SetQueue(paths, 0, false, source)
|
||||
q.DropSourceForPlaylist(7)
|
||||
|
||||
if got := q.GetState().Source; got != source {
|
||||
t.Errorf("source = %+v, want %+v unchanged for a different playlist", got, source)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,186 +0,0 @@
|
||||
// Package riff reads the chunk layout of a RIFF/WAVE container.
|
||||
//
|
||||
// It exists because both halves of WAV tagging need it and neither can
|
||||
// import the other: backend/tagwriter writes a WAV's tags into a RIFF
|
||||
// "id3 " chunk and already imports backend/metadata, which is what has
|
||||
// to read them back out. backend/tagtotals is the precedent.
|
||||
//
|
||||
// The two readers here are deliberately different. Parse holds every
|
||||
// chunk's data in memory, which is what rewriting a file needs; a WAV's
|
||||
// audio *is* the "data" chunk, so doing that on the scan path would
|
||||
// read every library file in full. ID3Chunk seeks over what it is not
|
||||
// looking for instead. Both walk the same headers.
|
||||
package riff
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Sentinel errors describing a container this package will not read.
|
||||
var (
|
||||
ErrRF64NotSupported = errors.New("RF64 files are not yet supported")
|
||||
ErrNotRIFF = errors.New("not a RIFF file")
|
||||
ErrNotWAVE = errors.New("not a WAVE file")
|
||||
ErrNoID3Chunk = errors.New("no ID3 chunk in RIFF file")
|
||||
)
|
||||
|
||||
// Chunk holds a single RIFF sub-chunk (ID + raw data).
|
||||
type Chunk struct {
|
||||
ID [4]byte
|
||||
Data []byte
|
||||
}
|
||||
|
||||
// IsID3 reports whether id is that of an ID3v2 RIFF chunk. Both
|
||||
// lowercase "id3 " and uppercase "ID3 " are accepted.
|
||||
func IsID3(id [4]byte) bool {
|
||||
return strings.ToLower(string(id[:3])) == "id3"
|
||||
}
|
||||
|
||||
// Parse reads every RIFF sub-chunk from r, in order, starting at the
|
||||
// reader's current position. It rejects RF64 files and non-WAVE
|
||||
// containers with descriptive errors. The parser is lenient: it
|
||||
// tolerates a missing final padding byte and ignores the declared
|
||||
// RIFF size.
|
||||
func Parse(r io.Reader) ([]Chunk, error) {
|
||||
if err := readContainer(r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var chunks []Chunk
|
||||
|
||||
for {
|
||||
id, size, err := nextHeader(r)
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Copied rather than allocated up front, as ID3Chunk does: the
|
||||
// size is four bytes off the file, so a truncated one is free to
|
||||
// declare a chunk larger than the whole of itself.
|
||||
var data bytes.Buffer
|
||||
if _, err := io.CopyN(&data, r, int64(size)); err != nil {
|
||||
return nil, fmt.Errorf("read chunk data for %q: %w", id, err)
|
||||
}
|
||||
|
||||
chunks = append(chunks, Chunk{ID: id, Data: data.Bytes()})
|
||||
|
||||
// Odd-length chunks have a padding byte. Lenient: if the
|
||||
// read fails (e.g. EOF), just break rather than error.
|
||||
if size%2 != 0 {
|
||||
var pad [1]byte
|
||||
|
||||
if _, err := r.Read(pad[:]); err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return chunks, nil
|
||||
}
|
||||
|
||||
// ID3Chunk returns the payload of the ID3v2 chunk of the RIFF/WAVE
|
||||
// container at the reader's current position, seeking over every other
|
||||
// chunk rather than reading it. It returns ErrNoID3Chunk when the
|
||||
// container carries no such chunk, and leaves the read position
|
||||
// unspecified either way.
|
||||
func ID3Chunk(r io.ReadSeeker) ([]byte, error) {
|
||||
if err := readContainer(r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for {
|
||||
id, size, err := nextHeader(r)
|
||||
if errors.Is(err, io.EOF) {
|
||||
return nil, ErrNoID3Chunk
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if !IsID3(id) {
|
||||
// Odd-length chunks carry a padding byte. Seeking past
|
||||
// the end of the file is not an error; the next header
|
||||
// read is what reports the end.
|
||||
if _, err := r.Seek(int64(size)+int64(size%2), io.SeekCurrent); err != nil {
|
||||
return nil, fmt.Errorf("skip chunk %q: %w", id, err)
|
||||
}
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
// Copied rather than allocated up front: a truncated file is
|
||||
// free to declare a chunk larger than the whole of itself.
|
||||
var data bytes.Buffer
|
||||
if _, err := io.CopyN(&data, r, int64(size)); err != nil {
|
||||
return nil, fmt.Errorf("read chunk data for %q: %w", id, err)
|
||||
}
|
||||
|
||||
return data.Bytes(), nil
|
||||
}
|
||||
}
|
||||
|
||||
// readContainer consumes the 12-byte RIFF/WAVE header at the reader's
|
||||
// current position.
|
||||
func readContainer(r io.Reader) error {
|
||||
var magic [4]byte
|
||||
if _, err := io.ReadFull(r, magic[:]); err != nil {
|
||||
return fmt.Errorf("read RIFF magic: %w", err)
|
||||
}
|
||||
|
||||
if string(magic[:]) == "RF64" {
|
||||
return ErrRF64NotSupported
|
||||
}
|
||||
|
||||
if string(magic[:]) != "RIFF" {
|
||||
return fmt.Errorf("%w: got %q", ErrNotRIFF, magic)
|
||||
}
|
||||
|
||||
// Read (and discard) RIFF size — lenient, do not enforce.
|
||||
var riffSize uint32
|
||||
if err := binary.Read(r, binary.LittleEndian, &riffSize); err != nil {
|
||||
return fmt.Errorf("read RIFF size: %w", err)
|
||||
}
|
||||
|
||||
var form [4]byte
|
||||
if _, err := io.ReadFull(r, form[:]); err != nil {
|
||||
return fmt.Errorf("read WAVE form type: %w", err)
|
||||
}
|
||||
|
||||
if string(form[:]) != "WAVE" {
|
||||
return fmt.Errorf("%w: got %q", ErrNotWAVE, form)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// nextHeader reads one sub-chunk header. It returns io.EOF once the
|
||||
// chunks are exhausted, including for a header cut short.
|
||||
func nextHeader(r io.Reader) ([4]byte, uint32, error) {
|
||||
var id [4]byte
|
||||
|
||||
_, err := io.ReadFull(r, id[:])
|
||||
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
return id, 0, io.EOF
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return id, 0, fmt.Errorf("read chunk ID: %w", err)
|
||||
}
|
||||
|
||||
var size uint32
|
||||
if err := binary.Read(r, binary.LittleEndian, &size); err != nil {
|
||||
return id, 0, fmt.Errorf("read chunk size for %q: %w", id, err)
|
||||
}
|
||||
|
||||
return id, size, nil
|
||||
}
|
||||
@@ -1,254 +0,0 @@
|
||||
package riff_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"runtime"
|
||||
"testing"
|
||||
|
||||
"yellowjacket/backend/riff"
|
||||
)
|
||||
|
||||
// chunk is one sub-chunk to put in a test container.
|
||||
type chunk struct {
|
||||
id string
|
||||
data []byte
|
||||
}
|
||||
|
||||
// buildRIFF assembles a container from magic, form type and chunks,
|
||||
// padding odd-length chunks the way a writer must.
|
||||
func buildRIFF(magic, form string, chunks []chunk) []byte {
|
||||
var body bytes.Buffer
|
||||
|
||||
body.WriteString(form)
|
||||
|
||||
for _, c := range chunks {
|
||||
body.WriteString(c.id)
|
||||
_ = binary.Write(&body, binary.LittleEndian, uint32(len(c.data)))
|
||||
body.Write(c.data)
|
||||
|
||||
if len(c.data)%2 != 0 {
|
||||
body.WriteByte(0)
|
||||
}
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
|
||||
out.WriteString(magic)
|
||||
_ = binary.Write(&out, binary.LittleEndian, uint32(body.Len()))
|
||||
out.Write(body.Bytes())
|
||||
|
||||
return out.Bytes()
|
||||
}
|
||||
|
||||
func TestID3Chunk_FindsTheTagPastTheAudio(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
chunks []chunk
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "after an odd-length chunk",
|
||||
chunks: []chunk{
|
||||
{id: "fmt ", data: make([]byte, 16)},
|
||||
{id: "LIST", data: []byte("INFOodd")},
|
||||
{id: "data", data: make([]byte, 200)},
|
||||
{id: "id3 ", data: []byte("ID3vTAG")},
|
||||
},
|
||||
want: "ID3vTAG",
|
||||
},
|
||||
{
|
||||
// The chunk ID is written both ways in the wild, and the
|
||||
// writer accepts either, so the reader must too.
|
||||
name: "uppercase ID3",
|
||||
chunks: []chunk{
|
||||
{id: "data", data: make([]byte, 8)},
|
||||
{id: "ID3 ", data: []byte("upper")},
|
||||
},
|
||||
want: "upper",
|
||||
},
|
||||
{
|
||||
name: "first chunk",
|
||||
chunks: []chunk{
|
||||
{id: "id3 ", data: []byte("first")},
|
||||
{id: "data", data: make([]byte, 8)},
|
||||
},
|
||||
want: "first",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
r := bytes.NewReader(buildRIFF("RIFF", "WAVE", tc.chunks))
|
||||
|
||||
got, err := riff.ID3Chunk(r)
|
||||
if err != nil {
|
||||
t.Fatalf("ID3Chunk: %v", err)
|
||||
}
|
||||
|
||||
if string(got) != tc.want {
|
||||
t.Errorf("chunk data: got %q, want %q", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestID3Chunk_RejectsWhatItCannotRead(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
bytes []byte
|
||||
want error
|
||||
}{
|
||||
{
|
||||
name: "no ID3 chunk",
|
||||
bytes: buildRIFF("RIFF", "WAVE", []chunk{{id: "data", data: []byte{1, 2}}}),
|
||||
want: riff.ErrNoID3Chunk,
|
||||
},
|
||||
{
|
||||
name: "no chunks at all",
|
||||
bytes: buildRIFF("RIFF", "WAVE", nil),
|
||||
want: riff.ErrNoID3Chunk,
|
||||
},
|
||||
{
|
||||
name: "not RIFF",
|
||||
bytes: []byte("ID3\x03\x00\x00\x00\x00\x00\x00\x00\x00"),
|
||||
want: riff.ErrNotRIFF,
|
||||
},
|
||||
{
|
||||
name: "not WAVE",
|
||||
bytes: buildRIFF("RIFF", "AVI ", []chunk{{id: "id3 ", data: []byte("x")}}),
|
||||
want: riff.ErrNotWAVE,
|
||||
},
|
||||
{
|
||||
name: "RF64",
|
||||
bytes: buildRIFF("RF64", "WAVE", []chunk{{id: "id3 ", data: []byte("x")}}),
|
||||
want: riff.ErrRF64NotSupported,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, err := riff.ID3Chunk(bytes.NewReader(tc.bytes))
|
||||
if !errors.Is(err, tc.want) {
|
||||
t.Errorf("ID3Chunk error: got %v, want %v", err, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A file cut short mid-chunk is a file with no tag, not a reason to
|
||||
// allocate the size it claims: the declared size is four bytes any
|
||||
// truncation can leave saying 4 GB.
|
||||
func TestID3Chunk_ToleratesATruncatedFile(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
full := buildRIFF("RIFF", "WAVE", []chunk{
|
||||
{id: "data", data: make([]byte, 64)},
|
||||
{id: "id3 ", data: []byte("tag")},
|
||||
})
|
||||
|
||||
t.Run("cut inside the audio", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, err := riff.ID3Chunk(bytes.NewReader(full[:32]))
|
||||
if !errors.Is(err, riff.ErrNoID3Chunk) {
|
||||
t.Errorf("ID3Chunk error: got %v, want %v", err, riff.ErrNoID3Chunk)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("cut inside the tag", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if _, err := riff.ID3Chunk(bytes.NewReader(full[:len(full)-2])); err == nil {
|
||||
t.Error("ID3Chunk: got nil error for a truncated tag chunk")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Parse is the writer's half and reads every chunk into memory, which
|
||||
// is what preserving them needs.
|
||||
func TestParse_ReadsEveryChunkInOrder(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
raw := buildRIFF("RIFF", "WAVE", []chunk{
|
||||
{id: "fmt ", data: make([]byte, 16)},
|
||||
{id: "LIST", data: []byte("INFOodd")},
|
||||
{id: "id3 ", data: []byte("tag")},
|
||||
})
|
||||
|
||||
chunks, err := riff.Parse(bytes.NewReader(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("Parse: %v", err)
|
||||
}
|
||||
|
||||
want := []string{"fmt ", "LIST", "id3 "}
|
||||
if len(chunks) != len(want) {
|
||||
t.Fatalf("chunk count: got %d, want %d", len(chunks), len(want))
|
||||
}
|
||||
|
||||
for i, id := range want {
|
||||
if got := string(chunks[i].ID[:]); got != id {
|
||||
t.Errorf("chunk %d: got %q, want %q", i, got, id)
|
||||
}
|
||||
}
|
||||
|
||||
if !riff.IsID3(chunks[2].ID) || string(chunks[2].Data) != "tag" {
|
||||
t.Errorf("id3 chunk: got %q", chunks[2].Data)
|
||||
}
|
||||
|
||||
// The padding byte after an odd chunk is not part of its data.
|
||||
if string(chunks[1].Data) != "INFOodd" {
|
||||
t.Errorf("odd chunk data: got %q, want %q", chunks[1].Data, "INFOodd")
|
||||
}
|
||||
}
|
||||
|
||||
// A chunk size is four bytes read off the file, so a truncated or
|
||||
// malformed WAV is free to declare a chunk larger than the whole of
|
||||
// itself. Parse must grow with what arrives rather than with what was
|
||||
// claimed.
|
||||
//
|
||||
// This measures the allocation instead of the error because the error
|
||||
// is the same either way: a build sizing its buffer from the header
|
||||
// reports the truncation correctly, having asked the allocator for a
|
||||
// gigabyte on the way. Deliberately not parallel — TotalAlloc is
|
||||
// process-wide, and a test paused beside another one is measuring it
|
||||
// too.
|
||||
func TestParse_DoesNotAllocateWhatAChunkClaims(t *testing.T) {
|
||||
// Large enough that a header-sized buffer is unmistakable, in a
|
||||
// container of a few dozen bytes.
|
||||
const declared = 1 << 30
|
||||
|
||||
var raw bytes.Buffer
|
||||
|
||||
raw.WriteString("RIFF")
|
||||
_ = binary.Write(&raw, binary.LittleEndian, uint32(declared+12))
|
||||
raw.WriteString("WAVE")
|
||||
raw.WriteString("data")
|
||||
_ = binary.Write(&raw, binary.LittleEndian, uint32(declared))
|
||||
raw.WriteString("and then the file ends")
|
||||
|
||||
var before, after runtime.MemStats
|
||||
|
||||
runtime.GC()
|
||||
runtime.ReadMemStats(&before)
|
||||
|
||||
if _, err := riff.Parse(bytes.NewReader(raw.Bytes())); err == nil {
|
||||
t.Fatal("Parse: got nil error for a chunk larger than the file holding it")
|
||||
}
|
||||
|
||||
runtime.ReadMemStats(&after)
|
||||
|
||||
if grew := after.TotalAlloc - before.TotalAlloc; grew > 1<<20 {
|
||||
t.Errorf("Parse allocated %d bytes reading a %d-byte file whose chunk header claimed %d",
|
||||
grew, raw.Len(), declared)
|
||||
}
|
||||
}
|
||||
@@ -13,10 +13,9 @@ import (
|
||||
// indistinguishable from never having written one. So these assert the
|
||||
// round trip through the *reader the scan uses*, not the bytes.
|
||||
//
|
||||
// WAV was the exception until #104 -- dhowden/tag has no RIFF reader,
|
||||
// so metadata.ExtractTags could not see a WAV's ID3 chunk and this
|
||||
// case read the chunk itself, which is a test of the writer wearing
|
||||
// the shape of a round trip. All four go through the scanner now.
|
||||
// WAV is the exception and it is not this change's: dhowden/tag has no
|
||||
// RIFF reader at all, so metadata.ExtractTags cannot see a WAV's ID3
|
||||
// chunk -- which is why every other test here reads that chunk itself.
|
||||
func TestWriteTotals_RoundTripsInEveryFormat(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -92,7 +91,7 @@ func TestWriteTotals_RoundTripsInEveryFormat(t *testing.T) {
|
||||
},
|
||||
{
|
||||
name: "wav",
|
||||
read: viaScanner,
|
||||
read: readWavID3Tags,
|
||||
write: func(t *testing.T, dir string) string {
|
||||
t.Helper()
|
||||
|
||||
|
||||
+105
-11
@@ -8,22 +8,116 @@ import (
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
id3v2 "github.com/bogem/id3v2/v2"
|
||||
|
||||
"yellowjacket/backend/fileutil"
|
||||
"yellowjacket/backend/riff"
|
||||
)
|
||||
|
||||
// errFileTooLargeForWAV is the one RIFF error that belongs to the
|
||||
// writer; reading rejects a container in backend/riff.
|
||||
var errFileTooLargeForWAV = errors.New("file too large for WAV format (>4GB)")
|
||||
// Sentinel errors for WAV RIFF operations.
|
||||
var (
|
||||
errRF64NotSupported = errors.New("RF64 files are not yet supported")
|
||||
errNotRIFF = errors.New("not a RIFF file")
|
||||
errNotWAVE = errors.New("not a WAVE file")
|
||||
errFileTooLargeForWAV = errors.New("file too large for WAV format (>4GB)")
|
||||
)
|
||||
|
||||
// riffChunk holds a single RIFF sub-chunk (ID + raw data).
|
||||
type riffChunk struct {
|
||||
id [4]byte
|
||||
data []byte
|
||||
}
|
||||
|
||||
// parseRIFF reads all RIFF sub-chunks from r. It rejects RF64 files
|
||||
// and non-WAVE containers with descriptive errors. The parser is
|
||||
// lenient on read: it tolerates missing padding bytes and ignores
|
||||
// the declared RIFF size.
|
||||
func parseRIFF(r io.ReadSeeker) ([]riffChunk, error) {
|
||||
// Read 4-byte container magic.
|
||||
var magic [4]byte
|
||||
if _, err := io.ReadFull(r, magic[:]); err != nil {
|
||||
return nil, fmt.Errorf("read RIFF magic: %w", err)
|
||||
}
|
||||
|
||||
if string(magic[:]) == "RF64" {
|
||||
return nil, errRF64NotSupported
|
||||
}
|
||||
|
||||
if string(magic[:]) != "RIFF" {
|
||||
return nil, fmt.Errorf("%w: got %q", errNotRIFF, magic)
|
||||
}
|
||||
|
||||
// Read (and discard) RIFF size — lenient, do not enforce.
|
||||
var riffSize uint32
|
||||
if err := binary.Read(r, binary.LittleEndian, &riffSize); err != nil {
|
||||
return nil, fmt.Errorf("read RIFF size: %w", err)
|
||||
}
|
||||
|
||||
// Read 4-byte form type.
|
||||
var form [4]byte
|
||||
if _, err := io.ReadFull(r, form[:]); err != nil {
|
||||
return nil, fmt.Errorf("read WAVE form type: %w", err)
|
||||
}
|
||||
|
||||
if string(form[:]) != "WAVE" {
|
||||
return nil, fmt.Errorf("%w: got %q", errNotWAVE, form)
|
||||
}
|
||||
|
||||
// Read sub-chunks until EOF.
|
||||
var chunks []riffChunk
|
||||
|
||||
for {
|
||||
var chunkID [4]byte
|
||||
|
||||
_, err := io.ReadFull(r, chunkID[:])
|
||||
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
break
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read chunk ID: %w", err)
|
||||
}
|
||||
|
||||
var chunkSize uint32
|
||||
if err := binary.Read(r, binary.LittleEndian, &chunkSize); err != nil {
|
||||
return nil, fmt.Errorf("read chunk size for %q: %w", chunkID, err)
|
||||
}
|
||||
|
||||
data := make([]byte, chunkSize)
|
||||
if _, err := io.ReadFull(r, data); err != nil {
|
||||
return nil, fmt.Errorf("read chunk data for %q: %w", chunkID, err)
|
||||
}
|
||||
|
||||
chunks = append(chunks, riffChunk{id: chunkID, data: data})
|
||||
|
||||
// Odd-length chunks have a padding byte. Lenient: if the
|
||||
// read fails (e.g. EOF), just break rather than error.
|
||||
if chunkSize%2 != 0 {
|
||||
var pad [1]byte
|
||||
|
||||
if _, err := r.Read(pad[:]); err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return chunks, nil
|
||||
}
|
||||
|
||||
// isID3ChunkID returns true if id represents an ID3v2 RIFF chunk.
|
||||
// Both lowercase "id3 " and uppercase "ID3 " are accepted.
|
||||
func isID3ChunkID(id [4]byte) bool {
|
||||
s := strings.ToLower(string(id[:3]))
|
||||
|
||||
return s == "id3"
|
||||
}
|
||||
|
||||
// writeRIFF writes a complete RIFF/WAVE container to w, preserving
|
||||
// the given chunks in order and appending the id3Data as the final
|
||||
// "id3 " chunk. Returns errFileTooLargeForWAV if the result would
|
||||
// exceed the 4 GB RIFF limit.
|
||||
func writeRIFF(w io.Writer, chunks []riff.Chunk, id3Data []byte) error {
|
||||
func writeRIFF(w io.Writer, chunks []riffChunk, id3Data []byte) error {
|
||||
// Calculate total RIFF payload size:
|
||||
// 4 bytes (WAVE form type)
|
||||
// + for each preserved chunk: 8 (header) + len(data) + padding
|
||||
@@ -31,7 +125,7 @@ func writeRIFF(w io.Writer, chunks []riff.Chunk, id3Data []byte) error {
|
||||
riffPayload := uint64(4)
|
||||
|
||||
for _, c := range chunks {
|
||||
sz := uint64(len(c.Data))
|
||||
sz := uint64(len(c.data))
|
||||
riffPayload += 8 + sz
|
||||
|
||||
if sz%2 != 0 {
|
||||
@@ -68,7 +162,7 @@ func writeRIFF(w io.Writer, chunks []riff.Chunk, id3Data []byte) error {
|
||||
|
||||
// Write each preserved chunk.
|
||||
for _, c := range chunks {
|
||||
if err := writeChunk(w, c.ID, c.Data); err != nil {
|
||||
if err := writeChunk(w, c.id, c.data); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -131,7 +225,7 @@ func writeWavTags(
|
||||
return fmt.Errorf("open wav for reading: %w", err)
|
||||
}
|
||||
|
||||
allChunks, err := riff.Parse(f)
|
||||
allChunks, err := parseRIFF(f)
|
||||
|
||||
// Close immediately — we need the handle released before
|
||||
// AtomicWrite creates the replacement file.
|
||||
@@ -143,13 +237,13 @@ func writeWavTags(
|
||||
|
||||
// Separate preserved chunks from existing ID3 data.
|
||||
var (
|
||||
preserved []riff.Chunk
|
||||
preserved []riffChunk
|
||||
existingID3 []byte
|
||||
)
|
||||
|
||||
for _, c := range allChunks {
|
||||
if riff.IsID3(c.ID) {
|
||||
existingID3 = c.Data
|
||||
if isID3ChunkID(c.id) {
|
||||
existingID3 = c.data
|
||||
} else {
|
||||
preserved = append(preserved, c)
|
||||
}
|
||||
|
||||
+17
-103
@@ -12,7 +12,6 @@ import (
|
||||
id3v2 "github.com/bogem/id3v2/v2"
|
||||
|
||||
"yellowjacket/backend/metadata"
|
||||
"yellowjacket/backend/riff"
|
||||
)
|
||||
|
||||
// createTestWAV builds a minimal valid WAV file with an optional
|
||||
@@ -271,88 +270,6 @@ func TestWriteWavTags_PartialUpdate(t *testing.T) {
|
||||
assertStrField(t, "Composer", meta.Composer, "Original Composer")
|
||||
}
|
||||
|
||||
// The writer has always been correct and the reader could not see it:
|
||||
// a WAV tagged by this app scanned as an untagged file, so editing
|
||||
// tags, autotagging a folder or importing a WAV download all appeared
|
||||
// to work and changed nothing the library could show (#104). So this
|
||||
// asserts the write through metadata.ExtractTags -- the reader the
|
||||
// scan uses -- rather than through the id3 chunk.
|
||||
func TestWriteWavTags_ReadBackByTheScanner(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
path := createTestWAV(t, dir, "scanner.wav", nil)
|
||||
art := tinyJPEG(t)
|
||||
|
||||
changes := TagChanges{
|
||||
FieldTitle: "Some Song",
|
||||
FieldArtist: "Some Artist",
|
||||
FieldAlbum: "Some Album",
|
||||
FieldAlbumArtist: "Some Album Artist",
|
||||
FieldGenre: "Rock",
|
||||
FieldYear: 2024,
|
||||
FieldTrackNumber: 3,
|
||||
FieldComposer: "Some Composer",
|
||||
FieldCoverArt: art,
|
||||
}
|
||||
|
||||
if err := writeWavTags(testLogger(), path, changes); err != nil {
|
||||
t.Fatalf("writeWavTags: %v", err)
|
||||
}
|
||||
|
||||
meta, err := metadata.ExtractTags(path)
|
||||
if err != nil {
|
||||
t.Fatalf("ExtractTags: %v", err)
|
||||
}
|
||||
|
||||
if meta.TagReadWarning != nil {
|
||||
t.Errorf("TagReadWarning: %v", meta.TagReadWarning)
|
||||
}
|
||||
|
||||
assertStrField(t, "Title", meta.Title, "Some Song")
|
||||
assertStrField(t, "Artist", meta.Artist, "Some Artist")
|
||||
assertStrField(t, "Album", meta.Album, "Some Album")
|
||||
assertStrField(t, "AlbumArtist", meta.AlbumArtist, "Some Album Artist")
|
||||
assertStrField(t, "Genre", meta.Genre, "Rock")
|
||||
assertStrField(t, "Composer", meta.Composer, "Some Composer")
|
||||
assertStrField(t, "FileFormat", meta.FileFormat, "WAV")
|
||||
assertIntField(t, "Year", meta.Year, 2024)
|
||||
assertIntField(t, "TrackNumber", meta.TrackNumber, 3)
|
||||
|
||||
if !strings.HasPrefix(meta.TagFormat, "ID3v2") {
|
||||
t.Errorf("TagFormat: got %q, want an ID3v2 version", meta.TagFormat)
|
||||
}
|
||||
|
||||
if meta.Picture == nil {
|
||||
t.Fatal("expected cover art, got nil")
|
||||
}
|
||||
|
||||
if !bytes.Equal(meta.Picture.Data, art) {
|
||||
t.Errorf("picture data mismatch: got %d bytes, want %d",
|
||||
len(meta.Picture.Data), len(art))
|
||||
}
|
||||
}
|
||||
|
||||
// An untagged WAV is a file with no tags, not a file with a problem:
|
||||
// the scanner falls back to the filename and must not be handed a
|
||||
// warning to surface about it.
|
||||
func TestUntaggedWav_ReadsAsEmptyWithoutAWarning(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
path := createTestWAV(t, t.TempDir(), "bare.wav", nil)
|
||||
|
||||
meta, err := metadata.ExtractTags(path)
|
||||
if err != nil {
|
||||
t.Fatalf("ExtractTags: %v", err)
|
||||
}
|
||||
|
||||
if meta.TagReadWarning != nil {
|
||||
t.Errorf("TagReadWarning: %v", meta.TagReadWarning)
|
||||
}
|
||||
|
||||
assertStrField(t, "Title", meta.Title, "")
|
||||
}
|
||||
|
||||
func TestWriteWavTags_ChunkPreservation(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -365,7 +282,7 @@ func TestWriteWavTags_ChunkPreservation(t *testing.T) {
|
||||
t.Fatalf("open original: %v", err)
|
||||
}
|
||||
|
||||
origChunks, err := riff.Parse(origFile)
|
||||
origChunks, err := parseRIFF(origFile)
|
||||
_ = origFile.Close()
|
||||
|
||||
if err != nil {
|
||||
@@ -375,7 +292,7 @@ func TestWriteWavTags_ChunkPreservation(t *testing.T) {
|
||||
// Record original chunk data by ID string.
|
||||
origData := map[string][]byte{}
|
||||
for _, c := range origChunks {
|
||||
origData[string(c.ID[:])] = c.Data
|
||||
origData[string(c.id[:])] = c.data
|
||||
}
|
||||
|
||||
// Write a tag to trigger RIFF rewrite.
|
||||
@@ -392,7 +309,7 @@ func TestWriteWavTags_ChunkPreservation(t *testing.T) {
|
||||
t.Fatalf("open after write: %v", err)
|
||||
}
|
||||
|
||||
newChunks, err := riff.Parse(newFile)
|
||||
newChunks, err := parseRIFF(newFile)
|
||||
_ = newFile.Close()
|
||||
|
||||
if err != nil {
|
||||
@@ -403,7 +320,7 @@ func TestWriteWavTags_ChunkPreservation(t *testing.T) {
|
||||
origNonID3 := 0
|
||||
|
||||
for _, c := range origChunks {
|
||||
if !riff.IsID3(c.ID) {
|
||||
if !isID3ChunkID(c.id) {
|
||||
origNonID3++
|
||||
}
|
||||
}
|
||||
@@ -411,7 +328,7 @@ func TestWriteWavTags_ChunkPreservation(t *testing.T) {
|
||||
newNonID3 := 0
|
||||
|
||||
for _, c := range newChunks {
|
||||
if !riff.IsID3(c.ID) {
|
||||
if !isID3ChunkID(c.id) {
|
||||
newNonID3++
|
||||
}
|
||||
}
|
||||
@@ -442,17 +359,17 @@ func TestWriteWavTags_ChunkPreservation(t *testing.T) {
|
||||
// in chunks and its data matches want byte-for-byte.
|
||||
func checkChunkPreserved(
|
||||
t *testing.T,
|
||||
chunks []riff.Chunk,
|
||||
chunks []riffChunk,
|
||||
idStr string,
|
||||
want []byte,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
for _, c := range chunks {
|
||||
if string(c.ID[:]) == idStr {
|
||||
if !bytes.Equal(c.Data, want) {
|
||||
if string(c.id[:]) == idStr {
|
||||
if !bytes.Equal(c.data, want) {
|
||||
t.Errorf("chunk %q data changed: got %d bytes, want %d",
|
||||
idStr, len(c.Data), len(want))
|
||||
idStr, len(c.data), len(want))
|
||||
}
|
||||
|
||||
return
|
||||
@@ -513,7 +430,7 @@ func TestWriteWavTags_RejectsRF64(t *testing.T) {
|
||||
buf.WriteString("WAVE")
|
||||
|
||||
// Minimal ds64 chunk (required for RF64 but we just need
|
||||
// enough bytes for riff.Parse to hit the RF64 rejection).
|
||||
// enough bytes for parseRIFF to hit the RF64 rejection).
|
||||
buf.WriteString("ds64")
|
||||
_ = binary.Write(&buf, binary.LittleEndian, uint32(28)) //nolint:mnd
|
||||
buf.Write(make([]byte, 28)) //nolint:mnd
|
||||
@@ -537,12 +454,9 @@ func TestWriteWavTags_RejectsRF64(t *testing.T) {
|
||||
|
||||
// readWavID3Tags extracts ID3v2 metadata from a WAV file by parsing
|
||||
// the RIFF structure and reading the id3 chunk with bogem/id3v2.
|
||||
//
|
||||
// metadata.ExtractTags reads a WAV since #104 and is what the round
|
||||
// trips assert through. This stays for the two cases that are about
|
||||
// the bytes rather than about the scan: a tag with every frame
|
||||
// cleared, which no reader reports as anything, and the chunk
|
||||
// preservation test, which is already parsing the container itself.
|
||||
// dhowden/tag's ReadFrom does not support WAV files, and its
|
||||
// ReadID3v2Tags fails on empty tags (after clearing all frames).
|
||||
// Using bogem/id3v2.ParseReader handles all cases correctly.
|
||||
func readWavID3Tags(
|
||||
t *testing.T,
|
||||
path string,
|
||||
@@ -556,17 +470,17 @@ func readWavID3Tags(
|
||||
|
||||
defer func() { _ = f.Close() }()
|
||||
|
||||
chunks, err := riff.Parse(f)
|
||||
chunks, err := parseRIFF(f)
|
||||
if err != nil {
|
||||
t.Fatalf("riff.Parse: %v", err)
|
||||
t.Fatalf("parseRIFF: %v", err)
|
||||
}
|
||||
|
||||
// Find the id3 chunk.
|
||||
var id3Data []byte
|
||||
|
||||
for _, c := range chunks {
|
||||
if riff.IsID3(c.ID) {
|
||||
id3Data = c.Data
|
||||
if isID3ChunkID(c.id) {
|
||||
id3Data = c.data
|
||||
|
||||
break
|
||||
}
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 83 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 285 KiB |
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user