Compare commits
39
Commits
5b85d51b39
...
7de1b4edc1
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7de1b4edc1 | ||
|
|
ff687f0bd9 | ||
|
|
62bb40fc4d | ||
|
|
ba35858208 | ||
|
|
7c3c0e25b9 | ||
|
|
0ca37a31a6 | ||
|
|
c48123f7a3 | ||
|
|
213640c9a8 | ||
|
|
ccacd67a21 | ||
|
|
5ca6cad45a | ||
|
|
65333857e2 | ||
|
|
cbd82a5a74 | ||
|
|
e190fd75b9 | ||
|
|
d0d86f85d5 | ||
|
|
e7950006c5 | ||
|
|
01bc5f2094 | ||
|
|
f15846f8fb | ||
|
|
aead8eaef4 | ||
|
|
0001135f3a | ||
|
|
08da4f2774 | ||
|
|
d3fc2b9237 | ||
|
|
a181a98ce3 | ||
|
|
16886c92cf | ||
|
|
0a8b93d76d | ||
|
|
5b161f50c9 | ||
|
|
4ae5ffc928 | ||
|
|
91775be2b7 | ||
|
|
967d9bdae0 | ||
|
|
20c2e74412 | ||
|
|
37f75d50e5 | ||
|
|
b3299844c0 | ||
|
|
62d66cd97f | ||
|
|
19267a9148 | ||
|
|
17fffffa85 | ||
|
|
e58b604ae3 | ||
|
|
b59bf1ca53 | ||
|
|
497a8dac18 | ||
|
|
cf92f10566 | ||
|
|
0a87675dde |
@@ -0,0 +1,420 @@
|
||||
---
|
||||
name: playwright-cli
|
||||
description: Automate browser interactions, test web pages and work with Playwright tests.
|
||||
allowed-tools: Bash(playwright-cli:*) Bash(npx:*) Bash(npm:*)
|
||||
---
|
||||
|
||||
# Browser Automation with playwright-cli
|
||||
|
||||
## Quick start
|
||||
|
||||
```bash
|
||||
# open new browser
|
||||
playwright-cli open
|
||||
# navigate to a page
|
||||
playwright-cli goto https://playwright.dev
|
||||
# interact with the page using refs from the snapshot
|
||||
playwright-cli click e15
|
||||
playwright-cli type "page.click"
|
||||
playwright-cli press Enter
|
||||
# take a screenshot (rarely used, as snapshot is more common)
|
||||
playwright-cli screenshot
|
||||
# close the browser
|
||||
playwright-cli close
|
||||
```
|
||||
|
||||
## Commands
|
||||
|
||||
### Core
|
||||
|
||||
```bash
|
||||
playwright-cli open
|
||||
# open and navigate right away
|
||||
playwright-cli open https://example.com/
|
||||
playwright-cli goto https://playwright.dev
|
||||
playwright-cli type "search query"
|
||||
playwright-cli click e3
|
||||
playwright-cli dblclick e7
|
||||
# --submit presses Enter after filling the element
|
||||
playwright-cli fill e5 "user@example.com" --submit
|
||||
playwright-cli drag e2 e8
|
||||
# drop files or data onto an element (from outside the page)
|
||||
playwright-cli drop e4 --path=./image.png
|
||||
playwright-cli drop e4 --data="text/plain=hello world"
|
||||
playwright-cli hover e4
|
||||
playwright-cli select e9 "option-value"
|
||||
playwright-cli upload ./document.pdf
|
||||
playwright-cli check e12
|
||||
playwright-cli uncheck e12
|
||||
playwright-cli snapshot
|
||||
# search the snapshot for text or a regexp, returns matching nodes with surrounding context
|
||||
playwright-cli find "Sign in"
|
||||
playwright-cli find --regex "Sign (in|up)"
|
||||
# wrap the regexp in slashes to add flags, e.g. /i for case-insensitive
|
||||
playwright-cli find --regex "/sign (in|up)/i"
|
||||
playwright-cli eval "document.title"
|
||||
playwright-cli eval "el => el.textContent" e5
|
||||
# get element id, class, or any attribute not visible in the snapshot
|
||||
playwright-cli eval "el => el.id" e5
|
||||
playwright-cli eval "el => el.getAttribute('data-testid')" e5
|
||||
playwright-cli dialog-accept
|
||||
playwright-cli dialog-accept "confirmation text"
|
||||
playwright-cli dialog-dismiss
|
||||
playwright-cli resize 1920 1080
|
||||
playwright-cli close
|
||||
```
|
||||
|
||||
### Navigation
|
||||
|
||||
```bash
|
||||
playwright-cli go-back
|
||||
playwright-cli go-forward
|
||||
playwright-cli reload
|
||||
```
|
||||
|
||||
### Keyboard
|
||||
|
||||
```bash
|
||||
playwright-cli press Enter
|
||||
playwright-cli press ArrowDown
|
||||
playwright-cli keydown Shift
|
||||
playwright-cli keyup Shift
|
||||
```
|
||||
|
||||
### Mouse
|
||||
|
||||
```bash
|
||||
playwright-cli mousemove 150 300
|
||||
playwright-cli mousedown
|
||||
playwright-cli mousedown right
|
||||
playwright-cli mouseup
|
||||
playwright-cli mouseup right
|
||||
playwright-cli mousewheel 0 100
|
||||
```
|
||||
|
||||
### Save as
|
||||
|
||||
```bash
|
||||
playwright-cli screenshot
|
||||
playwright-cli screenshot e5
|
||||
playwright-cli screenshot --filename=page.png
|
||||
playwright-cli screenshot --hires
|
||||
playwright-cli pdf --filename=page.pdf
|
||||
```
|
||||
|
||||
### Tabs
|
||||
|
||||
```bash
|
||||
playwright-cli tab-list
|
||||
playwright-cli tab-new
|
||||
playwright-cli tab-new https://example.com/page
|
||||
playwright-cli tab-close
|
||||
playwright-cli tab-close 2
|
||||
playwright-cli tab-select 0
|
||||
```
|
||||
|
||||
### Storage
|
||||
|
||||
```bash
|
||||
playwright-cli state-save
|
||||
playwright-cli state-save auth.json
|
||||
playwright-cli state-load auth.json
|
||||
|
||||
# Cookies
|
||||
playwright-cli cookie-list
|
||||
playwright-cli cookie-list --domain=example.com
|
||||
playwright-cli cookie-get session_id
|
||||
playwright-cli cookie-set session_id abc123
|
||||
playwright-cli cookie-set session_id abc123 --domain=example.com --httpOnly --secure
|
||||
playwright-cli cookie-delete session_id
|
||||
playwright-cli cookie-clear
|
||||
|
||||
# LocalStorage
|
||||
playwright-cli localstorage-list
|
||||
playwright-cli localstorage-get theme
|
||||
playwright-cli localstorage-set theme dark
|
||||
playwright-cli localstorage-delete theme
|
||||
playwright-cli localstorage-clear
|
||||
|
||||
# SessionStorage
|
||||
playwright-cli sessionstorage-list
|
||||
playwright-cli sessionstorage-get step
|
||||
playwright-cli sessionstorage-set step 3
|
||||
playwright-cli sessionstorage-delete step
|
||||
playwright-cli sessionstorage-clear
|
||||
```
|
||||
|
||||
### Network
|
||||
|
||||
```bash
|
||||
playwright-cli route "**/*.jpg" --status=404
|
||||
playwright-cli route "https://api.example.com/**" --body='{"mock": true}'
|
||||
playwright-cli route-list
|
||||
playwright-cli unroute "**/*.jpg"
|
||||
playwright-cli unroute
|
||||
```
|
||||
|
||||
### DevTools
|
||||
|
||||
```bash
|
||||
playwright-cli console
|
||||
playwright-cli console warning
|
||||
playwright-cli requests
|
||||
playwright-cli request 5
|
||||
playwright-cli run-code "async page => await page.context().grantPermissions(['geolocation'])"
|
||||
playwright-cli run-code --filename=script.js
|
||||
playwright-cli tracing-start
|
||||
playwright-cli tracing-stop
|
||||
playwright-cli video-start video.webm
|
||||
playwright-cli video-chapter "Chapter Title" --description="Details" --duration=2000
|
||||
playwright-cli video-stop
|
||||
|
||||
# annotate each subsequent action (click, type, ...) with a callout naming the action and highlighting the target
|
||||
playwright-cli video-show-actions --duration=600 --position=top-right
|
||||
playwright-cli video-hide-actions
|
||||
|
||||
# launch the dashboard for UI review / design feedback — user annotates the page, you receive the annotated screenshot, snapshot, and notes
|
||||
playwright-cli show --annotate
|
||||
|
||||
# generate a Playwright locator for an element from its ref or selector
|
||||
playwright-cli generate-locator e5 --raw
|
||||
|
||||
# show a persistent highlight overlay for an element, optionally with a custom style
|
||||
playwright-cli highlight e5
|
||||
playwright-cli highlight e5 --style="outline: 3px dashed red"
|
||||
# hide a single element highlight, or all page highlights when no target is given
|
||||
playwright-cli highlight e5 --hide
|
||||
playwright-cli highlight --hide
|
||||
```
|
||||
|
||||
## Raw output
|
||||
|
||||
The global `--raw` option strips page status, generated code, and snapshot sections from the output, returning only the result value. Use it to pipe command output into other tools. Commands that don't produce output return nothing.
|
||||
|
||||
```bash
|
||||
playwright-cli --raw eval "JSON.stringify(performance.timing)" | jq '.loadEventEnd - .navigationStart'
|
||||
playwright-cli --raw eval "JSON.stringify([...document.querySelectorAll('a')].map(a => a.href))" > links.json
|
||||
playwright-cli --raw snapshot > before.yml
|
||||
playwright-cli click e5
|
||||
playwright-cli --raw snapshot > after.yml
|
||||
diff before.yml after.yml
|
||||
TOKEN=$(playwright-cli --raw cookie-get session_id)
|
||||
playwright-cli --raw localstorage-get theme
|
||||
```
|
||||
|
||||
For structured output wrapping every reply as JSON, pass --json
|
||||
```bash
|
||||
playwright-cli list --json
|
||||
```
|
||||
|
||||
## Open parameters
|
||||
```bash
|
||||
# Use specific browser when creating session
|
||||
playwright-cli open --browser=chrome
|
||||
playwright-cli open --browser=firefox
|
||||
playwright-cli open --browser=webkit
|
||||
playwright-cli open --browser=msedge
|
||||
|
||||
# Emulate a generic mobile device (Pixel 10 for Chromium, iPhone 17 for WebKit).
|
||||
# Prefer this when a mobile layout is acceptable: mobile pages are usually
|
||||
# lighter, so snapshots are smaller and cheaper.
|
||||
playwright-cli open --mobile
|
||||
playwright-cli open --device="iPhone 15"
|
||||
|
||||
# Use persistent profile (by default profile is in-memory)
|
||||
playwright-cli open --persistent
|
||||
# Use persistent profile with custom directory
|
||||
playwright-cli open --profile=/path/to/profile
|
||||
|
||||
# Connect to browser via Playwright Extension
|
||||
playwright-cli attach --extension=chrome
|
||||
|
||||
# Connect to a running Chrome or Edge by channel name
|
||||
playwright-cli attach --cdp=chrome
|
||||
playwright-cli attach --cdp=msedge
|
||||
|
||||
# Connect to a running browser via CDP endpoint
|
||||
playwright-cli attach --cdp=http://localhost:9222
|
||||
|
||||
# Start with config file
|
||||
playwright-cli open --config=my-config.json
|
||||
|
||||
# Close the browser
|
||||
playwright-cli close
|
||||
# Detach from an attached browser (leaves the external browser running)
|
||||
playwright-cli -s=msedge detach
|
||||
# Delete user data for the default session
|
||||
playwright-cli delete-data
|
||||
```
|
||||
|
||||
## URLs with `&` on Windows
|
||||
|
||||
On Windows, `cmd.exe` and PowerShell treat `&` as a command separator, so URLs with multiple query parameters get truncated before `playwright-cli` runs. Escape `&` with `^&` in `cmd.exe`, or use `--%` in PowerShell:
|
||||
|
||||
```batch
|
||||
playwright-cli goto "https://example.com/?a=1^&b=2"
|
||||
```
|
||||
|
||||
```powershell
|
||||
playwright-cli --% goto "https://example.com/?a=1&b=2"
|
||||
```
|
||||
|
||||
## Snapshots
|
||||
|
||||
After each command, playwright-cli provides a snapshot of the current browser state.
|
||||
|
||||
```bash
|
||||
> playwright-cli goto https://example.com
|
||||
### Page
|
||||
- Page URL: https://example.com/
|
||||
- Page Title: Example Domain
|
||||
### Snapshot
|
||||
[Snapshot](.playwright-cli/page-2026-02-14T19-22-42-679Z.yml)
|
||||
```
|
||||
|
||||
You can also take a snapshot on demand using `playwright-cli snapshot` command. All the options below can be combined as needed.
|
||||
|
||||
```bash
|
||||
# default - save to a file with timestamp-based name
|
||||
playwright-cli snapshot
|
||||
|
||||
# save to file, use when snapshot is a part of the workflow result
|
||||
playwright-cli snapshot --filename=after-click.yaml
|
||||
|
||||
# snapshot an element instead of the whole page
|
||||
playwright-cli snapshot "#main"
|
||||
|
||||
# limit snapshot depth for efficiency, take a partial snapshot afterwards
|
||||
playwright-cli snapshot --depth=4
|
||||
playwright-cli snapshot e34
|
||||
|
||||
# include each element's bounding box as [box=x,y,width,height]
|
||||
playwright-cli snapshot --boxes
|
||||
|
||||
# search a large snapshot instead of capturing it all — returns matching nodes
|
||||
# with 3 lines of context around each match (like grep -C)
|
||||
playwright-cli find "Add to cart"
|
||||
playwright-cli find --regex "\\$[0-9]+\\.[0-9]{2}"
|
||||
```
|
||||
|
||||
## Targeting elements
|
||||
|
||||
By default, use refs from the snapshot to interact with page elements.
|
||||
|
||||
```bash
|
||||
# get snapshot with refs
|
||||
playwright-cli snapshot
|
||||
|
||||
# interact using a ref
|
||||
playwright-cli click e15
|
||||
```
|
||||
|
||||
You can also use css selectors or Playwright locators.
|
||||
|
||||
```bash
|
||||
# css selector
|
||||
playwright-cli click "#main > button.submit"
|
||||
|
||||
# role locator
|
||||
playwright-cli click "getByRole('button', { name: 'Submit' })"
|
||||
|
||||
# test id
|
||||
playwright-cli click "getByTestId('submit-button')"
|
||||
```
|
||||
|
||||
## Browser Sessions
|
||||
|
||||
```bash
|
||||
# create new browser session named "mysession" with persistent profile
|
||||
playwright-cli -s=mysession open example.com --persistent
|
||||
# same with manually specified profile directory (use when requested explicitly)
|
||||
playwright-cli -s=mysession open example.com --profile=/path/to/profile
|
||||
playwright-cli -s=mysession click e6
|
||||
playwright-cli -s=mysession close # stop a named browser
|
||||
playwright-cli -s=mysession delete-data # delete user data for persistent session
|
||||
|
||||
playwright-cli list
|
||||
# Close all browsers
|
||||
playwright-cli close-all
|
||||
# Forcefully kill all browser processes
|
||||
playwright-cli kill-all
|
||||
```
|
||||
|
||||
## Installation
|
||||
|
||||
If global `playwright-cli` command is not available, try a local version via `npx playwright cli`:
|
||||
|
||||
```bash
|
||||
npx --no-install playwright --version
|
||||
```
|
||||
|
||||
When local version is available, use `npx playwright cli` in all commands. Otherwise, install `playwright-cli` as a global command:
|
||||
|
||||
```bash
|
||||
npm install -g @playwright/cli@latest
|
||||
```
|
||||
|
||||
## Example: Form submission
|
||||
|
||||
```bash
|
||||
playwright-cli open https://example.com/form
|
||||
playwright-cli snapshot
|
||||
|
||||
playwright-cli fill e1 "user@example.com"
|
||||
playwright-cli fill e2 "password123"
|
||||
playwright-cli click e3
|
||||
playwright-cli snapshot
|
||||
playwright-cli close
|
||||
```
|
||||
|
||||
## Example: Multi-tab workflow
|
||||
|
||||
```bash
|
||||
playwright-cli open https://example.com
|
||||
playwright-cli tab-new https://example.com/other
|
||||
playwright-cli tab-list
|
||||
playwright-cli tab-select 0
|
||||
playwright-cli snapshot
|
||||
playwright-cli close
|
||||
```
|
||||
|
||||
## Example: Debugging with DevTools
|
||||
|
||||
```bash
|
||||
playwright-cli open https://example.com
|
||||
playwright-cli click e4
|
||||
playwright-cli fill e7 "test"
|
||||
playwright-cli console
|
||||
playwright-cli requests
|
||||
playwright-cli close
|
||||
```
|
||||
|
||||
```bash
|
||||
playwright-cli open https://example.com
|
||||
playwright-cli tracing-start
|
||||
playwright-cli click e4
|
||||
playwright-cli fill e7 "test"
|
||||
playwright-cli tracing-stop
|
||||
playwright-cli close
|
||||
```
|
||||
|
||||
## Example: Interactive session
|
||||
|
||||
Ask the user for UI review or design feedback. The user draws boxes on the live page and types comments; you receive the annotated screenshot, the snapshot of the marked region, and the user's notes. Use this whenever the user asks for "UI review", "design feedback", or to "ask the user what they think / want / mean":
|
||||
|
||||
```bash
|
||||
playwright-cli open https://example.com
|
||||
playwright-cli show --annotate
|
||||
```
|
||||
|
||||
## Specific tasks
|
||||
|
||||
* **Running and Debugging Playwright tests** [references/playwright-tests.md](references/playwright-tests.md)
|
||||
* **Request mocking** [references/request-mocking.md](references/request-mocking.md)
|
||||
* **Running Playwright code** [references/running-code.md](references/running-code.md)
|
||||
* **Browser session management** [references/session-management.md](references/session-management.md)
|
||||
* **Storage state (cookies, localStorage)** [references/storage-state.md](references/storage-state.md)
|
||||
* **Test generation (plan / generate / heal)** [references/test-generation.md](references/test-generation.md)
|
||||
* **Tracing** [references/tracing.md](references/tracing.md)
|
||||
* **Video recording** [references/video-recording.md](references/video-recording.md)
|
||||
* **Inspecting element attributes** [references/element-attributes.md](references/element-attributes.md)
|
||||
@@ -0,0 +1,23 @@
|
||||
# Inspecting Element Attributes
|
||||
|
||||
When the snapshot doesn't show an element's `id`, `class`, `data-*` attributes, or other DOM properties, use `eval` to inspect them.
|
||||
|
||||
## Examples
|
||||
|
||||
```bash
|
||||
playwright-cli snapshot
|
||||
# snapshot shows a button as e7 but doesn't reveal its id or data attributes
|
||||
|
||||
# get the element's id
|
||||
playwright-cli eval "el => el.id" e7
|
||||
|
||||
# get all CSS classes
|
||||
playwright-cli eval "el => el.className" e7
|
||||
|
||||
# get a specific attribute
|
||||
playwright-cli eval "el => el.getAttribute('data-testid')" e7
|
||||
playwright-cli eval "el => el.getAttribute('aria-label')" e7
|
||||
|
||||
# get a computed style property
|
||||
playwright-cli eval "el => getComputedStyle(el).display" e7
|
||||
```
|
||||
@@ -0,0 +1,39 @@
|
||||
# Running Playwright Tests
|
||||
|
||||
To run Playwright tests, use the `npx playwright test` command, or a package manager script. To avoid opening the interactive html report, use `PLAYWRIGHT_HTML_OPEN=never` environment variable.
|
||||
|
||||
```bash
|
||||
# Run all tests
|
||||
PLAYWRIGHT_HTML_OPEN=never npx playwright test
|
||||
|
||||
# Run all tests through a custom npm script
|
||||
PLAYWRIGHT_HTML_OPEN=never npm run special-test-command
|
||||
```
|
||||
|
||||
# Debugging Playwright Tests
|
||||
|
||||
To debug a failing Playwright test, run it with `--debug=cli` option. This command will pause the test at the start and print the debugging instructions.
|
||||
|
||||
**IMPORTANT**: run the command in the background and check the output until "Debugging Instructions" is printed. Make sure to stop the command after you have finished.
|
||||
|
||||
Once instructions containing a session name are printed, use `playwright-cli` to attach the session and explore the page.
|
||||
|
||||
```bash
|
||||
# Run the test
|
||||
PLAYWRIGHT_HTML_OPEN=never npx playwright test --debug=cli
|
||||
# ...
|
||||
# ... debugging instructions for "tw-abcdef" session ...
|
||||
# ...
|
||||
|
||||
# Attach to the test
|
||||
playwright-cli attach tw-abcdef
|
||||
```
|
||||
|
||||
Keep the test running in the background while you explore and look for a fix.
|
||||
The test is paused at the start, so you should step over or pause at a particular location
|
||||
where the problem is most likely to be.
|
||||
|
||||
Every action you perform with `playwright-cli` generates corresponding Playwright TypeScript code.
|
||||
This code appears in the output and can be copied directly into the test. Most of the time, a specific locator or an expectation should be updated, but it could also be a bug in the app. Use your judgement.
|
||||
|
||||
After fixing the test, stop the background test run. Rerun to check that test passes.
|
||||
@@ -0,0 +1,87 @@
|
||||
# Request Mocking
|
||||
|
||||
Intercept, mock, modify, and block network requests.
|
||||
|
||||
## CLI Route Commands
|
||||
|
||||
```bash
|
||||
# Mock with custom status
|
||||
playwright-cli route "**/*.jpg" --status=404
|
||||
|
||||
# Mock with JSON body
|
||||
playwright-cli route "**/api/users" --body='[{"id":1,"name":"Alice"}]' --content-type=application/json
|
||||
|
||||
# Mock with custom headers
|
||||
playwright-cli route "**/api/data" --body='{"ok":true}' --header="X-Custom: value"
|
||||
|
||||
# Remove headers from requests
|
||||
playwright-cli route "**/*" --remove-header=cookie,authorization
|
||||
|
||||
# List active routes
|
||||
playwright-cli route-list
|
||||
|
||||
# Remove a route or all routes
|
||||
playwright-cli unroute "**/*.jpg"
|
||||
playwright-cli unroute
|
||||
```
|
||||
|
||||
## URL Patterns
|
||||
|
||||
```
|
||||
**/api/users - Exact path match
|
||||
**/api/*/details - Wildcard in path
|
||||
**/*.{png,jpg,jpeg} - Match file extensions
|
||||
**/search?q=* - Match query parameters
|
||||
```
|
||||
|
||||
## Advanced Mocking with run-code
|
||||
|
||||
For conditional responses, request body inspection, response modification, or delays:
|
||||
|
||||
### Conditional Response Based on Request
|
||||
|
||||
```bash
|
||||
playwright-cli run-code "async page => {
|
||||
await page.route('**/api/login', route => {
|
||||
const body = route.request().postDataJSON();
|
||||
if (body.username === 'admin') {
|
||||
route.fulfill({ body: JSON.stringify({ token: 'mock-token' }) });
|
||||
} else {
|
||||
route.fulfill({ status: 401, body: JSON.stringify({ error: 'Invalid' }) });
|
||||
}
|
||||
});
|
||||
}"
|
||||
```
|
||||
|
||||
### Modify Real Response
|
||||
|
||||
```bash
|
||||
playwright-cli run-code "async page => {
|
||||
await page.route('**/api/user', async route => {
|
||||
const response = await route.fetch();
|
||||
const json = await response.json();
|
||||
json.isPremium = true;
|
||||
await route.fulfill({ response, json });
|
||||
});
|
||||
}"
|
||||
```
|
||||
|
||||
### Simulate Network Failures
|
||||
|
||||
```bash
|
||||
playwright-cli run-code "async page => {
|
||||
await page.route('**/api/offline', route => route.abort('internetdisconnected'));
|
||||
}"
|
||||
# Options: connectionrefused, timedout, connectionreset, internetdisconnected
|
||||
```
|
||||
|
||||
### Delayed Response
|
||||
|
||||
```bash
|
||||
playwright-cli run-code "async page => {
|
||||
await page.route('**/api/slow', async route => {
|
||||
await new Promise(r => setTimeout(r, 3000));
|
||||
route.fulfill({ body: JSON.stringify({ data: 'loaded' }) });
|
||||
});
|
||||
}"
|
||||
```
|
||||
@@ -0,0 +1,241 @@
|
||||
# Running Custom Playwright Code
|
||||
|
||||
Use `run-code` to execute arbitrary Playwright code for advanced scenarios not covered by CLI commands.
|
||||
|
||||
## Syntax
|
||||
|
||||
```bash
|
||||
playwright-cli run-code "async page => {
|
||||
// Your Playwright code here
|
||||
// Access page.context() for browser context operations
|
||||
}"
|
||||
```
|
||||
|
||||
You can also load the function from a file:
|
||||
|
||||
```bash
|
||||
playwright-cli run-code --filename=./my-script.js
|
||||
```
|
||||
|
||||
|
||||
The code must be a single function expression, it is wrapped in `(...)` and evaluated.
|
||||
import/export/require syntax is not supported.
|
||||
|
||||
## Geolocation
|
||||
|
||||
```bash
|
||||
# Grant geolocation permission and set location
|
||||
playwright-cli run-code "async page => {
|
||||
await page.context().grantPermissions(['geolocation']);
|
||||
await page.context().setGeolocation({ latitude: 37.7749, longitude: -122.4194 });
|
||||
}"
|
||||
|
||||
# Set location to London
|
||||
playwright-cli run-code "async page => {
|
||||
await page.context().grantPermissions(['geolocation']);
|
||||
await page.context().setGeolocation({ latitude: 51.5074, longitude: -0.1278 });
|
||||
}"
|
||||
|
||||
# Clear geolocation override
|
||||
playwright-cli run-code "async page => {
|
||||
await page.context().clearPermissions();
|
||||
}"
|
||||
```
|
||||
|
||||
## Permissions
|
||||
|
||||
```bash
|
||||
# Grant multiple permissions
|
||||
playwright-cli run-code "async page => {
|
||||
await page.context().grantPermissions([
|
||||
'geolocation',
|
||||
'notifications',
|
||||
'camera',
|
||||
'microphone'
|
||||
]);
|
||||
}"
|
||||
|
||||
# Grant permissions for specific origin
|
||||
playwright-cli run-code "async page => {
|
||||
await page.context().grantPermissions(['clipboard-read'], {
|
||||
origin: 'https://example.com'
|
||||
});
|
||||
}"
|
||||
```
|
||||
|
||||
## Media Emulation
|
||||
|
||||
```bash
|
||||
# Emulate dark color scheme
|
||||
playwright-cli run-code "async page => {
|
||||
await page.emulateMedia({ colorScheme: 'dark' });
|
||||
}"
|
||||
|
||||
# Emulate light color scheme
|
||||
playwright-cli run-code "async page => {
|
||||
await page.emulateMedia({ colorScheme: 'light' });
|
||||
}"
|
||||
|
||||
# Emulate reduced motion
|
||||
playwright-cli run-code "async page => {
|
||||
await page.emulateMedia({ reducedMotion: 'reduce' });
|
||||
}"
|
||||
|
||||
# Emulate print media
|
||||
playwright-cli run-code "async page => {
|
||||
await page.emulateMedia({ media: 'print' });
|
||||
}"
|
||||
```
|
||||
|
||||
## Wait Strategies
|
||||
|
||||
```bash
|
||||
# Wait for network idle
|
||||
playwright-cli run-code "async page => {
|
||||
await page.waitForLoadState('networkidle');
|
||||
}"
|
||||
|
||||
# Wait for specific element
|
||||
playwright-cli run-code "async page => {
|
||||
await page.locator('.loading').waitFor({ state: 'hidden' });
|
||||
}"
|
||||
|
||||
# Wait for function to return true
|
||||
playwright-cli run-code "async page => {
|
||||
await page.waitForFunction(() => window.appReady === true);
|
||||
}"
|
||||
|
||||
# Wait with timeout
|
||||
playwright-cli run-code "async page => {
|
||||
await page.locator('.result').waitFor({ timeout: 10000 });
|
||||
}"
|
||||
```
|
||||
|
||||
## Frames and Iframes
|
||||
|
||||
```bash
|
||||
# Work with iframe
|
||||
playwright-cli run-code "async page => {
|
||||
const frame = page.locator('iframe#my-iframe').contentFrame();
|
||||
await frame.locator('button').click();
|
||||
}"
|
||||
|
||||
# Get all frames
|
||||
playwright-cli run-code "async page => {
|
||||
const frames = page.frames();
|
||||
return frames.map(f => f.url());
|
||||
}"
|
||||
```
|
||||
|
||||
## File Downloads
|
||||
|
||||
```bash
|
||||
# Handle file download
|
||||
playwright-cli run-code "async page => {
|
||||
const downloadPromise = page.waitForEvent('download');
|
||||
await page.getByRole('link', { name: 'Download' }).click();
|
||||
const download = await downloadPromise;
|
||||
await download.saveAs('./downloaded-file.pdf');
|
||||
return download.suggestedFilename();
|
||||
}"
|
||||
```
|
||||
|
||||
## Clipboard
|
||||
|
||||
```bash
|
||||
# Read clipboard (requires permission)
|
||||
playwright-cli run-code "async page => {
|
||||
await page.context().grantPermissions(['clipboard-read']);
|
||||
return await page.evaluate(() => navigator.clipboard.readText());
|
||||
}"
|
||||
|
||||
# Write to clipboard
|
||||
playwright-cli run-code "async page => {
|
||||
await page.evaluate(text => navigator.clipboard.writeText(text), 'Hello clipboard!');
|
||||
}"
|
||||
```
|
||||
|
||||
## Page Information
|
||||
|
||||
```bash
|
||||
# Get page title
|
||||
playwright-cli run-code "async page => {
|
||||
return await page.title();
|
||||
}"
|
||||
|
||||
# Get current URL
|
||||
playwright-cli run-code "async page => {
|
||||
return page.url();
|
||||
}"
|
||||
|
||||
# Get page content
|
||||
playwright-cli run-code "async page => {
|
||||
return await page.content();
|
||||
}"
|
||||
|
||||
# Get viewport size
|
||||
playwright-cli run-code "async page => {
|
||||
return page.viewportSize();
|
||||
}"
|
||||
```
|
||||
|
||||
## JavaScript Execution
|
||||
|
||||
```bash
|
||||
# Execute JavaScript and return result
|
||||
playwright-cli run-code "async page => {
|
||||
return await page.evaluate(() => {
|
||||
return {
|
||||
userAgent: navigator.userAgent,
|
||||
language: navigator.language,
|
||||
cookiesEnabled: navigator.cookieEnabled
|
||||
};
|
||||
});
|
||||
}"
|
||||
|
||||
# Pass arguments to evaluate
|
||||
playwright-cli run-code "async page => {
|
||||
const multiplier = 5;
|
||||
return await page.evaluate(m => document.querySelectorAll('li').length * m, multiplier);
|
||||
}"
|
||||
```
|
||||
|
||||
## Error Handling
|
||||
|
||||
```bash
|
||||
# Try-catch in run-code
|
||||
playwright-cli run-code "async page => {
|
||||
try {
|
||||
await page.getByRole('button', { name: 'Submit' }).click({ timeout: 1000 });
|
||||
return 'clicked';
|
||||
} catch (e) {
|
||||
return 'element not found';
|
||||
}
|
||||
}"
|
||||
```
|
||||
|
||||
## Complex Workflows
|
||||
|
||||
```bash
|
||||
# Login and save state
|
||||
playwright-cli run-code "async page => {
|
||||
await page.goto('https://example.com/login');
|
||||
await page.getByRole('textbox', { name: 'Email' }).fill('user@example.com');
|
||||
await page.getByRole('textbox', { name: 'Password' }).fill('secret');
|
||||
await page.getByRole('button', { name: 'Sign in' }).click();
|
||||
await page.waitForURL('**/dashboard');
|
||||
await page.context().storageState({ path: 'auth.json' });
|
||||
return 'Login successful';
|
||||
}"
|
||||
|
||||
# Scrape data from multiple pages
|
||||
playwright-cli run-code "async page => {
|
||||
const results = [];
|
||||
for (let i = 1; i <= 3; i++) {
|
||||
await page.goto(\`https://example.com/page/\${i}\`);
|
||||
const items = await page.locator('.item').allTextContents();
|
||||
results.push(...items);
|
||||
}
|
||||
return results;
|
||||
}"
|
||||
```
|
||||
@@ -0,0 +1,225 @@
|
||||
# Browser Session Management
|
||||
|
||||
Run multiple isolated browser sessions concurrently with state persistence.
|
||||
|
||||
## Named Browser Sessions
|
||||
|
||||
Use `-s` flag to isolate browser contexts:
|
||||
|
||||
```bash
|
||||
# Browser 1: Authentication flow
|
||||
playwright-cli -s=auth open https://app.example.com/login
|
||||
|
||||
# Browser 2: Public browsing (separate cookies, storage)
|
||||
playwright-cli -s=public open https://example.com
|
||||
|
||||
# Commands are isolated by browser session
|
||||
playwright-cli -s=auth fill e1 "user@example.com"
|
||||
playwright-cli -s=public snapshot
|
||||
```
|
||||
|
||||
## Browser Session Isolation Properties
|
||||
|
||||
Each browser session has independent:
|
||||
- Cookies
|
||||
- LocalStorage / SessionStorage
|
||||
- IndexedDB
|
||||
- Cache
|
||||
- Browsing history
|
||||
- Open tabs
|
||||
|
||||
## Browser Session Commands
|
||||
|
||||
```bash
|
||||
# List all browser sessions
|
||||
playwright-cli list
|
||||
|
||||
# Stop a browser session (close the browser)
|
||||
playwright-cli close # stop the default browser
|
||||
playwright-cli -s=mysession close # stop a named browser
|
||||
|
||||
# Stop all browser sessions
|
||||
playwright-cli close-all
|
||||
|
||||
# Forcefully kill all daemon processes (for stale/zombie processes)
|
||||
playwright-cli kill-all
|
||||
|
||||
# Delete browser session user data (profile directory)
|
||||
playwright-cli delete-data # delete default browser data
|
||||
playwright-cli -s=mysession delete-data # delete named browser data
|
||||
```
|
||||
|
||||
## Environment Variable
|
||||
|
||||
Set a default browser session name via environment variable:
|
||||
|
||||
```bash
|
||||
export PLAYWRIGHT_CLI_SESSION="mysession"
|
||||
playwright-cli open example.com # Uses "mysession" automatically
|
||||
```
|
||||
|
||||
## Common Patterns
|
||||
|
||||
### Concurrent Scraping
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
# Scrape multiple sites concurrently
|
||||
|
||||
# Start all browsers
|
||||
playwright-cli -s=site1 open https://site1.com &
|
||||
playwright-cli -s=site2 open https://site2.com &
|
||||
playwright-cli -s=site3 open https://site3.com &
|
||||
wait
|
||||
|
||||
# Take snapshots from each
|
||||
playwright-cli -s=site1 snapshot
|
||||
playwright-cli -s=site2 snapshot
|
||||
playwright-cli -s=site3 snapshot
|
||||
|
||||
# Cleanup
|
||||
playwright-cli close-all
|
||||
```
|
||||
|
||||
### A/B Testing Sessions
|
||||
|
||||
```bash
|
||||
# Test different user experiences
|
||||
playwright-cli -s=variant-a open "https://app.com?variant=a"
|
||||
playwright-cli -s=variant-b open "https://app.com?variant=b"
|
||||
|
||||
# Compare
|
||||
playwright-cli -s=variant-a screenshot
|
||||
playwright-cli -s=variant-b screenshot
|
||||
```
|
||||
|
||||
### Persistent Profile
|
||||
|
||||
By default, browser profile is kept in memory only. Use `--persistent` flag on `open` to persist the browser profile to disk:
|
||||
|
||||
```bash
|
||||
# Use persistent profile (auto-generated location)
|
||||
playwright-cli open https://example.com --persistent
|
||||
|
||||
# Use persistent profile with custom directory
|
||||
playwright-cli open https://example.com --profile=/path/to/profile
|
||||
```
|
||||
|
||||
## Attaching to a Running Browser
|
||||
|
||||
Use `attach` to connect to a browser that is already running, instead of launching a new one.
|
||||
|
||||
### Attach by channel name
|
||||
|
||||
Connect to a running Chrome or Edge instance by its channel name. The browser must have remote debugging enabled — navigate to `chrome://inspect/#remote-debugging` in the target browser and check "Allow remote debugging for this browser instance".
|
||||
|
||||
```bash
|
||||
# Attach to Chrome
|
||||
playwright-cli attach --cdp=chrome
|
||||
|
||||
# Attach to Chrome Canary
|
||||
playwright-cli attach --cdp=chrome-canary
|
||||
|
||||
# Attach to Microsoft Edge
|
||||
playwright-cli attach --cdp=msedge
|
||||
|
||||
# Attach to Edge Dev
|
||||
playwright-cli attach --cdp=msedge-dev
|
||||
```
|
||||
|
||||
Supported channels: `chrome`, `chrome-beta`, `chrome-dev`, `chrome-canary`, `msedge`, `msedge-beta`, `msedge-dev`, `msedge-canary`.
|
||||
|
||||
When `--session` is not provided, the session is named after the channel (e.g. `--cdp=msedge` creates a session called `msedge`), so parallel attaches to Chrome and Edge don't collide on `default`. Pass `--session=<name>` to override.
|
||||
|
||||
### Attach via CDP endpoint
|
||||
|
||||
Connect to a browser that exposes a Chrome DevTools Protocol endpoint:
|
||||
|
||||
```bash
|
||||
playwright-cli attach --cdp=http://localhost:9222
|
||||
```
|
||||
|
||||
### Attach via browser extension
|
||||
|
||||
Connect to a browser with the Playwright extension installed:
|
||||
|
||||
```bash
|
||||
playwright-cli attach --extension
|
||||
```
|
||||
|
||||
### Detach
|
||||
|
||||
Tear down an attached session without affecting the external browser:
|
||||
|
||||
```bash
|
||||
# Detach the default attached session
|
||||
playwright-cli detach
|
||||
|
||||
# Detach a specific attached session
|
||||
playwright-cli -s=msedge detach
|
||||
```
|
||||
|
||||
`detach` only works on sessions created via `attach`. For sessions created via `open`, use `close`.
|
||||
|
||||
## Default Browser Session
|
||||
|
||||
When `-s` is omitted, commands use the default browser session:
|
||||
|
||||
```bash
|
||||
# These use the same default browser session
|
||||
playwright-cli open https://example.com
|
||||
playwright-cli snapshot
|
||||
playwright-cli close # Stops default browser
|
||||
```
|
||||
|
||||
## Browser Session Configuration
|
||||
|
||||
Configure a browser session with specific settings when opening:
|
||||
|
||||
```bash
|
||||
# Open with config file
|
||||
playwright-cli open https://example.com --config=.playwright/my-cli.json
|
||||
|
||||
# Open with specific browser
|
||||
playwright-cli open https://example.com --browser=firefox
|
||||
|
||||
# Open in headed mode
|
||||
playwright-cli open https://example.com --headed
|
||||
|
||||
# Open with persistent profile
|
||||
playwright-cli open https://example.com --persistent
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
|
||||
### 1. Name Browser Sessions Semantically
|
||||
|
||||
```bash
|
||||
# GOOD: Clear purpose
|
||||
playwright-cli -s=github-auth open https://github.com
|
||||
playwright-cli -s=docs-scrape open https://docs.example.com
|
||||
|
||||
# AVOID: Generic names
|
||||
playwright-cli -s=s1 open https://github.com
|
||||
```
|
||||
|
||||
### 2. Always Clean Up
|
||||
|
||||
```bash
|
||||
# Stop browsers when done
|
||||
playwright-cli -s=auth close
|
||||
playwright-cli -s=scrape close
|
||||
|
||||
# Or stop all at once
|
||||
playwright-cli close-all
|
||||
|
||||
# If browsers become unresponsive or zombie processes remain
|
||||
playwright-cli kill-all
|
||||
```
|
||||
|
||||
### 3. Delete Stale Browser Data
|
||||
|
||||
```bash
|
||||
# Remove old browser data to free disk space
|
||||
playwright-cli -s=oldsession delete-data
|
||||
```
|
||||
@@ -0,0 +1,275 @@
|
||||
# Storage Management
|
||||
|
||||
Manage cookies, localStorage, sessionStorage, and browser storage state.
|
||||
|
||||
## Storage State
|
||||
|
||||
Save and restore complete browser state including cookies and storage.
|
||||
|
||||
### Save Storage State
|
||||
|
||||
```bash
|
||||
# Save to auto-generated filename (storage-state-{timestamp}.json)
|
||||
playwright-cli state-save
|
||||
|
||||
# Save to specific filename
|
||||
playwright-cli state-save my-auth-state.json
|
||||
```
|
||||
|
||||
### Restore Storage State
|
||||
|
||||
```bash
|
||||
# Load storage state from file
|
||||
playwright-cli state-load my-auth-state.json
|
||||
|
||||
# Reload page to apply cookies
|
||||
playwright-cli open https://example.com
|
||||
```
|
||||
|
||||
### Storage State File Format
|
||||
|
||||
The saved file contains:
|
||||
|
||||
```json
|
||||
{
|
||||
"cookies": [
|
||||
{
|
||||
"name": "session_id",
|
||||
"value": "abc123",
|
||||
"domain": "example.com",
|
||||
"path": "/",
|
||||
"expires": 1893456000,
|
||||
"httpOnly": true,
|
||||
"secure": true,
|
||||
"sameSite": "Lax"
|
||||
}
|
||||
],
|
||||
"origins": [
|
||||
{
|
||||
"origin": "https://example.com",
|
||||
"localStorage": [
|
||||
{ "name": "theme", "value": "dark" },
|
||||
{ "name": "user_id", "value": "12345" }
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
## Cookies
|
||||
|
||||
### List All Cookies
|
||||
|
||||
```bash
|
||||
playwright-cli cookie-list
|
||||
```
|
||||
|
||||
### Filter Cookies by Domain
|
||||
|
||||
```bash
|
||||
playwright-cli cookie-list --domain=example.com
|
||||
```
|
||||
|
||||
### Filter Cookies by Path
|
||||
|
||||
```bash
|
||||
playwright-cli cookie-list --path=/api
|
||||
```
|
||||
|
||||
### Get Specific Cookie
|
||||
|
||||
```bash
|
||||
playwright-cli cookie-get session_id
|
||||
```
|
||||
|
||||
### Set a Cookie
|
||||
|
||||
```bash
|
||||
# Basic cookie
|
||||
playwright-cli cookie-set session abc123
|
||||
|
||||
# Cookie with options
|
||||
playwright-cli cookie-set session abc123 --domain=example.com --path=/ --httpOnly --secure --sameSite=Lax
|
||||
|
||||
# Cookie with expiration (Unix timestamp)
|
||||
playwright-cli cookie-set remember_me token123 --expires=1893456000
|
||||
```
|
||||
|
||||
### Delete a Cookie
|
||||
|
||||
```bash
|
||||
playwright-cli cookie-delete session_id
|
||||
```
|
||||
|
||||
### Clear All Cookies
|
||||
|
||||
```bash
|
||||
playwright-cli cookie-clear
|
||||
```
|
||||
|
||||
### Advanced: Multiple Cookies or Custom Options
|
||||
|
||||
For complex scenarios like adding multiple cookies at once, use `run-code`:
|
||||
|
||||
```bash
|
||||
playwright-cli run-code "async page => {
|
||||
await page.context().addCookies([
|
||||
{ name: 'session_id', value: 'sess_abc123', domain: 'example.com', path: '/', httpOnly: true },
|
||||
{ name: 'preferences', value: JSON.stringify({ theme: 'dark' }), domain: 'example.com', path: '/' }
|
||||
]);
|
||||
}"
|
||||
```
|
||||
|
||||
## Local Storage
|
||||
|
||||
### List All localStorage Items
|
||||
|
||||
```bash
|
||||
playwright-cli localstorage-list
|
||||
```
|
||||
|
||||
### Get Single Value
|
||||
|
||||
```bash
|
||||
playwright-cli localstorage-get token
|
||||
```
|
||||
|
||||
### Set Value
|
||||
|
||||
```bash
|
||||
playwright-cli localstorage-set theme dark
|
||||
```
|
||||
|
||||
### Set JSON Value
|
||||
|
||||
```bash
|
||||
playwright-cli localstorage-set user_settings '{"theme":"dark","language":"en"}'
|
||||
```
|
||||
|
||||
### Delete Single Item
|
||||
|
||||
```bash
|
||||
playwright-cli localstorage-delete token
|
||||
```
|
||||
|
||||
### Clear All localStorage
|
||||
|
||||
```bash
|
||||
playwright-cli localstorage-clear
|
||||
```
|
||||
|
||||
### Advanced: Multiple Operations
|
||||
|
||||
For complex scenarios like setting multiple values at once, use `run-code`:
|
||||
|
||||
```bash
|
||||
playwright-cli run-code "async page => {
|
||||
await page.evaluate(() => {
|
||||
localStorage.setItem('token', 'jwt_abc123');
|
||||
localStorage.setItem('user_id', '12345');
|
||||
localStorage.setItem('expires_at', Date.now() + 3600000);
|
||||
});
|
||||
}"
|
||||
```
|
||||
|
||||
## Session Storage
|
||||
|
||||
### List All sessionStorage Items
|
||||
|
||||
```bash
|
||||
playwright-cli sessionstorage-list
|
||||
```
|
||||
|
||||
### Get Single Value
|
||||
|
||||
```bash
|
||||
playwright-cli sessionstorage-get form_data
|
||||
```
|
||||
|
||||
### Set Value
|
||||
|
||||
```bash
|
||||
playwright-cli sessionstorage-set step 3
|
||||
```
|
||||
|
||||
### Delete Single Item
|
||||
|
||||
```bash
|
||||
playwright-cli sessionstorage-delete step
|
||||
```
|
||||
|
||||
### Clear sessionStorage
|
||||
|
||||
```bash
|
||||
playwright-cli sessionstorage-clear
|
||||
```
|
||||
|
||||
## IndexedDB
|
||||
|
||||
### List Databases
|
||||
|
||||
```bash
|
||||
playwright-cli run-code "async page => {
|
||||
return await page.evaluate(async () => {
|
||||
const databases = await indexedDB.databases();
|
||||
return databases;
|
||||
});
|
||||
}"
|
||||
```
|
||||
|
||||
### Delete Database
|
||||
|
||||
```bash
|
||||
playwright-cli run-code "async page => {
|
||||
await page.evaluate(() => {
|
||||
indexedDB.deleteDatabase('myDatabase');
|
||||
});
|
||||
}"
|
||||
```
|
||||
|
||||
## Common Patterns
|
||||
|
||||
### Authentication State Reuse
|
||||
|
||||
```bash
|
||||
# Step 1: Login and save state
|
||||
playwright-cli open https://app.example.com/login
|
||||
playwright-cli snapshot
|
||||
playwright-cli fill e1 "user@example.com"
|
||||
playwright-cli fill e2 "password123"
|
||||
playwright-cli click e3
|
||||
|
||||
# Save the authenticated state
|
||||
playwright-cli state-save auth.json
|
||||
|
||||
# Step 2: Later, restore state and skip login
|
||||
playwright-cli state-load auth.json
|
||||
playwright-cli open https://app.example.com/dashboard
|
||||
# Already logged in!
|
||||
```
|
||||
|
||||
### Save and Restore Roundtrip
|
||||
|
||||
```bash
|
||||
# Set up authentication state
|
||||
playwright-cli open https://example.com
|
||||
playwright-cli eval "() => { document.cookie = 'session=abc123'; localStorage.setItem('user', 'john'); }"
|
||||
|
||||
# Save state to file
|
||||
playwright-cli state-save my-session.json
|
||||
|
||||
# ... later, in a new session ...
|
||||
|
||||
# Restore state
|
||||
playwright-cli state-load my-session.json
|
||||
playwright-cli open https://example.com
|
||||
# Cookies and localStorage are restored!
|
||||
```
|
||||
|
||||
## Security Notes
|
||||
|
||||
- Never commit storage state files containing auth tokens
|
||||
- Add `*.auth-state.json` to `.gitignore`
|
||||
- Delete state files after automation completes
|
||||
- Use environment variables for sensitive data
|
||||
- By default, sessions run in-memory mode which is safer for sensitive operations
|
||||
@@ -0,0 +1,433 @@
|
||||
# Test generation (plan → generate → heal)
|
||||
|
||||
End-to-end workflow for authoring and maintaining Playwright tests with `playwright-cli`. Every `playwright-cli` action emits the equivalent Playwright TypeScript, and that generated code is the raw material for every test. The sections below can be used independently:
|
||||
|
||||
- **How generation works** — the core mechanic everything else relies on: actions become TypeScript, plus how to add assertions.
|
||||
- **Plan** — explore the app, produce a spec file describing what to test.
|
||||
- **Generate** — turn a spec into Playwright test files. Update the spec if it's vague or stale.
|
||||
- **Heal** — diagnose failing tests, fix the code, reconcile the spec with reality.
|
||||
|
||||
Plan / generate / heal lean on the same mechanic: run `npx playwright test --debug=cli` in the background, then `playwright-cli attach tw-XXXX` to drive the paused page interactively. See [playwright-tests.md](playwright-tests.md) for the debug/attach mechanics.
|
||||
|
||||
---
|
||||
|
||||
## 0. How generation works
|
||||
|
||||
Every action you perform with `playwright-cli` generates corresponding Playwright TypeScript code. This code appears in the output and can be copied directly into your test files.
|
||||
|
||||
```bash
|
||||
# Start a session
|
||||
playwright-cli open https://example.com/login
|
||||
|
||||
# Take a snapshot to see elements
|
||||
playwright-cli snapshot
|
||||
# Output shows: e1 [textbox "Email"], e2 [textbox "Password"], e3 [button "Sign In"]
|
||||
|
||||
# Fill form fields - generates code automatically
|
||||
playwright-cli fill e1 "user@example.com"
|
||||
# Ran Playwright code:
|
||||
# await page.getByRole('textbox', { name: 'Email' }).fill('user@example.com');
|
||||
|
||||
playwright-cli fill e2 "password123"
|
||||
# Ran Playwright code:
|
||||
# await page.getByRole('textbox', { name: 'Password' }).fill('password123');
|
||||
|
||||
playwright-cli click e3
|
||||
# Ran Playwright code:
|
||||
# await page.getByRole('button', { name: 'Sign In' }).click();
|
||||
```
|
||||
|
||||
### Building a test file
|
||||
|
||||
Collect the generated code into a Playwright test:
|
||||
|
||||
```typescript
|
||||
import { test, expect } from '@playwright/test';
|
||||
|
||||
test('login flow', async ({ page }) => {
|
||||
// Generated code from playwright-cli session:
|
||||
await page.goto('https://example.com/login');
|
||||
await page.getByRole('textbox', { name: 'Email' }).fill('user@example.com');
|
||||
await page.getByRole('textbox', { name: 'Password' }).fill('password123');
|
||||
await page.getByRole('button', { name: 'Sign In' }).click();
|
||||
|
||||
// Add assertions
|
||||
await expect(page).toHaveURL(/.*dashboard/);
|
||||
});
|
||||
```
|
||||
|
||||
### Use semantic locators
|
||||
|
||||
The generated code uses role-based locators when possible, which are more resilient:
|
||||
|
||||
```typescript
|
||||
// Generated (good - semantic)
|
||||
await page.getByRole('button', { name: 'Submit' }).click();
|
||||
|
||||
// Avoid (fragile - CSS selectors)
|
||||
await page.locator('#submit-btn').click();
|
||||
```
|
||||
|
||||
### Explore before recording
|
||||
|
||||
Take snapshots to understand the page structure before recording actions:
|
||||
|
||||
```bash
|
||||
playwright-cli open https://example.com
|
||||
playwright-cli snapshot
|
||||
# Review the element structure
|
||||
playwright-cli click e5
|
||||
```
|
||||
|
||||
### Add assertions manually
|
||||
|
||||
Generated code captures actions but not assertions. Add expectations in your test using one of the recommended matchers:
|
||||
|
||||
- `toBeVisible()` — element is rendered and visible
|
||||
- `toHaveText(text)` — element text content matches
|
||||
- `toHaveValue(value) / toBeEmpty()` — input/select value matches
|
||||
- `toBeChecked() / toBeUnchecked()` — checkbox state matches
|
||||
- `toMatchAriaSnapshot(snapshot)` — page (or locator) matches a partial accessibility snapshot
|
||||
|
||||
Use `playwright-cli generate-locator <target>` to produce the locator expression for the assertion, and the snapshot/eval commands to capture the expected value.
|
||||
|
||||
When asserting text content, make sure that generated locator does not contain text from the element itself. `getByTestId()` or `getByLabel()` usually work well with asserting text. When locator is text-based, prefer `toBeVisible()` instead.
|
||||
|
||||
Snapshot to be matched does not have to contain all the information - only capture what's necessary for the assertion. You can use regular expressions for unstable values.
|
||||
|
||||
```bash
|
||||
# Get a stable locator for an element ref to use in the assertion
|
||||
playwright-cli --raw generate-locator e5
|
||||
# getByRole('button', { name: 'Submit' })
|
||||
|
||||
# Capture expected text content for toHaveText
|
||||
playwright-cli --raw eval "el => el.textContent" e5
|
||||
|
||||
# Capture expected input value for toHaveValue/toBeEmpty
|
||||
playwright-cli --raw eval "el => el.value" e5
|
||||
|
||||
# Capture expected aria snapshot for toMatchAriaSnapshot/toBeChecked
|
||||
# (whole page, or use a ref to scope to a region)
|
||||
playwright-cli --raw snapshot
|
||||
playwright-cli --raw snapshot e5
|
||||
```
|
||||
|
||||
```typescript
|
||||
// Generated action
|
||||
await page.getByRole('button', { name: 'Submit' }).click();
|
||||
|
||||
// Manual assertions using the outputs above:
|
||||
await expect(page.getByRole('alert', { name: 'Success' })).toBeVisible();
|
||||
await expect(page.getByTestId('main-header')).toHaveText('Welcome, user');
|
||||
await expect(page.getByRole('textbox', { name: 'Email' })).toHaveValue('user@example.com');
|
||||
await expect(page.getByRole('checkbox', { name: 'Enable notifications' })).toBeChecked();
|
||||
|
||||
// toMatchAriaSnapshot on the whole page, finds a matching region
|
||||
await expect(page).toMatchAriaSnapshot(`
|
||||
- heading "Welcome, user"
|
||||
- link /\\d+ new messages?/
|
||||
- button "Sign out"
|
||||
`);
|
||||
|
||||
// toMatchAriaSnapshot scoped to a region
|
||||
await expect(page.getByRole('navigation')).toMatchAriaSnapshot(`
|
||||
- link "Home"
|
||||
- link /\\d+ new messages?/
|
||||
- link "Profile"
|
||||
`);
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 1. Planning
|
||||
|
||||
Goal: produce a spec file (e.g. `specs/<feature>.plan.md`) that enumerates the scenarios to test. **Always** write the spec to a file.
|
||||
|
||||
### 1.1 Prerequisite: workspace
|
||||
|
||||
Check the workspace has Playwright installed before anything else:
|
||||
|
||||
```bash
|
||||
# Either of these confirms a workspace:
|
||||
test -f playwright.config.ts || test -f playwright.config.js
|
||||
npx --no-install playwright --version
|
||||
```
|
||||
|
||||
If there is no Playwright install, bootstrap one and let the user pick the defaults:
|
||||
|
||||
```bash
|
||||
npm init playwright@latest
|
||||
```
|
||||
|
||||
### 1.2 Prerequisite: seed test
|
||||
|
||||
A **seed test** is a minimal test that lands the page in the state every scenario starts from: navigation to the app, any required login, feature flags, etc. Scenarios assume a fresh start *after* the seed. `--debug=cli` pauses *inside* this test, so the seed is where every planning and generation session begins.
|
||||
|
||||
Minimum viable seed:
|
||||
|
||||
```ts
|
||||
// tests/seed.spec.ts
|
||||
import { test } from '@playwright/test';
|
||||
|
||||
test('seed', async ({ page }) => {
|
||||
await page.goto('https://example.com/');
|
||||
});
|
||||
```
|
||||
|
||||
Preferred — push navigation into a fixture so scenario tests reuse it:
|
||||
|
||||
```ts
|
||||
// tests/fixtures.ts
|
||||
import { test as baseTest } from '@playwright/test';
|
||||
export { expect } from '@playwright/test';
|
||||
|
||||
export const test = baseTest.extend({
|
||||
page: async ({ page }, use) => {
|
||||
await page.goto('https://example.com/');
|
||||
await use(page);
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
```ts
|
||||
// tests/seed.spec.ts
|
||||
import { test } from './fixtures';
|
||||
|
||||
test('seed', async ({ page }) => {
|
||||
// Fixture already navigates. This empty body tells agents where to start.
|
||||
});
|
||||
```
|
||||
|
||||
If no seed exists, create one that at least navigates to the app.
|
||||
|
||||
### 1.3 Explore the app
|
||||
|
||||
Launch the app via the seed in the background and attach:
|
||||
|
||||
```bash
|
||||
PLAYWRIGHT_HTML_OPEN=never npx playwright test tests/seed.spec.ts --debug=cli
|
||||
# wait for "Debugging Instructions" and the session name tw-XXXX
|
||||
playwright-cli attach tw-XXXX
|
||||
```
|
||||
|
||||
Resume so the seed runs, then probe the app:
|
||||
|
||||
```bash
|
||||
playwright-cli resume # resume so that seed test runs fully
|
||||
playwright-cli snapshot # inventory of interactive elements
|
||||
playwright-cli click e5 # follow a flow
|
||||
playwright-cli eval "location.href" # read URL / state
|
||||
playwright-cli show --annotate # ask the user to point at something
|
||||
```
|
||||
|
||||
Map out:
|
||||
|
||||
- Interactive surfaces (forms, buttons, lists, filters, modals).
|
||||
- Primary user journeys end-to-end.
|
||||
- Edge cases: empty states, validation errors, very long input, boundary values.
|
||||
- Persistence: reload, local/session storage, URL fragments.
|
||||
- Navigation: which controls change the URL, back/forward behaviour.
|
||||
|
||||
**Important**: Do not just open the app url with playwright-cli, always go through the test to capture any custom setup done there.
|
||||
**Important**: Stop the background test when done exploring.
|
||||
|
||||
### 1.4 Write the spec file
|
||||
|
||||
Save under `specs/<feature>.plan.md`. Use this structure:
|
||||
|
||||
```markdown
|
||||
# <Feature> Test Plan
|
||||
|
||||
## Application Overview
|
||||
|
||||
<One paragraph describing what the feature does and why it matters.>
|
||||
|
||||
## Test Scenarios
|
||||
|
||||
### 1. <Group Name>
|
||||
|
||||
**Seed:** `tests/seed.spec.ts`
|
||||
|
||||
#### 1.1. <kebab-case-scenario-name>
|
||||
|
||||
**File:** `tests/<group>/<kebab-case-scenario-name>.spec.ts`
|
||||
|
||||
**Steps:**
|
||||
1. <Concrete user step>
|
||||
- expect: <observable outcome>
|
||||
- expect: <another observable outcome>
|
||||
2. <Next step>
|
||||
- expect: <outcome>
|
||||
|
||||
#### 1.2. <next-scenario>
|
||||
...
|
||||
|
||||
### 2. <Next Group>
|
||||
|
||||
**Seed:** `tests/seed.spec.ts`
|
||||
...
|
||||
```
|
||||
|
||||
Guidelines:
|
||||
|
||||
- Each scenario is independent and starts from the seed's fresh state — never chain scenarios.
|
||||
- Scenario names are kebab-case and match the test file name (`should-add-single-todo` → `should-add-single-todo.spec.ts`).
|
||||
- Cover happy path, edge cases, validation, negative flows, persistence.
|
||||
- Write steps at the user level ("Type 'Buy milk' into the input"), not the API level ("call `fill`").
|
||||
- Put observable outcomes in `- expect:` bullets; each becomes an assertion during generation.
|
||||
|
||||
---
|
||||
|
||||
## 2. Generate
|
||||
|
||||
Goal: take a spec file and produce Playwright test files. Optionally update the spec if it has drifted.
|
||||
|
||||
### 2.1 Inputs
|
||||
|
||||
- **Spec file**, e.g. `specs/basic-operations.plan.md`.
|
||||
- **Target**: either a single scenario (e.g. `1.2`), a whole group (`1`), or all.
|
||||
- **Seed file**, read from the `**Seed:**` line of the scenario's group.
|
||||
|
||||
### 2.2 Generate one scenario
|
||||
|
||||
For each target scenario, in sequence (never in parallel — scenarios share the seed session):
|
||||
|
||||
```bash
|
||||
PLAYWRIGHT_HTML_OPEN=never npx playwright test <seed-file> --debug=cli # background
|
||||
playwright-cli attach tw-XXXX
|
||||
# resume
|
||||
```
|
||||
|
||||
**Do not** just open the app url with playwright-cli, always go through the test to capture any custom setup done there.
|
||||
|
||||
Walk the scenario's `Steps:` one by one with `playwright-cli`, treating the spec as the plan and the live app as the source of truth. If a step is vague ("click the button" — which button?), references an element that no longer exists, or contradicts the app's actual behaviour, use your judgement: update the spec to match what the app really does, then keep going. Editing the spec mid-generation is expected.
|
||||
|
||||
Every action prints the equivalent Playwright TypeScript (see [How generation works](#0-how-generation-works)):
|
||||
|
||||
```bash
|
||||
playwright-cli snapshot # find refs
|
||||
playwright-cli fill e3 "John Doe" # -> page.getByRole('textbox', {...}).fill(...)
|
||||
playwright-cli press Enter
|
||||
playwright-cli click e7
|
||||
```
|
||||
|
||||
For each `- expect:` bullet, add an explicit assertion. See [How generation works](#0-how-generation-works) for details.
|
||||
|
||||
Collect the generated code and write the test file at the path given in the spec:
|
||||
|
||||
```ts
|
||||
// spec: specs/basic-operations.plan.md
|
||||
// seed: tests/seed.spec.ts
|
||||
import { test, expect } from './fixtures'; // or '@playwright/test' if no fixtures file
|
||||
|
||||
test.describe('Signing in and out', () => {
|
||||
test('should sign in', async ({ page }) => {
|
||||
// 1. Navigate to the application
|
||||
// (handled by the seed fixture)
|
||||
|
||||
// 2. Type 'John Doe' into the username field
|
||||
await page.getByRole('textbox', { name: 'username' }).fill('John Doe');
|
||||
|
||||
// 3. Type password
|
||||
await page.getByRole('textbox', { name: 'password' }).fill('TestPassword');
|
||||
|
||||
// 4. Press Enter to submit
|
||||
await page.getByRole('textbox', { name: 'password' }).press('Enter');
|
||||
|
||||
await expect(page.getByRole('heading')).toContainText('Welcome, John Doe!');
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
Rules:
|
||||
|
||||
- **One test per file.** File path, describe name, and test name come verbatim from the spec (minus the ordinal).
|
||||
- Prefix each numbered step with a `// N. <step text>` comment before its actions.
|
||||
- Use the describe group name verbatim from the spec (no `1.` ordinal).
|
||||
- Import from `./fixtures` if the project has one; otherwise `@playwright/test`.
|
||||
- **Important**: close the CLI session and stop the background test before moving to the next scenario.
|
||||
|
||||
### 2.3 Generate multiple scenarios
|
||||
|
||||
Loop 2.2 over the targeted scenarios one at a time, restarting the seed between each so every test starts from a clean page. This is safe to parallelise due to unique generated session names - just make sure each test run is stopped.
|
||||
|
||||
### 2.4 Run generated tests
|
||||
|
||||
After generation, run the new tests once:
|
||||
|
||||
```bash
|
||||
PLAYWRIGHT_HTML_OPEN=never npx playwright test tests/<group>/<scenario>.spec.ts
|
||||
```
|
||||
|
||||
Any failure goes to Section 3.
|
||||
|
||||
---
|
||||
|
||||
## 3. Heal
|
||||
|
||||
Goal: fix failing tests, and update the spec if the app's intended behaviour changed.
|
||||
|
||||
### 3.1 Find failing tests
|
||||
|
||||
```bash
|
||||
PLAYWRIGHT_HTML_OPEN=never npx playwright test
|
||||
```
|
||||
|
||||
Record the list of failing `<file>:<line>` entries and process them one at a time. Do not attempt parallel fixes — shared state and the single CLI session make that fragile.
|
||||
|
||||
### 3.2 Debug one failure
|
||||
|
||||
Run the single failing test in debug mode in the background, then attach:
|
||||
|
||||
```bash
|
||||
PLAYWRIGHT_HTML_OPEN=never npx playwright test tests/<group>/<scenario>.spec.ts:<line> --debug=cli
|
||||
# wait for "Debugging Instructions" and the tw-XXXX session name
|
||||
playwright-cli attach tw-XXXX
|
||||
```
|
||||
|
||||
The test is paused at the start. Step forward or run to until just before the failing action or assertion, then diagnose:
|
||||
|
||||
```bash
|
||||
playwright-cli snapshot # did the element change / move / rename?
|
||||
playwright-cli console # app-side errors?
|
||||
playwright-cli requests # failed request? wrong payload?
|
||||
playwright-cli show --annotate # ask the user to point somewhere
|
||||
```
|
||||
|
||||
Common causes: selector drift, new wrapper element, label/ARIA rename, timing (transition, async load), assertion text updated in the app, test data leaking between runs.
|
||||
|
||||
Rehearse the corrected interaction with `playwright-cli` — the generated code in the output is what you paste back into the test.
|
||||
|
||||
### 3.3 Apply the fix
|
||||
|
||||
Edit the test file: update the locator, assertion, step order, or inputs to match the corrected behaviour. Stop the background debug run. Rerun the single test to confirm green.
|
||||
|
||||
Never skip hooks or add sleeps as a fix. Never use `networkidle`.
|
||||
|
||||
### 3.4 Reconcile with the spec
|
||||
|
||||
Open the spec referenced by the `// spec:` header in the test file and locate the scenario that matches the test.
|
||||
|
||||
- **Fix was purely technical** (locator drift, better assertion shape) and the spec's user-level behaviour still matches the app → leave the spec alone.
|
||||
- **Fix changed user-visible steps, inputs, order, or expected outcomes** that the spec describes → update the spec to match reality. Keep the scenario id and file path stable; only the step / expect lines change.
|
||||
- **Unclear whether the app change is intentional** (spec is stale) **or a regression** (test was right, app is wrong) → **stop and ask the user**. Provide:
|
||||
- the scenario id (e.g. `2.3`),
|
||||
- the spec lines that no longer match,
|
||||
- the observed app behaviour (quote a snapshot excerpt or a concrete outcome).
|
||||
|
||||
Only after the user answers, either update the spec (intentional change) or file/flag the test as covering a bug (regression).
|
||||
|
||||
### 3.5 Iteration and giving up
|
||||
|
||||
- Fix failures one at a time; rerun after each.
|
||||
- If after thorough investigation you are confident the test is correct but the app is wrong *and* the user has confirmed it's a bug: mark the test `test.fixme(...)` with a comment pointing at the user's decision or issue link. Never silently skip.
|
||||
|
||||
---
|
||||
|
||||
## Cross-references
|
||||
|
||||
| For... | See |
|
||||
|---|---|
|
||||
| `--debug=cli` / attach mechanics | [playwright-tests.md](playwright-tests.md) |
|
||||
| Mocking requests during exploration/generation | [request-mocking.md](request-mocking.md) |
|
||||
| Managing the CLI browser session | [session-management.md](session-management.md) |
|
||||
@@ -0,0 +1,139 @@
|
||||
# Tracing
|
||||
|
||||
Capture detailed execution traces for debugging and analysis. Traces include DOM snapshots, screenshots, network activity, and console logs.
|
||||
|
||||
## Basic Usage
|
||||
|
||||
```bash
|
||||
# Start trace recording
|
||||
playwright-cli tracing-start
|
||||
|
||||
# Perform actions
|
||||
playwright-cli open https://example.com
|
||||
playwright-cli click e1
|
||||
playwright-cli fill e2 "test"
|
||||
|
||||
# Stop trace recording
|
||||
playwright-cli tracing-stop
|
||||
```
|
||||
|
||||
## Trace Output Files
|
||||
|
||||
When you start tracing, Playwright creates a `traces/` directory with several files:
|
||||
|
||||
### `trace-{timestamp}.trace`
|
||||
|
||||
**Action log** - The main trace file containing:
|
||||
- Every action performed (clicks, fills, navigations)
|
||||
- DOM snapshots before and after each action
|
||||
- Screenshots at each step
|
||||
- Timing information
|
||||
- Console messages
|
||||
- Source locations
|
||||
|
||||
### `trace-{timestamp}.network`
|
||||
|
||||
**Network log** - Complete network activity:
|
||||
- All HTTP requests and responses
|
||||
- Request headers and bodies
|
||||
- Response headers and bodies
|
||||
- Timing (DNS, connect, TLS, TTFB, download)
|
||||
- Resource sizes
|
||||
- Failed requests and errors
|
||||
|
||||
### `resources/`
|
||||
|
||||
**Resources directory** - Cached resources:
|
||||
- Images, fonts, stylesheets, scripts
|
||||
- Response bodies for replay
|
||||
- Assets needed to reconstruct page state
|
||||
|
||||
## What Traces Capture
|
||||
|
||||
| Category | Details |
|
||||
|----------|---------|
|
||||
| **Actions** | Clicks, fills, hovers, keyboard input, navigations |
|
||||
| **DOM** | Full DOM snapshot before/after each action |
|
||||
| **Screenshots** | Visual state at each step |
|
||||
| **Network** | All requests, responses, headers, bodies, timing |
|
||||
| **Console** | All console.log, warn, error messages |
|
||||
| **Timing** | Precise timing for each operation |
|
||||
|
||||
## Use Cases
|
||||
|
||||
### Debugging Failed Actions
|
||||
|
||||
```bash
|
||||
playwright-cli tracing-start
|
||||
playwright-cli open https://app.example.com
|
||||
|
||||
# This click fails - why?
|
||||
playwright-cli click e5
|
||||
|
||||
playwright-cli tracing-stop
|
||||
# Open trace to see DOM state when click was attempted
|
||||
```
|
||||
|
||||
### Analyzing Performance
|
||||
|
||||
```bash
|
||||
playwright-cli tracing-start
|
||||
playwright-cli open https://slow-site.com
|
||||
playwright-cli tracing-stop
|
||||
|
||||
# View network waterfall to identify slow resources
|
||||
```
|
||||
|
||||
### Capturing Evidence
|
||||
|
||||
```bash
|
||||
# Record a complete user flow for documentation
|
||||
playwright-cli tracing-start
|
||||
|
||||
playwright-cli open https://app.example.com/checkout
|
||||
playwright-cli fill e1 "4111111111111111"
|
||||
playwright-cli fill e2 "12/25"
|
||||
playwright-cli fill e3 "123"
|
||||
playwright-cli click e4
|
||||
|
||||
playwright-cli tracing-stop
|
||||
# Trace shows exact sequence of events
|
||||
```
|
||||
|
||||
## Trace vs Video vs Screenshot
|
||||
|
||||
| Feature | Trace | Video | Screenshot |
|
||||
|---------|-------|-------|------------|
|
||||
| **Format** | .trace file | .webm video | .png/.jpeg image |
|
||||
| **DOM inspection** | Yes | No | No |
|
||||
| **Network details** | Yes | No | No |
|
||||
| **Step-by-step replay** | Yes | Continuous | Single frame |
|
||||
| **File size** | Medium | Large | Small |
|
||||
| **Best for** | Debugging | Demos | Quick capture |
|
||||
|
||||
## Best Practices
|
||||
|
||||
### 1. Start Tracing Before the Problem
|
||||
|
||||
```bash
|
||||
# Trace the entire flow, not just the failing step
|
||||
playwright-cli tracing-start
|
||||
playwright-cli open https://example.com
|
||||
# ... all steps leading to the issue ...
|
||||
playwright-cli tracing-stop
|
||||
```
|
||||
|
||||
### 2. Clean Up Old Traces
|
||||
|
||||
Traces can consume significant disk space:
|
||||
|
||||
```bash
|
||||
# Remove traces older than 7 days
|
||||
find .playwright-cli/traces -mtime +7 -delete
|
||||
```
|
||||
|
||||
## Limitations
|
||||
|
||||
- Traces add overhead to automation
|
||||
- Large traces can consume significant disk space
|
||||
- Some dynamic content may not replay perfectly
|
||||
@@ -0,0 +1,143 @@
|
||||
# Video Recording
|
||||
|
||||
Capture browser automation sessions as video for debugging, documentation, or verification. Produces WebM (VP8/VP9 codec).
|
||||
|
||||
## Basic Recording
|
||||
|
||||
```bash
|
||||
# Open browser first
|
||||
playwright-cli open
|
||||
|
||||
# Start recording
|
||||
playwright-cli video-start demo.webm
|
||||
|
||||
# Add a chapter marker for section transitions
|
||||
playwright-cli video-chapter "Getting Started" --description="Opening the homepage" --duration=2000
|
||||
|
||||
# Navigate and perform actions
|
||||
playwright-cli goto https://example.com
|
||||
playwright-cli snapshot
|
||||
playwright-cli click e1
|
||||
|
||||
# Add another chapter
|
||||
playwright-cli video-chapter "Filling Form" --description="Entering test data" --duration=2000
|
||||
playwright-cli fill e2 "test input"
|
||||
|
||||
# Stop and save
|
||||
playwright-cli video-stop
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
|
||||
### 1. Use Descriptive Filenames
|
||||
|
||||
```bash
|
||||
# Include context in filename
|
||||
playwright-cli video-start recordings/login-flow-2024-01-15.webm
|
||||
playwright-cli video-start recordings/checkout-test-run-42.webm
|
||||
```
|
||||
|
||||
### 2. Record entire hero scripts.
|
||||
|
||||
When recording a video for the user or as a proof of work, it is best to create a code snippet and execute it with run-code.
|
||||
It allows inserting appropriate pauses between the actions and annotating the video. There are new Playwright APIs for that.
|
||||
|
||||
1) Perform scenario using CLI and take note of all locators and actions. You'll need those locators to request their bounding boxes for highlight.
|
||||
2) Create a file with the intended script for video (below). Use pressSequentially w/ delay for nice typing, make reasonable pauses.
|
||||
3) Use playwright-cli run-code --filename your-script.js
|
||||
|
||||
**Important**: Overlays are `pointer-events: none` — they do not interfere with page interactions. You can safely keep sticky overlays visible while clicking, filling, or performing any actions on the page.
|
||||
|
||||
```js
|
||||
async page => {
|
||||
await page.screencast.start({ path: 'video.webm', size: { width: 1280, height: 800 } });
|
||||
await page.goto('https://demo.playwright.dev/todomvc');
|
||||
|
||||
// Show a chapter card — blurs the page and shows a dialog.
|
||||
// Blocks until duration expires, then auto-removes.
|
||||
// Use this for simple use cases, but always feel free to hand-craft your own beautiful
|
||||
// overlay via await page.screencast.showOverlay().
|
||||
await page.screencast.showChapter('Adding Todo Items', {
|
||||
description: 'We will add several items to the todo list.',
|
||||
duration: 2000,
|
||||
});
|
||||
|
||||
// Perform action
|
||||
await page.getByRole('textbox', { name: 'What needs to be done?' }).pressSequentially('Walk the dog', { delay: 60 });
|
||||
await page.getByRole('textbox', { name: 'What needs to be done?' }).press('Enter');
|
||||
await page.waitForTimeout(1000);
|
||||
|
||||
// Show next chapter
|
||||
await page.screencast.showChapter('Verifying Results', {
|
||||
description: 'Checking the item appeared in the list.',
|
||||
duration: 2000,
|
||||
});
|
||||
|
||||
// Add a sticky annotation that stays while you perform actions.
|
||||
// Overlays are pointer-events: none, so they won't block clicks.
|
||||
const annotation = await page.screencast.showOverlay(`
|
||||
<div style="position: absolute; top: 8px; right: 8px;
|
||||
padding: 6px 12px; background: rgba(0,0,0,0.7);
|
||||
border-radius: 8px; font-size: 13px; color: white;">
|
||||
✓ Item added successfully
|
||||
</div>
|
||||
`);
|
||||
|
||||
// Perform more actions while the annotation is visible
|
||||
await page.getByRole('textbox', { name: 'What needs to be done?' }).pressSequentially('Buy groceries', { delay: 60 });
|
||||
await page.getByRole('textbox', { name: 'What needs to be done?' }).press('Enter');
|
||||
await page.waitForTimeout(1500);
|
||||
|
||||
// Remove the annotation when done
|
||||
await annotation.dispose();
|
||||
|
||||
// You can also highlight relevant locators and provide contextual annotations.
|
||||
const bounds = await page.getByText('Walk the dog').boundingBox();
|
||||
await page.screencast.showOverlay(`
|
||||
<div style="position: absolute;
|
||||
top: ${bounds.y}px;
|
||||
left: ${bounds.x}px;
|
||||
width: ${bounds.width}px;
|
||||
height: ${bounds.height}px;
|
||||
border: 1px solid red;">
|
||||
</div>
|
||||
<div style="position: absolute;
|
||||
top: ${bounds.y + bounds.height + 5}px;
|
||||
left: ${bounds.x + bounds.width / 2}px;
|
||||
transform: translateX(-50%);
|
||||
padding: 6px;
|
||||
background: #808080;
|
||||
border-radius: 10px;
|
||||
font-size: 14px;
|
||||
color: white;">Check it out, it is right above this text
|
||||
</div>
|
||||
`, { duration: 2000 });
|
||||
|
||||
await page.screencast.stop();
|
||||
}
|
||||
```
|
||||
|
||||
Embrace creativity, overlays are powerful.
|
||||
|
||||
### Overlay API Summary
|
||||
|
||||
| Method | Use Case |
|
||||
|--------|----------|
|
||||
| `page.screencast.showChapter(title, { description?, duration?, styleSheet? })` | Full-screen chapter card with blurred backdrop — ideal for section transitions |
|
||||
| `page.screencast.showOverlay(html, { duration? })` | Custom HTML overlay — use for callouts, labels, highlights |
|
||||
| `disposable.dispose()` | Remove a sticky overlay added without duration |
|
||||
| `page.screencast.hideOverlays()` / `page.screencast.showOverlays()` | Temporarily hide/show all overlays |
|
||||
|
||||
## Tracing vs Video
|
||||
|
||||
| Feature | Video | Tracing |
|
||||
|---------|-------|---------|
|
||||
| Output | WebM file | Trace file (viewable in Trace Viewer) |
|
||||
| Shows | Visual recording | DOM snapshots, network, console, actions |
|
||||
| Use case | Demos, documentation | Debugging, analysis |
|
||||
| Size | Larger | Smaller |
|
||||
|
||||
## Limitations
|
||||
|
||||
- Recording adds slight overhead to automation
|
||||
- Large recordings can consume significant disk space
|
||||
@@ -52,8 +52,14 @@ jobs:
|
||||
- name: Publish to the Gitea Arch registry
|
||||
run: |
|
||||
cd /build/yellowjacket/packaging/arch
|
||||
pkg=$(ls yellowjacket-*.pkg.tar.zst)
|
||||
echo "Uploading $pkg"
|
||||
curl --fail-with-body --user "${OWNER}:${PACKAGE_TOKEN}" \
|
||||
--upload-file "$pkg" \
|
||||
"${SERVER_URL}/api/packages/${OWNER}/arch/${ARCH_REPO}"
|
||||
# makepkg also produces a -debug package (detached symbols); end users
|
||||
# don't need it, so publish only the runtime package(s).
|
||||
for pkg in yellowjacket-*.pkg.tar.zst; do
|
||||
case "$pkg" in
|
||||
yellowjacket-debug-*) continue ;;
|
||||
esac
|
||||
echo "Uploading $pkg"
|
||||
curl --fail-with-body --user "${OWNER}:${PACKAGE_TOKEN}" \
|
||||
--upload-file "$pkg" \
|
||||
"${SERVER_URL}/api/packages/${OWNER}/arch/${ARCH_REPO}"
|
||||
done
|
||||
|
||||
@@ -0,0 +1,309 @@
|
||||
name: CI
|
||||
|
||||
# The other three workflows package and publish; none of them test
|
||||
# anything, so a green tick on this repo used to mean "the Arch package
|
||||
# built", which is not the question anyone was asking. This is the
|
||||
# workflow that gates.
|
||||
#
|
||||
# Both jobs were prototyped end to end in a bare ubuntu:24.04 container
|
||||
# before being written here, so every step below is a transcription of
|
||||
# something observed working rather than something expected to.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: ['**']
|
||||
pull_request:
|
||||
workflow_dispatch:
|
||||
|
||||
# A newer push supersedes an older one on the same ref. Job 2 binds
|
||||
# :34115, so overlapping runs on one runner would fight over the port.
|
||||
concurrency:
|
||||
group: ci-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
GO_VERSION: '1.25.0'
|
||||
# Shared by all three Playwright consumers (@playwright/cli, e2e/'s
|
||||
# @playwright/test, frontend/'s Vitest provider). See the browsers
|
||||
# step in job 2 for why that is not the whole story.
|
||||
PLAYWRIGHT_BROWSERS_PATH: /cache/ms-playwright
|
||||
# Best-effort pnpm store reuse; pnpm reads npm_config_* for its own
|
||||
# config keys. If it ever stops honouring this we lose cache warmth
|
||||
# and nothing else.
|
||||
npm_config_store_dir: /cache/pnpm-store
|
||||
|
||||
jobs:
|
||||
# ---------------------------------------------------------------- #
|
||||
# Job 1: everything that does not need a display. #
|
||||
# ---------------------------------------------------------------- #
|
||||
check:
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
# Not golang:1.25 — this job runs `make ui-test`, which is Vitest
|
||||
# *browser* mode and needs a Chromium and its system libraries
|
||||
# anyway, so the "fast job needs no browser" split does not hold.
|
||||
# Not the Playwright image either: e2e/ pins @playwright/test
|
||||
# ^1.56 and frontend/ pins playwright ^1.62, so a prebuilt browser
|
||||
# set matches at most one of them. Ubuntu 24.04 is also what
|
||||
# Playwright's WebKit build links against, which job 2 needs.
|
||||
image: ubuntu:24.04
|
||||
# GOMODCACHE / GOCACHE / GOLANGCI_LINT_CACHE are already mounted
|
||||
# and exported for every job by the runner's container.options, so
|
||||
# only the Node-side caches are listed here. The runner's
|
||||
# valid_volumes allows anything under the cache root.
|
||||
volumes:
|
||||
- /home/logan/docker/gitea/data/runner/cache/tool:/cache/tool
|
||||
- /home/logan/docker/gitea/data/runner/cache/ms-playwright:/cache/ms-playwright
|
||||
- /home/logan/docker/gitea/data/runner/cache/pnpm-store:/cache/pnpm-store
|
||||
env:
|
||||
PACKAGE_TOKEN: ${{ secrets.PACKAGE_TOKEN }}
|
||||
SERVER_URL: ${{ github.server_url }}
|
||||
REPO: ${{ github.repository }}
|
||||
SHA: ${{ github.sha }}
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
steps:
|
||||
- name: System packages
|
||||
run: |
|
||||
set -eu
|
||||
apt-get update -qq
|
||||
# libwebkit2gtk-4.1-dev and libasound2-dev are not optional:
|
||||
# the app is cgo, and without alsa.pc oto/v3 fails at
|
||||
# `pkg-config --cflags -- alsa` before anything is compiled.
|
||||
apt-get install -y -qq --no-install-recommends \
|
||||
ca-certificates curl git jq build-essential pkg-config \
|
||||
libwebkit2gtk-4.1-dev libgtk-3-dev libasound2-dev ffmpeg
|
||||
|
||||
# Cloned by hand rather than with actions/checkout: that is a JS
|
||||
# action and needs node inside the job container before any step
|
||||
# has had a chance to install it. Same approach as the other
|
||||
# three workflows in this directory.
|
||||
- name: Clone repo at this commit
|
||||
run: |
|
||||
set -eu
|
||||
git clone --quiet \
|
||||
"https://x-access-token:${PACKAGE_TOKEN}@${SERVER_URL#https://}/${REPO}.git" /src
|
||||
git -C /src checkout --quiet --detach "$SHA"
|
||||
git -C /src log --oneline -1
|
||||
# make bindings-check compares against the work tree, so git
|
||||
# has to be willing to operate on a directory it does not own.
|
||||
git config --global --add safe.directory /src
|
||||
|
||||
- name: Go toolchain
|
||||
run: |
|
||||
set -eu
|
||||
if [ ! -x /cache/tool/go/bin/go ] || ! /cache/tool/go/bin/go version | grep -q "$GO_VERSION"; then
|
||||
mkdir -p /cache/tool && rm -rf /cache/tool/go
|
||||
curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" | tar -C /cache/tool -xz
|
||||
fi
|
||||
echo "/cache/tool/go/bin" >> "$GITHUB_PATH"
|
||||
/cache/tool/go/bin/go version
|
||||
|
||||
- name: Node toolchain
|
||||
run: |
|
||||
set -eu
|
||||
curl -fsSL https://deb.nodesource.com/setup_22.x | bash -
|
||||
apt-get install -y -qq --no-install-recommends nodejs
|
||||
corepack enable
|
||||
node --version
|
||||
|
||||
- name: Vitest provider browser
|
||||
working-directory: /src/frontend
|
||||
run: |
|
||||
set -eu
|
||||
pnpm install --frozen-lockfile
|
||||
npx playwright install --with-deps chromium
|
||||
|
||||
# main.go embeds the built frontend (`//go:embed all:frontend/dist`),
|
||||
# so *every* Go typecheck needs it to exist first — lint, test and
|
||||
# bindings-check all fail with "pattern all:frontend/dist: no
|
||||
# matching files found" on a fresh clone. This never bites locally
|
||||
# because anyone who has run the app once has a dist/ lying around,
|
||||
# which is exactly why CI has to do it explicitly.
|
||||
- name: Build the frontend
|
||||
working-directory: /src/frontend
|
||||
run: pnpm build
|
||||
|
||||
# `make lint` and `make test` each run all three build
|
||||
# configurations (app / indexbuild / dev) with matching tag sets.
|
||||
- name: Lint
|
||||
working-directory: /src
|
||||
run: make lint
|
||||
|
||||
- name: Test
|
||||
working-directory: /src
|
||||
run: make test
|
||||
|
||||
- name: Typecheck the frontend
|
||||
working-directory: /src/frontend
|
||||
run: npx tsc --noEmit
|
||||
|
||||
- name: Component and store suite
|
||||
working-directory: /src
|
||||
run: make ui-test
|
||||
|
||||
# frontend/wailsjs is generated by `wails`, not by `go generate`,
|
||||
# so the codegen pre-commit hook does not cover it.
|
||||
- name: Bindings are current
|
||||
working-directory: /src
|
||||
run: make bindings-check
|
||||
|
||||
# Every `make <target>` named under .pi/**/*.md must exist, so an
|
||||
# agent is never sent at a command that was renamed away.
|
||||
- name: Documented make targets exist
|
||||
working-directory: /src
|
||||
run: make skill-check
|
||||
|
||||
# ---------------------------------------------------------------- #
|
||||
# Job 2: the real app, under a virtual display. #
|
||||
# ---------------------------------------------------------------- #
|
||||
e2e:
|
||||
runs-on: ubuntu-latest
|
||||
needs: check
|
||||
container:
|
||||
image: ubuntu:24.04
|
||||
volumes:
|
||||
- /home/logan/docker/gitea/data/runner/cache/tool:/cache/tool
|
||||
- /home/logan/docker/gitea/data/runner/cache/ms-playwright:/cache/ms-playwright
|
||||
- /home/logan/docker/gitea/data/runner/cache/pnpm-store:/cache/pnpm-store
|
||||
env:
|
||||
PACKAGE_TOKEN: ${{ secrets.PACKAGE_TOKEN }}
|
||||
SERVER_URL: ${{ github.server_url }}
|
||||
REPO: ${{ github.repository }}
|
||||
SHA: ${{ github.sha }}
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
# The explore artifact is stubbed with a dead address, exactly as
|
||||
# scripts/seed-sandbox.sh does it. Serving a real cut-down
|
||||
# artifact would mean building one under the indexbuild tag from
|
||||
# dump state this runner does not have, and no spec asserts on
|
||||
# explore content, so it would buy nothing. Note that
|
||||
# dev-headless.sh does *not* set this itself — only seed-sandbox
|
||||
# does — so the run would otherwise fetch the real artifact over
|
||||
# the network. It is also worth ~8x on suite wall clock: the
|
||||
# testctl DB restore spec copies every table, and the real
|
||||
# artifact makes that table set enormous.
|
||||
YJ_CORE_INDEX_URL: 'http://127.0.0.1:1/none.tar.zst'
|
||||
steps:
|
||||
- name: System packages
|
||||
run: |
|
||||
set -eu
|
||||
apt-get update -qq
|
||||
apt-get install -y -qq --no-install-recommends \
|
||||
ca-certificates curl git jq build-essential pkg-config \
|
||||
libwebkit2gtk-4.1-dev libgtk-3-dev libasound2-dev \
|
||||
xvfb dbus dbus-x11 ffmpeg libasound2t64
|
||||
|
||||
- name: Clone repo at this commit
|
||||
run: |
|
||||
set -eu
|
||||
git clone --quiet \
|
||||
"https://x-access-token:${PACKAGE_TOKEN}@${SERVER_URL#https://}/${REPO}.git" /src
|
||||
git -C /src checkout --quiet --detach "$SHA"
|
||||
git config --global --add safe.directory /src
|
||||
|
||||
- name: Go toolchain
|
||||
run: |
|
||||
set -eu
|
||||
if [ ! -x /cache/tool/go/bin/go ] || ! /cache/tool/go/bin/go version | grep -q "$GO_VERSION"; then
|
||||
mkdir -p /cache/tool && rm -rf /cache/tool/go
|
||||
curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" | tar -C /cache/tool -xz
|
||||
fi
|
||||
echo "/cache/tool/go/bin" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Node toolchain
|
||||
run: |
|
||||
set -eu
|
||||
curl -fsSL https://deb.nodesource.com/setup_22.x | bash -
|
||||
apt-get install -y -qq --no-install-recommends nodejs
|
||||
corepack enable
|
||||
|
||||
# scripts/seed-sandbox.sh drives the real AddLibrary binding
|
||||
# through playwright-cli, so the CLI has to be on PATH.
|
||||
- name: Playwright CLI
|
||||
run: npm install -g @playwright/cli
|
||||
|
||||
- name: Browsers
|
||||
working-directory: /src/e2e
|
||||
run: |
|
||||
set -eu
|
||||
# PLAYWRIGHT_BROWSERS_PATH unifies the *location*, not the
|
||||
# *revisions*: @playwright/cli bundles its own playwright-core
|
||||
# pinned to a different Chromium build than @playwright/test,
|
||||
# so each installs its own into the shared directory. Drop
|
||||
# either line and the other fails with "Browser chromium is
|
||||
# not installed; expected executable at ...".
|
||||
pnpm install --frozen-lockfile
|
||||
npx playwright install --with-deps chromium webkit
|
||||
playwright-cli install-browser chromium
|
||||
|
||||
# oto/v3 talks to libasound directly, and a container has no
|
||||
# PulseAudio socket to fall back on. ALSA's null plugin advances
|
||||
# its pointer on a timer rather than discarding instantly, so beep
|
||||
# is consumed at real-time rate and the elapsed clock actually
|
||||
# moves — which playback.spec.ts asserts. Measured: InitSpeaker
|
||||
# succeeds in ~36 ms and all six playback specs pass. Without
|
||||
# this, app.go joins the failure into startupErr and everything
|
||||
# except playback still works, so the suite fails looking like
|
||||
# flake rather than like a missing dependency.
|
||||
- name: Null audio sink
|
||||
run: |
|
||||
printf 'pcm.!default { type null }\nctl.!default { type null }\n' > /etc/asound.conf
|
||||
|
||||
- name: Fixtures and seed
|
||||
working-directory: /src
|
||||
run: |
|
||||
set -eu
|
||||
make testdata
|
||||
# A seed is built by *running the app* and driving the real
|
||||
# AddLibrary binding — never by writing config.toml and DB
|
||||
# rows, which would be a second description of a valid YJ_HOME.
|
||||
make sandbox-seed NAME=default
|
||||
|
||||
# dev-headless daemonises (writes .dev/app.pid and returns), which
|
||||
# is why Playwright's webServer cannot supervise it and why this is
|
||||
# a step of its own. e2e/'s globalSetup checks /__test/health.
|
||||
- name: Start the app headless
|
||||
working-directory: /src
|
||||
run: make dev-headless SEED=default
|
||||
|
||||
- name: E2E — chromium
|
||||
working-directory: /src
|
||||
run: make e2e
|
||||
|
||||
# Playwright's Linux WebKit links Ubuntu 24.04 libraries that Arch
|
||||
# does not provide, so this cannot run on a dev machine at all: CI
|
||||
# is the only place we get any signal about the WebKit2GTK renderer
|
||||
# we actually ship. Required rather than advisory because it was
|
||||
# measured green (19/19) in this exact container before being
|
||||
# enabled, and because nothing in e2e/ compares pixels — every
|
||||
# assertion is an event payload, a testid, an attribute or backend
|
||||
# state, so a WebKit failure here is an engine bug, not baseline
|
||||
# noise. It costs ~11 s.
|
||||
- name: E2E — webkit
|
||||
working-directory: /src
|
||||
env:
|
||||
YJ_E2E_WEBKIT: '1'
|
||||
run: make e2e E2E_ARGS="--project=webkit"
|
||||
|
||||
# The app log is the only place a hung binding call explains
|
||||
# itself, so put it in the job log where `gitea_ci job_logs` can
|
||||
# reach it without downloading an artifact.
|
||||
- name: App log on failure
|
||||
if: failure()
|
||||
working-directory: /src
|
||||
run: tail -n 200 .dev/app.log || true
|
||||
|
||||
- name: Upload traces and screenshots
|
||||
if: failure()
|
||||
continue-on-error: true
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: e2e-report-${{ github.run_id }}
|
||||
path: |
|
||||
/src/e2e/playwright-report/
|
||||
/src/.dev/app.log
|
||||
retention-days: 7
|
||||
|
||||
- name: Stop the app
|
||||
if: always()
|
||||
working-directory: /src
|
||||
run: make dev-stop || true
|
||||
@@ -0,0 +1,86 @@
|
||||
name: Sync Homebrew formula
|
||||
|
||||
# On every version tag, recompute the release tarball checksum and push an
|
||||
# updated Formula/yellowjacket.rb into the Homebrew tap repo. Keeping the tap
|
||||
# in a separate repo (github.com/Shadow-Puppet/homebrew-yellowjacket) is what
|
||||
# lets users install with a single command:
|
||||
#
|
||||
# brew install shadow-puppet/yellowjacket/yellowjacket
|
||||
#
|
||||
# (`shadow-puppet/yellowjacket` is shorthand for the homebrew-yellowjacket repo;
|
||||
# brew auto-taps it, so no separate `brew tap` step is needed.)
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "v*"
|
||||
|
||||
jobs:
|
||||
sync-formula:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
# GitHub PAT (or fine-grained token) with write access to the tap repo.
|
||||
TAP_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }}
|
||||
# Gitea source that serves the release tarball referenced by the formula.
|
||||
SOURCE_TARBALL_BASE: https://git.ljones.me/yonlu/yellowjacket/archive
|
||||
# separate GitHub tap repo the formula is published to.
|
||||
TAP_REPO: Shadow-Puppet/homebrew-yellowjacket
|
||||
steps:
|
||||
- name: Check out source (for the canonical formula)
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Compute version and tarball checksum
|
||||
run: |
|
||||
set -euo pipefail
|
||||
TAG="${GITHUB_REF_NAME}" # e.g. v1.3.0
|
||||
VERSION="${TAG#v}" # e.g. 1.3.0
|
||||
TARBALL="${SOURCE_TARBALL_BASE}/${TAG}.tar.gz"
|
||||
|
||||
echo "Fetching ${TARBALL}"
|
||||
# Retry briefly: the tag archive can lag a few seconds behind the push.
|
||||
for attempt in 1 2 3 4 5; do
|
||||
if curl -fSsL "$TARBALL" -o release.tar.gz; then
|
||||
break
|
||||
fi
|
||||
echo "attempt ${attempt} failed, retrying..."
|
||||
sleep 5
|
||||
done
|
||||
|
||||
SHA256="$(sha256sum release.tar.gz | cut -d' ' -f1)"
|
||||
echo "version=${VERSION} sha256=${SHA256}"
|
||||
|
||||
echo "VERSION=${VERSION}" >> "$GITHUB_ENV"
|
||||
echo "SHA256=${SHA256}" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Render the formula with the new version and checksum
|
||||
run: |
|
||||
set -euo pipefail
|
||||
src="packaging/homebrew/Formula/yellowjacket.rb"
|
||||
# Rewrite only the two managed lines; the interpolated url picks up the
|
||||
# new version automatically.
|
||||
sed -E \
|
||||
-e "s|^ version \".*\"| version \"${VERSION}\"|" \
|
||||
-e "s|^ sha256 \".*\"| sha256 \"${SHA256}\"|" \
|
||||
"$src" > yellowjacket.rb
|
||||
echo "----- rendered formula -----"
|
||||
cat yellowjacket.rb
|
||||
|
||||
- name: Push to the Homebrew tap repo
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git clone "https://x-access-token:${TAP_TOKEN}@github.com/${TAP_REPO}.git" tap
|
||||
mkdir -p tap/Formula
|
||||
cp yellowjacket.rb tap/Formula/yellowjacket.rb
|
||||
|
||||
cd tap
|
||||
git config user.name "yellowjacket-ci"
|
||||
git config user.email "yj@yellowjacket.app"
|
||||
|
||||
if git diff --quiet; then
|
||||
echo "Formula already up to date; nothing to push."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
git add Formula/yellowjacket.rb
|
||||
git commit -m "yellowjacket ${VERSION}"
|
||||
git push origin HEAD:main
|
||||
@@ -0,0 +1,162 @@
|
||||
name: Search index maintenance
|
||||
|
||||
# indexbuild decides what to do from the index's own state, so every
|
||||
# trigger below runs the same command:
|
||||
#
|
||||
# no completed import -> build (first run, or resume a partial one)
|
||||
# import older than 6mo -> rebuild (re-import from the newest dump)
|
||||
# otherwise -> refresh (fold in new incremental listens)
|
||||
#
|
||||
# A refresh is cheap and no-ops when nothing new has been published, so
|
||||
# running it on every push to main is safe.
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
schedule:
|
||||
# Weekly update pass. The 6-month rebuild is triggered by the same
|
||||
# command when it notices the import has aged out.
|
||||
- cron: '0 4 * * 1'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
mode:
|
||||
description: 'auto | build | refresh | rebuild'
|
||||
required: false
|
||||
default: 'auto'
|
||||
budget:
|
||||
description: 'Max build time this run'
|
||||
required: false
|
||||
default: '3h'
|
||||
artists:
|
||||
description: 'Top artists in the core artifact'
|
||||
required: false
|
||||
default: '50000'
|
||||
|
||||
# Runs share one persistent working directory, so they must not overlap.
|
||||
# A push landing mid-build waits rather than corrupting the checkpoint.
|
||||
concurrency:
|
||||
group: search-index
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
maintain-index:
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
# CGO is not needed: the project uses the pure-Go modernc sqlite
|
||||
# driver, and neither command imports the Wails app.
|
||||
image: golang:1.25
|
||||
# This host path must exist on the runner and be listed verbatim in
|
||||
# act_runner's container.valid_volumes. It holds explore-staging/
|
||||
# (counts.bin + state.json) and yj.db — the checkpoint that makes
|
||||
# resuming possible. Losing it means re-downloading ~205GB.
|
||||
volumes:
|
||||
- /srv/yellowjacket/index-cache:/cache
|
||||
env:
|
||||
YJ_HOME: /cache
|
||||
CGO_ENABLED: '0'
|
||||
PACKAGE_TOKEN: ${{ secrets.PACKAGE_TOKEN }}
|
||||
SERVER_URL: ${{ github.server_url }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.repository }}
|
||||
SHA: ${{ github.sha }}
|
||||
MODE: ${{ inputs.mode || 'auto' }}
|
||||
BUDGET: ${{ inputs.budget || '3h' }}
|
||||
ARTISTS: ${{ inputs.artists || '50000' }}
|
||||
steps:
|
||||
# Cloned by hand rather than with actions/checkout: that is a JS
|
||||
# action and needs node inside the job container, which the golang
|
||||
# image does not carry. Same approach as arch-package.yml.
|
||||
- name: Clone repo at this commit
|
||||
run: |
|
||||
set -eu
|
||||
git clone --quiet \
|
||||
"https://x-access-token:${PACKAGE_TOKEN}@${SERVER_URL#https://}/${REPO}.git" \
|
||||
/src
|
||||
git -C /src checkout --quiet --detach "$SHA"
|
||||
git -C /src log --oneline -1
|
||||
|
||||
- name: Verify the cache volume
|
||||
run: |
|
||||
set -eu
|
||||
mkdir -p /cache
|
||||
# A RAM-backed cache would defeat the point: the checkpoint has
|
||||
# to outlive the job, and the import wants real disk headroom.
|
||||
fstype=$(stat -f -c %T /cache || echo unknown)
|
||||
echo "cache fstype: $fstype"
|
||||
case "$fstype" in
|
||||
tmpfs|ramfs)
|
||||
echo "::error::/cache is RAM-backed; use a disk-backed host path."
|
||||
exit 1 ;;
|
||||
esac
|
||||
df -h /cache
|
||||
|
||||
- name: Build tools
|
||||
working-directory: /src
|
||||
# The dump importer is behind the `indexbuild` tag so it is not
|
||||
# linked into the app binary; cmd/indexbuild carries the same tag
|
||||
# and will not build without it.
|
||||
run: |
|
||||
go build -tags indexbuild -o /usr/local/bin/ ./cmd/indexbuild
|
||||
go build -o /usr/local/bin/ ./cmd/indexexport
|
||||
|
||||
- name: Maintain index
|
||||
id: maintain
|
||||
run: |
|
||||
set +e
|
||||
indexbuild -mode "$MODE" -budget "$BUDGET"
|
||||
code=$?
|
||||
set -e
|
||||
case "$code" in
|
||||
0) ;;
|
||||
3) echo "::notice::Build checkpointed with work remaining — rerun to continue." ;;
|
||||
*) exit "$code" ;;
|
||||
esac
|
||||
|
||||
# Publishing only on `changed` keeps identical artifacts from
|
||||
# accumulating when a refresh finds nothing new.
|
||||
- name: Export core artifact
|
||||
if: steps.maintain.outputs.complete == 'true' && steps.maintain.outputs.changed == 'true'
|
||||
run: |
|
||||
set -eu
|
||||
command -v zstd >/dev/null 2>&1 || { apt-get update -qq && apt-get install -y -qq zstd; }
|
||||
indexexport -o /tmp/core-index.db -artists "$ARTISTS"
|
||||
zstd -19 -T0 -q -f /tmp/core-index.db -o /tmp/core-index.db.zst
|
||||
sha256sum /tmp/core-index.db.zst | tee /tmp/core-index.db.zst.sha256
|
||||
ls -lh /tmp/core-index.db.zst
|
||||
|
||||
- name: Publish to the Gitea package registry
|
||||
if: steps.maintain.outputs.complete == 'true' && steps.maintain.outputs.changed == 'true'
|
||||
run: |
|
||||
set -eu
|
||||
pkg="${SERVER_URL}/api/packages/${OWNER}/generic/yellowjacket-core-index"
|
||||
|
||||
# Published twice: under a dated version for history, and under
|
||||
# the fixed "latest" version the client fetches. Clients cannot
|
||||
# discover the newest dated version on their own — the package
|
||||
# listing API requires a token, while a plain file GET does not
|
||||
# — so "latest" is what makes an anonymous first run possible.
|
||||
#
|
||||
# A generic package rejects re-uploading a filename that already
|
||||
# exists, so "latest" is deleted before being rewritten. It is
|
||||
# absent on the very first publish, hence the tolerated 404.
|
||||
curl --silent --show-error --user "${OWNER}:${PACKAGE_TOKEN}" \
|
||||
--request DELETE "${pkg}/latest" || true
|
||||
|
||||
for version in "$(date -u +%Y%m%d)" latest; do
|
||||
for f in core-index.db.zst core-index.db.zst.sha256; do
|
||||
echo "Uploading $f -> $version"
|
||||
curl --fail-with-body --user "${OWNER}:${PACKAGE_TOKEN}" \
|
||||
--upload-file "/tmp/$f" "${pkg}/${version}/${f}"
|
||||
done
|
||||
done
|
||||
|
||||
- name: Summary
|
||||
if: always()
|
||||
run: |
|
||||
echo "complete=${{ steps.maintain.outputs.complete }}"
|
||||
echo "changed=${{ steps.maintain.outputs.changed }}"
|
||||
if [ "${{ steps.maintain.outputs.complete }}" != "true" ]; then
|
||||
echo "Build incomplete — rerun to continue from the checkpoint."
|
||||
echo "Progress lives in /cache/data/explore-staging."
|
||||
elif [ "${{ steps.maintain.outputs.changed }}" != "true" ]; then
|
||||
echo "Nothing new to publish."
|
||||
fi
|
||||
+14
@@ -3,6 +3,20 @@ node_modules
|
||||
build
|
||||
test_data
|
||||
test.db
|
||||
|
||||
# ── Agent development harness (plan 005) ──
|
||||
# playwright-cli scratch output (snapshots, console logs, screenshots)
|
||||
.playwright-cli/
|
||||
# headless app process state: pid file + captured stdout
|
||||
.dev/
|
||||
# Playwright spec output: traces, screenshots and videos of failures
|
||||
e2e/test-results/
|
||||
e2e/playwright-report/
|
||||
|
||||
# Vitest browser-mode scratch (screenshot diffs, failure captures).
|
||||
# The committed baselines under frontend/test/**/__screenshots__ stay.
|
||||
frontend/.vitest-attachments/
|
||||
|
||||
.aider*
|
||||
lefthook-local.yml
|
||||
|
||||
|
||||
+118
@@ -0,0 +1,118 @@
|
||||
# Work log
|
||||
|
||||
Temporal memory: what happened and what's next. Structure lives in
|
||||
`CLAUDE.md`, operational instructions in `.pi/skills/yellowjacket-dev/`,
|
||||
measured discoveries in `.planning/NOTES.md`. Don't duplicate those here.
|
||||
|
||||
## Current state
|
||||
|
||||
Plan 005 (agent development harness) is **complete — all seven
|
||||
phases**. Everything from phase 1 onward is still **uncommitted**: one
|
||||
large but coherent working-tree diff, nothing pushed.
|
||||
|
||||
All four tiers verified green from a cold, cleaned state:
|
||||
`make ui-test` 313 passed, `make lint` 0 issues × 3 configurations,
|
||||
`make test` green × 3 passes, `make e2e` 19 passed. Both CI jobs
|
||||
verified green in a bare `ubuntu:24.04` container, including 19/19 on
|
||||
WebKit.
|
||||
|
||||
**Committed and pushed** as `5ca6cad` (the harness) + `ccacd67` (a CI
|
||||
fix), and **green on the real runner**: job `check` ~4 min, job `e2e`
|
||||
~3 min with 19/19 chromium *and* 19/19 webkit. One commit rather than
|
||||
seven because the working tree was the end state, not per-phase
|
||||
snapshots — `Makefile`, `CLAUDE.md` and `lefthook.yml` are touched by
|
||||
nearly every phase, so a split would have been fabricated history.
|
||||
|
||||
Still unverified, because no run has failed yet: the
|
||||
`actions/upload-artifact` step (`continue-on-error`, so it cannot mask
|
||||
a real failure) and whether pnpm honours `npm_config_store_dir` for
|
||||
store caching. Worth checking the next time a spec legitimately fails.
|
||||
|
||||
- [ ] `gitea_ci`'s `job_logs` returns 404 on Gitea 1.27.1 — the endpoint
|
||||
is not exposed. Logs come from the VPS instead: `zstdcat` the file
|
||||
under `gitea/actions_log/<owner>/<repo>/<xx>/<task_id>.log.zst`,
|
||||
and note `zstdcat` is not in the gitea container, so
|
||||
`docker cp` it out first. Job status is `action_run_job.status`
|
||||
(1 success, 2 failure, 4 skipped, 5 waiting, 6 running).
|
||||
Probably belongs in the `gitea` skill, not here.
|
||||
|
||||
Open items deliberately not fixed: WAV tags are write-only
|
||||
(`TestWAVTagsAreNotReadableYet`), `themeStore.loadFromBackend`'s failure
|
||||
handler cannot recover, `backend/playlist` has no CRUD suite.
|
||||
|
||||
## Log
|
||||
|
||||
### 2026-08-11 — cold skill run, then phase 7 (CI)
|
||||
|
||||
- **Followed the skill cold first**, as the last session asked. It
|
||||
works: app up from a wiped `.dev/`, an undocumented flow driven
|
||||
(queue panel + shuffle, asserted on `QueueModeChanged`), stopped —
|
||||
~1 minute, no dead ends. One real config bug: `outputDir` in
|
||||
`.playwright/cli.config.json` resolves against **cwd**, not the
|
||||
config file's directory (only `initScript` does that), so snapshots
|
||||
were landing above the repo and a *stale* one from the previous
|
||||
session answered `ls -t` instead. That cost a DOM walk to disprove a
|
||||
regression that did not exist. Four smaller doc gaps fixed
|
||||
(`sandbox-seed` already runs `testdata`; `ui-setup`/`e2e-setup` were
|
||||
undocumented prerequisites; `snapshot` prints a path; `dev-stop`
|
||||
leaves the browser open), plus `dev-headless.sh`'s own banner, which
|
||||
was suggesting the bare `window.go` call its next paragraph warns
|
||||
against.
|
||||
- **Built both CI jobs as container scripts before writing any YAML**,
|
||||
then transcribed the YAML back out and re-ran it to prove the
|
||||
transcription. Push-and-see is a bad loop on a self-hosted runner.
|
||||
- **It found a real bug immediately**: `make lint` omitted
|
||||
`webkit2_41` on all three passes, so it was linting configurations
|
||||
nothing builds. Invisible on Arch (which still ships
|
||||
`webkit2gtk-4.0.pc`), fatal on Ubuntu 24.04. Tag sets now match
|
||||
`make test`.
|
||||
- **Both open decisions settled by measurement**: ALSA `null` PCM for
|
||||
audio (no daemon; the elapsed clock really advances), dead-address
|
||||
stub for the explore artifact (and setting it for the *app* run, not
|
||||
just seeding, is worth 8x on suite wall clock). **WebKit is a
|
||||
required step** — it had never been run anywhere, so one throwaway
|
||||
container run replaced a coin flip with 19/19 at +11 s.
|
||||
|
||||
### 2026-08-10 — phase 6, pi affordances
|
||||
|
||||
- Added `.pi/skills/yellowjacket-dev/` as a directory rather than a flat
|
||||
file: only the description is always in context, so `SKILL.md` stays
|
||||
short enough that reading it whole is never a decision, and the deeper
|
||||
material sits in `references/{harness,fixtures,ui-tier,schema-change}.md`.
|
||||
- Settled the CLAUDE.md-vs-skill split **grammatically, not topically**,
|
||||
because a topical split is what rots — every new fact gets two
|
||||
plausible homes. Three docs, three tenses: NOTES.md is past
|
||||
(measured, dated, append-only), CLAUDE.md is present (what the system
|
||||
is), the skill is imperative (what to run). A new paragraph's tense
|
||||
decides where it goes.
|
||||
- The five gotchas (binding timeouts, first-run wizard, `pkill -f`,
|
||||
seeds-by-running, WebKit-is-CI-only) went **inline in SKILL.md**, not
|
||||
into a reference: you need them before the failure, not after.
|
||||
- Trimmed CLAUDE.md's "Fixtures and the headless harness" section by
|
||||
about half — the command sequences and gotchas it was carrying are now
|
||||
the skill's, and leaving both would have created exactly the duplicate
|
||||
description this repo has a standing rule against.
|
||||
- Added `make skill-check` / `scripts/skill-check.sh` + a pre-commit
|
||||
hook: every command in `.pi/**/*.md` must be a real `make` target, so
|
||||
the Makefile stays the source of truth for invocation and a renamed
|
||||
target fails a commit instead of misleading an agent later. Verified
|
||||
it fails (it caught its own not-yet-created target) and passes.
|
||||
- Added the `/e2e` prompt template: promoting a hand-driven
|
||||
`playwright-cli` session into a spec is a transcription with four
|
||||
fixed substitutions (refs → testids, sleeps → `waitForEvent`, raw
|
||||
`window.go` → `callBinding`, short fixture → `LONG_TRACK`), plus three
|
||||
runs — pass, pass again, pass after a DB restore — because the usual
|
||||
failure is a spec depending on state the hand-driving left behind.
|
||||
- One shell trap: under `set -euo pipefail`, `x="$(make -pqRr | …)"`
|
||||
fails the whole assignment, because `make -q` exits non-zero when a
|
||||
target is out of date and `pipefail` propagates it.
|
||||
|
||||
### Earlier
|
||||
|
||||
Phases 1–5 of plan 005: fixture generator and manifest, headless launch
|
||||
and seeds, the event bridge + `data-testid` pass + `backend/testctl` +
|
||||
`e2e/`, the Vitest component tier + `make bindings-check`, and the
|
||||
`events.Emit` wrapper with its in-process service-event tests. Recaps
|
||||
and the five "verified end to end" blocks are in
|
||||
`.planning/plans/active/005-agent-development-harness.md`; the lessons
|
||||
are in `.planning/NOTES.md`.
|
||||
@@ -0,0 +1,55 @@
|
||||
---
|
||||
description: Promote a hand-driven playwright-cli session into a committed spec in e2e/
|
||||
argument-hint: "[name of the flow]"
|
||||
---
|
||||
Promote the flow I just drove by hand into a committed Playwright spec.
|
||||
Flow: ${@:-infer it from the playwright-cli commands in this session}
|
||||
|
||||
This is a transcription with fixed substitutions, not a fresh test.
|
||||
Work from what actually happened in this session, not from what the UI
|
||||
looks like it should do.
|
||||
|
||||
**1. Recover the flow.** List the `playwright-cli` calls made this
|
||||
session, in order, and the assertion each one was really checking. Then,
|
||||
before writing anything, ask the running app what fired:
|
||||
|
||||
```
|
||||
playwright-cli -s=yj eval "() => window.__yjEvents.names()"
|
||||
```
|
||||
|
||||
Await the events that are actually in that list. Do not guess event
|
||||
names from `backend/events/`.
|
||||
|
||||
**2. Substitute, one for one.**
|
||||
|
||||
- `click e15` → a role or `data-testid` selector. Snapshot refs are
|
||||
per-snapshot and meaningless in a spec. If the only stable selector
|
||||
would be structural, add a `data-testid` to the Lit component and
|
||||
re-run `make ui-test`.
|
||||
- any sleep, or "it looked settled" → `waitForEvent(app, 'X')`.
|
||||
- `window.go.…` → `callBinding(app, path, args)`, which times out.
|
||||
- a short fixture track → `LONG_TRACK`, if the flow needs playback to
|
||||
still be running on the next line. Every other fixture is 2–6 s.
|
||||
- `getByRole('button', { name })` → add `exact: true`.
|
||||
|
||||
**3. Place it.** `e2e/specs/<area>.spec.ts`, importing `test`, `expect`
|
||||
and the helpers from `../support/fixtures.js` — never `@playwright/test`
|
||||
directly. Match the surrounding specs' comment style: say what the test
|
||||
is protecting against, not what the lines do.
|
||||
|
||||
**4. Prove it is a spec and not a recording.** Three runs, in order:
|
||||
|
||||
```
|
||||
make e2e E2E_ARGS='--grep "<name>"' # it passes
|
||||
make e2e E2E_ARGS='--grep "<name>"' # again — catches dependence on
|
||||
# state the first run left
|
||||
```
|
||||
|
||||
then once more after restoring the database through `/__test/`, which
|
||||
catches dependence on state *my hand-driving* left behind — the single
|
||||
most likely way a promoted spec passes here and fails in CI. Leave the
|
||||
database as you found it: snapshot/restore, or reset in `beforeEach`.
|
||||
|
||||
**5. Then the whole suite:** `make e2e`. If the promotion turned up a
|
||||
new trap, append it to `.planning/NOTES.md`; if it turned up a bug,
|
||||
tell me rather than asserting the broken behaviour.
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"skills": ["../.claude/skills"]
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
---
|
||||
name: yellowjacket-dev
|
||||
description: Operating YellowJacket's development harness — which of the four test tiers to use for a given change, how to run the app headless and drive it with playwright-cli, seed and sandbox lifecycle, the three build-tag passes, and the failure modes that waste a cycle if you meet them cold. Use whenever building, running, testing or debugging this repo.
|
||||
---
|
||||
|
||||
# Working on YellowJacket
|
||||
|
||||
`CLAUDE.md` says what this system **is**. This skill says what to
|
||||
**run**. `.planning/NOTES.md` records what we **measured** and when.
|
||||
Keep them in those three tenses: if something here is wrong, fix it
|
||||
here and add the discovery to `NOTES.md` — do not add a corrective
|
||||
paragraph to `CLAUDE.md`.
|
||||
|
||||
Every command below is a `make` target on purpose. The Makefile is the
|
||||
source of truth for *how* to invoke something; this file only decides
|
||||
*which* and *in what order*. `make skill-check` fails if a target named
|
||||
here has disappeared.
|
||||
|
||||
## Read this part before you fail
|
||||
|
||||
Five things cost a cycle each the first time. They are here, not in a
|
||||
reference, because you need them *before* the failure, not after.
|
||||
|
||||
- **Time out every binding call.** A bound Go method called with wrong
|
||||
argument types makes the backend log `error parsing arguments` and
|
||||
**never fire the callback**, so the promise hangs forever. Use
|
||||
`window.__yjEvents.call(path, args, ms)` (browser) or `callBinding`
|
||||
(specs), never a bare `window.go.…`. When one hangs anyway,
|
||||
`make dev-logs` — `.dev/app.log` is the only place the reason appears.
|
||||
- **Nothing is clickable on a fresh `YJ_HOME`.** `<first-run-wizard>`
|
||||
intercepts all pointer events until a library exists, and the click
|
||||
fails with a Playwright interception error that reads like a selector
|
||||
bug. Use a seed unless you are *testing* the wizard, in which case
|
||||
`make dev-headless-fresh`.
|
||||
- **Never `pkill -f`.** The pattern matches the invoking shell's own
|
||||
command line, killing it and silently dropping the rest of your
|
||||
compound command. `make dev-stop` kills by saved PID.
|
||||
- **Seeds are produced by running the app**, never by hand-writing a
|
||||
`config.toml` and DB rows — a hand-built `YJ_HOME` is a second
|
||||
description of a valid one and will drift. `make sandbox-seed` drives
|
||||
the real `AddLibrary` binding and waits for the real scan.
|
||||
- **Playwright's WebKit does not run on Arch** (Ubuntu-only libs).
|
||||
`--browser=webkit` is CI-only; local work is Chromium.
|
||||
|
||||
## Which tier
|
||||
|
||||
Four tiers. Start at the cheapest one that can see your change, and
|
||||
only climb when it cannot.
|
||||
|
||||
| You changed | Run | Cost |
|
||||
|---|---|---|
|
||||
| A Lit component, a store, the shortcut service | `make ui-test` | ~2 s, no app |
|
||||
| …and it renders differently | `make ui-visual` | + 6 baselines, opt-in |
|
||||
| Any Go code | `make test` | 3 passes, ~2 min |
|
||||
| A service that emits events | `make test` — assert on the payload, see `backend/queue/emit_test.go` | in-process, no app |
|
||||
| A bound method or a bound struct field | `make bindings` then `make ui-test` | ~1.5 s + 2 s |
|
||||
| A user-visible flow across frontend *and* backend | `make e2e` (needs the app up) | ~1 min |
|
||||
| Something you cannot predict — exploring | `make dev-headless SEED=default` + `playwright-cli` | interactive |
|
||||
| A `.sql` or `.templ` file | `make generate`, then the checklist in [references/schema-change.md](references/schema-change.md) | |
|
||||
|
||||
Two targets are once-per-clone prerequisites that are **not**
|
||||
dependencies of the targets needing them, so on a fresh checkout each
|
||||
fails with a missing-browser error that reads like a broken test:
|
||||
`make ui-setup` before `make ui-test`, and `make e2e-setup` before
|
||||
`make e2e`. (`make testdata` *is* a dependency of `make test` and
|
||||
`make sandbox-seed`; run it by hand only when invoking `go test`
|
||||
directly, since anything using `internal/testfixtures` **skips**
|
||||
rather than fails without it — a green run without the library means
|
||||
less than it looks.)
|
||||
|
||||
Two rules about climbing:
|
||||
|
||||
- **A component test passing is not the app rendering.** If you touched
|
||||
anything in `frontend/src`, verify it in the real app too — start it
|
||||
headless, `screenshot --filename=/tmp/shot.png`, and *read the PNG*.
|
||||
- **Do not write an e2e spec first.** Drive the flow by hand, then
|
||||
promote it with `/e2e`. Specs written blind assert on selectors that
|
||||
do not exist.
|
||||
|
||||
Before a commit, the gate is `make lint`, `make test`, `make ui-test`
|
||||
and `make bindings-check` — all four are also lefthook hooks, so
|
||||
skipping them locally only defers the failure.
|
||||
|
||||
## Running the app
|
||||
|
||||
The app cannot be started without a display: `devserver.Run` ends in a
|
||||
blocking GTK window with no flag to suppress it. The harness gives it a
|
||||
virtual one and returns.
|
||||
|
||||
```bash
|
||||
make sandbox-seed NAME=default # once (~10 s; runs make testdata itself,
|
||||
# then builds a seed by running the app)
|
||||
make dev-headless SEED=default # starts in the background, returns when :34115 answers
|
||||
make dev-logs # tail .dev/app.log
|
||||
make dev-stop # SIGTERM, so shutdown hooks persist state
|
||||
```
|
||||
|
||||
Then drive it. Run `playwright-cli` **from the repo root** — it picks
|
||||
up `.playwright/cli.config.json` from the cwd, and writes its snapshots
|
||||
and console logs to `.playwright-cli/` relative to the cwd too. `playwright-cli`'s own skill covers the commands; what
|
||||
is specific here is that a session must be *named* so it survives
|
||||
across separate shell calls:
|
||||
|
||||
```bash
|
||||
playwright-cli -s=yj open http://localhost:34115
|
||||
playwright-cli -s=yj snapshot # a11y tree, pierces shadow DOM
|
||||
playwright-cli -s=yj screenshot --filename=/tmp/shot.png
|
||||
playwright-cli -s=yj eval "() => window.__yjEvents.names()"
|
||||
playwright-cli -s=yj eval "() => window.__yjEvents.call('queue.Queue.GetState', [], 5000)"
|
||||
playwright-cli -s=yj click e391 # ref from the snapshot
|
||||
playwright-cli -s=yj close # `make dev-stop` does not do this
|
||||
```
|
||||
|
||||
`snapshot` prints a *path*, not the tree — read the file it names, and
|
||||
check the timestamp, because a stale one from a previous session sits
|
||||
in the same directory.
|
||||
|
||||
`.playwright/cli.config.json` is picked up automatically: it sets the
|
||||
viewport, `data-testid`, and the init script that installs the event
|
||||
bridge. **Assert on an event, not a timeout** — half this app is
|
||||
push-driven. The bridge and the dev-only `/__test/` control surface are
|
||||
documented in [references/harness.md](references/harness.md).
|
||||
|
||||
Other `YJ_HOME`s exist for humans and block the terminal: `make dev`,
|
||||
`make sandbox <name>`, `make fresh-install`. Do not use them; you will
|
||||
never get the shell back.
|
||||
|
||||
## Go, and the three build configurations
|
||||
|
||||
`make test` and `make lint` already run all three. Spell them out only
|
||||
when iterating on a single package:
|
||||
|
||||
```bash
|
||||
go test -tags webkit2_41 ./backend/player/ # the app build
|
||||
go test -tags webkit2_41 -run TestName ./backend/player/
|
||||
go test -tags "webkit2_41 indexbuild" ./backend/explore/... ./cmd/... # dump importer
|
||||
go test -tags "webkit2_41 dev" ./backend/testctl/... # control surface
|
||||
```
|
||||
|
||||
Forgetting the tag gives a build error that looks like a missing
|
||||
package. Audio integration tests additionally need
|
||||
`YELLOWJACKET_INTEGRATION=1`.
|
||||
|
||||
Three things golangci-lint v2 will reject that are easy to write:
|
||||
a dynamic `fmt.Errorf` without a sentinel (`err113`), a `return` with
|
||||
no blank line before it (`nlreturn`), and a long `//nolint` comment on
|
||||
the same line as its statement (`golines` reflows it and breaks the
|
||||
directive) — put the directive on its own line above.
|
||||
|
||||
**Emit events through `events.Emit(ctx, …)`, never
|
||||
`runtime.EventsEmit`.** `TestNoDirectRuntimeEmits` walks the tree and
|
||||
fails the build otherwise, including in files no lint pass compiles.
|
||||
|
||||
## References
|
||||
|
||||
- [harness.md](references/harness.md) — the event bridge API, the
|
||||
`/__test/` endpoints, and the config traps.
|
||||
- [fixtures.md](references/fixtures.md) — the generated library, the
|
||||
manifest, and selecting fixtures by case.
|
||||
- [ui-tier.md](references/ui-tier.md) — how the Vitest tier fakes Wails,
|
||||
and what breaks in it.
|
||||
- [schema-change.md](references/schema-change.md) — the two-file
|
||||
schema/migration checklist.
|
||||
@@ -0,0 +1,67 @@
|
||||
# The fixture library
|
||||
|
||||
`test_data/music_library_test/` is **generated, not committed**:
|
||||
`make testdata` (~1 s) builds 31 deterministic tracks across MP3, FLAC,
|
||||
Ogg Vorbis and WAV. `make testdata-force` rebuilds unconditionally,
|
||||
`make testdata-clean` deletes it. `make test` and `make sandbox-seed`
|
||||
depend on it, so it is rarely run by hand.
|
||||
|
||||
Tests that need it fetch it through `internal/testfixtures` and skip
|
||||
themselves when it has not been generated.
|
||||
|
||||
## Select by case, never by path
|
||||
|
||||
```go
|
||||
m := testfixtures.Load(t)
|
||||
paths := m.Case(t, testfixtures.CaseCoverDedup)
|
||||
track := m.Track(t, rel)
|
||||
```
|
||||
|
||||
Cases: `cover-dedup`, `multi-disc`, `various-artists`, `flac-album`,
|
||||
`ogg-album`, `wav-tracks`, `partial-tags`, `unicode`, `duplicates`,
|
||||
`edge-lengths`, `broken`.
|
||||
|
||||
Two invariants worth not breaking:
|
||||
|
||||
- **The clean library is exactly 31 tracks**, because `sandbox-seed`
|
||||
verifies the scan against that count. Deliberately malformed files
|
||||
live in a *sibling* root, `test_data/music_library_broken/`
|
||||
(`m.BrokenPath()`), so the scanner never sees them.
|
||||
- **Tags are written by `backend/tagwriter`, not by ffmpeg** (which
|
||||
encodes with `-map_metadata -1`). Fixture and reader therefore cannot
|
||||
drift into agreeing with each other and disagreeing with reality.
|
||||
|
||||
The manifest (`test_data/music_library_test.manifest.json`, outside the
|
||||
scanned root) hashes the *spec* — paths, formats, durations, tags,
|
||||
cover identity — not the bytes, because ffmpeg stamps encoder version
|
||||
strings and identical specs produce different bytes on different builds.
|
||||
|
||||
## In e2e specs
|
||||
|
||||
- **Every fixture except one is 2–6 seconds.** A spec that starts
|
||||
playback and then clicks pause races the track ending and fails
|
||||
against a correct UI. Use `LONG_TRACK` (90 s, `edge-lengths`) exported
|
||||
from `e2e/support/fixtures.ts`.
|
||||
- **WAV tracks scan in untitled.** `backend/tagwriter` writes WAV tags
|
||||
into a RIFF `id3 ` chunk and `dhowden/tag` has no RIFF parser, so
|
||||
there is no "Field Recordings" artist in the Artists view. This is a
|
||||
known open bug pinned by `TestWAVTagsAreNotReadableYet`; do not
|
||||
"fix" a spec by asserting the broken behaviour elsewhere.
|
||||
|
||||
## Seeds
|
||||
|
||||
```bash
|
||||
make sandbox-seed NAME=default # build (boots a fresh YJ_HOME and drives the app)
|
||||
make sandbox-seeds # list
|
||||
make dev-headless SEED=default # restore into a run
|
||||
```
|
||||
|
||||
A seed is a tarred `YJ_HOME` produced by *running the app*: fresh home →
|
||||
real `AddLibrary` binding → poll until the real scan reports the
|
||||
manifest's track count → SIGTERM so shutdown hooks persist state → tar.
|
||||
Never hand-write one. Seeding points `YJ_CORE_INDEX_URL` at a dead
|
||||
address on purpose, so no seed depends on what the explore artifact
|
||||
server happened to be serving.
|
||||
|
||||
Rebuild a seed after any schema change, or the restored database is
|
||||
migrated on open in a way the seed's author never saw.
|
||||
@@ -0,0 +1,85 @@
|
||||
# The harness: event bridge and control surface
|
||||
|
||||
Two things ride on top of the headless app. Both exist only in dev
|
||||
builds; neither is reachable from a shipped binary.
|
||||
|
||||
## The event bridge (`.playwright/init-events.js`)
|
||||
|
||||
Loaded as an `initScript` by `.playwright/cli.config.json` and by
|
||||
`e2e/support/fixtures.ts`, so an exploratory session and a committed
|
||||
spec see an identical page. It records every backend event by wrapping
|
||||
`window.wails.EventsNotify` — the single choke point all 46 events pass
|
||||
through, whether or not the app subscribes to them.
|
||||
|
||||
```js
|
||||
window.__yjEvents.wait('LibraryScanComplete', { timeoutMs: 60000 })
|
||||
window.__yjEvents.names() // name -> count; use this to find out
|
||||
// what actually fired before asserting
|
||||
window.__yjEvents.last('QueueChanged')
|
||||
window.__yjEvents.since(seq)
|
||||
window.__yjEvents.reset() // drop the buffer
|
||||
window.__yjEvents.ready(20000) // resolves when a binding round-trips,
|
||||
// which is later than DOM-ready and true
|
||||
window.__yjEvents.call('queue.Queue.GetState', [], 5000)
|
||||
```
|
||||
|
||||
- **`wait` resolves against already-buffered events as well as future
|
||||
ones**, so there is no race between doing the thing and listening.
|
||||
- **Install exactly one recorder.** Listeners survive across `eval`
|
||||
calls; a second recorder double-counts. Call `reset()`, never
|
||||
re-register.
|
||||
- **`call` times out on purpose.** A binding with wrong argument types
|
||||
never fires its callback. A 5 s rejection naming `.dev/app.log` beats
|
||||
an infinite hang.
|
||||
|
||||
In specs, use the wrappers rather than `page.evaluate`:
|
||||
`waitForEvent`, `resetEvents`, `eventNames`, `callBinding`, and the
|
||||
`app` fixture (a page with the bridge installed and the backend
|
||||
actually answering) from `e2e/support/fixtures.ts`.
|
||||
|
||||
## The control surface (`backend/testctl`, mounted at `/__test/`)
|
||||
|
||||
Gated twice: behind the `dev` build tag (with a no-op `!dev` twin) and
|
||||
behind `YJ_TESTCTL=1`, which `scripts/dev-headless.sh` sets and
|
||||
`make dev` does not.
|
||||
|
||||
| Endpoint | Use |
|
||||
|---|---|
|
||||
| `GET /__test/health` | is this a seeded dev build, and which library |
|
||||
| `POST /__test/db/snapshot?name=X` | save the SQLite state |
|
||||
| `POST /__test/db/restore?name=X` | put it back (see below) |
|
||||
| `POST /__test/emit` `{name, data}` | force any backend event |
|
||||
| `POST /__test/sql` `{sql, args}` | read rows, or a write count |
|
||||
|
||||
`TestCtl` in `e2e/support/fixtures.ts` is the typed client.
|
||||
|
||||
- **`emit` is the fast way to render a push-driven view** without
|
||||
staging the work that would produce it — job progress, download
|
||||
progress, scan progress. It calls `events.Deliver`, which *errors*
|
||||
when the event reaches nobody, so a `200` means it really arrived.
|
||||
- **`restore` is slow** (~40 s in the suite) because it copies every
|
||||
table. Prefer snapshotting once and restoring only when a spec
|
||||
genuinely mutates state.
|
||||
|
||||
## Traps in the config
|
||||
|
||||
- **The two path keys in `.playwright/cli.config.json` resolve
|
||||
differently.** `initScript` is relative to the *config file's*
|
||||
directory (`"init-events.js"`, not `".playwright/init-events.js"`);
|
||||
`outputDir` is relative to the *shell's cwd*. Set `outputDir` to
|
||||
`".playwright-cli"` and run `playwright-cli` from the repo root, or
|
||||
snapshots land somewhere neither `.gitignore` nor your next `ls`
|
||||
will find, and you will read a stale one from a previous session
|
||||
and think a component regressed.
|
||||
- **`snapshot` writes a file, it does not print the tree.** The
|
||||
command prints a path under `outputDir`; read that. Only the tail
|
||||
is echoed.
|
||||
- **Three separate browser caches.** `playwright-cli`, `@playwright/test`
|
||||
(`make e2e-setup`) and the Vitest provider (`make ui-setup`) each
|
||||
download their own Chromium. One working is no guarantee for the next.
|
||||
- **`getByRole('button', { name })` matches substrings.** "Play" also
|
||||
matches "Add queue to playlist"; transport controls need
|
||||
`exact: true`.
|
||||
- **`e2e/` is its own npm package** with `"type": "module"`. Without
|
||||
that, Playwright transpiles the specs to CJS and every `import.meta`
|
||||
throws — reported, unhelpfully, as "No tests found".
|
||||
@@ -0,0 +1,52 @@
|
||||
# Changing the database schema
|
||||
|
||||
The reasoning — why there are two files, what the old 48-step migration
|
||||
chain got wrong, and when squashing is legitimate — is in `CLAUDE.md`
|
||||
under *Backend packages → database*. Read it once. This is the
|
||||
checklist.
|
||||
|
||||
A schema change needs **two** files, not one:
|
||||
|
||||
1. **`backend/database/sql/schemas/*.sql`** — `CREATE TABLE ... IF NOT
|
||||
EXISTS`, the literal target shape, what sqlc reads and what a fresh
|
||||
install gets verbatim. Add the new column **last** in the
|
||||
`CREATE TABLE`.
|
||||
2. **`backend/database/sql/migrations/NNNN_description.sql`** — the
|
||||
`ALTER TABLE ... ADD COLUMN` (and any index on it) that gets an
|
||||
existing database to the same shape. Schema files are a no-op against
|
||||
a table that already exists, so without this an upgrade never gets
|
||||
the column.
|
||||
|
||||
Then:
|
||||
|
||||
```bash
|
||||
make generate # sqlc + templ
|
||||
go test -tags webkit2_41 ./backend/database/ # migration + column-order tests
|
||||
make test
|
||||
```
|
||||
|
||||
Rebuild any seed you rely on (`make sandbox-seed NAME=default`) and
|
||||
delete your own dev `YJ_HOME` if you want to see the fresh-install path
|
||||
rather than the migrated one.
|
||||
|
||||
## The three ways this goes wrong
|
||||
|
||||
- **Column order must match between the two paths.** `ADD COLUMN`
|
||||
always appends, so a migrated column declared anywhere but last in
|
||||
`CREATE TABLE` leaves fresh and upgraded installs disagreeing on
|
||||
order — and sqlc binds `SELECT *` positionally, so one of them
|
||||
silently reads the wrong field.
|
||||
`TestMigrations_ColumnOrderMatchesFreshInstall` is the regression test.
|
||||
- **Do not put an index on a migrated column in `sql/schemas/`.**
|
||||
Schema files run *before* migrations, against a database that may not
|
||||
have the column yet, and the predicate fails. Declare the index in the
|
||||
migration, after the `ALTER TABLE`.
|
||||
- **Do not add a third description of the schema anywhere.** A
|
||||
migration's `ADD COLUMN` failing with "duplicate column name" against
|
||||
an already-current database is expected and tolerated, not an error to
|
||||
route around.
|
||||
|
||||
New queries go in `backend/database/sql/queries/`; generated Go lands in
|
||||
`backend/database/sql/sqlcgen/`, which is never edited by hand. Tests
|
||||
use `database.NewTestDB(t)`, built by the same `applySchema` production
|
||||
uses, so the two cannot diverge.
|
||||
@@ -0,0 +1,74 @@
|
||||
# The component and store tier (`make ui-test`)
|
||||
|
||||
313 tests in a real Chromium in ~2 s with no Wails, no backend, no
|
||||
seeded library and no virtual display. This is the cheapest coverage
|
||||
available and where the bulk of UI regression belongs.
|
||||
|
||||
```bash
|
||||
make ui-setup # once: the Vitest provider's own Chromium
|
||||
make ui-test # behaviour only
|
||||
make ui-watch
|
||||
make ui-visual # + toMatchScreenshot baselines (YJ_VISUAL=1)
|
||||
make ui-visual-update # re-record them
|
||||
make ui-test UI_ARGS='store/queue' # filter
|
||||
```
|
||||
|
||||
## How it works
|
||||
|
||||
`frontend/wailsjs/` is a pure passthrough — every binding is
|
||||
`window.go[svc][Type][Method](args)`, every runtime call is
|
||||
`window.runtime.X(...)`. So `frontend/test/support/wails-fake.ts`
|
||||
replaces **those two globals and nothing else**, and the tests then
|
||||
exercise the *real* generated bindings and the *real* store code. No
|
||||
module mocking, and no second description of the Wails layer.
|
||||
|
||||
```ts
|
||||
emit(Events.QueueChanged, payload); // push a backend event
|
||||
stub('queue.Queue.GetState', state); // a value, or a function of the args
|
||||
stubFailure('queue.Queue.SetQueue'); // reject, as a Go error does
|
||||
calls('queue.Queue.SetQueue'); // what the frontend called back with
|
||||
lastArgs('queue.Queue.SetQueue');
|
||||
const el = await fixture('now-playing'); // mount; shadow()/text() query it
|
||||
```
|
||||
|
||||
The dispatcher mirrors wails' own `desktop/events.js`, including
|
||||
`maxCallbacks` expiry and the fact that a frontend `EventsEmit`
|
||||
notifies local listeners *before* Go.
|
||||
|
||||
## Four things that will cost you time
|
||||
|
||||
- **Store singletons are constructed at module import**, before any test
|
||||
can stub. `test/setup.ts` therefore carries import-time defaults for
|
||||
the stores that read config in their constructor. Without one, a store
|
||||
caches `undefined` where Go would have sent `[]`, and components crash
|
||||
on `.length` — which reads exactly like a component bug and is not.
|
||||
Adding a store that reads config on construction means adding its
|
||||
default there.
|
||||
- **`vitest.config.mts`, not `.ts`** — it `mergeConfig`s the repo's
|
||||
`vite.config.mts` to reuse the `@go`/`@store`/`@components` aliases,
|
||||
and a `.ts` sibling cannot import it.
|
||||
- **Screenshots need the theme.** The setup file imports
|
||||
`@store/theme-store` for its side effect (it applies the `--yj-*`
|
||||
ramp to `:root`); without it a component renders white-on-white and
|
||||
the baseline is blank.
|
||||
- **`@lit-labs/virtualizer` never produces two identical frames**, so
|
||||
`toMatchScreenshot` on `<queue-panel>` fails with "could not capture a
|
||||
stable screenshot" rather than a diff. Assert on its rows instead.
|
||||
|
||||
Visual baselines are font-hinting and compositing sensitive, which is
|
||||
why they are opt-in: they only mean anything on the machine that
|
||||
recorded them.
|
||||
|
||||
## Bindings
|
||||
|
||||
`frontend/wailsjs/` is generated by `wails`, **not** by `go generate`,
|
||||
so the pre-commit codegen check does not cover it — a renamed Go bound
|
||||
method first shows up at runtime, as a call that never settles.
|
||||
|
||||
```bash
|
||||
make bindings-check # ~1.5 s, also a pre-commit hook
|
||||
make bindings # regenerate for real
|
||||
```
|
||||
|
||||
The generator rewrites `wailsjs/runtime/*` as mode 755 every run; that
|
||||
is churn, not drift, and the check ignores it.
|
||||
@@ -0,0 +1,667 @@
|
||||
# Notes
|
||||
|
||||
Gotchas, measured facts, and things already considered and rejected.
|
||||
Measurements carry the date they were taken — several of these are
|
||||
properties of someone else's server and can change.
|
||||
|
||||
## MetaBrainz caps a client at ~2 MB/s (measured 2026-07-29)
|
||||
|
||||
`data.metabrainz.org` serves a single client at roughly 2.1 MB/s, and
|
||||
**concurrency does not help**: one Range stream and four concurrent
|
||||
lanes delivered 32 MB at 2,111,195 B/s and 2,209,000 B/s respectively,
|
||||
while the same machine pulled 66.9 MB/s from a CDN. One of the four
|
||||
lanes starved to 0.5 MB/s. The lanes divide a fixed cap; they do not
|
||||
raise it.
|
||||
|
||||
Consequences:
|
||||
|
||||
- No client-side concurrency change will speed up a dump download.
|
||||
Pushing harder earns 503s (the reason `dumpLanes` is 4).
|
||||
- Stage 1 of a full import costs ~11.8 h at best (89 GB after column
|
||||
projection). Before projection it was 205 GB — about 27 h.
|
||||
|
||||
This is the entire reason the catalog is built centrally and shipped as
|
||||
an artifact rather than derived per install.
|
||||
|
||||
## Further stage-1 reductions, not yet taken
|
||||
|
||||
Both are CI-side options; neither is safe as a silent client default
|
||||
because each changes *what gets counted*.
|
||||
|
||||
- **Project `recording_mbid` only** (24.1% of row-group bytes instead of
|
||||
43.4%): ~49 GB, ~6.5 h. `canonical_musicbrainz_data.csv` already
|
||||
carries `recording_mbid`, `release_mbid`, `artist_mbids` and
|
||||
`release_group_mbid`, so release/RG/artist counts can be rolled up
|
||||
locally. Cost: listens with no recording MBID are dropped, and artist
|
||||
totals become "sum of their recordings" rather than direct attribution.
|
||||
- **Stride-sample members** (1-in-4): ~12 GB, ~1.6 h. The dump is flat
|
||||
numbered members (`0.parquet`, …). Sampling is viable because the
|
||||
counts only feed a *ranking* for a top-N cut. Must be a stride, never
|
||||
a prefix — if members are time-ordered a prefix biases hard toward one
|
||||
era.
|
||||
|
||||
## Incremental dump retention is 30 days (measured 2026-07-29)
|
||||
|
||||
The incremental directory held 30 dumps (series 2579–2610), and full
|
||||
dumps land roughly monthly. An artifact older than ~30 days cannot be
|
||||
topped up: the dailies bridging the gap are gone. That is a permanent
|
||||
undercount of that window, not corruption — but it pins the artifact
|
||||
republish cadence at monthly.
|
||||
|
||||
## Anonymous package download is UNVERIFIED
|
||||
|
||||
The client fetches the artifact from a fixed `latest` URL because Gitea's
|
||||
package *listing* API requires a token while a plain file GET appears not
|
||||
to — a probe of the not-yet-published artifact returned 404 rather than
|
||||
401. **That is suggestive, not proof.** No artifact has been published
|
||||
yet to test against. Confirm before relying on it.
|
||||
|
||||
Also worth deciding deliberately: every install pulling from a personal
|
||||
Gitea makes its bandwidth and uptime a user-facing dependency.
|
||||
|
||||
## Migrations came back (2026-08-08), scoped to avoid the old failure mode
|
||||
|
||||
The "no migration chain" design below lasted until a real `make sandbox`
|
||||
DB (schema pre-dating the `tagging_items.synthetic`/`parent_group_key`
|
||||
columns) hit `no such column: parent_group_key` — `IF NOT EXISTS` had
|
||||
silently no-op'd the `CREATE TABLE` on the existing table, columns and
|
||||
all. A database written by an older build genuinely needed an upgrade
|
||||
path; there wasn't one.
|
||||
|
||||
What came back is **not** the old 48-step chain. `sql/schemas/*.sql`
|
||||
stays the single source of truth for the current shape (still what sqlc
|
||||
reads, still what a fresh install gets verbatim). `sql/migrations/*.sql`
|
||||
holds small numbered files — `ALTER TABLE ADD COLUMN`, `CREATE INDEX`,
|
||||
etc. — that run after the schema files, tracked in `schema_migrations`,
|
||||
tolerating "duplicate column name" as a no-op so the exact same files run
|
||||
unconditionally on both a fresh database and an old one and converge on
|
||||
one shape. See the "Schema changes need two things, not one" section in
|
||||
CLAUDE.md for the column-order and index-placement gotchas this
|
||||
implies, and `backend/database/migrations_test.go` for the regression
|
||||
tests. Squashing `sql/migrations/` back into `sql/schemas/` and deleting
|
||||
the migration files is fine pre-1.0 (see CLAUDE.md); stop once real user
|
||||
databases exist.
|
||||
|
||||
The original decision this replaces, kept for why the old chain died:
|
||||
|
||||
`applySchema` created the whole schema from `sql/schemas/*.sql` on every
|
||||
open; all DDL was `IF NOT EXISTS`. A database written by an older build
|
||||
was not supported and there was no upgrade path, by design.
|
||||
|
||||
Two things that removal fixed, worth not reintroducing:
|
||||
|
||||
- The 48-step chain was ~3,700 of `database.go`'s 4,061 lines, plus
|
||||
helpers that existed only to serve it (`backupDatabase`,
|
||||
`readLibraryDirFromTOML`, `isDuplicateColumnErr`, …).
|
||||
- `sql/schemas/` had drifted badly from the real schema — it still
|
||||
described a `genre_recordings` table that migrations had renamed, and
|
||||
omitted `explore_index`, `http_cache`, `artist_images`,
|
||||
`similar_artist_map`, `release_to_rg`, `lyrics_index` and
|
||||
`artist_metadata` entirely. sqlc reads that directory, so it had been
|
||||
generating against a stale schema and silently missed columns such as
|
||||
`audio_files.modified_at`.
|
||||
|
||||
The new design's answer to this specific risk: `sql/schemas/` is
|
||||
never edited to describe something migrations already did elsewhere
|
||||
— it's edited to directly declare the target shape, and migrations
|
||||
exist only to carry an old on-disk database to that same shape. There
|
||||
is exactly one hand-maintained description of "what does the schema
|
||||
look like", same as before; migrations don't add a second one.
|
||||
|
||||
**When regenerating schema files from a live database, remember the seed
|
||||
rows.** `file_types` (the four supported extensions), `player_state` and
|
||||
`queue` each carry `INSERT OR IGNORE` rows that `sqlite_master` does not
|
||||
contain. Dropping them breaks every audio-file foreign key.
|
||||
|
||||
## ANALYZE runs after the catalog merge, not at schema creation
|
||||
|
||||
The old migration 45 ran `ANALYZE` once. With the migration chain gone
|
||||
there is no equivalent moment — an empty database has nothing to measure
|
||||
— so it runs at the end of the artifact import instead
|
||||
(`SearchIndex.analyzeIndex`). Without current statistics the planner
|
||||
mis-estimates the partial expression indexes on `explore_index`
|
||||
(`idx_explore_title_lower`, `idx_explore_artist_lower`) and scans a
|
||||
million rows for queries that should seek.
|
||||
|
||||
If another path ever populates the catalog, it needs the same call.
|
||||
|
||||
## Writers, not readers, are responsible for name quality
|
||||
|
||||
`resolveArtistName` falls back to returning the artist MBID when it
|
||||
cannot find a name. That is fine for a one-off render but must never be
|
||||
persisted — an MBID stored as a title is unsearchable and shows as a
|
||||
UUID in the UI.
|
||||
|
||||
This used to be defended at every read (`title != mbid` predicates) and
|
||||
in the upsert's conflict rules. Those defenses are gone; `AddFromCache`
|
||||
now refuses to write a name equal to the MBID and lets the upsert's
|
||||
"non-empty wins" rule fill it in when a real name arrives.
|
||||
`TestAddFromCacheNeverStoresMBIDAsName` guards this.
|
||||
|
||||
## Attached databases are invisible to the read pool
|
||||
|
||||
`database.DB` holds two handles: a single-writer connection and a
|
||||
separate query-only pool. `ATTACH` binds to one connection, so anything
|
||||
touching an attached database must use `ExecContext`/`QueryRowWriter`
|
||||
(the writer) — `QueryContext` routes to the pool, where the attachment
|
||||
does not exist and the query fails with "no such table".
|
||||
|
||||
## FTS triggers are defined in Go, not in the schema
|
||||
|
||||
`explore_index`'s three FTS sync triggers live in
|
||||
`exploreIndexFTSTriggers` in `database.go` rather than in
|
||||
`sql/schemas/explore_index.sql`, because the bulk-load path drops and
|
||||
recreates them (`SuspendExploreIndexFTS`). Defining them in both places
|
||||
would be two copies free to drift.
|
||||
|
||||
Bulk loads must suspend them: measured on a real import, assembly runs at
|
||||
~31 rows/s with the triggers attached and ~4,700 rows/s without.
|
||||
|
||||
## Explore "library only" toggle was removed (2026-08-06)
|
||||
|
||||
The Explore UI used to have a "library only" mode toggle
|
||||
(`frontend/src/store/explore-settings.ts`, `explore:libraryOnly` in
|
||||
localStorage) that filtered the Explore UI to owned content only. It was
|
||||
removed outright — the app now always shows full (network-enriched)
|
||||
Explore data. If offline/library-only mode is wanted again, it should be
|
||||
built from scratch rather than restored; the old implementation gated
|
||||
several component code paths in ad hoc ways. A separate
|
||||
`deep_catalog_enabled` backend flag briefly existed for the same idea and
|
||||
was removed earlier, when the dump importer left the app binary.
|
||||
|
||||
## The dev server is a real, drivable app (verified 2026-08-10)
|
||||
|
||||
`wails dev` binds an HTTP + WebSocket server on `localhost:34115`
|
||||
(`internal/frontend/devserver/devserver.go`) that serves the frontend
|
||||
with the generated bindings on `window.go` and bridges every call and
|
||||
every `runtime.EventsEmit` to the **same** Go backend the desktop
|
||||
window uses. A browser pointed at it is not a mock — measured:
|
||||
`queue.Queue.GetState()` returned real JSON, `SetVolume(42)` produced a
|
||||
real `VolumeChanged`. Multiple clients are supported by design
|
||||
(`notifyExcludingSender` fans events to the other web clients *and* the
|
||||
desktop frontend). See plan 005.
|
||||
|
||||
Four facts that cost time to find:
|
||||
|
||||
- **The GTK window cannot be suppressed.** `devserver.Run` ends in
|
||||
`d.Frontend.Run(ctx)` with no flag to skip it, so headless needs
|
||||
Xvfb. Nobody upstream has a way around this.
|
||||
- **Build the dev binary directly.** `app_dev.go` parses `-devserver`,
|
||||
`-assetdir`, `-loglevel` from `os.Args`, so
|
||||
`go build -tags "dev webkit2_41"` plus those flags gives the same
|
||||
server with no watcher, no reload broadcast and one PID.
|
||||
- **A binding call with wrong argument types hangs forever.** The
|
||||
backend logs `error parsing arguments` and never fires the callback,
|
||||
so the caller's promise never settles. Always use a timeout; the app
|
||||
log is the only place the reason shows up.
|
||||
- **`dbus-run-session` does not break audio, and fixes MPRIS.** It
|
||||
replaces the bus, not `/run/user/1000`, so PulseAudio still works and
|
||||
`org.mpris.MediaPlayer2.yellowjacket` registers on the private bus.
|
||||
|
||||
## Playwright's WebKit does not run on Arch (measured 2026-08-10)
|
||||
|
||||
`playwright-cli install-browser webkit` downloads fine and then fails to
|
||||
link: its Linux build wants Ubuntu 24.04 libraries (`libicu74`,
|
||||
`libWPEWebKit-2.0.so.1`, `libflite`) that Arch does not provide, and the
|
||||
dependency check emits `apt-get` advice. So `--browser=webkit` — the
|
||||
cheap way to approximate the WebKit2GTK renderer we actually ship — is a
|
||||
CI-only capability. Local browser work is Chromium, which is unaffected.
|
||||
|
||||
## The fixture library is generated, and generated by our own writers
|
||||
|
||||
`test_data/music_library_test/` is produced by `cmd/gentestdata`
|
||||
(`make testdata`), not committed — 31 tracks across MP3, FLAC, Ogg
|
||||
Vorbis and WAV, ~700 KB, ~1 s to build. Two rules keep it honest:
|
||||
|
||||
- **Tags are written by `backend/tagwriter`, not by ffmpeg.** ffmpeg
|
||||
only encodes (with `-map_metadata -1`); every tag comes from the same
|
||||
writers the app uses, so a fixture and the reader under test cannot
|
||||
drift into agreeing with each other and disagreeing with reality.
|
||||
`tagwriter.WriteFileTags` exists for this — it is the format switch
|
||||
`WriteUntrackedFileTags` already had, lifted out so tooling with no
|
||||
app to construct can call it.
|
||||
- **The manifest hash covers the spec, not the bytes.** ffmpeg stamps
|
||||
encoder version strings, so identical specs produce different bytes
|
||||
on different ffmpeg builds. `test_data/music_library_test.manifest.json`
|
||||
hashes paths, formats, durations, tags and cover identity instead,
|
||||
and lives *outside* the library root so the scanner never sees it.
|
||||
|
||||
Fixtures are selected in tests by *case* (`testfixtures.CaseCoverDedup`,
|
||||
`CaseUnicode`, `CaseDuplicates`, …) rather than by path. Deliberately
|
||||
malformed files live in a sibling root, `test_data/music_library_broken/`
|
||||
— the clean library's track count has to stay at exactly 31 for seeds
|
||||
to be verifiable, and a zero-byte `.flac` in the scanned tree used to
|
||||
get swept into `testFlacFiles` and fail the duration parser.
|
||||
|
||||
## WAV tags are write-only (found 2026-08-10)
|
||||
|
||||
`backend/tagwriter` writes WAV tags into a RIFF `id3 ` chunk, and
|
||||
`backend/metadata` reads through `dhowden/tag`, which recognises MP3,
|
||||
FLAC, OGG, MP4 and DSF and has **no RIFF parser at all**. So every tag
|
||||
the app writes to a WAV is invisible to the app that wrote it, and WAV
|
||||
tracks always scan in untitled — visible in the Artists view, where the
|
||||
fixture library's WAV tracks produce no "Field Recordings" artist.
|
||||
|
||||
The fix is small (unwrap the `id3 ` chunk and hand the payload to
|
||||
`tag.ReadFrom`) but was out of scope for plan 005.
|
||||
`TestWAVTagsAreNotReadableYet` asserts the gap so that fixing the
|
||||
reader turns into a failing test rather than nothing at all.
|
||||
|
||||
## The headless harness: how to run this app without a window
|
||||
|
||||
`make dev-headless [SEED=<name>]` starts the app in the background and
|
||||
returns; `make dev-stop`, `make dev-logs`. It runs the dev *binary*
|
||||
(`go build -tags "dev webkit2_41"`), not `wails dev`, under
|
||||
`dbus-run-session -- xvfb-run`. See plan 005 and
|
||||
`scripts/dev-headless.sh` for why each of those three is load-bearing.
|
||||
|
||||
**Seeds are built by running the app.** The first-run wizard's
|
||||
dismissal condition is not a config file — it is
|
||||
`GetAllLibrariesWithTrackCounts()` returning something — so
|
||||
`make sandbox-seed NAME=<n>` boots a fresh `YJ_HOME`, calls the real
|
||||
`AddLibrary` binding through `playwright-cli`, waits for the real scan
|
||||
to reach the manifest's track count, stops the app with SIGTERM so the
|
||||
shutdown hooks persist state, and tars the result. Never hand-write a
|
||||
`config.toml` and DB rows: that is a second description of a valid
|
||||
`YJ_HOME`, free to drift, exactly like the migration chain was.
|
||||
|
||||
Seeding points `YJ_CORE_INDEX_URL` at a dead address on purpose, so no
|
||||
seed depends on what the explore artifact server was serving that day.
|
||||
|
||||
Two parsing traps, both already paid for:
|
||||
|
||||
- `playwright-cli` echoes the evaluated source back after the result,
|
||||
so scraping its output for bare digits picks up numbers from your own
|
||||
JavaScript. Return a tagged sentinel (`'YJTRACKS' + '=' + n`) and
|
||||
grep for that.
|
||||
- Waiting on a fixed sleep or on a scan event is worse than waiting on
|
||||
the observable outcome. Polling the track count the app itself
|
||||
reports also validates the fixture manifest against the real scanner.
|
||||
|
||||
## Restoring a database needs foreign keys *off*, not deferred
|
||||
|
||||
`/__test/db/restore` (`backend/testctl`) copies every ordinary table
|
||||
out of an ATTACHed snapshot. The obvious implementation — one
|
||||
transaction with `PRAGMA defer_foreign_keys = ON` — fails at COMMIT
|
||||
with a bare `FOREIGN KEY constraint failed (787)` that names nothing.
|
||||
|
||||
Deferring postpones the *check*; it does not stop `ON DELETE CASCADE`
|
||||
from firing. Tables are copied in name order, which is not dependency
|
||||
order, so `DELETE FROM libraries` cascades away rows of a child table
|
||||
that was already restored earlier in the loop, and the final state is
|
||||
genuinely inconsistent.
|
||||
|
||||
`PRAGMA foreign_keys` is a no-op inside a transaction, so it has to be
|
||||
set on the connection around it. That is safe only because the writer
|
||||
is a single connection (`SetMaxOpenConns(1)`); the restore re-enables
|
||||
enforcement afterwards and runs `PRAGMA foreign_key_check`, so a bad
|
||||
restore is reported instead of left in place.
|
||||
`TestRestoreRoundTrip` pins it.
|
||||
|
||||
Two related traps in the same path: FTS5 virtual tables cannot be
|
||||
written with `SELECT *` and their shadow tables (`_data`, `_idx`,
|
||||
`_docsize`, `_config`) must be rebuilt rather than copied — but the
|
||||
prefix test that excludes them must not swallow `explore_index`, an
|
||||
ordinary table whose name is a prefix of two virtual ones.
|
||||
|
||||
## The event bridge hooks EventsNotify, not EventsOn
|
||||
|
||||
`.playwright/init-events.js` records backend events by wrapping
|
||||
`window.wails.EventsNotify`. That is the single choke point: wails'
|
||||
`ipc_websocket.js` does `case "n": window.wails.EventsNotify(msg)` and
|
||||
fans out to listeners from there, so one wrap captures all 46 events
|
||||
whether or not the app subscribes to them. Wrapping `EventsOn` would
|
||||
have needed 46 registrations and would have missed anything the app
|
||||
does not listen for.
|
||||
|
||||
`window.wails` does not exist when an initScript runs, so the script
|
||||
installs an accessor on `window` and wraps at assignment time (wails'
|
||||
`main.js` does a plain `window.wails = {...}`), then redefines the
|
||||
property as a plain value so nothing downstream can tell.
|
||||
|
||||
The buffer lives on `window.__yjEvents` with `wait()`, `reset()`,
|
||||
`names()`, a `ready()` that resolves only when a binding actually
|
||||
round-trips, and a `call()` that **times out** — a binding invoked with
|
||||
wrong argument types never fires its callback, and a 2s rejection
|
||||
naming `.dev/app.log` is worth more than an infinite hang.
|
||||
|
||||
## Small harness traps, each of which cost a cycle
|
||||
|
||||
- **Paths in `.playwright/cli.config.json` resolve against the config
|
||||
file's directory**, not the repo root. `".playwright/init-events.js"`
|
||||
becomes `.playwright/.playwright/init-events.js`.
|
||||
- **`playwright-cli` and `@playwright/test` have separate browser
|
||||
caches.** The CLI working is no guarantee `npx playwright test` can
|
||||
launch; it needs its own `npx playwright install chromium` (the
|
||||
runner wants `chrome-headless-shell`, which the CLI never fetched).
|
||||
- **`getByRole('button', { name: 'Play' })` also matches "Add queue to
|
||||
playlist".** Accessible-name matching is substring by default; the
|
||||
transport controls need `exact: true`.
|
||||
- **Every fixture track except one is 2–6 seconds.** A spec that plays
|
||||
a track and then clicks pause races the track ending and fails
|
||||
against a correct UI. Use the 90-second `Long Player`
|
||||
(`edge-lengths`), exported as `LONG_TRACK` from `e2e/support`.
|
||||
- **`e2e/` needs `"type": "module"`** or Playwright transpiles the
|
||||
specs to CJS and every `import.meta` in the support code throws
|
||||
"Cannot use 'import.meta' outside a module" — reported as
|
||||
"No tests found".
|
||||
|
||||
## The component tier fakes two globals, and that is all it fakes
|
||||
|
||||
`frontend/wailsjs/` is a pure passthrough: every generated binding is
|
||||
`window['go'][svc][Type][Method](args)` and every runtime call is
|
||||
`window.runtime.X(...)`. So `frontend/test/support/wails-fake.ts`
|
||||
replaces those two globals and nothing else, and the tests then run the
|
||||
*real* generated bindings and the *real* store code. No module mocking,
|
||||
and no second description of the Wails layer to drift from the first —
|
||||
the same discipline `sql/schemas/` and the seeds get.
|
||||
|
||||
The dispatcher mirrors wails' own `internal/frontend/runtime/desktop/
|
||||
events.js`, which matters in two places: listeners registered with
|
||||
`maxCallbacks` expire and are removed mid-iteration, and `EventsEmit`
|
||||
from the frontend notifies local JS listeners *before* it notifies Go
|
||||
(so a frontend emit is observable in-page).
|
||||
|
||||
Four things that cost time:
|
||||
|
||||
- **Store singletons are constructed at module import**, so the fake
|
||||
must be installed from `setupFiles`, and any store that reads config
|
||||
in its constructor loads before a test can stub it. `test/setup.ts`
|
||||
carries import-time defaults for exactly those. Without them a store
|
||||
caches `undefined` where Go would have sent `[]`, and four
|
||||
components then crash on `.length` — which reads as a component bug
|
||||
and is not one.
|
||||
- **`vitest.config.mts`, not `.ts`.** The repo's vite config is
|
||||
`vite.config.mts`; a `.ts` sibling cannot import it, and `mergeConfig`
|
||||
is how the `@go`/`@store`/`@components` aliases get reused rather
|
||||
than restated.
|
||||
- **Vitest 4 takes a provider factory, not a string.**
|
||||
`provider: playwright()` from `@vitest/browser-playwright`, which is
|
||||
a third package beyond `vitest` and `@vitest/browser`, and needs its
|
||||
own `npx playwright install chromium` — a third browser cache after
|
||||
`playwright-cli`'s and `@playwright/test`'s.
|
||||
- **Pre-bundle Web Awesome or Vite reloads mid-test.** Its components
|
||||
are one deep import per element; discovering them lazily makes Vite
|
||||
re-optimise and reload the page underneath a running test. The glob
|
||||
`@awesome.me/webawesome/dist/components/*/*.js` in `optimizeDeps.
|
||||
include` settles it.
|
||||
|
||||
Screenshots need the app's surface, not the default white page: the
|
||||
setup file imports `@store/theme-store` for its side effect (it applies
|
||||
the `--yj-*` ramp to `:root`) and sets the two `index.css` declarations
|
||||
that matter, or a component renders white-on-white and the baseline is
|
||||
blank. And a `@lit-labs/virtualizer` list never produces two identical
|
||||
frames, so `toMatchScreenshot` on `<queue-panel>` fails with "could not
|
||||
capture a stable screenshot" rather than a diff — assert its rows
|
||||
instead.
|
||||
|
||||
## Binding drift is now checked, and it is fast
|
||||
|
||||
`frontend/wailsjs/` is generated by `wails`, **not** by `go generate`,
|
||||
so the pre-commit codegen check never covered it: a renamed Go bound
|
||||
method or struct field first showed up at runtime, in a window, as a
|
||||
call that never settles. `wails generate module` (v2.10.2) rebuilds it
|
||||
in ~1.5 s, which is cheap enough to gate a commit on —
|
||||
`scripts/bindings-check.sh`, `make bindings-check`, and a `lefthook`
|
||||
pre-commit entry. Verified by renaming `queue.GetState` and watching it
|
||||
fail.
|
||||
|
||||
One quirk: the generator rewrites the three `wailsjs/runtime/` files as
|
||||
mode 755 every run. That is not drift, so the check compares with
|
||||
`git -c core.fileMode=false` and restores the modes afterwards.
|
||||
|
||||
## Emitting an event is now one call, and it cannot kill the process
|
||||
|
||||
`events.Emit(ctx, name, data...)` (`backend/events/emit.go`) is the
|
||||
only supported way to push a Wails event; `TestNoDirectRuntimeEmits`
|
||||
fails the build on any other `runtime.EventsEmit` in the tree.
|
||||
|
||||
The reason is that `runtime.getEvents` (wails `runtime.go:47`)
|
||||
`log.Fatalf`s — `os.Exit`, unrecoverable — whenever the context lacks
|
||||
its internal `"events"` value. That is any `context.Background()`, so
|
||||
in-process service tests were impossible and background workers that
|
||||
outlived their context could take the app down at launch.
|
||||
|
||||
Four packages had independently discovered this and hand-rolled a
|
||||
guard (`library.emit`, `download.emit`, `playlist.emitEvent`, and
|
||||
`autotagservice.emitEvent` with a whole `ctxReady` field). Nine other
|
||||
sites guarded on `ctx != nil`, **which does not help** — a non-nil
|
||||
context without the runtime is exactly the fatal case. The wrapper
|
||||
replicates wails' own precondition once and drops at debug level.
|
||||
|
||||
Three things worth knowing:
|
||||
|
||||
- **The test sink rides in the context**, `events.WithSink(ctx, rec)`,
|
||||
not in a package global. A global cannot survive `t.Parallel()` and
|
||||
would put a mutex on every production emit.
|
||||
- **`events.Deliver` is `Emit` that returns `ErrNoRuntime`**, and has
|
||||
exactly one caller: `/__test/emit` in `backend/testctl`. That
|
||||
endpoint exists to *impersonate* a backend emit, so answering `200`
|
||||
for an event that reached nobody would send you debugging the
|
||||
frontend for a backend no-op. Ordinary emitters want `Emit`.
|
||||
- **Enforcement is a walk of the tree, not a lint rule.**
|
||||
golangci-lint runs once per build configuration, so a stray emit in
|
||||
an `indexbuild`- or `dev`-tagged file is only visible to the pass
|
||||
that compiles it. One text walk sees all three, plus anything tagged
|
||||
out entirely. Two traps if you touch that test: the needle has to be
|
||||
built at runtime or the file matches itself, and it has to be the
|
||||
qualified selector (`.EventsEmit(`) or it matches the test's own
|
||||
function name.
|
||||
|
||||
What it unblocks is a fourth test tier — services, in-process, with no
|
||||
app: `backend/queue/emit_test.go`, `backend/config/emit_test.go`,
|
||||
`backend/playlist/emit_test.go` assert on the payload the *frontend
|
||||
receives*, which had never been covered. Two gotchas found writing
|
||||
them: `config.Save` refuses to write a config that was never
|
||||
`Load`ed (so a test that only calls `applyDefaults` sees its second
|
||||
setter fail, not its first), and `queue.SetQueue` resolves anything
|
||||
over `initialBatchSize` in a background phase, so assert with
|
||||
`rec.Wait` rather than immediately after the call.
|
||||
|
||||
## Docs are split by tense, and the split is checkable
|
||||
|
||||
Three places now describe this repo, and the rule for which one a new
|
||||
paragraph goes in is **grammatical, not topical** — a topical split
|
||||
("architecture here, testing there") is what rots, because every new
|
||||
fact has two plausible homes.
|
||||
|
||||
- `.planning/NOTES.md` — **past**: measured, dated, append-only.
|
||||
- `CLAUDE.md` — **present**: what the system is, and why.
|
||||
- `.pi/skills/yellowjacket-dev/` — **imperative**: what to run, in what
|
||||
order, and what it looks like when it fails.
|
||||
|
||||
So the skill carries the checklist for a schema change and CLAUDE.md
|
||||
carries the reasoning behind the two-file rule; the skill carries the
|
||||
headless lifecycle and CLAUDE.md carries only the invariant that seeds
|
||||
are produced by running the app. Phase 6 deleted about half of
|
||||
CLAUDE.md's harness section on those grounds.
|
||||
|
||||
`make skill-check` (`scripts/skill-check.sh`, pre-commit) makes it
|
||||
enforceable: every `make <target>` mentioned under `.pi/**/*.md` must
|
||||
exist. That is the actual anti-drift mechanism — the Makefile is the
|
||||
source of truth for *how* to invoke something and the skill only decides
|
||||
*which*, so a renamed target fails a commit instead of sending an agent
|
||||
confidently at a command that no longer exists. A skill that documents a
|
||||
command slightly wrong is worse than no skill.
|
||||
|
||||
One shell trap it cost: under `set -euo pipefail`,
|
||||
`x="$(make -pqRr | awk … )"` sinks the whole assignment, because
|
||||
`make -q` exits non-zero whenever a target is out of date and `pipefail`
|
||||
propagates that. Wrap it in `{ …; || true; }`.
|
||||
|
||||
## The skill was followed cold, and lost time in exactly one place
|
||||
|
||||
An agent that did not write `.pi/skills/yellowjacket-dev/` brought the
|
||||
app up from a wiped `.dev/` and no fixture library, drove a flow the
|
||||
skill does not describe (open the queue panel, toggle shuffle, assert
|
||||
on `QueueModeChanged`, confirm against `queue.Queue.GetState`) and
|
||||
stopped it — about a minute of wall clock, no dead ends. All four
|
||||
tiers then re-ran green from that cold state: 313 ui-test, 0 issues ×
|
||||
3 lint configurations, 3 test passes, 19/19 e2e.
|
||||
|
||||
The one expensive thing was a genuine config bug, not a doc error.
|
||||
`.playwright/cli.config.json` had `outputDir: "../.playwright-cli"`,
|
||||
written on the belief — which `references/harness.md` stated as a flat
|
||||
rule — that every path in that file resolves against the config file's
|
||||
directory. **Only `initScript` does.** `outputDir` resolves against the
|
||||
shell's cwd, so every snapshot and console log was landing in
|
||||
`/home/logan/Development/.playwright-cli`, one level *above* the repo:
|
||||
outside `.gitignore`, outside `find`, and invisible to the obvious
|
||||
`ls .playwright-cli/`. That directory still held a stale snapshot from
|
||||
the previous session, so the obvious `ls -t | head -1` returned it
|
||||
silently, and the transport buttons appeared to have lost their
|
||||
accessible names — a fabricated regression in phase 3's work that took
|
||||
a DOM walk to disprove. Reading a *stale* artifact is much worse than
|
||||
reading none, because it answers.
|
||||
|
||||
Four smaller corrections, all now in the skill:
|
||||
|
||||
- `make sandbox-seed` already depends on `make testdata`, so listing
|
||||
both made the fixture step look separately required. It also takes
|
||||
~10 s with warm caches, not the ~30 s claimed.
|
||||
- `make ui-setup` and `make e2e-setup` are once-per-clone
|
||||
prerequisites and are *not* dependencies of `make ui-test` /
|
||||
`make e2e`. The skill never mentioned them; on a fresh clone both
|
||||
fail with a missing-browser error that reads like a broken test.
|
||||
This matters for CI, which has no warm caches by definition.
|
||||
- `snapshot` prints a *path*, not the tree. Not said anywhere.
|
||||
- `make dev-stop` does not close the browser session;
|
||||
`playwright-cli -s=yj close` is a separate step.
|
||||
|
||||
And one place the tooling taught the opposite of the skill:
|
||||
`scripts/dev-headless.sh`'s own success banner suggested
|
||||
`eval "async () => await window.go.queue.Queue.GetState()"` — a bare
|
||||
`window.go` call with no timeout, which is precisely the hang the
|
||||
banner's next paragraph warns about. A gotcha documented in prose and
|
||||
contradicted by the copy-pasteable line three inches above it will lose
|
||||
every time; the banner now prints the `__yjEvents.call` form.
|
||||
|
||||
## An ALSA null PCM is enough for CI audio, and it clocks
|
||||
|
||||
Phase 7's job 2 needs playback to actually advance, because
|
||||
`e2e/specs/playback.spec.ts` asserts the elapsed clock moves — a
|
||||
missing audio device fails it in a way that reads like flake, since
|
||||
`app.go` joins `InitSpeaker` failure into `startupErr` and lets
|
||||
everything else work.
|
||||
|
||||
Measured locally, with PulseAudio made unreachable
|
||||
(`XDG_RUNTIME_DIR` pointed at an empty dir, `PULSE_SERVER=none`) and
|
||||
`ALSA_CONFIG_PATH` pointing at four lines:
|
||||
|
||||
```
|
||||
</usr/share/alsa/alsa.conf>
|
||||
pcm.!default { type null }
|
||||
ctl.!default { type null }
|
||||
```
|
||||
|
||||
`InitSpeaker` succeeded in 36 ms and all six playback/queue specs
|
||||
passed, including "the elapsed time advances". oto/v3 talks to
|
||||
libasound directly, and ALSA's `null` plugin advances its pointer on a
|
||||
timer rather than discarding instantly, so beep's stream is consumed at
|
||||
real-time rate. **No PipeWire, no PulseAudio and no daemon of any kind
|
||||
is required in the container** — one env var and a file.
|
||||
|
||||
Also found while setting this up: `scripts/dev-headless.sh` does *not*
|
||||
set `YJ_CORE_INDEX_URL`; only `scripts/seed-sandbox.sh` does. So a
|
||||
seeded run started by hand still reaches for the real explore artifact.
|
||||
Harmless locally, a network dependency and a minute of wall clock in
|
||||
CI — job 2 must set the dead-address override itself.
|
||||
|
||||
## CI was prototyped in a container before it was written, and it found a real bug
|
||||
|
||||
Both jobs of `.gitea/workflows/ci.yml` were built as shell scripts and
|
||||
run to green in a bare `ubuntu:24.04` container (`docker run -v
|
||||
repo:/src -v cache:/cache`) before a line of YAML existed, then the
|
||||
YAML was transcribed back out of the workflow and re-run in the same
|
||||
container to prove the transcription. That is worth the extra half
|
||||
hour on a self-hosted runner: the alternative is push-and-see, and a
|
||||
Gitea Actions run that never starts looks exactly like one that passed.
|
||||
|
||||
**`make lint` was linting three configurations nothing builds.** All
|
||||
three passes omitted `webkit2_41`, so wails resolved `webkit2gtk-4.0`.
|
||||
Arch still ships `webkit2gtk-4.0.pc`, so it passed locally and had
|
||||
done for the life of the repo; Ubuntu 24.04 dropped 4.0, and the
|
||||
**`dev` pass** fails there — wails' own `app_dev.go` is `dev`-tagged
|
||||
and drags in the 4.0 assetserver, which the other two passes never
|
||||
compile. The tag sets now match `make test` exactly
|
||||
(`webkit2_41`, `webkit2_41 indexbuild`, `webkit2_41 dev`). Still 0
|
||||
issues × 3 on Arch, and now 0 × 3 on Ubuntu too. Note what this means:
|
||||
"lint passes" and "the thing lint compiled is the thing we ship" were
|
||||
different claims, and only a second distro could tell them apart.
|
||||
|
||||
Five smaller container facts, all now comments in the workflow:
|
||||
|
||||
- **`libasound2-dev`, not just `libasound2t64`.** oto/v3 dies at
|
||||
`pkg-config --cflags -- alsa` before a line is compiled.
|
||||
- **`PLAYWRIGHT_BROWSERS_PATH` unifies the location, not the
|
||||
revisions.** `@playwright/cli` bundles its own `playwright-core`
|
||||
pinned to a different Chromium build than `e2e/`'s
|
||||
`@playwright/test`, so both must install into the shared directory.
|
||||
Installing one gives the other "Browser chromium is not installed;
|
||||
expected executable at …/chromium-1237/…". The "three separate
|
||||
browser caches" trap survives being pointed at one path.
|
||||
- **`git config --global --add safe.directory`** or `bindings-check`
|
||||
fails on a clone the container user does not own.
|
||||
- **The runner already mounts and exports `GOMODCACHE`, `GOCACHE` and
|
||||
`GOLANGCI_LINT_CACHE`** for every job via `container.options`, and
|
||||
`valid_volumes` is a glob over the cache root
|
||||
(`/home/logan/docker/gitea/data/runner/cache/**`), so new caches need
|
||||
no runner-side change. Only the Node-side ones had to be declared.
|
||||
- **The fixture hash is deterministic per ffmpeg, not across
|
||||
versions**: `5425fbb454a2` on Arch (ffmpeg n8.1.2), `599a8dd4f152`
|
||||
on Ubuntu 24.04. Nothing asserts a literal hash, so this is
|
||||
harmless — but a test that pinned one would be portable only by
|
||||
accident.
|
||||
|
||||
**Setting `YJ_CORE_INDEX_URL` for the *app* run, not just for seeding,
|
||||
is worth 8x on the suite.** `scripts/dev-headless.sh` never set it —
|
||||
only `seed-sandbox.sh` did — so a seeded local run still fetches the
|
||||
real explore artifact, and `testctl.spec.ts`'s restore then copies
|
||||
every table of a database full of catalogue: 42 s locally, versus a
|
||||
whole 19-spec suite in 7.3 s in CI with the artifact stubbed out.
|
||||
|
||||
## Playwright's WebKit passes, so it gates
|
||||
|
||||
19/19, in the same container, ~11 s on top of Chromium's ~7 s. It had
|
||||
never been run anywhere before — Arch cannot start it — so the honest
|
||||
default would have been advisory. Running it once in a throwaway
|
||||
container turned a coin flip into a decision: it is a **required**
|
||||
step.
|
||||
|
||||
Two things make that safe rather than brave. Nothing in `e2e/`
|
||||
compares pixels — every assertion is an event payload, a `data-testid`,
|
||||
an attribute or backend state, and the `toMatchScreenshot` baselines
|
||||
live in the Chromium-only Vitest tier — so a WebKit failure cannot be
|
||||
antialiasing noise; it is an engine difference in custom-element
|
||||
upgrade, a11y-tree shape or event ordering, which is exactly the
|
||||
WebKit2GTK signal we otherwise have no way to get. And it is cheap
|
||||
enough that the earlier plan to scope it (skip `testctl.spec.ts`,
|
||||
which tests Go and has no engine content) is not worth the
|
||||
complexity at 11 s.
|
||||
|
||||
## Every Go typecheck needs a built frontend, and a shared prototype dir hid it
|
||||
|
||||
`main.go` embeds the built assets (`//go:embed all:frontend/dist`), so
|
||||
`make lint`, `make test` and `make bindings-check` all fail on a fresh
|
||||
clone with `pattern all:frontend/dist: no matching files found
|
||||
(typecheck)` until `pnpm build` has run once. It never bites locally
|
||||
because anyone who has started the app has a `dist/` lying around, and
|
||||
it is not a Go dependency anything declares — which is why CI is the
|
||||
only place it shows up. Job 1 now builds the frontend before linting.
|
||||
|
||||
**The prototype missed it for an embarrassing and reusable reason.**
|
||||
Both job scripts were run against the *same* mounted directory, and
|
||||
job 2 runs `dev-headless`, which builds the frontend. So job 1 was
|
||||
silently consuming an artifact job 2 had produced on an earlier run,
|
||||
in an order CI never uses. A container proved the commands work; it did
|
||||
not prove the *inputs* were what CI would have, because the directory
|
||||
had accumulated state exactly the way a developer machine does.
|
||||
|
||||
The fix for the technique, not just the workflow: verify each job in a
|
||||
**fresh `git clone --no-hardlinks` of the pushed commit** (a plain
|
||||
`git clone` of a repo on the same filesystem fails with "Invalid
|
||||
cross-device link" into `/tmp` on a different device), not in an rsync
|
||||
of the working tree, and never two jobs in one directory. The
|
||||
distinction that matters is not clean-vs-dirty but *whose* dirt: a
|
||||
working-tree copy carries a developer's accumulated build output, which
|
||||
is the one thing CI is supposed to be checking you do not depend on.
|
||||
@@ -0,0 +1,379 @@
|
||||
# 001 — Ship a prebuilt "core" explore index
|
||||
|
||||
**Status:** complete
|
||||
**Branch:** cleanup/fresh-start-schema
|
||||
**Created:** 2026-07-25
|
||||
**Completed:** 2026-07-30
|
||||
|
||||
## Outcome
|
||||
|
||||
A fresh install downloads a 70.6 MB artifact and merges 1,076,133 rows
|
||||
in ~43 s, instead of streaming 205 GB over ~27 h. The dump importer that
|
||||
produces the artifact left the app binary entirely — it is behind the
|
||||
`indexbuild` build tag and runs only in CI.
|
||||
|
||||
Phase 5 landed differently than planned: rather than a user-facing
|
||||
setting gating the deep import, the deep import is simply not in the
|
||||
app. `deep_catalog_enabled` existed briefly and was removed with it.
|
||||
|
||||
Two things remain unverified or undone, both recorded in
|
||||
`.planning/NOTES.md`: anonymous package download on git.ljones.me has
|
||||
not been confirmed against a real published artifact, and installs whose
|
||||
index was built by older code (no `listens_applied_series`) have no
|
||||
rescue path — though with no migration chain, those databases are now
|
||||
unsupported anyway.
|
||||
|
||||
## Problem
|
||||
|
||||
A fresh install has no explore index. `StartIndexBuild()` is called
|
||||
unconditionally from two places in `app.go`, and `runDumpBuild` then
|
||||
downloads gigabytes from `data.metabrainz.org` before Explore can return
|
||||
anything beyond the user's own library:
|
||||
|
||||
| Stage | Source | Cost |
|
||||
|---|---|---|
|
||||
| Listen Counts | ListenBrainz spark full listens dump | **~205 GB streamed** — see below |
|
||||
| Catalog Import | MusicBrainz canonical dump (~2 GB `.tar.zst`) | scan ~30M CSV rows, assemble to budget |
|
||||
| Metadata Patch | MB/LB API | rate-limited at 3 req/s |
|
||||
| Listener Counts | LB API | rate-limited |
|
||||
|
||||
Measured 2026-07-25 against the live dump
|
||||
(`listenbrainz-spark-dump-2593-20260712-000004-full.tar`):
|
||||
|
||||
```
|
||||
content-length: 205073162240 # 205 GB
|
||||
accept-ranges: bytes
|
||||
```
|
||||
|
||||
The stage-1 reader skips non-`.parquet` tar members
|
||||
(`dumpcounts.go:317`), but a tar stream has no seek — skipped bytes
|
||||
still transit the wire. **So a first run on a fresh install pulls
|
||||
~205 GB.** Little of it touches disk (the counts map and checkpoint do,
|
||||
not the dump), but the bandwidth is real and it is per-user.
|
||||
|
||||
Consequences today:
|
||||
|
||||
- Every install pulls ~205 GB to derive a catalog that is **identical
|
||||
for everyone**. On a metered or slow connection this is untenable, and
|
||||
it is unconditional on first run.
|
||||
- **It refuses to start without 6 GB free** (`dumpMinStartFreeBytes`),
|
||||
and aborts below 2 GB (`dumpAbortFreeBytes`). This is what breaks
|
||||
`make fresh-install` on a tmpfs `/tmp`.
|
||||
- First-run Explore is empty for the length of the import.
|
||||
|
||||
The catalog half is **the same for everyone**. Only the local half
|
||||
(`PopulateLocalCrossReferences`, `BackfillLibraryDiscographies`) is
|
||||
per-user. Deriving the shared half on each machine is the waste this
|
||||
plan removes.
|
||||
|
||||
## Goal
|
||||
|
||||
Ship a prebuilt core index so a fresh install has a usable Explore
|
||||
immediately, and the runtime build collapses to the local half plus
|
||||
incremental refresh. The full dump import becomes an opt-in "deep
|
||||
catalog" upgrade rather than a prerequisite.
|
||||
|
||||
## Sizing evidence
|
||||
|
||||
Measured 2026-07-25 with a synthetic harness against the real schema and
|
||||
migrations (2.15M-row full run exceeded a 15-minute budget, so this is a
|
||||
200K-row calibration, `VACUUM`ed):
|
||||
|
||||
| Metric | Value |
|
||||
|---|---|
|
||||
| 200,000 rows, with FTS | 85.2 MB |
|
||||
| Cost per row | ~426 B |
|
||||
| zstd -19 | 29.6 MB (2.9x) |
|
||||
|
||||
Extrapolating to the current budgets (`keepRecordings` 1.5M +
|
||||
`keepReleaseGroup` 400K + `keepArtists` 250K = 2.15M rows):
|
||||
|
||||
| Tier | Rows | On disk | zstd -19 |
|
||||
|---|---|---|---|
|
||||
| Full budget | 2.15M | **~900 MB** | ~310 MB |
|
||||
| Core (proposed) | 500K | ~210 MB | **~72 MB** |
|
||||
| Minimal | 250K | ~105 MB | ~36 MB |
|
||||
|
||||
**This corrects an earlier figure.** A ~93 MB index was recorded in the
|
||||
2026-07-16 audit note; that measured the *legacy tier-crawl* index, not
|
||||
the dump-built one. The dump build targets an order of magnitude more
|
||||
rows. Shipping the full index is not viable as a casual download —
|
||||
which is exactly why this plan is scoped to a *core* subset.
|
||||
|
||||
⚠️ Two caveats on these numbers:
|
||||
|
||||
- The harness used a 14-word vocabulary, so its FTS measured only 7% of
|
||||
total size. Real titles have a far larger vocabulary and the real FTS
|
||||
share will be materially higher. **Treat the totals as a floor.**
|
||||
- Row width was estimated from the schema (3 UUIDs at 36 chars dominate);
|
||||
`aliases` was left empty and is populated for real artists.
|
||||
|
||||
Re-measure against a genuine dump-built index before committing to a
|
||||
tier size.
|
||||
|
||||
## What "core" should mean
|
||||
|
||||
`dumpcatalog.go` already has graded per-artist coverage (S2) —
|
||||
`perArtistArtistBudget = 10_000` split into tiers A/B/C with per-tier
|
||||
track and release-group caps. The core index should reuse that machinery
|
||||
rather than invent a second notion of importance:
|
||||
|
||||
- **Artists:** top ~50K by listen count.
|
||||
- **Release groups + recordings:** the S2 per-artist slice for those
|
||||
artists (tier A/B/C caps as they stand).
|
||||
- **Excluded:** the global long tail below the per-artist selection.
|
||||
|
||||
Anything not covered still works — it just resolves through the existing
|
||||
lazy paths (`EnsureArtistDiscography`, `AddFromCache`), which is the
|
||||
behaviour non-covered artists already get today.
|
||||
|
||||
## Distribution: download on first run, not `go:embed`
|
||||
|
||||
**Both packaging paths build from source** — the Homebrew formula builds
|
||||
from a release tarball, the Arch `PKGBUILD` clones the tag. So:
|
||||
|
||||
- Committing the artifact to git bloats the repo and every source tarball.
|
||||
- `go:embed` makes a from-source build require the artifact at build
|
||||
time, so source builds would have to download it anyway — and
|
||||
`build-prod` runs UPX over the binary, which would be pathological
|
||||
with a 70 MB+ embedded blob.
|
||||
|
||||
So "ship with the app" should mean **fetch a prebuilt artifact on first
|
||||
run** from a versioned URL. CI already publishes binary packages to the
|
||||
Gitea package registry (`.gitea/workflows/arch-package.yml`), so there is
|
||||
an existing place to host it.
|
||||
|
||||
Import path: download `.zst` → decompress → `ATTACH` → `INSERT INTO
|
||||
explore_index SELECT ...` through the **existing** `upsertBatch` conflict
|
||||
rules, which already do the right thing (non-empty wins, highest
|
||||
popularity wins, never clobber a good value with an empty one).
|
||||
|
||||
## Artifact contents
|
||||
|
||||
Ship the global catalog columns only. These are **per-user** and must be
|
||||
zeroed in the artifact, then recomputed locally by
|
||||
`PopulateLocalCrossReferences`:
|
||||
|
||||
- `in_library`, `is_similar`
|
||||
- `local_artist_id`, `local_release_group_id`, `local_recording_id`
|
||||
|
||||
`discog_fetched` should ship as `1` for artists whose S2 slice is
|
||||
included, so the backfill doesn't redundantly re-fetch them.
|
||||
|
||||
Also decide per-table whether to include: `similar_artist_map`,
|
||||
`artist_metadata`, `release_to_rg`. `release_to_rg` in particular may
|
||||
rival the index in size — measure before including.
|
||||
|
||||
**Resolved: the artifact ships no FTS.** Rows are inserted into the
|
||||
client's own `explore_index`, whose `AFTER INSERT` trigger populates
|
||||
`explore_index_fts` as a side effect — so shipping a search index would
|
||||
be pure redundant weight. `cmd/indexexport` builds the artifact without
|
||||
FTS or triggers accordingly.
|
||||
|
||||
## Update strategy
|
||||
|
||||
- **Popularity drift** — `dumpincremental.go` already implements
|
||||
incremental listens-dump refresh (`RefreshListenCounts`, weekly
|
||||
cadence). It applies unchanged on top of a shipped baseline, provided
|
||||
`listens_applied_series` is stamped in the artifact so deltas resume
|
||||
from the right point.
|
||||
- **Catalog additions** — new releases arrive via the existing lazy
|
||||
per-artist fetches. A refreshed artifact per app release is enough;
|
||||
no separate cadence needed.
|
||||
- **Schema changes** — `schema_version` exists on `explore_index` but is
|
||||
noted as dead in the audit. Either wire it up or version the artifact
|
||||
filename against the migration number, so an old artifact can't be
|
||||
imported into a newer schema.
|
||||
|
||||
## Build pipeline: build and cache in Gitea CI
|
||||
|
||||
The import is unusually well suited to running as a **series of
|
||||
time-boxed CI jobs against a persistent cache**, because the resumability
|
||||
already exists:
|
||||
|
||||
- Stage 1 streams over a `resumableReader` that reconnects with HTTP
|
||||
`Range` requests, and the live dump advertises `accept-ranges: bytes`.
|
||||
- `counts.bin` checkpoints `Offset` (absolute byte position) and
|
||||
`MemberIdx`, and the applier merges results **in member order** so
|
||||
"every checkpoint is a contiguous prefix of the stream"
|
||||
(`dumpcounts.go`).
|
||||
- Stage 2's canonical scan is deliberately restartable wholesale — "cheap
|
||||
enough to simply restart after an interruption" (`dumpcatalog.go`).
|
||||
|
||||
So a job that hits a runner time limit resumes at its exact byte offset
|
||||
on the next run. **No single multi-hour job is required** — schedule
|
||||
N bounded runs and let them converge.
|
||||
|
||||
What it needs:
|
||||
|
||||
1. **A persistent volume for `explore-staging/` + the DB.** `act_runner`
|
||||
uses the Docker backend and job containers are ephemeral, so bind-mount
|
||||
a host path (or a named Docker volume) and point `YJ_HOME` at it.
|
||||
Prefer this over the Actions cache — cache entries are size-capped and
|
||||
awkward at GB scale, and this is a self-hosted runner anyway.
|
||||
2. **A headless entrypoint** — currently the import only runs from the
|
||||
app lifecycle (`StartIndexBuild` via `OnDomReady`). This is a real gap,
|
||||
but a small one: `NewSearchIndex(db, lb, artistImg, logger)` takes no
|
||||
Wails dependency, and the single `runtime.EventsEmit` in
|
||||
`searchindex.go` sits inside `emitStatus`, which already early-returns
|
||||
when `runtimeCtx == nil`. A `cmd/indexbuild` that opens the DB and
|
||||
calls `StartBuild(context.Background())` — never `SetContext` — should
|
||||
work. Verify `scheduleChampionRebuild` in the `StartBuild` defer is
|
||||
also Wails-free.
|
||||
3. **Triggers.** `indexbuild` decides its own mode from index state, so
|
||||
every trigger runs the same command: push to `main` and a weekly cron
|
||||
both land on a cheap refresh (which no-ops when nothing new is
|
||||
published), and the 3-month rebuild fires when the command notices the
|
||||
import has aged out.
|
||||
|
||||
Then export: subset to core, zero the personal columns, stamp
|
||||
`dump_import_done` / `listens_applied_series` / schema version, `VACUUM`,
|
||||
`zstd -19`, checksum, publish to the Gitea package registry (the Arch
|
||||
workflow already authenticates against it with `PACKAGE_TOKEN`).
|
||||
|
||||
**Be a good citizen about the 205 GB.** Rebuild on the dump cadence
|
||||
(the audit notes a 90-day re-import cadence), never per-commit. Once a
|
||||
baseline exists, the ~180 MB daily incremental dumps already wired in
|
||||
`dumpincremental.go` keep popularity fresh — so the 205 GB is genuinely
|
||||
one-time per rebuild, not per refresh. Also check the runner's own
|
||||
egress if it is self-hosted on a home connection.
|
||||
|
||||
## Licensing
|
||||
|
||||
- MusicBrainz canonical dump is **CC0** — redistribution fine.
|
||||
- ListenBrainz-derived listen counts need their dump licence checked
|
||||
before redistribution, plus attribution in-app either way.
|
||||
- Note the derived counts already differ from LB API values (no MLHD+
|
||||
history) — a known, accepted divergence, but worth stating wherever
|
||||
the numbers are surfaced.
|
||||
|
||||
## Risks
|
||||
|
||||
- **Artifact staleness vs app version** — a user on an old release gets
|
||||
an old catalog. Mitigated by incremental refresh + lazy fetches.
|
||||
- **Download failure / offline install** — must degrade to today's
|
||||
behaviour (local library search), not a broken Explore. The failure is
|
||||
now visible in the Jobs panel, which helps.
|
||||
- **Users who want the full catalog** — keep the existing dump import as
|
||||
an explicit opt-in, gated behind a setting. Note that no such setting
|
||||
exists today: `StartIndexBuild()` is unconditional, and Library Only
|
||||
mode is frontend-`localStorage` only with no backend wiring.
|
||||
|
||||
## Phasing
|
||||
|
||||
1. ✅ **Headless entrypoint.** `cmd/indexbuild` — resumable, budgeted
|
||||
(`-budget 3h`), signal-aware, exit 3 = "more work remains". Verified
|
||||
to run without Wails; builds with `CGO_ENABLED=0` and no build tags.
|
||||
2. ✅ **Export tooling.** `cmd/indexexport` — top-N artists plus a
|
||||
per-artist window of their release groups and recordings, personal
|
||||
columns dropped, metadata stamped, vacuumed. Verified against a
|
||||
synthetic index: no personal columns leak, no orphaned rows, caps
|
||||
respected.
|
||||
3. ✅ **One real build.** Superseded by a real dump-built index that
|
||||
already existed on the dev machine (`dump_import_done` 2026-07-17).
|
||||
Measured 2026-07-29 — these replace every extrapolation above:
|
||||
|
||||
| | rows | on disk |
|
||||
|---|---|---|
|
||||
| `explore_index` | 2,052,168 (227,359 artists / 400,675 RGs / 1,424,134 recordings) | 383 MB |
|
||||
| its indexes | | 395 MB |
|
||||
| FTS | | 80 MB |
|
||||
|
||||
187 B/row for the shippable table, 418 B/row all-in — so the ~900 MB
|
||||
full-budget estimate was right. Two real exports:
|
||||
|
||||
| tier | rows | artifact | zstd -19 |
|
||||
|---|---|---|---|
|
||||
| 50K artists (default) | 1,076,133 | 191.5 MB | **70.6 MB** |
|
||||
| 25K artists / 10 RG / 20 rec | 620,973 | 110.6 MB | **37.8 MB** |
|
||||
|
||||
`release_to_rg` was empty in that index — it predates the code that
|
||||
persists it — so its size is still unmeasured.
|
||||
4. ✅ **Import path.** `backend/explore/artifactfetch.go` (download,
|
||||
Range-resume, sha256, zstd) and `artifactimport.go` (validate, ATTACH,
|
||||
batched merge, FTS rebuild, meta stamping). Reported in the Jobs panel
|
||||
under its own two stages. Measured end to end on the real 50K-artist
|
||||
artifact against a disk-backed DB: **1,076,133 rows merged in 43.2s**
|
||||
(24,900 rows/s), yielding a 455 MB `yj.db`, FTS populated and
|
||||
searchable. In-memory the same merge runs in 28.3s.
|
||||
5. ✅ **Gate the dump build.** `deep_catalog_enabled` in
|
||||
`explore_index_meta` (beside `index_build_paused` — it is build state,
|
||||
read at one decision point). Off by default; exposed as
|
||||
`DeepCatalogEnabled` / `SetDeepCatalogEnabled` on the explore Service.
|
||||
An interrupted dump import resumes regardless of the setting, so the
|
||||
gate never discards a checkpoint that already cost hours.
|
||||
|
||||
## Measured 2026-07-29: why the client cannot fix this itself
|
||||
|
||||
`data.metabrainz.org` caps a client at ~2.1 MB/s. One Range stream and
|
||||
four concurrent Range lanes both delivered 32 MB at the same aggregate
|
||||
rate (2,111,195 B/s vs 2,209,000 B/s) while the same machine pulled
|
||||
66.9 MB/s from a CDN. **Parallelism buys nothing** — the four lanes just
|
||||
divide the same cap, and one of them starved to 0.5 MB/s.
|
||||
|
||||
So stage 1 costs, unavoidably:
|
||||
|
||||
| | bytes | wall clock |
|
||||
|---|---|---|
|
||||
| Whole tar (what shipped before column projection) | 205 GB | ~27 h |
|
||||
| Column projection, 3 columns (43.4%) | 89 GB | ~11.8 h |
|
||||
| `recording_mbid` only (24.1%), rolled up via canonical | 49 GB | ~6.5 h |
|
||||
| + 1-in-4 member stride sample | 12 GB | ~1.6 h |
|
||||
|
||||
The last two are CI-side options, not client defaults: recording-only
|
||||
drops listens carrying no recording MBID and re-derives artist totals as
|
||||
a sum over recordings, and sampling trades exact counts for a ranking.
|
||||
Both are only safe because the selection they feed is a top-N cut.
|
||||
|
||||
## Distribution: the "latest" version trick
|
||||
|
||||
The client cannot enumerate package versions — Gitea's package listing
|
||||
API requires a token, while an anonymous file GET does not (a probe of a
|
||||
non-existent artifact returns 404, not 401). So `index-artifact.yml`
|
||||
publishes each artifact twice: under a dated version for history, and
|
||||
under a fixed `latest` version that the client fetches from a
|
||||
predictable URL. Generic packages reject overwriting an existing
|
||||
filename, so `latest` is DELETEd before each rewrite.
|
||||
|
||||
⚠️ **Unverified:** that anonymous package *download* is actually enabled
|
||||
on git.ljones.me. The 404-vs-401 probe is suggestive, not proof — no
|
||||
artifact has been published yet to test against. Confirm before relying
|
||||
on it, and note that every install pulling from a personal Gitea makes
|
||||
its bandwidth and uptime a user-facing dependency.
|
||||
|
||||
## Incremental retention bounds artifact staleness
|
||||
|
||||
The incremental dump directory holds 30 dumps (series 2579–2610 as of
|
||||
2026-07-29) and full dumps land roughly monthly. An artifact older than
|
||||
~30 days therefore cannot be topped up: the dailies bridging the gap are
|
||||
gone. That is a permanent undercount of that window's listens, not
|
||||
corruption — but it pins the republish cadence at monthly.
|
||||
|
||||
## Upgrade path for indexes built by older code
|
||||
|
||||
The dev machine's index has `dump_import_done` set but **no**
|
||||
`listens_applied_series` and an empty `release_to_rg`, because it was
|
||||
built before the code that writes them. That combination is a dead end:
|
||||
`RefreshListenCounts` bails with "no baseline series recorded", and
|
||||
`runDumpBuild` short-circuits on the done marker, so popularity can
|
||||
never update again. Current code writes both, so this affects only
|
||||
pre-existing installs — but the artifact import is the natural place to
|
||||
rescue them, since merging one stamps a fresh baseline series.
|
||||
6. ✅ **CI wiring.** `.gitea/workflows/index-artifact.yml` — push +
|
||||
weekly cron + manual, concurrency-guarded, publishes only when
|
||||
`complete && changed` so identical artifacts don't accumulate.
|
||||
Runner-side prerequisites are in place (cache dir + `valid_volumes`
|
||||
on the VPS runner).
|
||||
|
||||
Step 3 is the gate on everything downstream — and it is worth doing
|
||||
regardless of whether the artifact ever ships, since it is the only way
|
||||
to get real numbers for the index.
|
||||
|
||||
## Related
|
||||
|
||||
- `backend/explore/dumpimport.go` — stage orchestration, disk floors
|
||||
- `backend/explore/dumpcatalog.go` — budgets, S2 per-artist tiers
|
||||
- `backend/explore/dumpincremental.go` — incremental refresh (update path)
|
||||
- `backend/explore/searchindex.go` — `upsertBatch` conflict rules,
|
||||
`PopulateLocalCrossReferences`
|
||||
- Migration 26 in `backend/database/database.go` — `explore_index` schema
|
||||
@@ -0,0 +1,155 @@
|
||||
# 002 — Data lifecycle architecture
|
||||
|
||||
**Status:** completed (first tranche); follow-ups tracked below
|
||||
**Branch:** main
|
||||
**Created:** 2026-07-26
|
||||
|
||||
## Problem
|
||||
|
||||
An audit of asset and row cleanup found five leaks, four of which shared
|
||||
one root cause: **deletion logic was hand-written per call site and lived
|
||||
far from the thing being deleted.** `RemoveLibrary` knew about ten tables
|
||||
because someone enumerated them once; migration 32 added an eleventh and
|
||||
nothing noticed. Files written by `explore` had no cleanup counterpart
|
||||
anywhere. A function that evicted expired cache rows was written and
|
||||
never called.
|
||||
|
||||
Findings, in severity order:
|
||||
|
||||
1. **`RemoveLibrary` was broken for any scanned library.** `tagging_items`
|
||||
holds `FOREIGN KEY(library_id) REFERENCES libraries(id)` with no
|
||||
`ON DELETE` clause and was never cleared, so `DELETE FROM libraries`
|
||||
failed with `FOREIGN KEY constraint failed (787)` and rolled back the
|
||||
whole removal. Every scanned library has `tagging_items` rows (the
|
||||
scan upserts one per album folder), so this fired on essentially every
|
||||
real removal. `RemoveLibrary` had zero test coverage.
|
||||
2. **Artist images were never deleted by anything.** No `os.Remove` in
|
||||
`explore`, no `DELETE FROM artist_images` in the codebase. Unbounded
|
||||
in the number of artists ever browsed in Explore, most of whom are not
|
||||
in the library.
|
||||
3. **Cover art size variants leaked on removal.** Only the base
|
||||
`cover_art.file_path` was unlinked; the `_sm/_md/_lg` files beside it
|
||||
are derived filenames, not rows, so three files per cover survived.
|
||||
4. **`http_cache` was never pruned.** `Cache.Evict()` existed with no
|
||||
callers. Reads filter on `expires_at`, so expired rows were inert but
|
||||
accumulated for the life of the install.
|
||||
5. **Cover-art proxy cache was never pruned.** No eviction, no size cap.
|
||||
|
||||
## Approach
|
||||
|
||||
Rather than patch five holes, classify the data so the *class* of bug
|
||||
becomes hard to write. Everything persisted falls on two axes —
|
||||
regenerability and cost of regeneration — which collapse to four kinds:
|
||||
|
||||
| Kind | Regenerable? | Deletion policy |
|
||||
|---|---|---|
|
||||
| **Owned** — projection of the user's files | Yes, by rescan | Follows the files |
|
||||
| **Authored** — user-created, no other copy | **No** | Explicit user action only |
|
||||
| **Derived** — computed from owned | Yes, cheaply | Free; must never block owned deletion |
|
||||
| **Cache** — network or dump sourced | Yes, expensively | TTL/age eviction, never cascade |
|
||||
|
||||
The classification is not just vocabulary — it produces the right fix for
|
||||
each finding. Finding 1 is derived data acting as a referential parent of
|
||||
owned data, which the taxonomy makes categorically illegal. Finding 2 is
|
||||
cache data that never needed owner-linked cleanup at all; it wants age
|
||||
eviction. Finding 3 is derived data that must be swept against a live set
|
||||
rather than tracked individually.
|
||||
|
||||
A Go interface was considered and rejected: the only polymorphic consumer
|
||||
is the janitor, the substrates have nothing in common (SQL rows, an FTS
|
||||
virtual table, a view, three directories of JPEGs, a 900 MB index), and
|
||||
provenance is a static fact better enforced by package boundaries than by
|
||||
methods an implementation may lie about. A declarative catalog gets the
|
||||
same benefit for a tenth of the cost.
|
||||
|
||||
## What shipped
|
||||
|
||||
**`backend/datamap`** — the catalog. Every table, view, and asset
|
||||
directory declared with its `Kind`, its `Lifetime` (`cascade`, `set-null`,
|
||||
`swept`, `retained`), and a note explaining the classification. Plain data
|
||||
with no service dependencies, so tests can assert it against a live
|
||||
schema. FTS5 shadow tables resolve to their parent.
|
||||
|
||||
Tests that give it teeth (`backend/datamap/datamap_test.go`):
|
||||
|
||||
- `TestCatalogCoversSchema` — every table in `sqlite_master` is claimed by
|
||||
exactly one entry. **A new table fails the build until somebody states
|
||||
what it is and how it dies.**
|
||||
- `TestCatalogHasNoStaleEntries` — the reverse, catching drift.
|
||||
- `TestNoActionForeignKeysAreDeclaredSwept` — a `NO ACTION` foreign key
|
||||
blocks its parent's deletion, so its table must declare `swept`. This is
|
||||
the exact shape of finding 1, now caught at CI time.
|
||||
- `TestLifetimesMatchSchema` — declared cascade/set-null must match what
|
||||
SQLite actually enforces.
|
||||
- `TestAuthoredCascadesAreDeliberate` — authored data is unrecoverable, so
|
||||
a cascade onto it needs an explicit exemption.
|
||||
|
||||
**`backend/maintenance`** — the janitor. A registry of named jobs with
|
||||
per-job minimum intervals, run at startup-idle and on a 6h tick. Policies
|
||||
follow the taxonomy: derived data sweeps against a live set, cache data
|
||||
ages out. Registered in one place (`app.go: startJanitor`) so the full set
|
||||
of janitorial work is a single visible list.
|
||||
|
||||
Jobs: `http-cache-evict` (6h), `covers-sweep` (24h, live set from
|
||||
`cover_art` expanded via `CoverArtFileSet`), `artist-images-sweep` (24h,
|
||||
keeps art for library artists indefinitely, evicts browsed-artist art
|
||||
after 90d), `cover-art-proxy-sweep` (24h, 30d age eviction).
|
||||
|
||||
The covers sweep refuses to act on an empty live set — that means the
|
||||
query failed to see the table, not that every cover is garbage.
|
||||
|
||||
**Leak tests** (`backend/library/leak_test.go`) — driven by the catalog
|
||||
rather than a hardcoded list, so new tables are covered the moment they
|
||||
are catalogued:
|
||||
|
||||
- `TestRemoveLibraryLeavesNoOwnedOrDerivedRows` — removing the only
|
||||
library leaves no owned or derived rows, except those in
|
||||
`staleTolerated` with a written reason.
|
||||
- `TestRemoveLibraryPreservesAuthoredData` — authored data survives.
|
||||
- `TestSweptTablesAreActuallySwept` — a table declaring `swept` that
|
||||
nothing sweeps is caught.
|
||||
|
||||
All three were verified to fail when the finding-1 fix is reverted.
|
||||
|
||||
**Fixes** — `tagging_items` cleared inside the removal transaction
|
||||
(`crud.go` step 17); `CoverArtFileSet` expands originals to variants and
|
||||
the legacy `_thumb` name; `Cache.Evict` logic moved into a registered job.
|
||||
|
||||
**Incidental:** `Library.emit` — `runtime.EventsEmit` calls `log.Fatalf`
|
||||
on a context without a Wails runtime, which killed the test binary and
|
||||
made the whole package untestable. All ten emits in the package now route
|
||||
through a nil-safe helper. This also removes a real crash risk for
|
||||
background workers that outlive their context.
|
||||
|
||||
## Follow-ups
|
||||
|
||||
**`audio_files` is a mixed-kind table.** `play_count`, `last_played`, and
|
||||
`tag_status` are *authored* data living in an *owned* table. Orphan
|
||||
cleanup treats the whole row as regenerable, which is why renaming a file
|
||||
destroys its play count — the row is deleted and re-imported fresh. This
|
||||
is the strongest argument for splitting authored per-track state into its
|
||||
own table keyed by something more stable than a path. Related: an
|
||||
audio-stream content hash (excluding tag blocks, so it survives
|
||||
retagging) would let a rename be recognised as the same file. Deliberately
|
||||
out of scope here; it is a schema change plus a rename-detection pass, not
|
||||
a cleanup fix.
|
||||
|
||||
**Cascade adoption.** Fourteen of nineteen foreign keys are `NO ACTION`.
|
||||
Converting them to `CASCADE` would delete a lot of hand-written orphan
|
||||
sweeps, but SQLite cannot add `ON DELETE` via `ALTER TABLE` — each needs
|
||||
the 12-step table rebuild. Note the ordering constraint: cascades delete
|
||||
rows silently, so any code that collects file paths *before* deleting rows
|
||||
(as `RemoveLibrary` does for cover art) breaks under cascade. Mark-and-
|
||||
sweep must land first; the two compose, cascade plus path-collection does
|
||||
not.
|
||||
|
||||
**Consolidate the ten orphan sweeps.** `DELETE ... WHERE id NOT IN (...)`
|
||||
appears ten times across `crud.go`, `dbsync.go`, `smartplaylist.go`, and
|
||||
`database.go`. One shared `sweepOrphans(tx)` would shrink the surface where
|
||||
a new table can be forgotten. Worth doing opportunistically rather than as
|
||||
a big-bang refactor.
|
||||
|
||||
**Storage settings pane.** The catalog knows every table and directory and
|
||||
its kind; the janitor already computes bytes freed. A settings pane showing
|
||||
per-kind disk usage with "clear cache" and "rebuild derived data" buttons
|
||||
is now mostly a UI job.
|
||||
@@ -0,0 +1,279 @@
|
||||
# 003 — Download clients
|
||||
|
||||
**Status:** implemented (v1); follow-ups tracked below
|
||||
**Branch:** main
|
||||
**Created:** 2026-07-27
|
||||
|
||||
## Problem
|
||||
|
||||
YellowJacket can find music (`explore`), identify it (`autotag`), and
|
||||
manage it (`library`) — but it can't acquire it. The one gap between
|
||||
"you're missing this album" and "you own this album" is filled today by
|
||||
the user alt-tabbing to some other tool.
|
||||
|
||||
The naive fix is an HTTP client for Soulseek and a shell-out to yt-dlp.
|
||||
That produces two bespoke code paths with duplicated queueing, retry,
|
||||
staging and import logic, and a third service means a third copy. The
|
||||
services users want to connect are also not the same *kind* of thing —
|
||||
some search, some transfer bytes, some are entire automation systems we
|
||||
delegate to — so a single `DownloadClient` interface would be a lie that
|
||||
every adapter partially implements.
|
||||
|
||||
## The role decomposition
|
||||
|
||||
Every candidate integration fills one or two of three roles:
|
||||
|
||||
| Service | Searches | Transports | Delegates |
|
||||
|---|---|---|---|
|
||||
| slskd (Soulseek) | ✅ | ✅ | |
|
||||
| yt-dlp | ✅ | ✅ | |
|
||||
| Lidarr | | | ✅ |
|
||||
| Prowlarr | ✅ | | |
|
||||
| qBittorrent / Transmission | | ✅ | |
|
||||
| SABnzbd / NZBGet | | ✅ | |
|
||||
|
||||
So: three small interfaces, not one big one. A provider implements
|
||||
whichever it supports and declares that in a capability struct, the same
|
||||
way `jobs.Caps` lets the frontend render controls without switching on
|
||||
`Kind`.
|
||||
|
||||
```go
|
||||
// Searcher turns a request into ranked candidates.
|
||||
type Searcher interface {
|
||||
Search(ctx context.Context, req Request) ([]Candidate, error)
|
||||
}
|
||||
|
||||
// Transporter moves a candidate's bytes to a local staging directory.
|
||||
type Transporter interface {
|
||||
Grab(ctx context.Context, c Candidate, dst string, p ProgressFunc) (Result, error)
|
||||
Cancel(ctx context.Context, grabID string) error
|
||||
}
|
||||
|
||||
// Delegator hands the whole request to an external manager and
|
||||
// reports back when files land.
|
||||
type Delegator interface {
|
||||
Request(ctx context.Context, req Request) (string, error)
|
||||
Poll(ctx context.Context, externalID string) (DelegateStatus, error)
|
||||
}
|
||||
```
|
||||
|
||||
A `Provider` is the registry entry: identity, config, health check, caps,
|
||||
plus whichever of the three it satisfies. Search-only providers
|
||||
(Prowlarr) are paired with a transport at grab time by protocol match
|
||||
(`torrent` → qBittorrent, `usenet` → SABnzbd); providers that do both
|
||||
are self-pairing.
|
||||
|
||||
## v1 decisions (settled)
|
||||
|
||||
- **On-demand only.** User-initiated "find this album" from an Explore
|
||||
artist/album page or a missing-album row. No wanted list, no artist
|
||||
monitoring, no quality-cutoff upgrades. The queue and pipeline built
|
||||
here are exactly what monitoring would later sit on top of — see
|
||||
Deferred.
|
||||
- **Soulseek via slskd's REST API**, not a native protocol client. Same
|
||||
adapter shape as everything else, no wire protocol, no credentials in
|
||||
our process, fully testable against an `httptest` server. A native
|
||||
provider can slot in behind `Searcher`/`Transporter` later with no
|
||||
pipeline changes.
|
||||
- **Stage → autotag → import.** Downloads land in a staging directory,
|
||||
are matched against the intended release with the existing `autotag`
|
||||
scorer, tagged, then moved into the library and scanned. Never write
|
||||
into the library root directly.
|
||||
- **All four provider families in v1**, sequenced so each phase proves a
|
||||
different role shape (see Phases).
|
||||
|
||||
## Pipeline
|
||||
|
||||
```
|
||||
Request (MBID-anchored where possible)
|
||||
└─> fan-out Search across enabled providers (per-provider timeout)
|
||||
└─> merge + rank Candidates
|
||||
└─> user picks (or auto-pick above confidence threshold)
|
||||
└─> Grab into staging/<request-id>/
|
||||
└─> verify (audio decodes, expected track count)
|
||||
└─> autotag against the intended release
|
||||
└─> tagwriter writes tags
|
||||
└─> move into library layout
|
||||
└─> targeted incremental scan
|
||||
```
|
||||
|
||||
The `Request` should carry a release-group or release MBID whenever the
|
||||
user started from an Explore page, because that anchor is what makes the
|
||||
autotag step reliable instead of a second guess. Free-text requests are
|
||||
supported but flagged lower-confidence, and never auto-pick.
|
||||
|
||||
Staging lives under the user data dir, not the library. Partial grabs are
|
||||
resumable where the provider supports it and swept on startup where it
|
||||
doesn't.
|
||||
|
||||
## Candidate ranking
|
||||
|
||||
Two independent scores, kept separate:
|
||||
|
||||
1. **Match confidence** — does this candidate contain the release the
|
||||
user asked for? Reuse `autotag`'s distance/alignment machinery on the
|
||||
candidate's *filenames* (Soulseek gives paths, not tags), against the
|
||||
expected tracklist from the explore index.
|
||||
2. **Source quality** — format (FLAC > V0 > 320 > lower), bitrate,
|
||||
completeness (file count vs. expected track count), source health
|
||||
(slskd queue length and upload slots; seeders for torrents), and a
|
||||
user-set per-provider priority.
|
||||
|
||||
Ranking presents both, because they trade off — a perfectly-matched
|
||||
128kbps rip should lose to a well-matched FLAC, and the user should be
|
||||
able to see why. Reusing `autotag.ScoreBreakdown`'s "explain the ranking"
|
||||
pattern here is deliberate.
|
||||
|
||||
## Persistence
|
||||
|
||||
New tables (migration TBD, next free number):
|
||||
|
||||
- `download_providers` — id, kind, name, enabled, priority, config blob
|
||||
(JSON), `created_at`. Non-secret config only.
|
||||
- `download_requests` — id, source (`explore-album`, `explore-artist`,
|
||||
`manual`), release_mbid / release_group_mbid, free-text query,
|
||||
requested_at, state, resolved_download_id.
|
||||
- `download_items` — one row per grab attempt: request_id, provider_id,
|
||||
candidate JSON, state, bytes/total, staging path, error, timestamps.
|
||||
|
||||
**Secrets** (slskd API key, Lidarr/Prowlarr API keys, qBittorrent
|
||||
password) do not go in the TOML config or the DB in plaintext. Use the OS
|
||||
keyring where available with a clearly-labelled encrypted-file fallback,
|
||||
and never log a config value from a provider's secret field. Open
|
||||
question below on the exact library.
|
||||
|
||||
## Jobs integration
|
||||
|
||||
Add `jobs.KindDownload`. One job per request (not per file), with
|
||||
`Stages` for search → grab → import so the existing detail panel renders
|
||||
the pipeline for free. `Caps{Cancellable: true}`; pausable only for
|
||||
providers that can resume. Per-provider concurrency caps and a global
|
||||
cap, both configurable — hammering a Soulseek peer with eight parallel
|
||||
transfers gets you queued or banned.
|
||||
|
||||
## Frontend
|
||||
|
||||
- New `download-providers` section in `config-page` (HTMX + templ, same
|
||||
as existing settings) for provider CRUD, test-connection, priority.
|
||||
- New `download-picker` Lit component: the ranked-candidate dialog,
|
||||
invoked from Explore album/artist pages and from a missing-album row.
|
||||
- `download-store.ts` subscribing to the existing `JobsChanged` event —
|
||||
no new event channel needed for progress.
|
||||
|
||||
## Phases
|
||||
|
||||
Each phase is independently shippable and proves a distinct role shape.
|
||||
|
||||
1. **Core.** Interfaces, registry, `Request`/`Candidate`/`Result` types,
|
||||
staging dir, ranking, the stage→autotag→import tail, jobs wiring,
|
||||
schema, secret storage. Ships with a fake provider and full test
|
||||
coverage of the pipeline. No real network.
|
||||
2. **yt-dlp.** Subprocess provider: search + transport, no server for the
|
||||
user to run, so it's the fastest path to an end-to-end working
|
||||
feature. Proves the local-subprocess shape (binary discovery,
|
||||
version checks, stdout progress parsing, sandboxing the arg list).
|
||||
3. **slskd.** Remote search + transport over REST. Proves the remote
|
||||
HTTP shape and is the highest-value source. This is where filename-
|
||||
based match confidence earns its keep.
|
||||
4. **Lidarr.** Delegate. Proves the fire-and-poll shape, where we don't
|
||||
own the transfer and the "import" step is really "detect what Lidarr
|
||||
already imported and reconcile".
|
||||
5. **Prowlarr + qBittorrent/SABnzbd.** Proves split search/transport
|
||||
pairing — the one case where two providers cooperate on a single
|
||||
request.
|
||||
|
||||
## Risks and constraints
|
||||
|
||||
- **No bundled credentials, no default-on providers, no preconfigured
|
||||
indexers.** Every provider is off until the user configures it. The
|
||||
app ships the ability to connect to services the user already runs.
|
||||
- **yt-dlp is a moving target.** Pin a minimum version, check it at
|
||||
provider-enable time, and fail with a clear message rather than
|
||||
parsing garbage output.
|
||||
- **Filename-only matching is genuinely hard.** Soulseek results are
|
||||
`\Music\Album (1997) [FLAC]\01 - Track.flac` at best. Budget real
|
||||
effort for the path-parsing heuristics; `autotag/normalize.go` is the
|
||||
starting point.
|
||||
- **Partial and failed grabs must never reach the library.** The import
|
||||
step is the only writer into library paths, and it runs after
|
||||
verification. Staging sweep on startup.
|
||||
- **Tests must not hit the network.** `httptest` servers for slskd/
|
||||
Lidarr/Prowlarr, a stub binary for yt-dlp.
|
||||
|
||||
## Deferred
|
||||
|
||||
- Wanted list with background retry (the natural next plan).
|
||||
- Artist monitoring + auto-grab of new releases — cheap once the wanted
|
||||
list exists, because `explore`'s dump index already knows the full
|
||||
discography and `library` already knows what's owned.
|
||||
- Quality profiles and upgrade-if-better.
|
||||
- Native Soulseek protocol client.
|
||||
- Transmission/Deluge/NZBGet (same shape as their shipped siblings —
|
||||
add on demand).
|
||||
- Internet Archive / Bandcamp-collection providers: cheap REST adapters,
|
||||
worth adding once the core is proven.
|
||||
|
||||
## Resolved questions
|
||||
|
||||
1. **Secret storage.** No keyring dependency was added. Credentials go
|
||||
in a 0600 JSON file in the user data directory (`download-secrets.json`),
|
||||
keyed by provider row ID. This is deliberately *not* encryption — a
|
||||
key stored beside the data it unlocks protects nothing, and claiming
|
||||
otherwise would be worse than being clear about it. What the file
|
||||
mode buys is protection from other local users and from the config
|
||||
file being pasted into a bug report. `SecretStore` is an interface so
|
||||
an OS keyring backend can be added later without touching any
|
||||
provider.
|
||||
2. **Auto-pick.** Implemented behind `Downloads.AutoPick`, default off.
|
||||
It requires an MBID-anchored request, match ≥ 0.85, quality ≥ 0.5,
|
||||
and ≥ 0.08 of daylight over second place. Free-text requests can
|
||||
never auto-pick, because there is no tracklist to be right about.
|
||||
3. **Library layout.** Configurable path template, default
|
||||
`{albumartist}/{album}/{track} {title}`. Segments are sanitized for
|
||||
Windows-reserved characters and trailing dots/spaces so a library
|
||||
synced between platforms does not produce unopenable files. Existing
|
||||
files are never overwritten — a collision gets a numbered variant,
|
||||
because the file already there may be a better copy the user owns.
|
||||
4. **Entry point.** "Find this album" on the Explore album page, shown
|
||||
only when a client is connected and the album is not already owned.
|
||||
The artist-discography right-click is not wired up yet.
|
||||
|
||||
## What shipped
|
||||
|
||||
All five phases, ~4,500 lines with tests, `make lint` clean and the full
|
||||
backend suite green (including under `-race`).
|
||||
|
||||
**Core** (`backend/download/`): `Searcher`/`Transporter`/`Delegator`
|
||||
interfaces with capability-driven composition; `Request`/`Candidate`/
|
||||
`Result` types; provider registry with self-registering adapters;
|
||||
two-axis ranking; staging area with escape-guards and startup sweep;
|
||||
verify → tag → import tail; jobs integration under `KindDownload`;
|
||||
three tables catalogued in `datamap`.
|
||||
|
||||
**Providers**: yt-dlp (subprocess; assembles albums from per-track
|
||||
searches, since a "full album" video cannot be imported as tracks),
|
||||
slskd (remote search + transport, peer-health scoring, collects from the
|
||||
daemon's own downloads folder), Lidarr (delegate; reconciles in place
|
||||
rather than moving files out from under a system still managing them),
|
||||
Prowlarr (search-only) paired at grab time with qBittorrent or SABnzbd.
|
||||
|
||||
**Frontend**: `download-store.ts`, `download-picker` + `candidate-row`
|
||||
(two meters, not one blended score), `download-clients` settings section
|
||||
rendering its forms from backend descriptors so a new adapter needs no
|
||||
frontend change.
|
||||
|
||||
## Follow-ups
|
||||
|
||||
- **Resume across restart.** Live transfers are currently marked failed
|
||||
on startup and their staging swept, because the transports do not
|
||||
survive the process. slskd and qBittorrent can both resume in
|
||||
principle; the item rows already carry what would be needed.
|
||||
- ~~**Per-provider concurrency caps.**~~ Done in 004: per-kind defaults
|
||||
(slskd 1, yt-dlp 2, torrent/usenet 4) with a per-provider override,
|
||||
and the provider's slot is taken before the global one.
|
||||
- **Prowlarr candidates score blind.** Indexer results carry no file
|
||||
list, so match scoring has only the release title. Fetching the
|
||||
torrent metadata before ranking would fix this and is the single
|
||||
biggest ranking improvement available.
|
||||
- ~~Wanted list, artist monitoring~~ — done in 004. Quality profiles
|
||||
and upgrade-if-better remain deferred.
|
||||
@@ -0,0 +1,163 @@
|
||||
# 004 — Wanted list
|
||||
|
||||
**Status:** implemented
|
||||
**Branch:** main
|
||||
**Created:** 2026-07-29
|
||||
**Follows:** 003-download-clients
|
||||
|
||||
## Problem
|
||||
|
||||
Plan 003 shipped a request as a heavyweight row: library, anchors,
|
||||
cached tracklist, state machine, error text, cascading items. That is
|
||||
the right shape for *one attempt to acquire something* and the wrong
|
||||
shape for *the user wanting something*, and 003 used it for both.
|
||||
|
||||
The consequences showed up immediately. A request that found nothing was
|
||||
marked `failed`, which is a lie — the album exists, no source had it
|
||||
today. Retrying meant the user remembering to press a button. Wanting an
|
||||
artist's future releases was not expressible at all. And a user who
|
||||
acquired an album by other means kept a failed row about it forever.
|
||||
|
||||
## The model
|
||||
|
||||
A **want** is an MBID, what that MBID names, and retry bookkeeping.
|
||||
That is all.
|
||||
|
||||
```
|
||||
download_wants(mbid, entity, library_id, scope, secondary, state,
|
||||
parent_id, attempts, last_error, next_try_at,
|
||||
external_ids)
|
||||
```
|
||||
|
||||
`entity` is the only type distinction, and it carries all the policy:
|
||||
|
||||
| entity | meaning |
|
||||
|---|---|
|
||||
| `artist` | a subscription. Never satisfied; each pass expands the discography into child wants |
|
||||
| `release-group` | an album in the abstract — any release satisfies it |
|
||||
| `release` | one specific edition |
|
||||
| `recording` | one track |
|
||||
|
||||
`UNIQUE(mbid, library_id)` is load-bearing: it is what makes artist
|
||||
expansion idempotent, so a subscription can re-run every pass and add
|
||||
only what is genuinely new.
|
||||
|
||||
Requests did not go away — they became what they always were, the
|
||||
ephemeral record of one attempt, with a nullable `want_id` back-link.
|
||||
The lifetimes are now opposite and explicit: **a request is history, a
|
||||
want is intent.**
|
||||
|
||||
### Nothing here fails
|
||||
|
||||
There is no `failed` want state. An attempt can fail; a want cannot. A
|
||||
want that found nothing gets `attempts + 1`, a reason the user can read,
|
||||
and a longer backoff — 6h doubling to a 7-day ceiling, jittered so a
|
||||
list added in one sitting does not come due in one burst.
|
||||
|
||||
### Satisfaction is ownership, not download
|
||||
|
||||
A want retires when the *library* owns what it names, however it got
|
||||
there — bought, ripped, copied in. Inferring satisfaction from our own
|
||||
completed downloads would keep hunting for music already on disk.
|
||||
|
||||
### Artist scope defaults to `future`
|
||||
|
||||
Following an artist takes new releases only, and skips compilations,
|
||||
live albums and remixes. `all` backfills the discography, and the user
|
||||
can widen it from the wanted list. Subscribing should not silently queue
|
||||
forty albums.
|
||||
|
||||
## The reconciler
|
||||
|
||||
A 6-hourly loop (plus on-demand, plus a 3-minute startup delay so the
|
||||
explore index has loaded). Four steps, in this order:
|
||||
|
||||
1. **Expand** artist subscriptions into album wants — first, so step 2
|
||||
sees them this pass rather than next.
|
||||
2. **Retire** wants the library already owns.
|
||||
3. **Sync** to clients that keep their own list.
|
||||
4. **Attempt** a bounded batch (25) of due wants.
|
||||
|
||||
Everything the loop needs about music comes through a four-method
|
||||
`CatalogPort`, adapted to the explore index in `backend/downloadcatalog.go`
|
||||
— the composition root, so neither package learns about the other.
|
||||
|
||||
### Unattended grabs, and what stops them
|
||||
|
||||
`Manager.Attempt` is `Start` without the parking: it searches, and grabs
|
||||
only if `AutoPickable` clears. When it does not, **nothing is
|
||||
persisted** — no request row. A want retried weekly for a year would
|
||||
otherwise leave fifty identical failed rows, none of them anything the
|
||||
user can act on.
|
||||
|
||||
`AutoPickable` gained one condition: an anchored request with an empty
|
||||
`Expected` is refused. An anchor with no tracklist behind it is an
|
||||
anchor in name only, and match then rests on album/artist text — exactly
|
||||
the evidence a wrong-album candidate also has. Nobody is watching a
|
||||
reconcile pass.
|
||||
|
||||
## Per-provider concurrency
|
||||
|
||||
`Downloads.MaxConcurrent` was the only limit, and was never actually
|
||||
applied (`SetMaxConcurrent` did not exist). Now:
|
||||
|
||||
- **slskd defaults to 1.** A Soulseek peer serves one file at a time
|
||||
from one person's upload slot; asking for more gets you queued behind
|
||||
everyone else at best. One is both the polite number and usually the
|
||||
fastest.
|
||||
- yt-dlp 2, torrent/usenet clients 4, overridable per provider via a
|
||||
`maxConcurrent` field that `Register` appends automatically to any
|
||||
descriptor declaring `CanTransport`.
|
||||
- A grab takes its **provider's** slot before the global one, so a queue
|
||||
on a busy slskd cannot sit on a global slot a usenet transfer could
|
||||
have used. The transport is resolved before either slot is taken;
|
||||
delegates take neither, since the transfer is happening inside another
|
||||
system that is doing its own limiting.
|
||||
|
||||
## The Lister role
|
||||
|
||||
The fourth role, alongside Searcher/Transporter/Delegator. Lidarr
|
||||
already models a want — a monitored artist or album — and it is always
|
||||
on, where a desktop player is not. A subscription mirrored there keeps
|
||||
working while the app is closed.
|
||||
|
||||
- `artist` → Lidarr artist, `monitor: future|missing` per scope
|
||||
- `release-group`/`release` → monitored album
|
||||
- `recording` → not pushed. Lidarr cannot say "one track", and
|
||||
monitoring the album to get it downloads far more than was asked.
|
||||
|
||||
Sync is push-only in the loop; pulling happens only when the user
|
||||
explicitly imports ("adopt the artists Lidarr already monitors", which
|
||||
arrive at `future` scope). Removal **unmonitors**, never deletes — the
|
||||
user's Lidarr may predate this app.
|
||||
|
||||
## Frontend
|
||||
|
||||
- `Wanted` view in the sidebar: Following / Looking for / Paused /
|
||||
Found, with pause, remove, scope toggle and "Check now".
|
||||
- "Want this" on the album page, "Follow for new releases" on the artist
|
||||
page. The want button shows whether or not a client is connected —
|
||||
wanting is durable and stays queued until one exists.
|
||||
- `WantedListChanged` event, since a background pass changes the list
|
||||
without the UI doing anything.
|
||||
|
||||
## Files
|
||||
|
||||
`backend/download/want.go`, `wantstore.go`, `reconcile.go`,
|
||||
`provider_lidarr_list.go`; `backend/downloadcatalog.go`;
|
||||
schema `download_wants.sql` + migration 48 for the two new
|
||||
`download_requests` columns; `frontend/src/components/wanted-view/`.
|
||||
|
||||
## Deferred
|
||||
|
||||
- **Release-group wants are not retired by ownership of a specific
|
||||
release.** The library indexes release groups and recordings, not
|
||||
editions, so a `release` want is only satisfied by its own download
|
||||
completing.
|
||||
- **No recording lookup on the explore index**, so a track want relies
|
||||
on the title the UI passed in. A want added as a bare recording MBID
|
||||
has no tracklist and waits.
|
||||
- Quality profiles and upgrade-if-better (from 003).
|
||||
- Resume across restart (from 003) — still the largest gap, and it now
|
||||
matters more: an unattended grab that dies on restart is retried by
|
||||
the reconciler, but from zero bytes.
|
||||
@@ -0,0 +1,175 @@
|
||||
# 005 — Agent development harness
|
||||
|
||||
**Status:** implemented
|
||||
**Branch:** main
|
||||
**Created:** 2026-08-10
|
||||
**Shipped:** 2026-08-11 (`5ca6cad`, `ccacd67`)
|
||||
**Follows:** 004-wanted-list
|
||||
|
||||
## Problem
|
||||
|
||||
A coding agent could develop this repo's Go packages competently and
|
||||
could not develop the *application* at all. It could read 66k lines of
|
||||
backend, run 31k lines of tests and lint two build configurations. It
|
||||
could not start the app, see a window, click anything, or find out
|
||||
whether a change to a Lit component rendered.
|
||||
|
||||
The gap was not missing tests. Every path to running YellowJacket ended
|
||||
in a blocking GTK window — `make dev`, `make sandbox <n>` and
|
||||
`make fresh-install` all launch a WebKit window and never return the
|
||||
shell. So 265 bound methods across 11 services, 46 backend events, 33
|
||||
component directories, 13 reactive stores and a 357-line keyboard
|
||||
shortcut service had exactly one form of verification available:
|
||||
`tsc --noEmit`.
|
||||
|
||||
Three secondary facts made it worse. `test_data/music_library_test/` was
|
||||
referenced by three test files, gitignored, absent, and had no
|
||||
generator, so the audio path was unreachable from a clean clone. No
|
||||
workflow ran `make test` or `make lint` — gating existed only in
|
||||
`lefthook.yml`, which is local and `--no-verify`-skippable. And there
|
||||
was no `.pi/`, so none of the awkward invocations were wrapped in
|
||||
anything an agent could call.
|
||||
|
||||
## The unlock
|
||||
|
||||
`wails dev` already runs an HTTP + WebSocket dev server on
|
||||
`localhost:34115` (`internal/frontend/devserver/`). It serves the real
|
||||
frontend assets, injects the real generated bindings, and bridges every
|
||||
method call and every event over a websocket to the **same running Go
|
||||
backend** a desktop window attaches to. A plain Chromium pointed at
|
||||
that port gets a fully functional YellowJacket — not a mock, not a stub
|
||||
`wailsjs` layer. This is the sanctioned approach; Wails v3 ships a guide
|
||||
for it and the v2 community reached the same answer independently
|
||||
(discussion #4205).
|
||||
|
||||
The one caveat: `devserver.Run` still calls `d.Frontend.Run(ctx)`, which
|
||||
opens the GTK window and blocks, with no flag to suppress it. So the app
|
||||
needs a display — a virtual one.
|
||||
|
||||
## What shipped
|
||||
|
||||
117 files, ~14.6k lines. Four test tiers, cheapest first:
|
||||
|
||||
| Tier | Command | Cost | Needs the app? |
|
||||
|---|---|---|---|
|
||||
| Components and stores | `make ui-test` | ~2 s, 313 tests | no |
|
||||
| Services, in-process | `make test` | 3 passes | no |
|
||||
| Exploration | `make dev-headless` + `playwright-cli` | interactive | yes |
|
||||
| Frozen regressions | `make e2e` | ~20 s, 19 specs × 2 browsers | yes |
|
||||
|
||||
**Fixtures** (`cmd/gentestdata`, `make testdata`, `internal/testfixtures`).
|
||||
31 tracks across MP3/FLAC/OGG/WAV in ~1 s, deterministic, gitignored,
|
||||
covering the cases the app has code for: shared album art (dedup),
|
||||
missing and partial tags, unicode and RTL, multi-disc, various artists,
|
||||
a deliberate duplicate pair. Tests select by *case*
|
||||
(`CaseCoverDedup`, `CaseUnicode`, …) rather than by path, and skip
|
||||
themselves when the library has not been generated.
|
||||
|
||||
**Headless launch** (`scripts/dev-headless.sh`, `dev-stop.sh`,
|
||||
`seed-sandbox.sh`). `dbus-run-session -- xvfb-run -a` around the
|
||||
`dev`-tagged binary, backgrounded, writing `.dev/app.pid` and
|
||||
`.dev/app.log` and returning once `:34115` answers. The dev binary is
|
||||
run directly rather than through `wails dev`: `app_dev.go` parses
|
||||
`-devserver`/`-assetdir` from `os.Args`, so one process with a
|
||||
deterministic startup replaces a file watcher and rebuild supervisor an
|
||||
agent does not want. `dbus-run-session` is not incidental — a private
|
||||
session bus makes MPRIS actually register.
|
||||
|
||||
**Driving and seeing.** `.playwright/init-events.js` records every
|
||||
backend event on `window.__yjEvents` by wrapping
|
||||
`window.wails.EventsNotify`, the single choke point all 46 events pass
|
||||
through, so assertions await an event rather than a timeout. It also
|
||||
provides `ready()` and a `call()` that times out. `backend/testctl`
|
||||
mounts `/__test/` on the existing asset handler — `health`,
|
||||
`db/snapshot`, `db/restore`, `emit`, `sql` — gated twice, behind the
|
||||
`dev` build tag and behind `YJ_TESTCTL=1`. A `data-testid`/aria pass
|
||||
turned out to be mostly an accessibility fix: the five transport
|
||||
buttons had no accessible name at all.
|
||||
|
||||
**Component coverage** (`frontend/test/`, Vitest 4 browser mode).
|
||||
`frontend/wailsjs/` is a pure passthrough to `window.go` /
|
||||
`window.runtime`, so faking just those two globals runs the *real*
|
||||
generated bindings and the *real* store code — no module mocking, and
|
||||
no second description of the Wails layer free to drift.
|
||||
`make bindings-check` regenerates `frontend/wailsjs` in ~1.5 s and
|
||||
fails on a dirty tree, closing the gap where a renamed Go field first
|
||||
appeared at runtime in a window.
|
||||
|
||||
**`events.Emit`** (`backend/events/`). `runtime.getEvents` `log.Fatalf`s
|
||||
on any context lacking wails' internal `"events"` value — any
|
||||
`context.Background()` — so 35 emit sites could not run under test and a
|
||||
background worker could take the app down. All 35 now route through one
|
||||
wrapper that drops at debug level instead. Four packages had each
|
||||
hand-rolled the same guard; nine more guarded on `ctx != nil`, which
|
||||
does not help. The test sink rides in the context
|
||||
(`events.WithSink`), and `TestNoDirectRuntimeEmits` walks the tree —
|
||||
not a lint rule, because lint runs once per build configuration and
|
||||
would miss a stray emit in a tagged-out file.
|
||||
|
||||
**pi affordances** (`.pi/`). `skills/yellowjacket-dev/` is the
|
||||
operational manual; `prompts/e2e.md` promotes a hand-driven session
|
||||
into a spec; `journal.md` is the work log. `make skill-check` fails a
|
||||
commit if the skill cites a make target that does not exist.
|
||||
|
||||
**CI that gates** (`.gitea/workflows/ci.yml`). Two jobs in
|
||||
`ubuntu:24.04`: `check` (lint ×3, test ×3, `tsc --noEmit`, `ui-test`,
|
||||
`bindings-check`, `skill-check`) and `e2e` (Xvfb + private bus +
|
||||
fixtures + seed + `dev-headless` + Playwright on **Chromium and
|
||||
WebKit**). The other three workflows only package, so `gitea_ci`
|
||||
previously reported nothing about whether a push was healthy.
|
||||
|
||||
## Decisions worth keeping
|
||||
|
||||
- **The split between the three docs is grammatical, not topical.**
|
||||
`NOTES.md` past, `CLAUDE.md` present, the skill imperative. A topical
|
||||
split rots because every new fact gets two plausible homes.
|
||||
- **Seeds are produced by running the app**, never by hand-writing
|
||||
`config.toml` and DB rows — the same discipline `sql/schemas/` gets,
|
||||
for the same reason. A hand-built `YJ_HOME` is a second description
|
||||
of a valid one and will drift.
|
||||
- **The Makefile is the source of truth for *how* to invoke something**;
|
||||
the skill only decides *which* and *in what order*, and
|
||||
`make skill-check` enforces it.
|
||||
- **Verify in a fresh clone, not a copy of the working tree.** The CI
|
||||
prototype ran both jobs in one mounted directory and so consumed a
|
||||
`frontend/dist` an earlier job had built — hiding that `main.go`
|
||||
embeds it and every Go typecheck needs it. The question is not
|
||||
clean-vs-dirty but *whose* dirt.
|
||||
- **`make lint`'s tag sets must equal `make test`'s.** Without
|
||||
`webkit2_41` wails resolves `webkit2gtk-4.0`, which Arch ships and
|
||||
Ubuntu 24.04 does not, so lint was checking a configuration that only
|
||||
built on one distro. CI caught this on its first run.
|
||||
- **Playwright's WebKit gates** because it was measured (19/19) rather
|
||||
than assumed, and because nothing in `e2e/` compares pixels — so a
|
||||
failure is an engine difference, not baseline noise. It is the only
|
||||
WebKit2GTK signal obtainable, since it cannot start on Arch at all.
|
||||
|
||||
## Known blind spots
|
||||
|
||||
- **Xvfb is X11**, and `main.go` carries a Wayland-specific NVIDIA
|
||||
DMABuf workaround. CI never exercises that path. Acceptable — it is a
|
||||
crash workaround, not a feature — but it is a blind spot, not a
|
||||
surprise.
|
||||
- **Playwright's WebKit is not WebKit2GTK.** Closer than Chromium,
|
||||
still not the shipped renderer. A GTK-specific rendering bug can
|
||||
escape, and will for any view not in the smoke suite.
|
||||
- **The fixture hash is deterministic per ffmpeg, not across versions**
|
||||
(`5425fbb454a2` on Arch, `599a8dd4f152` on Ubuntu 24.04). Nothing
|
||||
asserts a literal hash; a test that did would be portable by accident.
|
||||
|
||||
## Left open, deliberately
|
||||
|
||||
- **WAV tags are write-only.** `backend/tagwriter` writes them into a
|
||||
RIFF `id3 ` chunk; `backend/metadata` reads through `dhowden/tag`,
|
||||
which has no RIFF parser, so every WAV scans in untitled. Found by
|
||||
the fixtures and pinned by `TestWAVTagsAreNotReadableYet`. The fix is
|
||||
small: unwrap the chunk, hand the payload to `tag.ReadFrom`.
|
||||
- **`themeStore.loadFromBackend`'s failure handler cannot recover** — it
|
||||
re-derives the colour ramp from the state that just failed it. One
|
||||
line; reachable only if the backend returns an empty accent.
|
||||
- **`backend/playlist` has no CRUD suite.** 2,900 lines; phase 5 added
|
||||
four emit-focused tests. Its own piece of work.
|
||||
- **Driving the real WebKit2GTK window.**
|
||||
`WEBKIT_INSPECTOR_SERVER` exposes WebKit's remote inspector, but the
|
||||
protocol is not CDP and Playwright cannot attach. A bespoke client is
|
||||
the only route and is not worth it.
|
||||
@@ -0,0 +1,89 @@
|
||||
# 006 — Orientation fixes: knowing where you are and what you're looking at
|
||||
|
||||
**Status:** implemented
|
||||
**Branch:** main
|
||||
**Created:** 2026-08-11
|
||||
**Follows:** 005-agent-development-harness
|
||||
|
||||
## Problem
|
||||
|
||||
Six reports from using the app, which turned out to be one theme with
|
||||
six faces: **the UI knew things it did not say.**
|
||||
|
||||
1. Some track names in the track list were links, most were not. The
|
||||
rule (has both a release-group *and* a recording MBID) was invisible,
|
||||
so the list looked randomly broken.
|
||||
2. Opening an album sometimes showed the full catalog tracklist and
|
||||
sometimes only the tracks the user owned, with nothing on screen
|
||||
distinguishing the two — or distinguishing either from "still
|
||||
fetching".
|
||||
3. The same on artist pages: a discography that was the artist's, or a
|
||||
discography that was the user's shelf, rendered identically.
|
||||
4. "Check now" on the requests list appeared to do nothing, because it
|
||||
honoured each request's retry backoff — a request searched an hour
|
||||
ago was not due, so a deliberate button press produced silence.
|
||||
5. Pressing **M** muted playback and left the volume indicator
|
||||
unchanged, because mute does not change the volume *number* and
|
||||
`VolumeChanged` carried nothing else.
|
||||
6. The home page did not exist. The sidebar had a Home item; it fell
|
||||
through to "Coming soon: home".
|
||||
|
||||
## What shipped
|
||||
|
||||
**Backend**
|
||||
|
||||
- `events.MuteChanged` (bool), emitted alongside `VolumeChanged` so the
|
||||
UI has something to react to when silence is the only thing that
|
||||
changed. `Player.Muted()` for symmetry; `MuteToggle` now takes the
|
||||
speaker lock and refuses politely when no streamer exists.
|
||||
- `download.Reconciler.RunNow` — a forced pass that ignores backoff,
|
||||
backed by a new `ListWantedDownloadRequests` query. `RunOnce` (the
|
||||
loop) still honours it: the backoff is a promise to the providers,
|
||||
not to the user, and a person pressing a button *is* the schedule.
|
||||
`Summary` gained `Waiting` and `NoProviders` so "nothing happened"
|
||||
can be reported with a reason.
|
||||
- `backend/home` — the shelf builder, with queries in
|
||||
`sql/queries/home.sql` that return album ids only, joined back to
|
||||
`GetAllAlbumsWithDetails` in Go rather than restating the album
|
||||
projection six times. A shelf with nothing behind it is omitted.
|
||||
|
||||
**Frontend**
|
||||
|
||||
- `explore-link.ts` rewritten: a name always goes somewhere. No MBID
|
||||
means the *library* page for the same album/artist (both detail views
|
||||
already accept a local id), resolved through the library store, with
|
||||
an untagged track highlighted by title instead of by recording MBID.
|
||||
Links now fire on a genuine single click only — see below.
|
||||
- `<catalog-scope-notice>` — one banner, four states (`catalog`,
|
||||
`loading`, `library`, `unavailable`), used by both detail pages. The
|
||||
album and artist pages grew an explicit `catalogPending` /
|
||||
`catalogLoaded` pair, because `loadingReleases` already meant
|
||||
"something is renderable" and a library stand-in satisfies that.
|
||||
- Artist page: an empty `BrowseReleaseGroups` no longer wipes the
|
||||
library-hydrated discography — an empty catalog answer means "not
|
||||
indexed yet", not "released nothing".
|
||||
- Downloads: a no-client banner, per-request "next check in …", honest
|
||||
idle summaries, and copy that says the retry schedule exists.
|
||||
- `<home-view>`: shelves as horizontal rows; a cover opens the album, a
|
||||
play button plays it.
|
||||
|
||||
## The one thing worth remembering
|
||||
|
||||
**Making every track name a link broke double-click-to-play**, and the
|
||||
e2e playback suite caught it: the title is the widest thing in a row,
|
||||
so the first click of the double-click landed on the link and navigated
|
||||
away. Fixed in one place — `singleClick()` in `explore-link.ts` holds
|
||||
the navigation for one double-click interval (250 ms) and drops it if a
|
||||
`dblclick` arrives, while leaving the dblclick itself to bubble to the
|
||||
row. Rows do not need to know links exist.
|
||||
|
||||
This is exactly the failure mode plan 005's e2e tier was built for; it
|
||||
was invisible before the change because the seeded fixture library has
|
||||
no MBIDs, so no track name was a link.
|
||||
|
||||
## Verification
|
||||
|
||||
`make lint` (3 configs), `make test` (3 passes), `make ui-test`
|
||||
(329 passing, up from 313), `make e2e` (23 passing, up from 19 — four
|
||||
new home-page specs), `tsc --noEmit`, and manual verification of all
|
||||
six items in the running app via `make dev-headless` + `playwright-cli`.
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"browser": {
|
||||
"browserName": "chromium",
|
||||
"launchOptions": {
|
||||
"channel": "chromium"
|
||||
},
|
||||
"contextOptions": {
|
||||
"viewport": { "width": 1440, "height": 900 }
|
||||
},
|
||||
"initScript": ["init-events.js"]
|
||||
},
|
||||
"testIdAttribute": "data-testid",
|
||||
"outputDir": ".playwright-cli",
|
||||
"console": {
|
||||
"level": "warning"
|
||||
},
|
||||
"timeouts": {
|
||||
"action": 10000,
|
||||
"navigation": 30000
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,302 @@
|
||||
/*
|
||||
* YellowJacket harness bridge — installed as a Playwright initScript, so
|
||||
* it runs in every page *before* any application script.
|
||||
*
|
||||
* Why this file exists: half of what this app does is push-driven. Scan
|
||||
* progress, job updates, download progress, WantedListChanged and 40-odd
|
||||
* other events arrive from Go whenever they arrive. An assertion that
|
||||
* sleeps and hopes is flaky; an assertion that awaits the event is not.
|
||||
*
|
||||
* Three things it provides on `window.__yjEvents`:
|
||||
*
|
||||
* record every backend -> frontend event, in order, with payloads
|
||||
* wait a promise that settles on a matching event (or rejects
|
||||
* with the list of events that *did* arrive, which is the
|
||||
* single most useful failure message this harness can give)
|
||||
* call a bound Go method that is guaranteed to settle: a binding
|
||||
* invoked with wrong argument types makes the backend log
|
||||
* "error parsing arguments" and never fire the callback, so
|
||||
* the in-page promise hangs forever. Timing out here fixes
|
||||
* that once instead of in every eval.
|
||||
*
|
||||
* WHERE IT HOOKS. Not EventsOn. Every backend event enters the page
|
||||
* at exactly one place — wails' ipc_websocket.js does
|
||||
*
|
||||
* case "n": window.wails.EventsNotify(message)
|
||||
*
|
||||
* and EventsNotify fans out to listeners from there. Wrapping that
|
||||
* single choke point captures all 46 events whether or not the app
|
||||
* subscribes to them, and needs one wrap rather than 46.
|
||||
*
|
||||
* `window.wails` does not exist yet when this script runs, so we install
|
||||
* an accessor on `window` and wrap at assignment time (wails' main.js
|
||||
* does a plain `window.wails = {...}`), then collapse the accessor back
|
||||
* to a data property so nothing downstream can tell.
|
||||
*
|
||||
* INSTALL EXACTLY ONCE. Listeners registered by one `eval` survive into
|
||||
* the next, so a recorder that re-registers double-counts. Tests call
|
||||
* `__yjEvents.reset()`; they never re-install.
|
||||
*/
|
||||
(() => {
|
||||
if (window.__yjEvents) {
|
||||
return;
|
||||
}
|
||||
|
||||
const LIMIT = 2000;
|
||||
|
||||
let seq = 0;
|
||||
const log = [];
|
||||
const waiters = new Set();
|
||||
|
||||
const summarize = () => {
|
||||
const counts = {};
|
||||
for (const e of log) {
|
||||
counts[e.name] = (counts[e.name] || 0) + 1;
|
||||
}
|
||||
return counts;
|
||||
};
|
||||
|
||||
const record = (name, data, dir) => {
|
||||
const entry = { seq: ++seq, name, data, dir, t: Date.now() };
|
||||
log.push(entry);
|
||||
if (log.length > LIMIT) {
|
||||
log.splice(0, log.length - LIMIT);
|
||||
}
|
||||
for (const w of Array.from(waiters)) {
|
||||
let hit = false;
|
||||
try {
|
||||
hit = w.test(entry);
|
||||
} catch {
|
||||
hit = false;
|
||||
}
|
||||
if (hit) {
|
||||
waiters.delete(w);
|
||||
clearTimeout(w.timer);
|
||||
w.resolve(entry);
|
||||
}
|
||||
}
|
||||
return entry;
|
||||
};
|
||||
|
||||
// `name` is a string, or "*" for any event. `match` is an optional
|
||||
// predicate over (data, entry) — only usable from an eval'd function,
|
||||
// which is how every harness call is written anyway.
|
||||
const makeTest = (name, match) => (entry) => {
|
||||
if (name && name !== "*" && entry.name !== name) {
|
||||
return false;
|
||||
}
|
||||
return match ? !!match(entry.data, entry) : true;
|
||||
};
|
||||
|
||||
const api = {
|
||||
version: 1,
|
||||
|
||||
/** Every recorded event, oldest first. */
|
||||
get log() {
|
||||
return log.slice();
|
||||
},
|
||||
|
||||
/** The sequence number of the most recent event. */
|
||||
get seq() {
|
||||
return seq;
|
||||
},
|
||||
|
||||
/** Drop the buffer. Does NOT touch the recorder or waiters. */
|
||||
reset() {
|
||||
const n = log.length;
|
||||
log.length = 0;
|
||||
return n;
|
||||
},
|
||||
|
||||
/** Every recorded event, optionally filtered by name. */
|
||||
all(name) {
|
||||
return name ? log.filter((e) => e.name === name) : log.slice();
|
||||
},
|
||||
|
||||
/** How many of `name` (or of everything) have arrived. */
|
||||
count(name) {
|
||||
return this.all(name).length;
|
||||
},
|
||||
|
||||
/** The most recent matching event, or null. */
|
||||
last(name) {
|
||||
const hits = this.all(name);
|
||||
return hits.length ? hits[hits.length - 1] : null;
|
||||
},
|
||||
|
||||
/** Everything after a sequence number — pairs with `.seq`. */
|
||||
since(n) {
|
||||
return log.filter((e) => e.seq > n);
|
||||
},
|
||||
|
||||
/** name -> count, for "what actually happened?" */
|
||||
names() {
|
||||
return summarize();
|
||||
},
|
||||
|
||||
/**
|
||||
* Settle on the next (or already-buffered) matching event.
|
||||
*
|
||||
* await __yjEvents.wait('LibraryScanComplete', { timeoutMs: 60000 })
|
||||
* await __yjEvents.wait('JobsChanged', { match: (d) => d.length > 0 })
|
||||
*
|
||||
* Rejects on timeout with the names that did arrive, because
|
||||
* "timed out waiting for X" without that list is a dead end.
|
||||
*/
|
||||
wait(name, opts) {
|
||||
const o = opts || {};
|
||||
const test = makeTest(name, o.match);
|
||||
const since = o.since || 0;
|
||||
|
||||
for (const entry of log) {
|
||||
if (entry.seq > since && test(entry)) {
|
||||
return Promise.resolve(entry);
|
||||
}
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const w = { test, resolve };
|
||||
w.timer = setTimeout(() => {
|
||||
waiters.delete(w);
|
||||
reject(
|
||||
new Error(
|
||||
`__yjEvents.wait(${JSON.stringify(name)}) timed out after ` +
|
||||
`${o.timeoutMs || 5000}ms; events seen: ` +
|
||||
JSON.stringify(summarize()),
|
||||
),
|
||||
);
|
||||
}, o.timeoutMs || 5000);
|
||||
waiters.add(w);
|
||||
});
|
||||
},
|
||||
|
||||
/**
|
||||
* Resolve when the backend is actually answering calls — not
|
||||
* when the DOM is ready, which is earlier and lies.
|
||||
*/
|
||||
async ready(timeoutMs) {
|
||||
const deadline = Date.now() + (timeoutMs || 15000);
|
||||
for (;;) {
|
||||
if (window.go?.queue?.Queue?.GetState) {
|
||||
try {
|
||||
await api.call("queue.Queue.GetState", [], 2000);
|
||||
return true;
|
||||
} catch {
|
||||
/* backend not up yet */
|
||||
}
|
||||
}
|
||||
if (Date.now() > deadline) {
|
||||
throw new Error("__yjEvents.ready timed out");
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 100));
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Call a bound Go method by dotted path, with a timeout.
|
||||
*
|
||||
* await __yjEvents.call('player.Player.SetVolume', [42])
|
||||
*
|
||||
* A binding called with the wrong argument types never fires its
|
||||
* callback — the reason appears only in .dev/app.log. Without a
|
||||
* timeout the caller waits forever; with one it gets told where
|
||||
* to look.
|
||||
*/
|
||||
call(path, args, timeoutMs) {
|
||||
const parts = String(path).split(".");
|
||||
let fn = window.go;
|
||||
for (const p of parts) {
|
||||
fn = fn?.[p];
|
||||
}
|
||||
if (typeof fn !== "function") {
|
||||
return Promise.reject(
|
||||
new Error(`__yjEvents.call: no such binding: ${path}`),
|
||||
);
|
||||
}
|
||||
|
||||
return Promise.race([
|
||||
Promise.resolve(fn(...(args || []))),
|
||||
new Promise((_, reject) =>
|
||||
setTimeout(
|
||||
() =>
|
||||
reject(
|
||||
new Error(
|
||||
`__yjEvents.call(${path}) did not settle in ` +
|
||||
`${timeoutMs || 10000}ms — almost always wrong ` +
|
||||
`argument types; check .dev/app.log for ` +
|
||||
`"error parsing arguments"`,
|
||||
),
|
||||
),
|
||||
timeoutMs || 10000,
|
||||
),
|
||||
),
|
||||
]);
|
||||
},
|
||||
};
|
||||
|
||||
Object.defineProperty(window, "__yjEvents", {
|
||||
value: api,
|
||||
configurable: false,
|
||||
enumerable: false,
|
||||
writable: false,
|
||||
});
|
||||
|
||||
// Wrap `obj[method]` once, routing every invocation through `tap`.
|
||||
const wrap = (obj, method, tap) => {
|
||||
const original = obj[method];
|
||||
if (typeof original !== "function" || original.__yjWrapped) {
|
||||
return;
|
||||
}
|
||||
const wrapped = function (...args) {
|
||||
try {
|
||||
tap(args);
|
||||
} catch {
|
||||
/* a broken recorder must never break the app */
|
||||
}
|
||||
return original.apply(this, args);
|
||||
};
|
||||
wrapped.__yjWrapped = true;
|
||||
obj[method] = wrapped;
|
||||
};
|
||||
|
||||
// Install an accessor that wraps on first assignment, then collapses
|
||||
// back into an ordinary property.
|
||||
const hookOnAssign = (name, onAssign) => {
|
||||
let value;
|
||||
Object.defineProperty(window, name, {
|
||||
configurable: true,
|
||||
enumerable: true,
|
||||
get: () => value,
|
||||
set: (v) => {
|
||||
value = v;
|
||||
try {
|
||||
onAssign(v);
|
||||
} catch {
|
||||
/* ditto */
|
||||
}
|
||||
Object.defineProperty(window, name, {
|
||||
value: v,
|
||||
configurable: true,
|
||||
enumerable: true,
|
||||
writable: true,
|
||||
});
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
// Inbound: every backend -> frontend event.
|
||||
hookOnAssign("wails", (w) => {
|
||||
wrap(w, "EventsNotify", ([message]) => {
|
||||
const parsed = JSON.parse(message);
|
||||
record(parsed.name, parsed.data, "in");
|
||||
});
|
||||
});
|
||||
|
||||
// Outbound: events the frontend emits, so a flow that round-trips
|
||||
// through Go is legible from one buffer.
|
||||
hookOnAssign("runtime", (r) => {
|
||||
wrap(r, "EventsEmit", (args) => {
|
||||
record(args[0], args.slice(1), "out");
|
||||
});
|
||||
});
|
||||
})();
|
||||
+103
@@ -1,3 +1,106 @@
|
||||
## [1.3.0](https://github.com/onion-4-dinner/yellowjacket/compare/v1.2.3...v1.3.0) (2026-03-20)
|
||||
|
||||
### Features
|
||||
|
||||
* **09-01:** add scan control events and cancelled metrics field ([c695024](https://github.com/onion-4-dinner/yellowjacket/commit/c695024241a7513b8fedb3fbf7ff364d0515b392))
|
||||
* **09-01:** add scan control fields and per-scan cancellable context ([cf22e52](https://github.com/onion-4-dinner/yellowjacket/commit/cf22e52a64850a80b9fcc63c21d81313e6bd56ab))
|
||||
* **09-02:** add frontend keyboard shortcut service, store, and controller ([40d4815](https://github.com/onion-4-dinner/yellowjacket/commit/40d48151dd798b57eed9f54a572ae4735356d09e))
|
||||
* **09-02:** add shortcuts config package with default bindings and Wails persistence ([6285ca9](https://github.com/onion-4-dinner/yellowjacket/commit/6285ca9dc4e6f211197e377d01c485b1ef65c300))
|
||||
* **09-03:** add scan control UI with pause/resume/cancel and confirmation dialog ([3914369](https://github.com/onion-4-dinner/yellowjacket/commit/391436927c826f2f17a4523be7829aefc04a6b12))
|
||||
* **09-04:** add keyboard shortcuts section to config page with conflict detection ([0451fb3](https://github.com/onion-4-dinner/yellowjacket/commit/0451fb38805ff2c27e43deb152daa892e733d2db))
|
||||
* **10-01:** implement migration 6 and pre-migration backup ([1179f56](https://github.com/onion-4-dinner/yellowjacket/commit/1179f56c3680112692e71e8dc7ce946446fa8a8a))
|
||||
* **10-01:** update SQL schema files for multi-library fresh installs ([535855b](https://github.com/onion-4-dinner/yellowjacket/commit/535855b383a457dd2be3298b4361313bef22b39d))
|
||||
* **10-02:** add migration 6 integration tests and NewTestDBWithLibrary helper ([bc15189](https://github.com/onion-4-dinner/yellowjacket/commit/bc151891b50e59e41da2e00dbfafbecaad11b4ac))
|
||||
* **10-02:** add sqlc queries for libraries and update playlist queries for phantom support ([02548dd](https://github.com/onion-4-dinner/yellowjacket/commit/02548dd55e59b28f3d6c8d9614f209140c979250))
|
||||
* **11-01:** per-library scan pipeline with queue coordinator ([943db1c](https://github.com/onion-4-dinner/yellowjacket/commit/943db1cf274bdf59daf28ab6c20f78ef5ef53105))
|
||||
* **11-02:** update config-page with per-library progress display and queue-aware cancel dialog ([d01591d](https://github.com/onion-4-dinner/yellowjacket/commit/d01591d6cc054a63b832c05a3164a72fdcaba342))
|
||||
* **11-02:** update library-manager with per-library progress and Scan All button ([d61f122](https://github.com/onion-4-dinner/yellowjacket/commit/d61f122b567e8ac2b30fa96c637cbebc14493c89))
|
||||
* **12-01:** add queue compaction method and wire removal hooks ([5995dfd](https://github.com/onion-4-dinner/yellowjacket/commit/5995dfd01d61cd4d2c0749eeeee2a1f93b739d68))
|
||||
* **12-01:** implement library CRUD methods and orphan cleanup pipeline ([bd44f83](https://github.com/onion-4-dinner/yellowjacket/commit/bd44f8306c9129b9420ad81938bcf8105a1cb55a))
|
||||
* **12-02:** make config sections collapsible with chevron dropdown ([12c6782](https://github.com/onion-4-dinner/yellowjacket/commit/12c678284c7582bd85cd52722f4d405b0bd0e20f))
|
||||
* **12-02:** remove Libraries sidebar nav item and view routing ([e199712](https://github.com/onion-4-dinner/yellowjacket/commit/e199712a56e1cb3c0fc43d3340abb892a6f5fa7b))
|
||||
* **12-02:** replace config-page library section with full library management UI ([ffc5d96](https://github.com/onion-4-dinner/yellowjacket/commit/ffc5d9639cf7c916a4f846590ae0d67cf13afe27))
|
||||
* **12-02:** selectable library list with checkbox scan targeting ([13a42ae](https://github.com/onion-4-dinner/yellowjacket/commit/13a42aea2287d7ed0ec9ff9856f52c1fa7767338))
|
||||
* **12-02:** show scan progress bar inline in library list entry ([df824c6](https://github.com/onion-4-dinner/yellowjacket/commit/df824c6989e92b2aefaa1ddf05b131ee319612d8))
|
||||
* **13-01:** add library-filtered Go query methods and FTS search ([5f7de50](https://github.com/onion-4-dinner/yellowjacket/commit/5f7de5060a5bc557b96203267de694ef366ed507))
|
||||
* **13-01:** add library-filtered sqlc queries for all browse views ([5cc58ce](https://github.com/onion-4-dinner/yellowjacket/commit/5cc58ce66ab70d8d5a570df5067f79ae2201037e))
|
||||
* **13-02:** add library filter dropdown and wire all views to respect active filter ([42b8cf9](https://github.com/onion-4-dinner/yellowjacket/commit/42b8cf9f52133499ffcd7363bd39dd0c1069e091))
|
||||
* **15-01:** migrate FTS5 search_index to contentless_delete=1 ([cb5155b](https://github.com/onion-4-dinner/yellowjacket/commit/cb5155b8906357ff77c5c579d57d02cf2eec6abe))
|
||||
* **15-02:** create backend/fileutil package with AtomicWrite ([4d64b5d](https://github.com/onion-4-dinner/yellowjacket/commit/4d64b5dcfe43951e8ec63383bbf72c99107c63c4))
|
||||
* **16-01:** add selectAll() to SelectionController and dispatch shortcut:select-all event ([f567762](https://github.com/onion-4-dinner/yellowjacket/commit/f5677628ef283b67370630b564f23178e43da3d2))
|
||||
* **16-01:** wire shortcut:select-all listener in track-list, queue-panel, and playlist-view ([906ea28](https://github.com/onion-4-dinner/yellowjacket/commit/906ea28751ce9f96fdeeb9410ab5f6518f09fcb9))
|
||||
* **16-02:** add go-flac dependencies and implement FLAC tag writer ([3642cbe](https://github.com/onion-4-dinner/yellowjacket/commit/3642cbe0d58f8912a786a4fc5380c40403add94a))
|
||||
* **16-03:** implement DB sync module for tag write pipeline ([2966079](https://github.com/onion-4-dinner/yellowjacket/commit/2966079625cd42412411429af02184d015526e9b))
|
||||
* **16-03:** WriteTrackTags pipeline with player safety, scan mutex, events, and app wiring ([64322f9](https://github.com/onion-4-dinner/yellowjacket/commit/64322f93538515d5a3e486dc14691b9c9dcf6f66))
|
||||
* **17-01:** add TrackMetadataChanged handler and remove selection gate on Track Details ([fc5cf70](https://github.com/onion-4-dinner/yellowjacket/commit/fc5cf70e4c1be3d3f1545c140db5202601a08109))
|
||||
* **17-01:** add WriteTrackTagsByPath and ImageFilePicker backend methods ([4235b4a](https://github.com/onion-4-dinner/yellowjacket/commit/4235b4a4d555882ce86628a88dd4e4eeee2c9097))
|
||||
* **17-02:** implement save flow, cover art editing, and error handling ([265a9ea](https://github.com/onion-4-dinner/yellowjacket/commit/265a9ea8ceba893f956a03546e9ac4189adc7716))
|
||||
* **18-01:** add BatchWriteProgress event constant ([3dba0e1](https://github.com/onion-4-dinner/yellowjacket/commit/3dba0e143c091327d305d39d2fa7a687ec47e172))
|
||||
* **18-01:** add BatchWriteTrackTags with progress, cancellation, and partial failure ([f557ffd](https://github.com/onion-4-dinner/yellowjacket/commit/f557ffd652179b7cf8f8ff4a06824f30edf08007))
|
||||
* **18-02:** add batch edit mode to track-details component ([6dab32b](https://github.com/onion-4-dinner/yellowjacket/commit/6dab32b36b497d54e8645e969aa79737ad3523ab))
|
||||
* **18-02:** wire batch track-details to all 4 view context menus ([656985a](https://github.com/onion-4-dinner/yellowjacket/commit/656985add92663440baebb871f8cd6d5723117fd))
|
||||
* **19-01:** implement WAV RIFF parser/writer and writeWavTags ([e6610ff](https://github.com/onion-4-dinner/yellowjacket/commit/e6610ff15e041213b6898ad48ff63b7060b312e7))
|
||||
* **20-01:** implement OGG Vorbis tag writer with custom page parser and CRC32 ([5e98c03](https://github.com/onion-4-dinner/yellowjacket/commit/5e98c036342b9e174abdc6d00db21c2e2901f18b))
|
||||
* **quick-17:** create playlist-details subpage component ([dc5c7d6](https://github.com/onion-4-dinner/yellowjacket/commit/dc5c7d6ca6cfbfac15546c048f1b33aaf47209c6))
|
||||
* **quick-18:** replace track-info with multi-column grid layout in playlist-details ([ce23177](https://github.com/onion-4-dinner/yellowjacket/commit/ce2317722870f932792dc6456a63235ff4611466))
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **09-05:** emit VolumeChanged event and persist state in ChangeVolume and MuteToggle ([bb3fd20](https://github.com/onion-4-dinner/yellowjacket/commit/bb3fd204f0895f357a14479b40754f397aae74c4))
|
||||
* **10-01:** move library_id index to migration 6 to fix existing DB startup ([75b2a34](https://github.com/onion-4-dinner/yellowjacket/commit/75b2a349ebd6fada5cbc92bfae9854cc2cd53c63))
|
||||
* **12-02:** claim orphaned tracks when adding library with matching path ([f60b6b5](https://github.com/onion-4-dinner/yellowjacket/commit/f60b6b525546ef77a3329fe92f03f336b7435a0e))
|
||||
* **12-02:** count failed saves as skipped so scan progress bar advances ([b36e472](https://github.com/onion-4-dinner/yellowjacket/commit/b36e472212957ff089f4f5d35f3978a754e23502))
|
||||
* **12-02:** delete artist_credit_artist before artist_credit in removal pipeline ([890284d](https://github.com/onion-4-dinner/yellowjacket/commit/890284ddb1d0fb95e423bddf27b40fb0db2d11e5))
|
||||
* **12-02:** dismiss inline rename on click outside ([9272b06](https://github.com/onion-4-dinner/yellowjacket/commit/9272b060bf98118e37f19a8c0834034691bfe6a2))
|
||||
* **12-02:** downgrade per-file save error to Debug, add warning count to scan summary ([cf18c39](https://github.com/onion-4-dinner/yellowjacket/commit/cf18c39dbd849d60218228cf1d2285ab2071e788))
|
||||
* **12-02:** invalidate library store cache on LibraryRemoved event ([b093fbb](https://github.com/onion-4-dinner/yellowjacket/commit/b093fbb10a24054c4ef62b0bd13f28d9bfe6f121))
|
||||
* **12-02:** keep Add Library button visible during scan ([649e516](https://github.com/onion-4-dinner/yellowjacket/commit/649e516aa30090665e9f10e89c1ccce378e36b96))
|
||||
* **12-02:** move Add Library button inline with scan buttons ([771345d](https://github.com/onion-4-dinner/yellowjacket/commit/771345dd9d3870b3a907e1cce09c7456ab7ccd85))
|
||||
* **12-02:** move scan buttons above library list, default to none selected ([ba3f840](https://github.com/onion-4-dinner/yellowjacket/commit/ba3f840a28fe2c6ca40c558305814d29c233d6e0))
|
||||
* **12-02:** refresh library track counts after scan completes ([1f872aa](https://github.com/onion-4-dinner/yellowjacket/commit/1f872aa005a9405d9bc1f64a4b1dd2f1f1d4a16c))
|
||||
* **12-02:** reorder orphan cleanup to delete FK children before recordings ([1d735c3](https://github.com/onion-4-dinner/yellowjacket/commit/1d735c3a5f5a78996d6ddbe5c787adf040fe2f21))
|
||||
* **12-02:** replace removed Scan() import with ScanAllLibraries() ([0559822](https://github.com/onion-4-dinner/yellowjacket/commit/05598224e4d5532d2e2a3a7e5d3b5411240b1024))
|
||||
* **12-02:** resolve phantom tracks caused by empty library root after TOML cleanup ([717e249](https://github.com/onion-4-dinner/yellowjacket/commit/717e249c368fd1cc8d5c8f945c352175708691cf))
|
||||
* **12-02:** serialize ScanWarning.Err as string instead of error interface ([ac8cbb3](https://github.com/onion-4-dinner/yellowjacket/commit/ac8cbb3296bd561a305627668c211dce7209df25))
|
||||
* **12-02:** soft scan claims orphaned library_id=0 tracks on startup ([1ad099a](https://github.com/onion-4-dinner/yellowjacket/commit/1ad099a9d35fc722475e238d3443fd5473566acd))
|
||||
* **12-02:** soft scan on launch — only scan libraries with changed file counts ([92c4d23](https://github.com/onion-4-dinner/yellowjacket/commit/92c4d23a9a1e545fab497816ee3dce43a181cded))
|
||||
* **12-02:** wait for scan to stop before library removal, surface errors in UI ([cf00498](https://github.com/onion-4-dinner/yellowjacket/commit/cf004986c95732d00208e83467267904ea3f2ef6))
|
||||
* **13-02:** auto-resolve phantom playlist tracks after library scan ([93262b9](https://github.com/onion-4-dinner/yellowjacket/commit/93262b9ae0f737d2893839ac585776207b3b44b6))
|
||||
* **13-02:** defer virtualizer event delegation until element exists ([f05d2bb](https://github.com/onion-4-dinner/yellowjacket/commit/f05d2bb603f5ea827164466fd0795a6c6e662529))
|
||||
* **13-02:** resolve phantom playlist tracks using M3U8 paths after scan ([9f595b7](https://github.com/onion-4-dinner/yellowjacket/commit/9f595b7ac10c2191b5469004901cbbc1331c1abb))
|
||||
* **14-01:** downgrade main-panel from contain:strict to layout+style+paint ([4b7d35d](https://github.com/onion-4-dinner/yellowjacket/commit/4b7d35d7ec4c8b14453a8f8250cd154b8c4c2537))
|
||||
* **14-perf:** fix scroll jumping and input latency ([3b2e189](https://github.com/onion-4-dinner/yellowjacket/commit/3b2e189e7d0e6d00393d087565190fd307774257))
|
||||
* **17-02:** fix cover art replace and remove ([d7c2965](https://github.com/onion-4-dinner/yellowjacket/commit/d7c2965752ae0ac9009d00f2431d5919a24558b7))
|
||||
* **17-02:** handle float64 numeric values from Wails JSON deserialization ([900db2e](https://github.com/onion-4-dinner/yellowjacket/commit/900db2e56cca254873a3a5a7a384008feac4211b))
|
||||
* **17-02:** refresh cover art URLs after save ([8cd4914](https://github.com/onion-4-dinner/yellowjacket/commit/8cd4914842f61c0c6b49e0216c7816e201a3c94a))
|
||||
* **17-02:** refresh track-details dialog data after successful save ([ffcdc41](https://github.com/onion-4-dinner/yellowjacket/commit/ffcdc41b0d4fad8ed428dbaa55f6cdd38c096822))
|
||||
* **18-02:** add field labels above title/artist/album inputs in batch edit mode ([9df2d67](https://github.com/onion-4-dinner/yellowjacket/commit/9df2d6764a0b0566dda33cff675debea4a61dea8))
|
||||
* **18-02:** add field labels to all track-details states (single/batch, read/edit) ([d430ad8](https://github.com/onion-4-dinner/yellowjacket/commit/d430ad884bfd38bea93389d8be730ff00388a7be))
|
||||
* **19-01:** add album_artist TPE2 mapping to applyTextChanges ([8f4c4a0](https://github.com/onion-4-dinner/yellowjacket/commit/8f4c4a0c2b14eeeaeccb972a40addb11f3d65437))
|
||||
* preserve scroll position in cached grid views ([54df917](https://github.com/onion-4-dinner/yellowjacket/commit/54df917ffdd69c4f7ffaeccf2d161261ca80d84e))
|
||||
* **queue-panel:** set flow layout _itemSize to match actual track item height ([288d9de](https://github.com/onion-4-dinner/yellowjacket/commit/288d9deae22d437fcd7857b368827db7b62c24f6))
|
||||
* **queue-panel:** suppress virtualizer scroll corrections during scrollbar drag ([0bd8cef](https://github.com/onion-4-dinner/yellowjacket/commit/0bd8cefa00dcae2f8bd9579de2aefd58e0a9e6c9))
|
||||
* **quick-19:** multi-root path resolution for playlist M3U8 tracks ([9144ded](https://github.com/onion-4-dinner/yellowjacket/commit/9144dedc2742925dc252d491763b4f2929238d0e))
|
||||
* **S21/T01:** fix all lint warnings and upgrade wsl to wsl_v5 ([f16157a](https://github.com/onion-4-dinner/yellowjacket/commit/f16157a2134cbeb1787ff851d4875d77f2f3f86b))
|
||||
|
||||
### Performance
|
||||
|
||||
* **12-02:** increase scan batch size from 50 to 300 ([21ea71e](https://github.com/onion-4-dinner/yellowjacket/commit/21ea71e2575d76258bd81d89ab8ac883aa3bed36))
|
||||
* **12-02:** skip FTS5 rebuild during library removal ([30f4461](https://github.com/onion-4-dinner/yellowjacket/commit/30f4461e6957e20d3dc607fa0886a75b5c21b3cf))
|
||||
* **14-01:** add CSS containment to app shell layout boundaries ([efa06f7](https://github.com/onion-4-dinner/yellowjacket/commit/efa06f7edf1e4acdc3d8865cad264403257ae40d))
|
||||
* **14-01:** add GPU promotion and containment to all scroll containers ([ac8a52e](https://github.com/onion-4-dinner/yellowjacket/commit/ac8a52e110f9f8ebdc3433b60594370352126a18))
|
||||
* **14-02:** replace innerHTML navigation with view caching system ([ad91043](https://github.com/onion-4-dinner/yellowjacket/commit/ad9104374a628342e0ea30cf409ff43de2c2f86e))
|
||||
* **14-03:** add notification batching to queue store and granular change tracking to library store ([d0c05dc](https://github.com/onion-4-dinner/yellowjacket/commit/d0c05dc1d43a4fe12cc07f3cff25375b08a74ba0))
|
||||
* **14-03:** eliminate per-item closure allocation in scroll render paths ([2f7ed70](https://github.com/onion-4-dinner/yellowjacket/commit/2f7ed7030425ed0ebb7a1a186917a79a7b26b850))
|
||||
* **14-04:** RAF-throttle scroll position saves and add overflow-anchor to queue panel ([6ca0b3c](https://github.com/onion-4-dinner/yellowjacket/commit/6ca0b3c5a84769af064ebe45a6eaac014d1a270a))
|
||||
* auto-detect NVIDIA+Wayland for DMABuf workaround ([915591a](https://github.com/onion-4-dinner/yellowjacket/commit/915591aea962beb60da2e96ac0f57307f646f675))
|
||||
* inline SVGs, memoize grid slices, batch store notifications ([a4eac39](https://github.com/onion-4-dinner/yellowjacket/commit/a4eac394cebefd29d0ebcb4b1e331444dcb8fbaf))
|
||||
* reduce software rendering overhead for NVIDIA+Wayland ([199c910](https://github.com/onion-4-dinner/yellowjacket/commit/199c91013fd806f6aefce49357df8a32b46faaa0))
|
||||
|
||||
### Refactoring
|
||||
|
||||
* **quick-17:** simplify playlist-view to navigate instead of expand ([955cd68](https://github.com/onion-4-dinner/yellowjacket/commit/955cd68be2dbf7a9071ef1c93084d687b59b6bd7))
|
||||
|
||||
## [1.2.2](https://github.com/onion-4-dinner/yellowjacket/compare/v1.2.1...v1.2.2) (2026-03-06)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
@@ -10,7 +10,6 @@ YellowJacket is a cross-platform desktop music player built with Go (backend) an
|
||||
|
||||
Active and historical plans live in `.planning/`:
|
||||
|
||||
- `.planning/ROADMAP.md` — vision, capability set, milestone sequence.
|
||||
- `.planning/NOTES.md` — gotchas, deferred items, open architecture questions, the "we already considered and rejected" list.
|
||||
- `.planning/plans/active/` — work currently in progress (read first).
|
||||
- `.planning/plans/pending/` — sequenced future work.
|
||||
@@ -18,18 +17,28 @@ Active and historical plans live in `.planning/`:
|
||||
|
||||
Numbering is sequential and stable across status moves (a plan keeps its `NNN-` prefix as it migrates between `pending → active → completed`). Abandoned plans are deleted; paused work stays in `pending/`.
|
||||
|
||||
The legacy `.gsd/` directory is a snapshot of the prior GSD-CLI planning system. It's gitignored and will be removed once nothing relies on it.
|
||||
|
||||
## Commands
|
||||
|
||||
```bash
|
||||
make dev # Hot-reload development (installs deps, generates code, cleans frontend)
|
||||
make dev-debug # Same as dev but with YJ_LOG_LEVEL=debug
|
||||
make dev-headless # Start headless in the background and return (SEED=<name> to seed)
|
||||
make dev-stop # Stop it (SIGTERM, so shutdown hooks run)
|
||||
make dev-logs # Tail .dev/app.log
|
||||
make testdata # Generate the deterministic fixture music library
|
||||
make sandbox-seed NAME=<n> # Build a seeded YJ_HOME by *running* the app
|
||||
make build-dev # Debug build with symbols
|
||||
make build-prod # Production build (stripped, UPX-compressed)
|
||||
make generate # Run code generators (sqlc + templ via go generate)
|
||||
make lint # golangci-lint v2 (strict)
|
||||
make test # All tests with race detector, 2min timeout
|
||||
make e2e # Playwright smoke suite against a running dev-headless app
|
||||
make e2e-setup # Install the e2e runner + its browser (once)
|
||||
make ui-test # Vitest component/store suite in a real browser (no app)
|
||||
make ui-visual # Same, including toMatchScreenshot comparisons
|
||||
make ui-setup # Install the Vitest provider's own Chromium (once)
|
||||
make bindings-check # Fail if frontend/wailsjs is stale vs the Go bindings
|
||||
make skill-check # Fail if .pi/ documents a make target that doesn't exist
|
||||
make lint # golangci-lint v2 (strict), all three build configurations
|
||||
make test # All tests with race detector, all three build configurations
|
||||
make vulncheck # govulncheck for CVEs
|
||||
make setup # Install go tools, frontend deps, git hooks (lefthook)
|
||||
```
|
||||
@@ -44,8 +53,83 @@ go test -tags webkit2_41 ./backend/player/ # Single package
|
||||
go test -tags webkit2_41 -run TestName ./backend/player/ # Single test
|
||||
```
|
||||
|
||||
The central index builder is behind a second tag and is **not** covered
|
||||
by the command above — `make test` runs both passes, but a manual run
|
||||
needs it spelled out:
|
||||
|
||||
```bash
|
||||
go test -tags "webkit2_41 indexbuild" ./backend/explore/... ./cmd/...
|
||||
```
|
||||
|
||||
`backend/testctl` is behind a third tag and needs its own pass too
|
||||
(`make test` runs all three):
|
||||
|
||||
```bash
|
||||
go test -tags "webkit2_41 dev" ./backend/testctl/...
|
||||
```
|
||||
|
||||
Audio playback integration tests require `YELLOWJACKET_INTEGRATION=1`.
|
||||
|
||||
### Fixtures and the headless harness
|
||||
|
||||
`test_data/music_library_test/` is **generated, not committed**: run
|
||||
`make testdata` (~1 s) before anything that needs audio. Tests reach it
|
||||
through `internal/testfixtures`, selecting files by *case*
|
||||
(`CaseCoverDedup`, `CaseUnicode`, `CaseDuplicates`, …) rather than by
|
||||
path, and skip themselves when it has not been generated.
|
||||
|
||||
The app itself can be run without a blocking window — `make
|
||||
dev-headless` — and driven with `playwright-cli` against the dev server
|
||||
on `:34115`, which is the real app with real bindings on `window.go`,
|
||||
bridged to the same Go backend a desktop window would use.
|
||||
|
||||
**The operational half of all this lives in the
|
||||
`yellowjacket-dev` skill** (`.pi/skills/yellowjacket-dev/`): which tier
|
||||
to reach for, the exact command sequences, seed lifecycle, and the
|
||||
failure modes worth knowing before you meet them. It is deliberately
|
||||
not repeated here — this section describes what exists, the skill says
|
||||
what to run.
|
||||
|
||||
Two things ride on top of the headless launch, both from plan 005
|
||||
phase 3:
|
||||
|
||||
- **The event bridge.** `.playwright/cli.config.json` loads
|
||||
`.playwright/init-events.js` as an `initScript`, which records every
|
||||
backend event on `window.__yjEvents`. Half this app is push-driven,
|
||||
so assertions **await an event, not a timeout**:
|
||||
`await window.__yjEvents.wait('LibraryScanComplete', {timeoutMs: 60000})`.
|
||||
It also provides `ready()` and `call('queue.Queue.GetState', [])`,
|
||||
which times out instead of hanging.
|
||||
- **The dev-only control surface**, `backend/testctl`, mounted at
|
||||
`/__test/` on the same port: `health`, `db/snapshot`, `db/restore`,
|
||||
`emit` (force any backend event, which renders push-driven views
|
||||
without staging the work that would produce them) and `sql`. It is
|
||||
compiled out of non-dev builds and additionally requires
|
||||
`YJ_TESTCTL=1`, which `dev-headless.sh` sets and `make dev` does not.
|
||||
|
||||
Frozen regression specs live in `e2e/` (its own npm package, so the
|
||||
Vitest browser mode does not share a package with the Playwright
|
||||
runner): `make e2e` against an already-running app.
|
||||
|
||||
**The cheapest tier needs none of that.** `make ui-test` runs 313
|
||||
Vitest tests in a real Chromium in ~2 s with no Wails, no backend, no
|
||||
seeded library and no virtual display, because `frontend/wailsjs/` is a
|
||||
pure passthrough to `window.go` / `window.runtime` and
|
||||
`frontend/test/support/wails-fake.ts` replaces just those two globals —
|
||||
so the tests exercise the real generated bindings and the real store
|
||||
code.
|
||||
|
||||
**`frontend/wailsjs/` is generated by `wails`, not `go generate`**, so
|
||||
the pre-commit codegen check does not cover it. `make bindings-check`
|
||||
(~1.5 s, also a pre-commit hook) regenerates it and fails on a dirty
|
||||
tree; `make bindings` regenerates it for real.
|
||||
|
||||
**Seeds are produced by running the app**, never by hand-writing a
|
||||
`config.toml` and DB rows — the same discipline `sql/schemas/` gets,
|
||||
for the same reason.
|
||||
|
||||
See `.planning/plans/active/005-agent-development-harness.md`.
|
||||
|
||||
## Architecture
|
||||
|
||||
**Wails app lifecycle** (`main.go` → `backend/app.go`): `YellowJacketApp` is the root struct bound to Wails. Its methods are callable from the frontend. Lifecycle hooks: `OnStartup` (init audio), `OnDomReady` (start library scan), `OnBeforeClose` (save window state), `OnShutdown` (persist player/queue state).
|
||||
@@ -55,17 +139,126 @@ Audio playback integration tests require `YELLOWJACKET_INTEGRATION=1`.
|
||||
- `queue` — Track queue with shuffle (Fisher-Yates), repeat modes, auto-advance, and session persistence.
|
||||
- `library` — Concurrent library scanning, metadata extraction, cover art deduplication, incremental rescan.
|
||||
- `database` — SQLite via pure-Go driver. Schema in `database/sql/schemas/`, queries in `database/sql/queries/`. **sqlc** generates Go code into `database/sql/sqlcgen/` — never edit that directory by hand.
|
||||
|
||||
**Schema changes need two things, not one.** `sql/schemas/*.sql` is
|
||||
`CREATE ... IF NOT EXISTS` and is what sqlc reads — it's the single
|
||||
source of truth for "what the schema looks like right now", and it's
|
||||
what a fresh install gets verbatim. But it's a no-op against a
|
||||
database that already has the table, so an existing install needs a
|
||||
matching file in `sql/schemas/../migrations/` (e.g.
|
||||
`NNNN_description.sql`, `ALTER TABLE ... ADD COLUMN ...` /
|
||||
`CREATE INDEX ...`) to actually reach that shape. Both run on every
|
||||
open, migrations after schema files, tracked in `schema_migrations`
|
||||
so each applies once; a migration's `ALTER TABLE ADD COLUMN` failing
|
||||
with "duplicate column name" on an already-current database is
|
||||
expected and tolerated, not an error.
|
||||
|
||||
A few things that bite if forgotten:
|
||||
- **Column order must match between the two paths.** `ALTER TABLE
|
||||
ADD COLUMN` always appends at the end, so a migrated column must
|
||||
also be declared *last* in the `CREATE TABLE` in `sql/schemas/`
|
||||
— otherwise a fresh install and an upgraded install disagree on
|
||||
column order, and a `SELECT *` query (sqlc binds those
|
||||
positionally) silently reads the wrong field on one of them. See
|
||||
`backend/database/migrations_test.go`'s
|
||||
`TestMigrations_ColumnOrderMatchesFreshInstall`, which is the
|
||||
regression test for exactly this.
|
||||
- **Don't put an index on a migrated column in `sql/schemas/`.**
|
||||
Schema files run before migrations, against a database that may
|
||||
not have that column yet — the index's predicate would fail
|
||||
(this is precisely the bug an earlier session shipped and a user
|
||||
hit at `make sandbox`). Declare it in the migration file instead,
|
||||
after the `ALTER TABLE` that adds the column.
|
||||
- This project **had** a 48-step migration chain before and tore it
|
||||
out (see `.planning/NOTES.md`, "No migration chain") because
|
||||
`sql/schemas/` had drifted from what the migrations actually
|
||||
produced and sqlc silently generated against the stale version.
|
||||
The design here avoids that by keeping `sql/schemas/` as the
|
||||
literal target shape (not a hand-maintained description of it)
|
||||
and letting migrations replay tolerantly against it — but the
|
||||
same drift is possible again if a schema change ships without
|
||||
updating both files. Don't reintroduce a *second* description of
|
||||
the schema anywhere else.
|
||||
- **Squashing is fine pre-1.0.** While this hasn't shipped to real
|
||||
users, periodically folding `sql/migrations/` into `sql/schemas/`
|
||||
and deleting the migration files (then wiping your own dev/sandbox
|
||||
DB) is a legitimate way to keep the migrations directory from
|
||||
accumulating dev-only churn — same effect as the old "just nuke
|
||||
it" workflow, opt-in instead of mandatory. Stop doing that once
|
||||
real user databases exist in the wild.
|
||||
- `metadata` — Tag extraction (ID3v2, Vorbis Comments, FLAC).
|
||||
- `config` — TOML-based settings. Settings page uses HTMX + templ for server-rendered HTML fragments.
|
||||
- `playlist` / `smartplaylist` — Playlist CRUD and rule-based smart playlists.
|
||||
- `mediacontrols` — MPRIS integration on Linux via D-Bus.
|
||||
- `system` — OS-specific paths (XDG on Linux, `%LOCALAPPDATA%` on Windows).
|
||||
- `explore` — Catalog search and browse over `explore_index`. See below.
|
||||
- `home` — The home page's "start listening" shelves. Each shelf is a
|
||||
*reason* (what you played last, what you never played, a genre you
|
||||
have depth in) rather than a filter, and carries the sentence that
|
||||
says so. Its queries (`sql/queries/home.sql`) return album ids only
|
||||
and are joined back to `GetAllAlbumsWithDetails` in Go, so the album
|
||||
projection has one definition. A shelf with nothing behind it is
|
||||
omitted, never rendered empty.
|
||||
- `profiling` — pprof server on `:6060`, compiled out in non-dev builds via build tags (`internal/dev/`).
|
||||
|
||||
**Explore catalog** (`backend/explore/`): the searchable MusicBrainz/
|
||||
ListenBrainz catalog in `explore_index`. Deriving it from the MetaBrainz
|
||||
dumps means streaming ~89 GB from a server that caps a client near
|
||||
2 MB/s — half a day, for a catalog identical for every user. So that
|
||||
work happens **once, centrally**, and users download the result:
|
||||
|
||||
- `cmd/indexbuild` builds the catalog from the dumps; `cmd/indexexport`
|
||||
cuts it down to a shippable core and stamps its provenance.
|
||||
`.gitea/workflows/index-artifact.yml` runs both and publishes the
|
||||
compressed artifact under a fixed `latest` version.
|
||||
- The app fetches and merges that artifact (`artifactfetch.go`,
|
||||
`artifactimport.go`) — about a minute, versus a day.
|
||||
- Everything the app does **not** need is behind the `indexbuild` build
|
||||
tag (`dumpimport.go`, `dumpcounts.go`, `dumpcatalog.go`,
|
||||
`dumpproject.go`, `dumpparallel.go`, `indexpatch.go`) so it is not
|
||||
linked into the binary. `dumpbuild_stub.go` is the app-side entry
|
||||
point; `dumpshared.go` holds what both sides use.
|
||||
- The app keeps popularity current with the daily incremental dumps
|
||||
(`dumpincremental.go`), and resolves artists outside the artifact's
|
||||
coverage lazily on first view.
|
||||
|
||||
**Frontend** (`frontend/`): Lit 3.2 web components + Web Awesome UI library + HTMX. State management via singleton reactive stores in `src/store/`. Wails bindings auto-generated in `frontend/wailsjs/` — don't edit by hand.
|
||||
|
||||
Two cross-cutting pieces of that UI are worth knowing before touching
|
||||
a list or a detail view:
|
||||
|
||||
- **`utils/explore-link.ts`** renders every track/album/artist name in
|
||||
the app. A name always navigates: to the MusicBrainz page when the
|
||||
entity is tagged, and to the *library* page for the same thing when
|
||||
it is not (`explore-album-details` and `explore-artist-details` both
|
||||
accept a local id instead of an MBID). It fires on a genuine single
|
||||
click only — the navigation is held for one double-click interval
|
||||
and dropped if a second click arrives, because the title is the
|
||||
widest thing in a row and double-clicking a row plays it. Rows do
|
||||
not need to know links exist.
|
||||
- **`<catalog-scope-notice>`** is how a detail page admits what it is
|
||||
showing: catalog data (silent), a library stand-in while a fetch is
|
||||
in flight, library-only because the entity has no MBID, or a failed/
|
||||
empty catalog answer with a retry. Both detail views track
|
||||
`catalogPending`/`catalogLoaded` separately from their loading flags,
|
||||
since "something is renderable" and "this is the catalog's answer"
|
||||
are different questions.
|
||||
|
||||
**Event-driven communication**: Backend emits events via Wails runtime; frontend stores subscribe to them. Event names are constants in `backend/events/`.
|
||||
|
||||
Emit through **`events.Emit(ctx, name, data...)`**, never
|
||||
`runtime.EventsEmit` — wails `log.Fatalf`s (unrecoverably) on any
|
||||
context that does not carry its runtime, which includes every
|
||||
`context.Background()`, so a direct call cannot run under test and can
|
||||
kill the app from a background worker. `TestNoDirectRuntimeEmits` fails
|
||||
the build on a direct call anywhere outside `backend/events`.
|
||||
|
||||
That wrapper is what makes services testable in-process: install a
|
||||
recorder with `events.WithSink(ctx, rec)` and assert on the payload the
|
||||
frontend would receive (`backend/queue/emit_test.go` is the model).
|
||||
`events.Deliver` is the same call returning an error instead of
|
||||
dropping, and has one legitimate caller — `/__test/emit`.
|
||||
|
||||
## Code Generation
|
||||
|
||||
Two generators run via `go generate ./...` (or `make generate`):
|
||||
@@ -82,8 +275,42 @@ Pre-commit hooks verify generated code is fresh — always run `make generate` a
|
||||
|
||||
## Testing
|
||||
|
||||
Tests use `database.NewTestDB(t)` for in-memory SQLite with full schema. Test audio fixtures live in `test_data/music_library_test/`. Table-driven tests are the norm.
|
||||
Tests use `database.NewTestDB(t)` for in-memory SQLite, built by the same
|
||||
`applySchema` production uses so the two cannot diverge. Test audio fixtures live in `test_data/music_library_test/`. Table-driven tests are the norm.
|
||||
|
||||
## Git Workflow
|
||||
|
||||
Direct push to `main` is blocked by lefthook — use feature branches and PRs. Pre-commit runs vet, lint, codegen check, and frontend typecheck in parallel. Pre-push runs the full test suite.
|
||||
Feature branches and PRs are the norm, but direct pushes to `main` are allowed. Pre-commit runs vet, lint, codegen check, and frontend typecheck in parallel. Pre-push runs the full test suite.
|
||||
|
||||
## CI
|
||||
|
||||
Four workflows in `.gitea/workflows/`. Three of them package and
|
||||
publish (`arch-package`, `homebrew-formula`, `index-artifact`); only
|
||||
`ci.yml` gates, and it is the one to look at when deciding whether a
|
||||
push was healthy.
|
||||
|
||||
Two jobs, both in an `ubuntu:24.04` container:
|
||||
|
||||
- **`check`** — no display: `make lint` and `make test` (three build
|
||||
configurations each), `tsc --noEmit`, `make ui-test`,
|
||||
`make bindings-check`, `make skill-check`.
|
||||
- **`e2e`** — under Xvfb and a private D-Bus: fixtures, a seed built by
|
||||
running the app, `make dev-headless`, then the Playwright suite
|
||||
against **both** Chromium and WebKit. Playwright's Linux WebKit links
|
||||
Ubuntu 24.04 libraries that Arch does not provide, so CI is the only
|
||||
place it can run, and it is the closest available approximation of
|
||||
the WebKit2GTK renderer that ships.
|
||||
|
||||
Two things the container needs that a developer machine does not. It
|
||||
has no PulseAudio socket, so `/etc/asound.conf` makes ALSA's `null`
|
||||
plugin the default device — that plugin advances its pointer on a
|
||||
timer, so playback is consumed at real-time rate and the elapsed clock
|
||||
moves, which `e2e/specs/playback.spec.ts` asserts. And
|
||||
`YJ_CORE_INDEX_URL` points at a dead address so no run fetches the real
|
||||
explore artifact, matching what `scripts/seed-sandbox.sh` already does.
|
||||
|
||||
**`make lint`'s tag sets must stay identical to `make test`'s.**
|
||||
Without `webkit2_41` wails resolves `webkit2gtk-4.0`, which Arch still
|
||||
ships and Ubuntu 24.04 does not — so a mismatch lints a configuration
|
||||
that only builds on one developer's distro, and says nothing about what
|
||||
ships.
|
||||
|
||||
@@ -2,11 +2,193 @@ VERSION ?= dev
|
||||
COMMIT ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo "unknown")
|
||||
LDFLAGS := -X 'main.version=$(VERSION)' -X 'main.commit=$(COMMIT)'
|
||||
|
||||
# YJ_HOME isolates the dev build's config + database from a packaged
|
||||
# install. Defaults to a sandbox under XDG data; override in .env to
|
||||
# point elsewhere (or unset it there to share the real user dirs).
|
||||
DEV_YJ_HOME ?= $(HOME)/.local/share/yellowjacket-dev
|
||||
|
||||
dev: setup generate clean
|
||||
if [ -f .env ]; then set -a; . ./.env; set +a; fi; go tool wails dev -tags webkit2_41 -loglevel Debug -v 2
|
||||
if [ -f .env ]; then set -a; . ./.env; set +a; fi; : "$${YJ_HOME:=$(DEV_YJ_HOME)}"; export YJ_HOME; go tool wails dev -tags webkit2_41 -loglevel Debug -v 2
|
||||
|
||||
dev-debug: setup generate clean
|
||||
if [ -f .env ]; then set -a; . ./.env; set +a; fi; YJ_LOG_LEVEL=debug go tool wails dev -tags webkit2_41 -loglevel Debug -v 2
|
||||
if [ -f .env ]; then set -a; . ./.env; set +a; fi; : "$${YJ_HOME:=$(DEV_YJ_HOME)}"; export YJ_HOME; YJ_LOG_LEVEL=debug go tool wails dev -tags webkit2_41 -loglevel Debug -v 2
|
||||
|
||||
# ── Headless harness (plan 005) ──────────────────────────────────────
|
||||
# The same dev server `make dev` runs, minus the blocking GTK window:
|
||||
# Xvfb gives it the display it insists on, and the script returns once
|
||||
# :34115 answers. This is the only entry point an agent can use, since
|
||||
# every other one blocks the terminal forever.
|
||||
dev-headless: ## Start the app headless in the background (SEED=<name> to seed)
|
||||
@./scripts/dev-headless.sh $(if $(SEED),--seed $(SEED),) $(HEADLESS_ARGS)
|
||||
|
||||
dev-headless-fresh: ## Same, but on an empty YJ_HOME (first-run wizard)
|
||||
@./scripts/dev-headless.sh --fresh $(HEADLESS_ARGS)
|
||||
|
||||
dev-stop: ## Stop the headless app (SIGTERM, so shutdown hooks run)
|
||||
@./scripts/dev-stop.sh
|
||||
|
||||
dev-logs: ## Tail the headless app log
|
||||
@tail -f .dev/app.log
|
||||
|
||||
# Seeds are produced by *running the app* — driving the real AddLibrary
|
||||
# binding and waiting for the real scan — never by hand-writing a
|
||||
# config.toml and DB rows. A hand-built seed is a second description
|
||||
# of a valid YJ_HOME and would drift from the real one.
|
||||
sandbox-seed: testdata ## Build a seeded YJ_HOME snapshot: make sandbox-seed NAME=<n>
|
||||
@./scripts/seed-sandbox.sh $(if $(NAME),--name $(NAME),)
|
||||
|
||||
sandbox-seeds: ## List built seeds
|
||||
@ls -1 .dev/seeds/*.tar 2>/dev/null | sed 's|.*/||; s|\.tar$$||' \
|
||||
|| echo " (none; build one with: make sandbox-seed NAME=default)"
|
||||
|
||||
# The specs drive the app that is *already* running: `make dev-headless`
|
||||
# daemonises, which is the opposite of what Playwright's `webServer`
|
||||
# supervises, and starting one per run would rebuild the frontend every
|
||||
# time. globalSetup fails with the exact commands to run if it is down.
|
||||
e2e: ## Run the Playwright smoke suite against a running dev-headless app
|
||||
@cd e2e && pnpm install --silent && npx playwright test $(E2E_ARGS)
|
||||
|
||||
e2e-setup: ## Install the e2e runner and its browser (once)
|
||||
@cd e2e && pnpm install && npx playwright install chromium
|
||||
|
||||
e2e-report: ## Open the HTML report from the last e2e run
|
||||
@cd e2e && npx playwright show-report
|
||||
|
||||
# The cheapest tier: components and stores in a real browser, with no
|
||||
# Wails, no backend, no seeded library and no virtual display. Lives in
|
||||
# frontend/ rather than e2e/ so the Vitest browser provider and the
|
||||
# Playwright runner cannot fight over versions or globs.
|
||||
ui-test: ## Run the Vitest component and store suite (frontend/)
|
||||
@cd frontend && pnpm install --silent && npx vitest run $(UI_ARGS)
|
||||
|
||||
ui-watch: ## Same suite, in watch mode
|
||||
@cd frontend && npx vitest
|
||||
|
||||
# Visual regression is opt-in: toMatchScreenshot baselines depend on
|
||||
# font hinting and compositing, so they only mean anything on the
|
||||
# machine (or container) that took them.
|
||||
ui-visual: ## Run the suite including screenshot comparisons
|
||||
@cd frontend && YJ_VISUAL=1 npx vitest run $(UI_ARGS)
|
||||
|
||||
ui-visual-update: ## Re-record the screenshot baselines
|
||||
@cd frontend && YJ_VISUAL=1 npx vitest run --update $(UI_ARGS)
|
||||
|
||||
ui-setup: ## Install the Vitest browser provider's own Chromium (once)
|
||||
@cd frontend && pnpm install && npx playwright install chromium
|
||||
|
||||
# frontend/wailsjs is generated by `wails`, NOT by `go generate`, so the
|
||||
# pre-commit codegen check does not cover it: a renamed Go struct field
|
||||
# currently surfaces at runtime, in a window. File modes are ignored
|
||||
# because `wails generate module` rewrites the runtime files as 755.
|
||||
bindings-check: ## Fail if frontend/wailsjs is stale against the Go bindings
|
||||
@./scripts/bindings-check.sh
|
||||
|
||||
# .pi/ documents commands, and a skill that documents a command wrongly
|
||||
# is worse than no skill: an agent runs it confidently. Every command
|
||||
# in there is a make target on purpose, so this is checkable.
|
||||
skill-check: ## Fail if .pi/ documents a make target that does not exist
|
||||
@./scripts/skill-check.sh
|
||||
|
||||
bindings: ## Regenerate frontend/wailsjs from the bound Go structs
|
||||
go tool wails generate module -tags webkit2_41
|
||||
@chmod 644 frontend/wailsjs/runtime/runtime.js \
|
||||
frontend/wailsjs/runtime/runtime.d.ts \
|
||||
frontend/wailsjs/runtime/package.json
|
||||
|
||||
.PHONY: dev-headless dev-headless-fresh dev-stop dev-logs \
|
||||
sandbox-seed sandbox-seeds e2e e2e-setup e2e-report \
|
||||
ui-test ui-watch ui-visual ui-visual-update ui-setup \
|
||||
bindings bindings-check skill-check
|
||||
|
||||
# Base directory for fresh-install sandboxes. Deliberately NOT $TMPDIR:
|
||||
# on most Linux distros /tmp is tmpfs (RAM-backed) and only a few GB, so
|
||||
# the search index dump import — which wants 6GB free before it will even
|
||||
# start, then streams multi-GB dumps through explore-staging/ — either
|
||||
# fails its precheck or eats that much RAM. XDG cache is disk-backed
|
||||
# everywhere and still throwaway.
|
||||
FRESH_HOME_BASE ?= $(if $(XDG_CACHE_HOME),$(XDG_CACHE_HOME),$(HOME)/.cache)
|
||||
|
||||
# fresh-install runs dev against a brand-new YJ_HOME so every launch
|
||||
# starts from a clean first-run state (no config.toml, no yj.db). The dir
|
||||
# is not cleaned up automatically, so you can inspect it afterward; the
|
||||
# printed path tells you where it is. Override the location with
|
||||
# FRESH_HOME_BASE=/some/disk make fresh-install.
|
||||
fresh-install: setup generate clean
|
||||
if [ -f .env ]; then set -a; . ./.env; set +a; fi; \
|
||||
mkdir -p "$(FRESH_HOME_BASE)"; \
|
||||
export YJ_HOME="$$(mktemp -d "$(FRESH_HOME_BASE)/yellowjacket-fresh.XXXXXX")"; \
|
||||
echo "==> fresh YJ_HOME=$$YJ_HOME"; \
|
||||
case "$$(findmnt -no FSTYPE -T "$$YJ_HOME" 2>/dev/null)" in \
|
||||
tmpfs|ramfs) echo "==> WARNING: $$YJ_HOME is RAM-backed; the search index import needs ~6GB of real disk. Set FRESH_HOME_BASE to a disk-backed path." ;; \
|
||||
esac; \
|
||||
go tool wails dev -tags webkit2_41 -loglevel Debug -v 2
|
||||
|
||||
# Named, persistent sandboxes: `make sandbox foo` runs dev against
|
||||
# $(FRESH_HOME_BASE)/yellowjacket-sandbox-foo, creating it on first use
|
||||
# and reusing it (never deleting) afterward, so you can keep several
|
||||
# long-lived states around — one with an imported search index, one with
|
||||
# a small library, etc. `make sandbox-foo` is the same thing.
|
||||
#
|
||||
# `make sandboxes` lists the ones that exist.
|
||||
#
|
||||
# `make sandbox-rm foo [bar ...]` deletes them again, after confirming.
|
||||
#
|
||||
# The bare words after `sandbox` / `sandbox-rm` are extra make goals, so
|
||||
# they need do-nothing rules to keep make from complaining. Those rules
|
||||
# exist only when one of those is the first goal, so typos in other
|
||||
# targets still fail loudly.
|
||||
SANDBOX_DIR = $(FRESH_HOME_BASE)/yellowjacket-sandbox
|
||||
ifneq (,$(filter $(firstword $(MAKECMDGOALS)),sandbox sandbox-rm))
|
||||
SANDBOX_ARGS := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS))
|
||||
SANDBOX_NAME := $(firstword $(SANDBOX_ARGS))
|
||||
$(foreach a,$(SANDBOX_ARGS),$(eval $(a):;@:))
|
||||
endif
|
||||
|
||||
sandbox: ## Run dev against a named, persistent YJ_HOME: make sandbox <name>
|
||||
@if [ -z "$(SANDBOX_NAME)" ]; then \
|
||||
echo "usage: make sandbox <name> (e.g. make sandbox foo)" >&2; exit 2; \
|
||||
fi
|
||||
@$(MAKE) --no-print-directory sandbox-$(SANDBOX_NAME)
|
||||
|
||||
sandbox-rm: ## Delete named sandboxes: make sandbox-rm <name> [name ...]
|
||||
@if [ -z "$(SANDBOX_ARGS)" ]; then \
|
||||
echo "usage: make sandbox-rm <name> [name ...]" >&2; exit 2; \
|
||||
fi
|
||||
@set -e; \
|
||||
targets=""; \
|
||||
for n in $(SANDBOX_ARGS); do \
|
||||
d="$(SANDBOX_DIR)-$$n"; \
|
||||
if [ -d "$$d" ]; then \
|
||||
echo " $$(du -sh "$$d" 2>/dev/null | cut -f1) $$d"; \
|
||||
targets="$$targets $$d"; \
|
||||
else \
|
||||
echo " (no such sandbox: $$n)" >&2; \
|
||||
fi; \
|
||||
done; \
|
||||
if [ -z "$$targets" ]; then exit 1; fi; \
|
||||
if [ "$(FORCE)" != "1" ]; then \
|
||||
printf "delete the above? [y/N] "; read -r ans; \
|
||||
case "$$ans" in y|Y|yes|YES) ;; *) echo "aborted"; exit 1 ;; esac; \
|
||||
fi; \
|
||||
rm -rf $$targets; \
|
||||
echo "==> removed"
|
||||
|
||||
sandbox-%: setup generate clean
|
||||
if [ -f .env ]; then set -a; . ./.env; set +a; fi; \
|
||||
export YJ_HOME="$(SANDBOX_DIR)-$*"; \
|
||||
mkdir -p "$$YJ_HOME"; \
|
||||
echo "==> sandbox '$*' YJ_HOME=$$YJ_HOME"; \
|
||||
case "$$(findmnt -no FSTYPE -T "$$YJ_HOME" 2>/dev/null)" in \
|
||||
tmpfs|ramfs) echo "==> WARNING: $$YJ_HOME is RAM-backed; the search index import needs ~6GB of real disk. Set FRESH_HOME_BASE to a disk-backed path." ;; \
|
||||
esac; \
|
||||
go tool wails dev -tags webkit2_41 -loglevel Debug -v 2
|
||||
|
||||
sandboxes: ## List existing named sandboxes
|
||||
@ls -d "$(SANDBOX_DIR)"-* 2>/dev/null \
|
||||
| sed 's|.*/yellowjacket-sandbox-| |' \
|
||||
|| echo " (none)"
|
||||
|
||||
.PHONY: sandbox sandbox-rm sandboxes
|
||||
|
||||
build-dev: generate
|
||||
go tool wails build -tags webkit2_41 -debug -clean -ldflags "$(LDFLAGS)"
|
||||
@@ -24,11 +206,45 @@ clean:
|
||||
generate:
|
||||
go generate ./...
|
||||
|
||||
lint:
|
||||
go tool golangci-lint run
|
||||
# The fixture library is generated, not committed: deterministic audio
|
||||
# across all four supported formats, tagged by backend/tagwriter so the
|
||||
# fixtures and the reader under test cannot drift. Regenerates only
|
||||
# when the spec's manifest hash has changed, so it is cheap to depend on.
|
||||
testdata: ## Generate the deterministic fixture music library
|
||||
go run ./cmd/gentestdata
|
||||
|
||||
test:
|
||||
testdata-force: ## Regenerate the fixture library unconditionally
|
||||
go run ./cmd/gentestdata -force
|
||||
|
||||
testdata-clean: ## Delete the generated fixture library
|
||||
rm -rf test_data/music_library_test test_data/music_library_broken \
|
||||
test_data/music_library_test.manifest.json
|
||||
|
||||
.PHONY: testdata testdata-force testdata-clean
|
||||
|
||||
# The tag sets must match `make test` exactly, or lint is checking three
|
||||
# configurations that nothing builds. webkit2_41 is not optional: without
|
||||
# it wails resolves webkit2gtk-4.0, which Ubuntu 24.04 no longer ships, so
|
||||
# the `dev` pass (wails' own app_dev.go is dev-tagged and pulls in the 4.0
|
||||
# assetserver) fails to typecheck anywhere but Arch.
|
||||
lint:
|
||||
go tool golangci-lint run --build-tags webkit2_41
|
||||
go tool golangci-lint run --build-tags "webkit2_41 indexbuild"
|
||||
go tool golangci-lint run --build-tags "webkit2_41 dev"
|
||||
|
||||
# Three passes: the app build, the `indexbuild` build that adds the
|
||||
# CI-only dump importer, and the `dev` build that adds profiling and
|
||||
# backend/testctl. Without the extra passes nothing would compile or
|
||||
# exercise backend/explore/dump*.go, cmd/indexbuild or the harness
|
||||
# control surface at all.
|
||||
test: testdata
|
||||
go test -tags webkit2_41 -race -count=1 -timeout 120s ./...
|
||||
go test -tags "webkit2_41 indexbuild" -race -count=1 -timeout 300s \
|
||||
./backend/explore/... ./cmd/...
|
||||
# backend/testctl only exists under the `dev` tag, so the pass above
|
||||
# does not compile it, let alone run it.
|
||||
go test -tags "webkit2_41 dev" -race -count=1 -timeout 120s \
|
||||
./backend/testctl/...
|
||||
|
||||
vulncheck:
|
||||
go tool govulncheck ./...
|
||||
|
||||
+207
@@ -10,6 +10,7 @@ import (
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
wailsruntime "github.com/wailsapp/wails/v2/pkg/runtime"
|
||||
|
||||
@@ -18,9 +19,14 @@ import (
|
||||
"yellowjacket/backend/config"
|
||||
"yellowjacket/backend/coverart"
|
||||
"yellowjacket/backend/database"
|
||||
"yellowjacket/backend/download"
|
||||
"yellowjacket/backend/events"
|
||||
"yellowjacket/backend/explore"
|
||||
"yellowjacket/backend/frontendutil"
|
||||
"yellowjacket/backend/home"
|
||||
"yellowjacket/backend/jobs"
|
||||
"yellowjacket/backend/library"
|
||||
"yellowjacket/backend/maintenance"
|
||||
"yellowjacket/backend/mediacontrols"
|
||||
"yellowjacket/backend/player"
|
||||
"yellowjacket/backend/playlist"
|
||||
@@ -28,6 +34,7 @@ import (
|
||||
"yellowjacket/backend/queue"
|
||||
"yellowjacket/backend/system"
|
||||
"yellowjacket/backend/tagwriter"
|
||||
"yellowjacket/backend/testctl"
|
||||
)
|
||||
|
||||
// YellowJacketApp is the main application struct for Wails.
|
||||
@@ -44,8 +51,13 @@ type YellowJacketApp struct {
|
||||
queue *queue.Queue
|
||||
explore *explore.Service
|
||||
autotag *autotagservice.Service
|
||||
downloads *download.Manager
|
||||
downloadSvc *download.Service
|
||||
wanted *download.Reconciler
|
||||
jobs *jobs.Registry
|
||||
mediaControls mediacontrols.Handler
|
||||
tagWriter *tagwriter.TagWriter
|
||||
janitor *maintenance.Runner
|
||||
appContext context.Context
|
||||
appConfig *config.Config
|
||||
startupErr error
|
||||
@@ -63,6 +75,7 @@ func NewYellowJacketApp(
|
||||
logger: logger,
|
||||
assetHandler: assetHandler,
|
||||
appContext: context.Background(),
|
||||
janitor: maintenance.NewRunner(logger),
|
||||
}
|
||||
|
||||
// create database
|
||||
@@ -120,6 +133,17 @@ func NewYellowJacketApp(
|
||||
yjApp.assetHandler.RegisterHandler("/artist-images/", artistImgHandler)
|
||||
}
|
||||
|
||||
// Dev-only /__test/ control surface: the residue of harness work the
|
||||
// browser cannot reach (snapshot/restore the DB mid-run, force a
|
||||
// backend event). Compiled out of non-dev builds entirely, and even
|
||||
// in a dev build it registers nothing unless YJ_TESTCTL=1. The
|
||||
// context is read lazily because it only exists after OnStartup.
|
||||
testctl.Register(yjApp.assetHandler, testctl.Deps{
|
||||
Logger: logger,
|
||||
DB: yjApp.database,
|
||||
Context: func() context.Context { return yjApp.appContext },
|
||||
})
|
||||
|
||||
// create playlist service
|
||||
yjApp.playlist = playlist.NewService(
|
||||
yjApp.logger, yjApp.database, yjApp.appConfig,
|
||||
@@ -148,6 +172,16 @@ func NewYellowJacketApp(
|
||||
yjApp.logger.WithGroup("explore"), yjApp.database,
|
||||
)
|
||||
|
||||
// create the background job registry and wire it into the
|
||||
// subsystems that run long jobs, so scans and index builds all
|
||||
// report through one surface.
|
||||
yjApp.jobs = jobs.NewRegistry(
|
||||
yjApp.logger.WithGroup("jobs"),
|
||||
jobs.NewStore(yjApp.database, yjApp.logger.WithGroup("jobs")),
|
||||
)
|
||||
yjApp.library.SetJobRegistry(yjApp.jobs)
|
||||
yjApp.explore.SetJobRegistry(yjApp.jobs)
|
||||
|
||||
// create autotag service (depends on explore + tagWriter)
|
||||
yjApp.autotag = autotagservice.NewService(
|
||||
yjApp.logger.WithGroup("autotag"),
|
||||
@@ -156,6 +190,16 @@ func NewYellowJacketApp(
|
||||
yjApp.tagWriter,
|
||||
)
|
||||
|
||||
// Create the download subsystem. Acquiring music is optional: a
|
||||
// failure here (unwritable data dir, say) must not stop the app
|
||||
// from playing the library the user already has, so it is logged
|
||||
// and the feature stays unavailable rather than fatal.
|
||||
if err := yjApp.initDownloads(); err != nil {
|
||||
yjApp.logger.Error(
|
||||
"download clients unavailable", "error", err,
|
||||
)
|
||||
}
|
||||
|
||||
yjApp.FEBindings = []any{
|
||||
yjApp.FrontendUtil,
|
||||
yjApp.appConfig,
|
||||
@@ -166,11 +210,59 @@ func NewYellowJacketApp(
|
||||
yjApp.tagWriter,
|
||||
yjApp.explore,
|
||||
yjApp.autotag,
|
||||
jobs.NewService(yjApp.jobs),
|
||||
home.NewService(
|
||||
yjApp.logger.WithGroup("home"),
|
||||
yjApp.database,
|
||||
yjApp.library,
|
||||
),
|
||||
}
|
||||
|
||||
if yjApp.downloadSvc != nil {
|
||||
yjApp.FEBindings = append(yjApp.FEBindings, yjApp.downloadSvc)
|
||||
}
|
||||
|
||||
return yjApp, nil
|
||||
}
|
||||
|
||||
// initDownloads builds the download subsystem: staging area, secret
|
||||
// store, importer and manager, plus the Wails-bound service.
|
||||
func (yj *YellowJacketApp) initDownloads() error {
|
||||
logger := yj.logger.WithGroup("download")
|
||||
|
||||
staging, err := download.NewStaging(logger)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not create download staging: %w", err)
|
||||
}
|
||||
|
||||
secrets, err := download.NewFileSecretStore()
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not create download secret store: %w", err)
|
||||
}
|
||||
|
||||
store := download.NewStore(yj.database)
|
||||
|
||||
importer := download.NewImporter(logger, staging, yj.tagWriter, yj.library)
|
||||
|
||||
yj.downloads = download.NewManager(
|
||||
logger, store, secrets, staging, importer, yj.library,
|
||||
)
|
||||
yj.downloads.SetJobRegistry(yj.jobs)
|
||||
|
||||
yj.downloadSvc = download.NewService(logger, yj.downloads, store, secrets)
|
||||
|
||||
// The wanted list needs the explore index to know what an artist
|
||||
// released and what the library already owns, so it is wired here
|
||||
// where both exist. The reconcile loop itself is not started until
|
||||
// the Wails runtime is up.
|
||||
yj.wanted = download.NewReconciler(
|
||||
logger, store, yj.downloads, newExploreCatalog(yj.explore),
|
||||
)
|
||||
yj.downloadSvc.SetReconciler(yj.wanted)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// playerAdapter wraps *player.Player to satisfy the tagwriter.PlayerStopper
|
||||
// interface, breaking the import cycle between tagwriter and player.
|
||||
type playerAdapter struct{ p *player.Player }
|
||||
@@ -181,6 +273,46 @@ func (a *playerAdapter) CurrentFilePath() string {
|
||||
|
||||
func (a *playerAdapter) StopAndRelease() { a.p.UnloadTrack() }
|
||||
|
||||
// initDownloadRuntime brings the download subsystem up once the Wails
|
||||
// runtime exists: it applies the user's import layout, builds providers
|
||||
// from stored config, and clears staging left by a previous run.
|
||||
//
|
||||
// Provider construction and the sweep both touch the network and the
|
||||
// filesystem, so they run in the background — a slow or unreachable
|
||||
// download client must not delay the window appearing.
|
||||
func (yj *YellowJacketApp) initDownloadRuntime(ctx context.Context) {
|
||||
cfg := yj.appConfig.Downloads
|
||||
if cfg == nil {
|
||||
cfg = &download.UserConfig{}
|
||||
cfg.ApplyDefaults()
|
||||
}
|
||||
|
||||
yj.downloads.SetImportOptions(download.ImportOptions{
|
||||
PathTemplate: cfg.PathTemplate,
|
||||
})
|
||||
yj.downloads.SetMaxConcurrent(cfg.MaxConcurrent)
|
||||
yj.downloads.SetPreferences(cfg.AutoDownloadPrefs())
|
||||
|
||||
go func() {
|
||||
if err := yj.downloads.Reload(ctx); err != nil {
|
||||
yj.logger.Warn("could not load download providers", "error", err)
|
||||
}
|
||||
|
||||
yj.downloads.Sweep(ctx)
|
||||
}()
|
||||
|
||||
if yj.wanted == nil {
|
||||
return
|
||||
}
|
||||
|
||||
yj.wanted.SetInterval(cfg.WantedInterval())
|
||||
yj.wanted.SetBatch(cfg.WantedBatch)
|
||||
yj.wanted.SetOnChange(func() {
|
||||
events.Emit(ctx, events.RequestsChanged)
|
||||
})
|
||||
yj.wanted.Start(ctx)
|
||||
}
|
||||
|
||||
// WindowConfig returns the window configuration for use by the host.
|
||||
func (yj *YellowJacketApp) WindowConfig() *config.WindowConfig {
|
||||
return yj.appConfig.Window
|
||||
@@ -202,6 +334,9 @@ func (yj *YellowJacketApp) OnStartup(ctx context.Context) {
|
||||
yj.playlist.EnsureDefaultPlaylist()
|
||||
// Recover playlists that lost tracks from a pre-fix FullRescan.
|
||||
go yj.playlist.RepopulateFromM3U()
|
||||
// Backfill snapshots for smart playlists created before
|
||||
// creation-time materialization existed.
|
||||
go yj.playlist.MaterializeUnmaterializedSmartPlaylists()
|
||||
|
||||
// Initialize speaker hardware (player struct created in
|
||||
// NewYellowJacketApp for Wails binding registration).
|
||||
@@ -216,6 +351,18 @@ func (yj *YellowJacketApp) OnStartup(ctx context.Context) {
|
||||
yj.tagWriter.SetContext(ctx)
|
||||
yj.explore.SetContext(ctx)
|
||||
yj.autotag.SetContext(ctx)
|
||||
yj.jobs.SetContext(ctx)
|
||||
|
||||
if yj.downloadSvc != nil {
|
||||
yj.downloadSvc.SetContext(ctx)
|
||||
yj.initDownloadRuntime(ctx)
|
||||
}
|
||||
|
||||
// Bring back jobs the user paused before the last shutdown, still
|
||||
// paused. Must run before the soft scan in OnDomReady, which
|
||||
// checks these records so it does not restart a paused library.
|
||||
yj.library.RestorePausedScans()
|
||||
yj.explore.AdoptPausedIndexBuild()
|
||||
|
||||
// Wire queue (created in NewYellowJacketApp for Wails binding)
|
||||
yj.queue.SetContext(ctx)
|
||||
@@ -253,6 +400,12 @@ func (yj *YellowJacketApp) OnStartup(ctx context.Context) {
|
||||
// with no API calls. Deep discographies stay lazy.
|
||||
yj.explore.PopulateLocalCrossReferences()
|
||||
|
||||
// Enrich any owned artists whose discography hasn't been
|
||||
// fetched yet so their wider catalogue is searchable offline
|
||||
// right after the scan. Background, bounded, resumable, and a
|
||||
// no-op once every owned artist is covered.
|
||||
yj.explore.BackfillLibraryDiscographies()
|
||||
|
||||
// Start (or resume) the dump-based index build. Skips
|
||||
// itself once the one-time import has completed, so this
|
||||
// is cheap on every startup.
|
||||
@@ -438,6 +591,11 @@ func (yj *YellowJacketApp) OnDomReady(ctx context.Context) {
|
||||
// to fill any remaining gaps.
|
||||
yj.explore.RebuildLyricsIndexIfNeeded()
|
||||
yj.explore.BackfillLibraryLyrics()
|
||||
|
||||
// Continue enriching any owned artists still missing their
|
||||
// discography (e.g. a prior run was capped or interrupted).
|
||||
// Cheap no-op once every owned artist is covered.
|
||||
yj.explore.BackfillLibraryDiscographies()
|
||||
}
|
||||
|
||||
// Kick off the autotag prefetch worker so any unscored
|
||||
@@ -446,5 +604,54 @@ func (yj *YellowJacketApp) OnDomReady(ctx context.Context) {
|
||||
// every app launch is fine; previously-scored items are
|
||||
// skipped (the worker filters score IS NULL).
|
||||
yj.autotag.StartBackgroundPrefetch()
|
||||
|
||||
// Start the janitor last: its sweeps compare against live data,
|
||||
// so running them after the scan and index work has settled
|
||||
// avoids deleting something a running import is about to
|
||||
// reference. Each job enforces its own minimum interval, so the
|
||||
// daily tick is a cheap no-op most of the time.
|
||||
yj.startJanitor()
|
||||
}()
|
||||
}
|
||||
|
||||
// janitorTick is how often the maintenance runner wakes up. Individual
|
||||
// jobs enforce their own minimum intervals, so most ticks do nothing.
|
||||
const janitorTick = 6 * time.Hour
|
||||
|
||||
// startJanitor registers the maintenance jobs and starts the background
|
||||
// runner. Every job is registered here rather than at each package's
|
||||
// init, so the full set of janitorial work is one visible list — a cache
|
||||
// that forgets to register is missing from this function, which is
|
||||
// harder to overlook than a function nobody calls.
|
||||
func (yj *YellowJacketApp) startJanitor() {
|
||||
coversDir, err := coverart.CoversDir()
|
||||
if err != nil {
|
||||
yj.logger.Warn("janitor: could not resolve covers directory",
|
||||
"err", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
dataDir, err := system.GetUserDataDirPath()
|
||||
if err != nil {
|
||||
yj.logger.Warn("janitor: could not resolve user data directory",
|
||||
"err", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
yj.janitor.Register(maintenance.ExpiredHTTPCacheJob(yj.database))
|
||||
yj.janitor.Register(maintenance.OrphanedCoverFilesJob(
|
||||
yj.database, coversDir, library.CoverArtFileSet,
|
||||
))
|
||||
yj.janitor.Register(maintenance.OrphanedArtistImagesJob(
|
||||
yj.database, filepath.Join(dataDir, explore.ArtistImageDirName),
|
||||
))
|
||||
yj.janitor.Register(maintenance.ExpiredProxyCacheJob(
|
||||
filepath.Join(dataDir, explore.CoverArtCacheDirName),
|
||||
))
|
||||
|
||||
yj.logger.Info("janitor started", "jobs", yj.janitor.JobNames())
|
||||
|
||||
yj.janitor.Start(yj.appContext, janitorTick)
|
||||
}
|
||||
|
||||
@@ -192,6 +192,16 @@ func rotateEndWord(s string) string {
|
||||
return s
|
||||
}
|
||||
|
||||
// TitleSimilarity exposes titleSimilarity for callers outside the
|
||||
// package that compare music metadata strings and should get the same
|
||||
// answer the tagger would. The download pipeline uses it to match
|
||||
// candidate filenames against an expected tracklist — Soulseek and
|
||||
// torrent results carry paths, not tags, so filename comparison is the
|
||||
// only signal available before the bytes arrive.
|
||||
func TitleSimilarity(a, b string) float64 {
|
||||
return titleSimilarity(a, b)
|
||||
}
|
||||
|
||||
// titleSimilarity returns a score in [0, 1] from stringDist. 1.0
|
||||
// means identical after normalization, 0.0 means fully dissimilar.
|
||||
func titleSimilarity(a, b string) float64 {
|
||||
|
||||
@@ -19,12 +19,14 @@ import (
|
||||
//
|
||||
// libraryID || 0 || normalized_parent_dir || 0 || disc_number
|
||||
//
|
||||
// where the parent directory is lower-cased. The folder is taken
|
||||
// as the album boundary — including the album tag string would
|
||||
// fragment albums whose tracks carry slightly different tags
|
||||
// (`Abbey Road` vs `Abbey Road (Remastered 2009)`, etc.). The
|
||||
// album name is still surfaced in `tagging_items.album_name` for
|
||||
// the review UI; it just doesn't decide grouping.
|
||||
// where the parent directory is lower-cased and disc_number is
|
||||
// normalized so an untagged disc (0) folds into disc 1 — see
|
||||
// normalizeDiscNumber. The folder is taken as the album boundary —
|
||||
// including the album tag string would fragment albums whose tracks
|
||||
// carry slightly different tags (`Abbey Road` vs `Abbey Road
|
||||
// (Remastered 2009)`, etc.). The album name is still surfaced in
|
||||
// `tagging_items.album_name` for the review UI; it just doesn't
|
||||
// decide grouping.
|
||||
//
|
||||
// Using SHA-1 matches the codebase's existing non-crypto
|
||||
// deterministic-key convention; collision risk at album-group
|
||||
@@ -41,7 +43,60 @@ func GroupKey(
|
||||
h.Write([]byte{0})
|
||||
h.Write([]byte(parentDir))
|
||||
h.Write([]byte{0})
|
||||
h.Write([]byte(strconv.Itoa(discNumber)))
|
||||
h.Write([]byte(strconv.Itoa(normalizeDiscNumber(discNumber))))
|
||||
|
||||
return hex.EncodeToString(h.Sum(nil))
|
||||
}
|
||||
|
||||
// SyntheticGroupKey returns a deterministic identifier for a
|
||||
// tag-clustered sub-group carved out of parentGroupKey by
|
||||
// SplitMixedFolder — same SHA-1-over-null-separated-fields shape as
|
||||
// GroupKey, but keyed on the cluster's (album, album-artist) tags
|
||||
// instead of a directory, since a synthetic group's tracks don't
|
||||
// share a directory boundary distinct from their siblings left
|
||||
// behind in the parent folder.
|
||||
func SyntheticGroupKey(parentGroupKey, albumName, albumArtist string) string {
|
||||
h := sha1.New() //nolint:gosec // see package doc — grouping only.
|
||||
h.Write([]byte(parentGroupKey))
|
||||
h.Write([]byte{0})
|
||||
h.Write([]byte(Normalize(albumName)))
|
||||
h.Write([]byte{0})
|
||||
h.Write([]byte(Normalize(albumArtist)))
|
||||
|
||||
return hex.EncodeToString(h.Sum(nil))
|
||||
}
|
||||
|
||||
// SyntheticTrackGroupKey returns a deterministic identifier for a
|
||||
// single leftover track carved out of a mixed-bag folder by
|
||||
// SplitMixedFolder's singleton fallback (autotag.SplitPlan). Keyed on
|
||||
// the track's own audio_files id rather than its tags — two
|
||||
// untagged leftover tracks would otherwise both normalize to the
|
||||
// same empty (album, album-artist) pair and collide under
|
||||
// SyntheticGroupKey.
|
||||
func SyntheticTrackGroupKey(parentGroupKey string, audioFileID int64) string {
|
||||
h := sha1.New() //nolint:gosec // see package doc — grouping only.
|
||||
h.Write([]byte(parentGroupKey))
|
||||
h.Write([]byte{0})
|
||||
h.Write([]byte("track"))
|
||||
h.Write([]byte{0})
|
||||
h.Write([]byte(strconv.FormatInt(audioFileID, 10)))
|
||||
|
||||
return hex.EncodeToString(h.Sum(nil))
|
||||
}
|
||||
|
||||
// normalizeDiscNumber folds a missing/invalid disc tag (<= 0) into
|
||||
// disc 1 for grouping purposes. Without this, a folder where only
|
||||
// some tracks carry an explicit "disc 1 of 1" tag — common when
|
||||
// files were ripped or re-tagged at different times — splits into
|
||||
// two tagging groups for what is really one single-disc album: the
|
||||
// untagged tracks hash to disc 0, the tagged ones to disc 1. A
|
||||
// genuine multi-disc release still separates correctly, since its
|
||||
// disc-2-and-up tracks carry an explicit non-zero, non-one disc
|
||||
// number.
|
||||
func normalizeDiscNumber(discNumber int) int {
|
||||
if discNumber <= 0 {
|
||||
return 1
|
||||
}
|
||||
|
||||
return discNumber
|
||||
}
|
||||
|
||||
@@ -88,6 +88,29 @@ func TestGroupKey_DistinctInputsDiffer(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGroupKey_UntaggedDiscFoldsIntoDiscOne(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// A folder where only some tracks carry an explicit disc tag must
|
||||
// not split: the untagged tracks (disc 0, dhowden/tag's zero value
|
||||
// for a missing frame) should group with the ones tagged disc 1.
|
||||
untagged := autotag.GroupKey(1, "/music/Artist/Album/01.mp3", 0)
|
||||
tagged := autotag.GroupKey(1, "/music/Artist/Album/02.mp3", 1)
|
||||
|
||||
if untagged != tagged {
|
||||
t.Fatalf(
|
||||
"disc 0 and disc 1 in the same folder should share a key, got %q vs %q",
|
||||
untagged, tagged,
|
||||
)
|
||||
}
|
||||
|
||||
// A genuine disc 2 must still separate from disc 1/untagged.
|
||||
discTwo := autotag.GroupKey(1, "/music/Artist/Album/01.mp3", 2)
|
||||
if discTwo == tagged {
|
||||
t.Fatalf("disc 2 should not share a key with disc 1, got %q", discTwo)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGroupKey_AmbiguityBoundary(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -39,14 +39,16 @@ func (r *LocalResolver) LocalTracksForGroup(
|
||||
out := make([]LocalTrack, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
out = append(out, LocalTrack{
|
||||
AudioFileID: row.ID,
|
||||
FilePath: row.FilePath,
|
||||
Title: row.Title,
|
||||
Artist: row.ArtistName,
|
||||
TrackNumber: int(row.TrackNumber),
|
||||
DiscNumber: int(row.DiscNumber),
|
||||
LengthMillis: row.LengthMilliseconds,
|
||||
RecordingMBID: row.RecordingMbid,
|
||||
AudioFileID: row.ID,
|
||||
FilePath: row.FilePath,
|
||||
Title: row.Title,
|
||||
Artist: row.ArtistName,
|
||||
TrackNumber: int(row.TrackNumber),
|
||||
DiscNumber: int(row.DiscNumber),
|
||||
LengthMillis: row.LengthMilliseconds,
|
||||
RecordingMBID: row.RecordingMbid,
|
||||
AlbumTag: row.AlbumName,
|
||||
AlbumArtistTag: row.AlbumArtist,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
+41
-1
@@ -61,6 +61,18 @@ type MBClient interface {
|
||||
query string,
|
||||
limit int,
|
||||
) ([]MBReleaseGroupHit, int, error)
|
||||
// SearchReleaseGroupsLocal searches the offline dump-derived
|
||||
// catalog for release groups matching albumName — no network
|
||||
// round-trip. ok is false when the local catalog isn't
|
||||
// populated yet (or the implementation has no offline index),
|
||||
// telling the caller to rely on the network cascade alone; ok
|
||||
// true with zero hits means the catalog was consulted and
|
||||
// genuinely has nothing.
|
||||
SearchReleaseGroupsLocal(
|
||||
ctx context.Context,
|
||||
albumName string,
|
||||
limit int,
|
||||
) (hits []MBReleaseGroupHit, ok bool)
|
||||
SearchRecordings(
|
||||
ctx context.Context,
|
||||
query string,
|
||||
@@ -128,12 +140,40 @@ func (r *MBResolver) ResolveMB(ctx context.Context, g Group) ([]Candidate, error
|
||||
}
|
||||
|
||||
nArtist := Normalize(groupArtist(g))
|
||||
steps := buildMBQueryCascade(nAlbum, nArtist, len(g.Tracks), vaLikely(g))
|
||||
|
||||
seen := make(map[string]bool)
|
||||
|
||||
var merged []Candidate
|
||||
|
||||
// Local-index pass: the offline dump-derived catalog covers
|
||||
// essentially every popular release group, so try it before
|
||||
// spending any rate-limited search calls. This never skips
|
||||
// BrowseReleases (the catalog doesn't carry per-release
|
||||
// tracklists) but it very often means the network Lucene
|
||||
// cascade below never has to run at all.
|
||||
if localHits, ok := r.client.SearchReleaseGroupsLocal(ctx, g.AlbumName, r.limit); ok {
|
||||
added := r.fanOutBrowse(ctx, g, localHits, "index", seen, &merged)
|
||||
|
||||
r.logger.Debug(
|
||||
"local index search done",
|
||||
"hits", len(localHits), "new_candidates", added,
|
||||
)
|
||||
|
||||
if added > 0 {
|
||||
ranked := RankCandidates(g, merged)
|
||||
if len(ranked) > 0 && ranked[0].Score >= cascadeSufficient {
|
||||
r.logger.Info(
|
||||
"MB cascade stopped — sufficient local-index candidate",
|
||||
"score", ranked[0].Score,
|
||||
)
|
||||
|
||||
return merged, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
steps := buildMBQueryCascade(nAlbum, nArtist, len(g.Tracks), vaLikely(g))
|
||||
|
||||
for _, step := range steps {
|
||||
hits, _, err := r.client.SearchReleaseGroups(ctx, step.query, r.limit)
|
||||
if err != nil {
|
||||
|
||||
@@ -23,6 +23,8 @@ type fakeMBClient struct {
|
||||
lookupRGs map[string]MBReleaseGroupHit
|
||||
searchRecs []MBRecordingHit
|
||||
recRelsByMBID map[string][]MBReleaseRef
|
||||
localHits []MBReleaseGroupHit
|
||||
localOK bool
|
||||
}
|
||||
|
||||
func (f *fakeMBClient) SearchReleaseGroups(
|
||||
@@ -36,6 +38,15 @@ func (f *fakeMBClient) SearchReleaseGroups(
|
||||
return hits, len(hits), nil
|
||||
}
|
||||
|
||||
// SearchReleaseGroupsLocal is a no-op by default (ok=false), so
|
||||
// existing cascade tests exercise the network path unchanged. Set
|
||||
// localHits / localOK on the fake to exercise the index-first path.
|
||||
func (f *fakeMBClient) SearchReleaseGroupsLocal(
|
||||
_ context.Context, _ string, _ int,
|
||||
) ([]MBReleaseGroupHit, bool) {
|
||||
return f.localHits, f.localOK
|
||||
}
|
||||
|
||||
func (f *fakeMBClient) BrowseReleases(
|
||||
_ context.Context, mbid string,
|
||||
) ([]MBRelease, error) {
|
||||
@@ -188,6 +199,89 @@ func TestMBResolver_CascadeStopsWhenSufficient(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMBResolver_LocalIndexSufficientSkipsNetworkSearch(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fake := &fakeMBClient{
|
||||
localOK: true,
|
||||
localHits: []MBReleaseGroupHit{
|
||||
{MBID: "rg1", Title: "Abbey Road"},
|
||||
},
|
||||
browseByMBID: map[string][]MBRelease{
|
||||
"rg1": {{
|
||||
MBID: "rel1", Title: "Abbey Road", Status: "Official",
|
||||
Tracks: []CandidateTrack{
|
||||
{Position: 1, Title: "Come Together", LengthMillis: 259000},
|
||||
},
|
||||
}},
|
||||
},
|
||||
}
|
||||
|
||||
r := NewMBResolver(fake, slog.New(slog.DiscardHandler))
|
||||
|
||||
cands, err := r.ResolveMB(context.Background(), abbeyRoadGroup())
|
||||
if err != nil {
|
||||
t.Fatalf("ResolveMB: %v", err)
|
||||
}
|
||||
|
||||
if len(cands) != 1 {
|
||||
t.Fatalf("expected 1 candidate, got %d", len(cands))
|
||||
}
|
||||
|
||||
if cands[0].Provenance != "index" {
|
||||
t.Errorf("provenance = %q, want 'index'", cands[0].Provenance)
|
||||
}
|
||||
|
||||
if len(fake.queries) != 0 {
|
||||
t.Errorf(
|
||||
"expected zero network search queries when the local index sufficed, got %d: %v",
|
||||
len(fake.queries), fake.queries,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMBResolver_LocalIndexThinFallsThroughToNetwork(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Local index is "ready" but has nothing plausible for this
|
||||
// album — the cascade must still fall through to the network
|
||||
// steps exactly as if there were no local index at all.
|
||||
fake := &fakeMBClient{
|
||||
localOK: true,
|
||||
localHits: nil,
|
||||
searchByStep: map[int][]MBReleaseGroupHit{
|
||||
1: {{MBID: "rg1", Title: "Abbey Road"}},
|
||||
},
|
||||
browseByMBID: map[string][]MBRelease{
|
||||
"rg1": {{
|
||||
MBID: "rel1", Title: "Abbey Road", Status: "Official",
|
||||
Tracks: []CandidateTrack{
|
||||
{Position: 1, Title: "Come Together", LengthMillis: 259000},
|
||||
},
|
||||
}},
|
||||
},
|
||||
}
|
||||
|
||||
r := NewMBResolver(fake, slog.New(slog.DiscardHandler))
|
||||
|
||||
cands, err := r.ResolveMB(context.Background(), abbeyRoadGroup())
|
||||
if err != nil {
|
||||
t.Fatalf("ResolveMB: %v", err)
|
||||
}
|
||||
|
||||
if len(cands) != 1 {
|
||||
t.Fatalf("expected 1 candidate, got %d", len(cands))
|
||||
}
|
||||
|
||||
if cands[0].Provenance != "no-track-count" {
|
||||
t.Errorf("provenance = %q, want 'no-track-count'", cands[0].Provenance)
|
||||
}
|
||||
|
||||
if len(fake.queries) != 2 { //nolint:mnd
|
||||
t.Errorf("expected the usual 2 network queries, got %d", len(fake.queries))
|
||||
}
|
||||
}
|
||||
|
||||
func TestMBResolver_CascadeContinuesPastMediocreHits(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
package autotag
|
||||
|
||||
// mixedBagMinTracks is the smallest folder IsMixedBag will flag.
|
||||
// Below this, artist/album divergence is just as likely to be
|
||||
// sampling noise (a 2-track folder with two different artists could
|
||||
// easily be a legitimate 2-track EP with a featured artist) as it is
|
||||
// a genuine junk-drawer folder.
|
||||
const mixedBagMinTracks = 4
|
||||
|
||||
// clusterMinSize is the smallest tag-matched group ClusterByAlbumArtist
|
||||
// will surface as a splittable cluster. A single track sharing no
|
||||
// album/artist with anything else in the folder gains nothing from
|
||||
// becoming its own one-track group — it stays in the leftover folder,
|
||||
// which the existing evidence-scaling (rank.go) already treats
|
||||
// appropriately harshly for a 1-track match.
|
||||
const clusterMinSize = 2
|
||||
|
||||
// IsMixedBag reports whether a group's local tracks look like an
|
||||
// unrelated pile of songs rather than one release: no artist
|
||||
// consensus AND no album consensus, across enough tracks that the
|
||||
// divergence isn't just noise. An explicit, non-VA album-artist tag
|
||||
// on the folder overrides the heuristic — a user (or a prior tagger)
|
||||
// who set a real album-artist meant this to read as one release.
|
||||
func IsMixedBag(g Group) bool {
|
||||
if len(g.Tracks) < mixedBagMinTracks {
|
||||
return false
|
||||
}
|
||||
|
||||
if g.AlbumArtist != "" && !isVAName(g.AlbumArtist) {
|
||||
return false
|
||||
}
|
||||
|
||||
return !hasTagConsensus(trackArtistTags(g.Tracks)) &&
|
||||
!hasTagConsensus(trackAlbumTags(g.Tracks))
|
||||
}
|
||||
|
||||
// hasTagConsensus reports whether every non-empty value in vals
|
||||
// normalizes to the same string. Empty values are ignored — missing
|
||||
// tags are unknown, not disagreement. A folder with zero non-empty
|
||||
// values has no consensus either way; callers only reach here after
|
||||
// already requiring enough tracks to matter.
|
||||
func hasTagConsensus(vals []string) bool {
|
||||
distinct := make(map[string]bool, 2) //nolint:mnd
|
||||
|
||||
for _, v := range vals {
|
||||
if v == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
distinct[Normalize(v)] = true
|
||||
|
||||
if len(distinct) > 1 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return len(distinct) == 1
|
||||
}
|
||||
|
||||
func trackArtistTags(tracks []LocalTrack) []string {
|
||||
out := make([]string, len(tracks))
|
||||
for i, t := range tracks {
|
||||
out[i] = t.Artist
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func trackAlbumTags(tracks []LocalTrack) []string {
|
||||
out := make([]string, len(tracks))
|
||||
for i, t := range tracks {
|
||||
out[i] = t.AlbumTag
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// TrackCluster is a set of local tracks sharing a non-empty (album,
|
||||
// album-artist) tag pair — a candidate sub-album hiding inside a
|
||||
// mixed-bag folder.
|
||||
type TrackCluster struct {
|
||||
AlbumName string
|
||||
AlbumArtist string
|
||||
Tracks []LocalTrack
|
||||
}
|
||||
|
||||
// ClusterByAlbumArtist groups tracks by normalized (album tag,
|
||||
// album-artist tag) and returns the clusters with at least
|
||||
// clusterMinSize members, in first-seen order (the caller typically
|
||||
// passes tracks already ordered by disc/track/path, so this stays
|
||||
// deterministic run to run). Tracks with no album tag, or whose
|
||||
// cluster never reaches clusterMinSize, are omitted — they belong in
|
||||
// the leftover folder, not a synthetic group of their own.
|
||||
func ClusterByAlbumArtist(tracks []LocalTrack) []TrackCluster {
|
||||
type key struct{ album, artist string }
|
||||
|
||||
index := make(map[key]int, 4) //nolint:mnd
|
||||
|
||||
var clusters []TrackCluster
|
||||
|
||||
for _, t := range tracks {
|
||||
album := Normalize(t.AlbumTag)
|
||||
if album == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
k := key{album: album, artist: Normalize(t.AlbumArtistTag)}
|
||||
|
||||
if i, ok := index[k]; ok {
|
||||
clusters[i].Tracks = append(clusters[i].Tracks, t)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
index[k] = len(clusters)
|
||||
clusters = append(clusters, TrackCluster{
|
||||
AlbumName: t.AlbumTag,
|
||||
AlbumArtist: t.AlbumArtistTag,
|
||||
Tracks: []LocalTrack{t},
|
||||
})
|
||||
}
|
||||
|
||||
out := clusters[:0]
|
||||
|
||||
for _, c := range clusters {
|
||||
if len(c.Tracks) >= clusterMinSize {
|
||||
out = append(out, c)
|
||||
}
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// SplitPlan returns the full set of synthetic groups a mixed-bag
|
||||
// folder should be torn into: ClusterByAlbumArtist's tag-matched
|
||||
// sub-albums, plus a one-track cluster for every track that didn't
|
||||
// share an (album, album-artist) pair with anything else in the
|
||||
// folder. Unlike ClusterByAlbumArtist alone — which leaves
|
||||
// unclustered tracks behind in the parent group, where they'd still
|
||||
// get folded into whatever partial-album match the scorer finds for
|
||||
// the rest of the pile — this guarantees every track leaves the
|
||||
// parent, so a folder of entirely unrelated singles (no two tracks
|
||||
// share an album tag) still gets torn apart instead of being scored
|
||||
// as one bogus album with a pile of "extra" tracks. Each singleton's
|
||||
// evidence-scaled score (rank.go) keeps it appropriately humble on
|
||||
// its own — it just no longer drags an unrelated release's score
|
||||
// down, or gets dragged down by one.
|
||||
func SplitPlan(tracks []LocalTrack) []TrackCluster {
|
||||
type key struct{ album, artist string }
|
||||
|
||||
index := make(map[key]int, 4) //nolint:mnd
|
||||
|
||||
var clusters []TrackCluster
|
||||
|
||||
// memberOf[i] is 1+the cluster index track i was assigned to (by
|
||||
// album/artist tag match), or 0 if it never matched anything.
|
||||
// Tracked by slice position rather than any LocalTrack field —
|
||||
// AudioFileID/FilePath are frequently zero-valued in this
|
||||
// package's own tests and would collide, wrongly treating
|
||||
// distinct untagged tracks as duplicates of one another.
|
||||
memberOf := make([]int, len(tracks))
|
||||
|
||||
for i, t := range tracks {
|
||||
album := Normalize(t.AlbumTag)
|
||||
if album == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
k := key{album: album, artist: Normalize(t.AlbumArtistTag)}
|
||||
|
||||
if ci, ok := index[k]; ok {
|
||||
clusters[ci].Tracks = append(clusters[ci].Tracks, t)
|
||||
memberOf[i] = ci + 1
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
index[k] = len(clusters)
|
||||
memberOf[i] = len(clusters) + 1
|
||||
clusters = append(clusters, TrackCluster{
|
||||
AlbumName: t.AlbumTag,
|
||||
AlbumArtist: t.AlbumArtistTag,
|
||||
Tracks: []LocalTrack{t},
|
||||
})
|
||||
}
|
||||
|
||||
// Clusters that never reached clusterMinSize don't survive as a
|
||||
// group; their sole member falls through to the singleton pass
|
||||
// below instead.
|
||||
kept := make([]TrackCluster, 0, len(clusters))
|
||||
keptIndex := make(map[int]int, len(clusters))
|
||||
|
||||
for oldIdx, c := range clusters {
|
||||
if len(c.Tracks) >= clusterMinSize {
|
||||
keptIndex[oldIdx] = len(kept)
|
||||
kept = append(kept, c)
|
||||
}
|
||||
}
|
||||
|
||||
for i, t := range tracks {
|
||||
if ci := memberOf[i] - 1; ci >= 0 {
|
||||
if _, ok := keptIndex[ci]; ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
kept = append(kept, TrackCluster{
|
||||
AlbumName: t.AlbumTag,
|
||||
AlbumArtist: t.AlbumArtistTag,
|
||||
Tracks: []LocalTrack{t},
|
||||
})
|
||||
}
|
||||
|
||||
return kept
|
||||
}
|
||||
@@ -0,0 +1,246 @@
|
||||
package autotag
|
||||
|
||||
import "testing"
|
||||
|
||||
func junkDrawerTracks() []LocalTrack {
|
||||
return []LocalTrack{
|
||||
{
|
||||
Title: "Song A", Artist: "Artist One",
|
||||
AlbumTag: "Album One", AlbumArtistTag: "Artist One",
|
||||
},
|
||||
{
|
||||
Title: "Song B", Artist: "Artist One",
|
||||
AlbumTag: "Album One", AlbumArtistTag: "Artist One",
|
||||
},
|
||||
{
|
||||
Title: "Song C", Artist: "Artist Two",
|
||||
AlbumTag: "Album Two", AlbumArtistTag: "Artist Two",
|
||||
},
|
||||
{
|
||||
Title: "Song D", Artist: "Artist Two",
|
||||
AlbumTag: "Album Two", AlbumArtistTag: "Artist Two",
|
||||
},
|
||||
{
|
||||
Title: "Song E", Artist: "Artist Three",
|
||||
AlbumTag: "Album Three", AlbumArtistTag: "Artist Three",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsMixedBag_DetectsJunkDrawer(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
g := Group{Tracks: junkDrawerTracks()}
|
||||
|
||||
if !IsMixedBag(g) {
|
||||
t.Fatal("expected a folder with no artist or album consensus to be flagged mixed-bag")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsMixedBag_RealAlbumNotFlagged(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
g := Group{
|
||||
AlbumArtist: "The Beatles",
|
||||
Tracks: []LocalTrack{
|
||||
{Title: "Come Together", Artist: "The Beatles"},
|
||||
{Title: "Something", Artist: "The Beatles"},
|
||||
{Title: "Maxwell's Silver Hammer", Artist: "The Beatles"},
|
||||
{Title: "Oh! Darling", Artist: "The Beatles"},
|
||||
},
|
||||
}
|
||||
|
||||
if IsMixedBag(g) {
|
||||
t.Fatal("a coherent single-artist album must not be flagged mixed-bag")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsMixedBag_ExplicitAlbumArtistOverridesHeuristic(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Per-track artists disagree (feat. credits, remixers, etc.) but
|
||||
// the folder carries a real album-artist tag — trust it.
|
||||
g := Group{
|
||||
AlbumArtist: "Some Artist",
|
||||
Tracks: []LocalTrack{
|
||||
{Title: "Track 1", Artist: "Some Artist"},
|
||||
{Title: "Track 2", Artist: "Some Artist feat. Guest"},
|
||||
{Title: "Track 3", Artist: "Someone Else"},
|
||||
{Title: "Track 4", Artist: "Some Artist"},
|
||||
},
|
||||
}
|
||||
|
||||
if IsMixedBag(g) {
|
||||
t.Fatal("explicit non-VA album-artist tag should override the divergence heuristic")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsMixedBag_VACompilationNotFlagged(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Various-artists compilation: artists diverge but every track
|
||||
// agrees on the album — this is vaLikely's case, not a junk
|
||||
// drawer, so IsMixedBag must require album divergence too.
|
||||
g := Group{
|
||||
Tracks: []LocalTrack{
|
||||
{Title: "Track 1", Artist: "Artist One", AlbumTag: "Now That's What I Call Music"},
|
||||
{Title: "Track 2", Artist: "Artist Two", AlbumTag: "Now That's What I Call Music"},
|
||||
{Title: "Track 3", Artist: "Artist Three", AlbumTag: "Now That's What I Call Music"},
|
||||
{Title: "Track 4", Artist: "Artist Four", AlbumTag: "Now That's What I Call Music"},
|
||||
},
|
||||
}
|
||||
|
||||
if IsMixedBag(g) {
|
||||
t.Fatal("a VA compilation with consistent album tags must not be flagged mixed-bag")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsMixedBag_TooFewTracksNotFlagged(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
g := Group{
|
||||
Tracks: []LocalTrack{
|
||||
{Title: "Track 1", Artist: "Artist One", AlbumTag: "Album One"},
|
||||
{Title: "Track 2", Artist: "Artist Two", AlbumTag: "Album Two"},
|
||||
},
|
||||
}
|
||||
|
||||
if IsMixedBag(g) {
|
||||
t.Fatal("a folder below mixedBagMinTracks must not be flagged, even if it diverges")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClusterByAlbumArtist_FindsSubAlbums(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tracks := junkDrawerTracks() // two 2-track clusters + one true singleton
|
||||
|
||||
clusters := ClusterByAlbumArtist(tracks)
|
||||
|
||||
if len(clusters) != 2 { //nolint:mnd
|
||||
t.Fatalf("expected 2 clusters (Album One, Album Two), got %d: %+v", len(clusters), clusters)
|
||||
}
|
||||
|
||||
for _, c := range clusters {
|
||||
if len(c.Tracks) != 2 { //nolint:mnd
|
||||
t.Errorf("cluster %q: expected 2 tracks, got %d", c.AlbumName, len(c.Tracks))
|
||||
}
|
||||
}
|
||||
|
||||
total := 0
|
||||
for _, c := range clusters {
|
||||
total += len(c.Tracks)
|
||||
}
|
||||
|
||||
if total != 4 { //nolint:mnd
|
||||
t.Errorf(
|
||||
"expected 4 clustered tracks total (Song E stays unclustered), got %d",
|
||||
total,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClusterByAlbumArtist_NoAlbumTagStaysUnclustered(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tracks := []LocalTrack{
|
||||
{Title: "Track 1", Artist: "Artist One"},
|
||||
{Title: "Track 2", Artist: "Artist One"},
|
||||
}
|
||||
|
||||
if clusters := ClusterByAlbumArtist(tracks); len(clusters) != 0 {
|
||||
t.Fatalf("tracks with no album tag must never cluster, got %+v", clusters)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClusterByAlbumArtist_DeterministicOrder(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tracks := junkDrawerTracks()
|
||||
|
||||
first := ClusterByAlbumArtist(tracks)
|
||||
second := ClusterByAlbumArtist(tracks)
|
||||
|
||||
if len(first) != len(second) {
|
||||
t.Fatalf("non-deterministic cluster count: %d vs %d", len(first), len(second))
|
||||
}
|
||||
|
||||
for i := range first {
|
||||
if first[i].AlbumName != second[i].AlbumName {
|
||||
t.Errorf(
|
||||
"non-deterministic cluster order at %d: %q vs %q",
|
||||
i,
|
||||
first[i].AlbumName,
|
||||
second[i].AlbumName,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if first[0].AlbumName != "Album One" {
|
||||
t.Errorf("expected first-seen cluster order, got %q first", first[0].AlbumName)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitPlan_ClustersPlusSingletonForEveryLeftover(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tracks := junkDrawerTracks() // two 2-track clusters + one true singleton (Song E)
|
||||
|
||||
plan := SplitPlan(tracks)
|
||||
|
||||
total := 0
|
||||
for _, c := range plan {
|
||||
total += len(c.Tracks)
|
||||
}
|
||||
|
||||
if total != len(tracks) {
|
||||
t.Fatalf("expected every track accounted for, got %d of %d", total, len(tracks))
|
||||
}
|
||||
|
||||
var singletons, clustered int
|
||||
|
||||
for _, c := range plan {
|
||||
switch len(c.Tracks) {
|
||||
case 1:
|
||||
singletons++
|
||||
case 2: //nolint:mnd
|
||||
clustered++
|
||||
default:
|
||||
t.Errorf("unexpected cluster size %d: %+v", len(c.Tracks), c)
|
||||
}
|
||||
}
|
||||
|
||||
if singletons != 1 {
|
||||
t.Errorf("expected exactly 1 singleton (Song E), got %d", singletons)
|
||||
}
|
||||
|
||||
if clustered != 2 { //nolint:mnd
|
||||
t.Errorf("expected exactly 2 clustered groups, got %d", clustered)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitPlan_AllUnrelatedTracksAllBecomeSingletons(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tracks := []LocalTrack{
|
||||
{Title: "Track 1", Artist: "Artist One", AlbumTag: "Album A"},
|
||||
{Title: "Track 2", Artist: "Artist Two", AlbumTag: "Album B"},
|
||||
{Title: "Track 3", Artist: "Artist Three"}, // no album tag at all
|
||||
}
|
||||
|
||||
plan := SplitPlan(tracks)
|
||||
|
||||
if len(plan) != len(tracks) {
|
||||
t.Fatalf(
|
||||
"expected every unrelated track to become its own singleton, got %d clusters for %d tracks",
|
||||
len(plan),
|
||||
len(tracks),
|
||||
)
|
||||
}
|
||||
|
||||
for _, c := range plan {
|
||||
if len(c.Tracks) != 1 {
|
||||
t.Errorf("expected singleton cluster, got %d tracks: %+v", len(c.Tracks), c)
|
||||
}
|
||||
}
|
||||
}
|
||||
+30
-3
@@ -33,6 +33,19 @@ const (
|
||||
// auto-accept entirely.
|
||||
evidenceFloor = 0.85
|
||||
evidenceFullTracks = 3
|
||||
|
||||
// Synthetic groups (SplitMixedFolder's tag-clustered sub-albums)
|
||||
// are, by construction, a SUBSET of a bigger folder: the folder
|
||||
// might not have every track from the release the cluster
|
||||
// belongs to. A candidate with more tracks than the synthetic
|
||||
// group is therefore expected, not a sign of a wrong match, so
|
||||
// its trackCountMatch penalty is softened relative to a real
|
||||
// folder (where a track-count gap usually does mean the wrong
|
||||
// release). A candidate with FEWER tracks than the group is
|
||||
// still scored by the normal (harsher) formula — that's a real
|
||||
// mismatch regardless of source.
|
||||
syntheticMissingPenaltyScale = 0.35
|
||||
syntheticTrackCountFloor = 0.55
|
||||
)
|
||||
|
||||
// vaNames are artist strings that signal "various artists" — used
|
||||
@@ -139,7 +152,7 @@ func ScoreCandidate(g Group, c Candidate) Candidate {
|
||||
|
||||
trackAgg := ((titleAvg*weightTitle + lengthAvg*weightLength) / trackWeightSum) * coverage
|
||||
|
||||
trackCountScore := trackCountMatch(len(targets), len(local))
|
||||
trackCountScore := trackCountMatch(len(targets), len(local), g.Synthetic)
|
||||
|
||||
// Artist fit: compare the folder's artist against the
|
||||
// candidate's release artist-credit. This is a SOFT signal, not
|
||||
@@ -347,8 +360,15 @@ func evidenceFactor(localTrackCount int) float64 {
|
||||
}
|
||||
|
||||
// trackCountMatch returns 1.0 when equal, 0.0 when off by >= 50%,
|
||||
// linear between.
|
||||
func trackCountMatch(a, b int) float64 {
|
||||
// linear between. When synthetic is true and the candidate (a) has
|
||||
// MORE tracks than the local group (b) — the group having fewer
|
||||
// tracks than the full release, exactly what's expected from a
|
||||
// tag-clustered subset of a folder — the penalty is softened instead
|
||||
// of using the normal harsh formula. Fewer candidate tracks than
|
||||
// local (b > a) always uses the normal formula: that pattern means
|
||||
// the group has tracks the candidate release doesn't, which is a
|
||||
// real mismatch however the group was built.
|
||||
func trackCountMatch(a, b int, synthetic bool) float64 {
|
||||
if a == 0 && b == 0 {
|
||||
return 1.0
|
||||
}
|
||||
@@ -357,6 +377,13 @@ func trackCountMatch(a, b int) float64 {
|
||||
return 0.0
|
||||
}
|
||||
|
||||
if synthetic && a > b {
|
||||
diff := a - b
|
||||
frac := float64(diff) / float64(a)
|
||||
|
||||
return max(1.0-frac*syntheticMissingPenaltyScale, syntheticTrackCountFloor)
|
||||
}
|
||||
|
||||
diff := a - b
|
||||
if diff < 0 {
|
||||
diff = -diff
|
||||
|
||||
@@ -49,6 +49,55 @@ func TestRankCandidates_PrefersExactTrackCountMatch(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestScoreCandidate_SyntheticGroupSoftensMissingTrackPenalty(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Two tracks pulled from a mixed-bag folder, tag-clustered as a
|
||||
// subset of a 5-track release — exactly what SplitMixedFolder
|
||||
// produces. A candidate release with the other 3 tracks the
|
||||
// folder simply never had must not be penalized nearly as hard
|
||||
// as a real folder missing 3 of 5 tracks would be.
|
||||
local := []autotag.LocalTrack{
|
||||
{Title: "A", TrackNumber: 1, LengthMillis: 200000},
|
||||
{Title: "B", TrackNumber: 2, LengthMillis: 200000},
|
||||
}
|
||||
|
||||
candidate := autotag.Candidate{
|
||||
ReleaseMBID: "full-release",
|
||||
Title: "Album",
|
||||
Status: "Official",
|
||||
Tracks: []autotag.CandidateTrack{
|
||||
{Position: 1, Title: "A", LengthMillis: 200000},
|
||||
{Position: 2, Title: "B", LengthMillis: 200000},
|
||||
{Position: 3, Title: "C", LengthMillis: 200000},
|
||||
{Position: 4, Title: "D", LengthMillis: 200000},
|
||||
{Position: 5, Title: "E", LengthMillis: 200000},
|
||||
},
|
||||
}
|
||||
|
||||
fromRealFolder := autotag.ScoreCandidate(
|
||||
autotag.Group{Tracks: local, Synthetic: false}, candidate,
|
||||
)
|
||||
fromSynthetic := autotag.ScoreCandidate(
|
||||
autotag.Group{Tracks: local, Synthetic: true}, candidate,
|
||||
)
|
||||
|
||||
if fromSynthetic.Breakdown.TrackCountFit <= fromRealFolder.Breakdown.TrackCountFit {
|
||||
t.Errorf(
|
||||
"synthetic track-count fit (%.3f) should exceed the real-folder fit (%.3f) for the same gap",
|
||||
fromSynthetic.Breakdown.TrackCountFit,
|
||||
fromRealFolder.Breakdown.TrackCountFit,
|
||||
)
|
||||
}
|
||||
|
||||
if fromSynthetic.Score <= fromRealFolder.Score {
|
||||
t.Errorf(
|
||||
"synthetic group score (%.3f) should exceed the real-folder score (%.3f)",
|
||||
fromSynthetic.Score, fromRealFolder.Score,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRankCandidates_PrefersOfficial(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -59,9 +59,16 @@ func Recommend(g Group, candidates []Candidate) Recommendation {
|
||||
|
||||
// Cap: missing or unmatched tracks mean the alignment itself is
|
||||
// incomplete, however good the matched tracks look (beets caps
|
||||
// these penalties at "medium" the same way).
|
||||
// these penalties at "medium" the same way). A synthetic
|
||||
// (tag-clustered) group is, by construction, a subset of a
|
||||
// bigger folder, so AlignmentMissing (the candidate has tracks
|
||||
// the group doesn't) is the expected shape rather than a defect
|
||||
// and doesn't cap the recommendation. AlignmentUnmatched (the
|
||||
// group has a track the candidate doesn't) is still a real
|
||||
// discrepancy regardless of source.
|
||||
for _, a := range top.Alignments {
|
||||
if a.Status == AlignmentMissing || a.Status == AlignmentUnmatched {
|
||||
if a.Status == AlignmentUnmatched ||
|
||||
(a.Status == AlignmentMissing && !g.Synthetic) {
|
||||
rec = minRecommendation(rec, RecommendationMedium)
|
||||
|
||||
break
|
||||
|
||||
@@ -97,6 +97,46 @@ func TestRecommend_AlignmentDefectsCapAtMedium(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecommend_SyntheticGroupMissingTracksDoNotCap(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
top := mkScoredCandidate("rg1", 0.95)
|
||||
top.Alignments = []TrackAlignment{
|
||||
{Status: AlignmentMatched},
|
||||
{Status: AlignmentMissing, LocalIndex: -1},
|
||||
}
|
||||
|
||||
g := fullGroup()
|
||||
g.Synthetic = true
|
||||
|
||||
if got := Recommend(g, []Candidate{top}); got != RecommendationStrong {
|
||||
t.Errorf(
|
||||
"synthetic group with only missing (not unmatched) tracks: Recommend = %q, want strong",
|
||||
got,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecommend_SyntheticGroupUnmatchedTracksStillCap(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
top := mkScoredCandidate("rg1", 0.95)
|
||||
top.Alignments = []TrackAlignment{
|
||||
{Status: AlignmentMatched},
|
||||
{Status: AlignmentUnmatched, LocalIndex: 1},
|
||||
}
|
||||
|
||||
g := fullGroup()
|
||||
g.Synthetic = true
|
||||
|
||||
if got := Recommend(g, []Candidate{top}); got != RecommendationMedium {
|
||||
t.Errorf(
|
||||
"synthetic group with an unmatched local track: Recommend = %q, want medium",
|
||||
got,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecommend_ThinEvidenceCapsAtMedium(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -104,6 +104,7 @@ func (s *Scorer) scoreGroup(
|
||||
AlbumName: item.AlbumName,
|
||||
AlbumArtist: item.AlbumArtist,
|
||||
Tracks: locals,
|
||||
Synthetic: item.Synthetic != 0,
|
||||
}
|
||||
|
||||
localHits, err := s.local.ResolveLocal(ctx, item.AlbumName)
|
||||
@@ -142,6 +143,7 @@ func (s *Scorer) scoreGroup(
|
||||
LocalTracks: locals,
|
||||
Candidates: candidates,
|
||||
Recommendation: Recommend(g, candidates),
|
||||
Synthetic: g.Synthetic,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -338,6 +338,12 @@ func (c *idFakeClient) LookupReleaseGroup(
|
||||
return autotag.MBReleaseGroupHit{}, nil
|
||||
}
|
||||
|
||||
func (c *idFakeClient) SearchReleaseGroupsLocal(
|
||||
_ context.Context, _ string, _ int,
|
||||
) ([]autotag.MBReleaseGroupHit, bool) {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func TestScorer_PersistScoreWritesTopMatch(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -462,3 +468,11 @@ func (c *countingMBClient) LookupReleaseGroup(
|
||||
|
||||
return autotag.MBReleaseGroupHit{}, nil
|
||||
}
|
||||
|
||||
// SearchReleaseGroupsLocal is not a network call — it never counts
|
||||
// against the zero-network-call assertions this fake exists for.
|
||||
func (c *countingMBClient) SearchReleaseGroupsLocal(
|
||||
_ context.Context, _ string, _ int,
|
||||
) ([]autotag.MBReleaseGroupHit, bool) {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
@@ -12,6 +12,15 @@ type LocalTrack struct {
|
||||
DiscNumber int
|
||||
LengthMillis int64
|
||||
RecordingMBID string
|
||||
|
||||
// AlbumTag/AlbumArtistTag are this track's OWN album tags (via
|
||||
// its release_group link), independent of the folder-level
|
||||
// Group.AlbumName/AlbumArtist below. A coherent album's tracks
|
||||
// all carry the same values here; a junk-drawer folder's don't.
|
||||
// Used only by SplitMixedFolder's clustering — the scorer itself
|
||||
// still ranks against Group.AlbumName/AlbumArtist.
|
||||
AlbumTag string
|
||||
AlbumArtistTag string
|
||||
}
|
||||
|
||||
// Group is the folder-level context candidates are ranked against:
|
||||
@@ -21,6 +30,14 @@ type Group struct {
|
||||
AlbumName string
|
||||
AlbumArtist string
|
||||
Tracks []LocalTrack
|
||||
|
||||
// Synthetic marks a group carved out of a mixed-bag folder by
|
||||
// SplitMixedFolder rather than corresponding to a real directory.
|
||||
// Its tracks are a tag-matched subset of a bigger folder, so a
|
||||
// candidate with MORE tracks than the group is expected, not a
|
||||
// sign of a bad match — see the synthetic-aware evidence/track-
|
||||
// count handling in rank.go and recommend.go.
|
||||
Synthetic bool
|
||||
}
|
||||
|
||||
// CandidateSource distinguishes candidates served from the local
|
||||
@@ -125,4 +142,5 @@ type GroupScore struct {
|
||||
LocalTracks []LocalTrack
|
||||
Candidates []Candidate // sorted by Score, descending
|
||||
Recommendation Recommendation
|
||||
Synthetic bool // true for a SplitMixedFolder-derived group
|
||||
}
|
||||
|
||||
@@ -19,8 +19,6 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
wailsruntime "github.com/wailsapp/wails/v2/pkg/runtime"
|
||||
|
||||
"yellowjacket/backend/autotag"
|
||||
"yellowjacket/backend/database"
|
||||
"yellowjacket/backend/database/sql/sqlcgen"
|
||||
@@ -85,13 +83,6 @@ type Service struct {
|
||||
exp *explore.Service
|
||||
logger *slog.Logger
|
||||
ctx context.Context
|
||||
// ctxReady reports whether ctx is the Wails lifecycle context set
|
||||
// via SetContext (rather than the context.Background() default). It
|
||||
// gates event emission: calling wailsruntime.EventsEmit with a
|
||||
// non-runtime context triggers log.Fatalf (os.Exit) inside Wails, so
|
||||
// a background worker that fires before OnStartup wires the context
|
||||
// would otherwise take the whole app down on launch.
|
||||
ctxReady bool
|
||||
|
||||
// Queue cursor — the group_key of the last item returned.
|
||||
// GetNextPending uses it to advance. Reset by StartAutotagQueue.
|
||||
@@ -177,7 +168,7 @@ func NewService(
|
||||
exp *explore.Service,
|
||||
tw *tagwriter.TagWriter,
|
||||
) *Service {
|
||||
mbAdapter := explore.NewAutotagClient(exp.MusicBrainz())
|
||||
mbAdapter := explore.NewAutotagClient(exp)
|
||||
scorer := autotag.NewScorer(db.Queries, mbAdapter, logger.WithGroup("autotag"))
|
||||
mbr := autotag.NewMBResolver(mbAdapter, logger.WithGroup("autotag-mb"))
|
||||
|
||||
@@ -210,32 +201,18 @@ func (s *Service) SetContext(ctx context.Context) {
|
||||
defer s.mu.Unlock()
|
||||
|
||||
s.ctx = ctx
|
||||
s.ctxReady = ctx != nil
|
||||
}
|
||||
|
||||
// emitEvent emits a Wails runtime event, but only when the stored
|
||||
// context actually carries the Wails runtime. Wails' EventsEmit calls
|
||||
// log.Fatalf — which os.Exit()s the process and cannot be recovered —
|
||||
// whenever the context lacks its internal "events" value (e.g. the
|
||||
// context.Background() default, or any non-lifecycle context). A
|
||||
// background worker (the prefetch/apply sweeps) that emits before, or
|
||||
// independently of, OnStartup wiring the real context would otherwise
|
||||
// take the whole app down on launch. We replicate Wails' own
|
||||
// precondition here so a not-yet-ready context degrades to a no-op
|
||||
// instead of a crash.
|
||||
// emitEvent emits a Wails runtime event under the service lock, which
|
||||
// the background prefetch/apply sweeps need because they can emit
|
||||
// before OnStartup has wired the real context. events.Emit tolerates
|
||||
// that; see its doc comment.
|
||||
func (s *Service) emitEvent(eventName string, data any) {
|
||||
s.mu.Lock()
|
||||
ready := s.ctxReady
|
||||
ctx := s.ctx
|
||||
s.mu.Unlock()
|
||||
|
||||
// hasWailsRuntime mirrors the check in wails/pkg/runtime.getEvents:
|
||||
// the runtime is present only when ctx.Value("events") is non-nil.
|
||||
if !ready || ctx == nil || ctx.Value("events") == nil {
|
||||
return
|
||||
}
|
||||
|
||||
wailsruntime.EventsEmit(ctx, eventName, data)
|
||||
events.Emit(ctx, eventName, data)
|
||||
}
|
||||
|
||||
// StartBackgroundPrefetch kicks off (or restarts) the prefetch
|
||||
@@ -307,6 +284,12 @@ func (s *Service) startPrefetch(libraryID int64) {
|
||||
s.mu.Unlock()
|
||||
}()
|
||||
|
||||
// Self-heal before enumerating: don't burn a scoring pass on rows
|
||||
// whose bookkeeping never ran or drifted (see ListPendingFolders).
|
||||
if err := s.db.Queries.PruneOrphanedTaggingItems(ctx); err != nil {
|
||||
s.logger.Warn("prefetch: prune orphaned items failed", "err", err)
|
||||
}
|
||||
|
||||
// Find all pending items missing a score. Ordered alphabetically
|
||||
// for stable progress reporting; libraryID=0 fans out to all.
|
||||
const maxPrefetch = 5000
|
||||
@@ -376,25 +359,30 @@ func (s *Service) startPrefetch(libraryID int64) {
|
||||
continue
|
||||
}
|
||||
|
||||
// A folder that looks like a pile of unrelated tracks gets
|
||||
// torn apart before scoring — otherwise the scorer treats
|
||||
// the whole pile as one album candidate and every track that
|
||||
// doesn't fit the best partial match gets counted as an
|
||||
// "extra" of it, rather than being matched on its own. The
|
||||
// original group key is gone once every track has moved to a
|
||||
// synthetic child, so score those instead of key.
|
||||
if newKeys := s.autoSplitMixedBag(ctx, key); len(newKeys) > 0 {
|
||||
for _, nk := range newKeys {
|
||||
s.scoreAndPersist(ctx, nk, "prefetch: score synthetic group")
|
||||
}
|
||||
|
||||
s.emitEvent(events.AutotagPrefetchProgress, map[string]any{
|
||||
"processed": i + 1,
|
||||
"total": total,
|
||||
})
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
// Local-first: the background sweep skips the MusicBrainz
|
||||
// cascade when a local candidate already scores well, so a
|
||||
// library with cross-library duplicates costs no network here.
|
||||
score, err := s.scorer.ScoreGroupLocalFirst(ctx, key)
|
||||
if err != nil {
|
||||
s.logger.Debug(
|
||||
"prefetch: score failed — skipping",
|
||||
"group_key", key, "err", err,
|
||||
)
|
||||
} else {
|
||||
s.cacheCandidates(key, score.Candidates)
|
||||
|
||||
if perr := s.scorer.PersistScore(ctx, score); perr != nil {
|
||||
s.logger.Debug(
|
||||
"prefetch: persist failed",
|
||||
"group_key", key, "err", perr,
|
||||
)
|
||||
}
|
||||
}
|
||||
s.scoreAndPersist(ctx, key, "prefetch: score failed — skipping")
|
||||
|
||||
s.emitEvent(events.AutotagPrefetchProgress, map[string]any{
|
||||
"processed": i + 1,
|
||||
@@ -409,6 +397,74 @@ func (s *Service) startPrefetch(libraryID int64) {
|
||||
s.logger.Info("autotag prefetch: done", "groups", total)
|
||||
}
|
||||
|
||||
// scoreAndPersist runs the cheap local-first score for one group and
|
||||
// caches + persists the result, logging (never failing the caller)
|
||||
// on error. failMsg labels the debug log line when scoring itself
|
||||
// errors.
|
||||
func (s *Service) scoreAndPersist(ctx context.Context, groupKey, failMsg string) {
|
||||
score, err := s.scorer.ScoreGroupLocalFirst(ctx, groupKey)
|
||||
if err != nil {
|
||||
s.logger.Debug(failMsg, "group_key", groupKey, "err", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
s.cacheCandidates(groupKey, score.Candidates)
|
||||
|
||||
if perr := s.scorer.PersistScore(ctx, score); perr != nil {
|
||||
s.logger.Debug(
|
||||
"prefetch: persist failed",
|
||||
"group_key", groupKey, "err", perr,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// autoSplitMixedBag detects a folder that looks like a pile of
|
||||
// unrelated tracks (autotag.IsMixedBag) and, if so, tears it apart
|
||||
// via the same clustering SplitMixedFolder uses (autotag.SplitPlan)
|
||||
// before the background sweep scores it — otherwise the scorer
|
||||
// treats the whole pile as one album candidate and every track that
|
||||
// doesn't fit the best partial match gets counted as an "extra" of
|
||||
// it. Returns the new synthetic group keys, or nil when the folder
|
||||
// isn't a mixed bag (or had nothing to split).
|
||||
func (s *Service) autoSplitMixedBag(ctx context.Context, groupKey string) []string {
|
||||
item, err := s.db.Queries.GetTaggingItem(ctx, groupKey)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
locals, err := s.scorer.LocalTracksForGroup(ctx, groupKey)
|
||||
if err != nil {
|
||||
s.logger.Debug("prefetch: auto-split load locals failed", "group_key", groupKey, "err", err)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
g := autotag.Group{AlbumName: item.AlbumName, AlbumArtist: item.AlbumArtist, Tracks: locals}
|
||||
if !autotag.IsMixedBag(g) {
|
||||
return nil
|
||||
}
|
||||
|
||||
clusters := autotag.SplitPlan(locals)
|
||||
if len(clusters) <= 1 {
|
||||
return nil
|
||||
}
|
||||
|
||||
newKeys, err := s.splitIntoSyntheticGroups(groupKey, item.LibraryID, clusters)
|
||||
if err != nil {
|
||||
s.logger.Warn("prefetch: auto-split failed", "group_key", groupKey, "err", err)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
s.logger.Info(
|
||||
"autotag prefetch: auto-split mixed-bag folder",
|
||||
"group_key", groupKey, "into", len(newKeys),
|
||||
)
|
||||
|
||||
return newKeys
|
||||
}
|
||||
|
||||
// PendingItem is a projection of tagging_items that's safe to hand
|
||||
// to the frontend. Score is dereferenced to 0 when NULL so TS sees
|
||||
// a plain number.
|
||||
@@ -430,6 +486,19 @@ type PendingItem struct {
|
||||
BestMatchReleaseMbid string `json:"bestMatchReleaseMbid"`
|
||||
Score float64 `json:"score"`
|
||||
Status string `json:"status"`
|
||||
// Synthetic marks a group SplitMixedFolder carved out of a
|
||||
// bigger folder by matching tags rather than a directory — the
|
||||
// review UI labels these distinctly since several may share the
|
||||
// same FolderSubPath.
|
||||
Synthetic bool `json:"synthetic"`
|
||||
// LikelyMixedBag is a cheap SQL-side approximation of autotag.
|
||||
// IsMixedBag, computed for the whole library in one pass by
|
||||
// ListPendingFolders (see ListLikelyMixedBagGroupKeys) rather
|
||||
// than hydrating every group's tracks in Go. It's a badge hint,
|
||||
// not a guarantee — ScoreView.MixedBag (computed from the real
|
||||
// track list when a folder is opened) is the authoritative check
|
||||
// that gates the SplitMixedFolder action itself.
|
||||
LikelyMixedBag bool `json:"likelyMixedBag"`
|
||||
}
|
||||
|
||||
// GetNextPending returns the next pending tagging item after the
|
||||
@@ -489,6 +558,15 @@ func (s *Service) GetNextPending() (*PendingItem, error) {
|
||||
func (s *Service) ListPendingFolders(libraryID int64) ([]PendingItem, error) {
|
||||
const maxFolders = 5000
|
||||
|
||||
// Self-heal before listing: a row whose bookkeeping (scan orphan
|
||||
// cleanup, maybeRebindTaggingGroup, SplitMixedFolder) never ran
|
||||
// or drifted otherwise lingers here indefinitely, showing as an
|
||||
// "old/nonexistent" entry with no folder path. Best-effort — a
|
||||
// failed prune shouldn't block the list itself.
|
||||
if err := s.db.Queries.PruneOrphanedTaggingItems(s.ctx); err != nil {
|
||||
s.logger.Warn("list pending folders: prune orphaned items failed", "err", err)
|
||||
}
|
||||
|
||||
rows, err := s.db.Queries.ListPendingTaggingItemsByScore(
|
||||
s.ctx,
|
||||
sqlcgen.ListPendingTaggingItemsByScoreParams{
|
||||
@@ -502,19 +580,32 @@ func (s *Service) ListPendingFolders(libraryID int64) ([]PendingItem, error) {
|
||||
return nil, fmt.Errorf("list pending folders: %w", err)
|
||||
}
|
||||
|
||||
mixedBagKeys, err := s.db.Queries.ListLikelyMixedBagGroupKeys(s.ctx)
|
||||
if err != nil {
|
||||
// A cheap badge hint isn't worth failing the whole list for.
|
||||
s.logger.Warn("list pending folders: mixed-bag triage failed", "err", err)
|
||||
}
|
||||
|
||||
mixedBag := make(map[string]bool, len(mixedBagKeys))
|
||||
for _, k := range mixedBagKeys {
|
||||
mixedBag[k] = true
|
||||
}
|
||||
|
||||
out := make([]PendingItem, 0, len(rows))
|
||||
|
||||
for _, row := range rows {
|
||||
item := PendingItem{
|
||||
GroupKey: row.GroupKey,
|
||||
LibraryID: row.LibraryID,
|
||||
LibraryName: row.LibraryName,
|
||||
FolderSubPath: folderSubPath(row.LibraryPath, row.SampleFilePath),
|
||||
TrackCount: row.TrackCount,
|
||||
AlbumName: row.AlbumName,
|
||||
AlbumArtist: row.AlbumArtist,
|
||||
DiscNumber: row.DiscNumber,
|
||||
Status: row.Status,
|
||||
GroupKey: row.GroupKey,
|
||||
LibraryID: row.LibraryID,
|
||||
LibraryName: row.LibraryName,
|
||||
FolderSubPath: folderSubPath(row.LibraryPath, row.SampleFilePath),
|
||||
TrackCount: row.TrackCount,
|
||||
AlbumName: row.AlbumName,
|
||||
AlbumArtist: row.AlbumArtist,
|
||||
DiscNumber: row.DiscNumber,
|
||||
Status: row.Status,
|
||||
Synthetic: row.Synthetic != 0,
|
||||
LikelyMixedBag: mixedBag[row.GroupKey],
|
||||
}
|
||||
|
||||
if row.BestMatchReleaseMbid.Valid {
|
||||
@@ -555,6 +646,7 @@ func (s *Service) GetPendingFolder(groupKey string) (*PendingItem, error) {
|
||||
AlbumArtist: row.AlbumArtist,
|
||||
DiscNumber: row.DiscNumber,
|
||||
Status: row.Status,
|
||||
Synthetic: row.Synthetic != 0,
|
||||
}
|
||||
|
||||
if row.BestMatchReleaseMbid.Valid {
|
||||
@@ -742,6 +834,15 @@ type ScoreView struct {
|
||||
// raw score it accounts for ambiguity (a rival release group
|
||||
// scoring nearly as high) and alignment defects.
|
||||
Recommendation string `json:"recommendation"`
|
||||
// MixedBag is true when this group's tracks look like an
|
||||
// unrelated pile rather than one release (autotag.IsMixedBag) —
|
||||
// the review UI offers SplitMixedFolder when set. Always false
|
||||
// for a group that's already Synthetic; a split group doesn't
|
||||
// get split again.
|
||||
MixedBag bool `json:"mixedBag"`
|
||||
// Synthetic mirrors PendingItem.Synthetic for the currently
|
||||
// open group.
|
||||
Synthetic bool `json:"synthetic"`
|
||||
}
|
||||
|
||||
// LocalTrackView mirrors autotag.LocalTrack.
|
||||
@@ -1151,6 +1252,156 @@ func (s *Service) RetagGroup(groupKey string) error {
|
||||
)
|
||||
}
|
||||
|
||||
// errNothingToSplit is returned by SplitMixedFolder when the
|
||||
// folder's tracks carry no repeated (album, album-artist) tag pair
|
||||
// to cluster on — nothing to split out.
|
||||
var errNothingToSplit = errors.New("autotag: no tag-matched sub-albums to split out")
|
||||
|
||||
// SplitMixedFolder is the "this folder is a pile of unrelated
|
||||
// tracks" escape hatch: it partitions the group's local tracks via
|
||||
// autotag.SplitPlan — tag-matched sub-albums (see
|
||||
// autotag.ClusterByAlbumArtist) plus a one-track cluster for every
|
||||
// track that didn't share an (album, album-artist) pair with
|
||||
// anything else — and carves each piece out into its own synthetic
|
||||
// tagging group, reassigning just those audio_files rows (no files
|
||||
// move on disk). Every track leaves the original group; nothing is
|
||||
// left behind to be scored as "extra tracks" of whichever piece
|
||||
// happens to match first. The synthetic groups are scored with
|
||||
// relaxed missing-track handling (rank.go, recommend.go), since
|
||||
// they're expected to be an incomplete subset of whatever release
|
||||
// they belong to.
|
||||
//
|
||||
// Returns the resulting PendingItems — the leftover original group
|
||||
// first (if anything remains in it), then the new synthetic groups
|
||||
// — so the frontend can splice them into the sidebar without a full
|
||||
// reload. Errors with errNothingToSplit when the folder is already
|
||||
// one coherent unit (SplitPlan produces a single cluster covering
|
||||
// every track); callers should treat that as "nothing to show", not
|
||||
// a failure.
|
||||
func (s *Service) SplitMixedFolder(groupKey string) ([]PendingItem, error) {
|
||||
item, err := s.db.Queries.GetTaggingItem(s.ctx, groupKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get tagging item: %w", err)
|
||||
}
|
||||
|
||||
locals, err := s.scorer.LocalTracksForGroup(s.ctx, groupKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load locals: %w", err)
|
||||
}
|
||||
|
||||
clusters := autotag.SplitPlan(locals)
|
||||
if len(clusters) <= 1 {
|
||||
return nil, errNothingToSplit
|
||||
}
|
||||
|
||||
newKeys, err := s.splitIntoSyntheticGroups(groupKey, item.LibraryID, clusters)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
out := make([]PendingItem, 0, len(newKeys)+1)
|
||||
|
||||
if leftover, err := s.GetPendingFolder(groupKey); err != nil {
|
||||
s.logger.Warn("split: reload leftover parent", "group_key", groupKey, "err", err)
|
||||
} else if leftover != nil {
|
||||
out = append(out, *leftover)
|
||||
}
|
||||
|
||||
for _, k := range newKeys {
|
||||
child, err := s.GetPendingFolder(k)
|
||||
if err != nil || child == nil {
|
||||
s.logger.Warn("split: reload synthetic group", "group_key", k, "err", err)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
out = append(out, *child)
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// splitIntoSyntheticGroups performs the actual DB migration inside a
|
||||
// single transaction: each cluster's tracks are reassigned onto a
|
||||
// deterministic synthetic group key, the parent's track count is
|
||||
// decremented per track moved, and the parent row is dropped if it
|
||||
// ends up empty. Returns the new group keys in cluster order.
|
||||
func (s *Service) splitIntoSyntheticGroups(
|
||||
parentKey string, libraryID int64, clusters []autotag.TrackCluster,
|
||||
) ([]string, error) {
|
||||
tx, err := s.db.BeginTx()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("begin split tx: %w", err)
|
||||
}
|
||||
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
q := s.db.Queries.WithTx(tx)
|
||||
newKeys := make([]string, 0, len(clusters))
|
||||
|
||||
for _, c := range clusters {
|
||||
var newKey string
|
||||
if len(c.Tracks) == 1 {
|
||||
// A lone leftover track from SplitPlan's singleton
|
||||
// fallback may carry an empty (or shared-but-coincidental)
|
||||
// album/album-artist tag — key on the track itself so two
|
||||
// untagged leftovers can't collide.
|
||||
newKey = autotag.SyntheticTrackGroupKey(parentKey, c.Tracks[0].AudioFileID)
|
||||
} else {
|
||||
newKey = autotag.SyntheticGroupKey(parentKey, c.AlbumName, c.AlbumArtist)
|
||||
}
|
||||
|
||||
newKeys = append(newKeys, newKey)
|
||||
|
||||
for _, t := range c.Tracks {
|
||||
if err := q.DecrementTaggingItemTrackCount(s.ctx, parentKey); err != nil {
|
||||
return nil, fmt.Errorf("decrement parent group: %w", err)
|
||||
}
|
||||
|
||||
upsertParams := sqlcgen.UpsertTaggingItemOnTrackAddParams{
|
||||
GroupKey: newKey,
|
||||
LibraryID: libraryID,
|
||||
AlbumName: c.AlbumName,
|
||||
AlbumArtist: c.AlbumArtist,
|
||||
DiscNumber: 0,
|
||||
}
|
||||
if err := q.UpsertTaggingItemOnTrackAdd(s.ctx, upsertParams); err != nil {
|
||||
return nil, fmt.Errorf("upsert synthetic group: %w", err)
|
||||
}
|
||||
|
||||
if err := q.SetAudioFileGroupKey(s.ctx, sqlcgen.SetAudioFileGroupKeyParams{
|
||||
GroupKey: newKey,
|
||||
ID: t.AudioFileID,
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("reassign track %d: %w", t.AudioFileID, err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := q.MarkTaggingItemSynthetic(s.ctx, sqlcgen.MarkTaggingItemSyntheticParams{
|
||||
ParentGroupKey: parentKey,
|
||||
GroupKey: newKey,
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("mark synthetic: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := q.DeleteTaggingItemIfEmpty(s.ctx, parentKey); err != nil {
|
||||
return nil, fmt.Errorf("cleanup leftover parent: %w", err)
|
||||
}
|
||||
|
||||
// The parent's cached candidates (if it still exists) no longer
|
||||
// reflect its track set now that some tracks moved out.
|
||||
if err := q.DeleteTaggingCandidates(s.ctx, parentKey); err != nil {
|
||||
s.logger.Warn("split: drop stale parent candidates", "group_key", parentKey, "err", err)
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return nil, fmt.Errorf("commit split: %w", err)
|
||||
}
|
||||
|
||||
return newKeys, nil
|
||||
}
|
||||
|
||||
// AckLibraryWarning records that the user has seen the first-
|
||||
// time-apply irreversibility warning for this library.
|
||||
func (s *Service) AckLibraryWarning(libraryID int64) error {
|
||||
@@ -1188,6 +1439,7 @@ func (s *Service) GetCandidatesForPasteURL(
|
||||
AlbumName: score.AlbumName,
|
||||
AlbumArtist: score.AlbumArtist,
|
||||
Tracks: score.LocalTracks,
|
||||
Synthetic: score.Synthetic,
|
||||
}, pasted)
|
||||
merged := append([]autotag.Candidate{scored}, score.Candidates...)
|
||||
score.Candidates = merged
|
||||
@@ -1357,16 +1609,26 @@ func extractReleaseMBID(url string) string {
|
||||
// top-ranked candidate; pass nil to skip cover art entirely (used
|
||||
// only by paths that don't need art).
|
||||
func scoreToView(s *autotag.GroupScore, exp *explore.Service) *ScoreView {
|
||||
group := autotag.Group{
|
||||
AlbumName: s.AlbumName,
|
||||
AlbumArtist: s.AlbumArtist,
|
||||
Tracks: s.LocalTracks,
|
||||
Synthetic: s.Synthetic,
|
||||
}
|
||||
|
||||
rec := s.Recommendation
|
||||
if rec == "" {
|
||||
// Paths that rebuild a GroupScore from cached candidates
|
||||
// don't run the scorer; derive the tier here.
|
||||
rec = autotag.Recommend(
|
||||
autotag.Group{Tracks: s.LocalTracks}, s.Candidates,
|
||||
)
|
||||
rec = autotag.Recommend(group, s.Candidates)
|
||||
}
|
||||
|
||||
out := &ScoreView{GroupKey: s.GroupKey, Recommendation: string(rec)}
|
||||
out := &ScoreView{
|
||||
GroupKey: s.GroupKey,
|
||||
Recommendation: string(rec),
|
||||
Synthetic: s.Synthetic,
|
||||
MixedBag: !s.Synthetic && autotag.IsMixedBag(group),
|
||||
}
|
||||
|
||||
for _, l := range s.LocalTracks {
|
||||
out.LocalTracks = append(out.LocalTracks, LocalTrackView{
|
||||
|
||||
@@ -0,0 +1,406 @@
|
||||
package autotagservice
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"testing"
|
||||
|
||||
"yellowjacket/backend/autotag"
|
||||
"yellowjacket/backend/database"
|
||||
"yellowjacket/backend/database/sql/sqlcgen"
|
||||
)
|
||||
|
||||
// newTestService builds a Service with just enough wired up for
|
||||
// SplitMixedFolder — no MB client, no tag writer. Constructed
|
||||
// directly (bypassing NewService) since this package's tests live
|
||||
// inside the package and don't need the explore/tagwriter
|
||||
// dependencies that method never touches.
|
||||
func newTestService(t *testing.T, db *database.DB) *Service {
|
||||
t.Helper()
|
||||
|
||||
logger := slog.New(slog.DiscardHandler)
|
||||
|
||||
return &Service{
|
||||
db: db,
|
||||
scorer: autotag.NewScorer(db.Queries, nil, logger),
|
||||
logger: logger,
|
||||
ctx: db.Ctx,
|
||||
}
|
||||
}
|
||||
|
||||
// seedMixedBagFolder drops one physical folder (single group_key)
|
||||
// containing two 2-track clusters (different album/album-artist tags
|
||||
// each) plus one leftover track with no album tag at all — the shape
|
||||
// SplitMixedFolder is meant to untangle.
|
||||
func seedMixedBagFolder(t *testing.T, db *database.DB, groupKey string, libraryID int64) {
|
||||
t.Helper()
|
||||
|
||||
ctx := db.Ctx
|
||||
q := db.Queries
|
||||
|
||||
addTrack := func(filePath, title, artist, album, albumArtist string, trackNum int) {
|
||||
ac, err := q.UpsertArtistCredit(ctx, artist)
|
||||
if err != nil {
|
||||
t.Fatalf("upsert artist credit: %v", err)
|
||||
}
|
||||
|
||||
rec, err := q.CreateRecordingFull(ctx, sqlcgen.CreateRecordingFullParams{
|
||||
Name: title,
|
||||
ArtistCreditID: ac.ID,
|
||||
TrackNumber: sql.NullInt64{Int64: int64(trackNum), Valid: true},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create recording: %v", err)
|
||||
}
|
||||
|
||||
if album != "" {
|
||||
albumArtistAC, err := q.UpsertArtistCredit(ctx, albumArtist)
|
||||
if err != nil {
|
||||
t.Fatalf("upsert album artist credit: %v", err)
|
||||
}
|
||||
|
||||
rg, err := q.UpsertReleaseGroup(ctx, sqlcgen.UpsertReleaseGroupParams{
|
||||
Name: album,
|
||||
AlbumArtistCreditID: sql.NullInt64{Int64: albumArtistAC.ID, Valid: true},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("upsert release group: %v", err)
|
||||
}
|
||||
|
||||
if _, err := q.CreateReleaseGroupRecording(
|
||||
ctx,
|
||||
sqlcgen.CreateReleaseGroupRecordingParams{
|
||||
ReleaseGroupID: rg.ID,
|
||||
RecordingID: rec.ID,
|
||||
TrackNumber: sql.NullInt64{Int64: int64(trackNum), Valid: true},
|
||||
},
|
||||
); err != nil {
|
||||
t.Fatalf("link release group recording: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := q.CreateAudioFileWithGroupKey(ctx, sqlcgen.CreateAudioFileWithGroupKeyParams{
|
||||
FilePath: filePath,
|
||||
LengthMilliseconds: 200000,
|
||||
FileTypeID: 0,
|
||||
RecordingID: rec.ID,
|
||||
Basename: filePath,
|
||||
LibraryID: libraryID,
|
||||
GroupKey: groupKey,
|
||||
TagStatus: "untagged",
|
||||
}); err != nil {
|
||||
t.Fatalf("create audio file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
addTrack("/junk/01.mp3", "Song A1", "Artist One", "Album One", "Artist One", 1)
|
||||
addTrack("/junk/02.mp3", "Song A2", "Artist One", "Album One", "Artist One", 2)
|
||||
addTrack("/junk/03.mp3", "Song B1", "Artist Two", "Album Two", "Artist Two", 1)
|
||||
addTrack("/junk/04.mp3", "Song B2", "Artist Two", "Album Two", "Artist Two", 2)
|
||||
addTrack("/junk/05.mp3", "Lone Song", "Artist Three", "", "", 1)
|
||||
|
||||
if _, err := db.ExecContext(`
|
||||
INSERT INTO tagging_items (group_key, library_id, track_count, album_name, album_artist, disc_number, status)
|
||||
VALUES (?, ?, 5, '', '', 0, 'pending')
|
||||
`, groupKey, libraryID); err != nil {
|
||||
t.Fatalf("insert tagging item: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// seedCoherentAlbum drops a single-artist, single-album folder — the
|
||||
// negative case for the mixed-bag triage query.
|
||||
func seedCoherentAlbum(t *testing.T, db *database.DB, groupKey string, libraryID int64) {
|
||||
t.Helper()
|
||||
|
||||
ctx := db.Ctx
|
||||
q := db.Queries
|
||||
|
||||
ac, err := q.UpsertArtistCredit(ctx, "The Beatles")
|
||||
if err != nil {
|
||||
t.Fatalf("upsert artist credit: %v", err)
|
||||
}
|
||||
|
||||
rg, err := q.UpsertReleaseGroup(ctx, sqlcgen.UpsertReleaseGroupParams{
|
||||
Name: "Abbey Road",
|
||||
AlbumArtistCreditID: sql.NullInt64{Int64: ac.ID, Valid: true},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("upsert release group: %v", err)
|
||||
}
|
||||
|
||||
titles := []string{"Come Together", "Something", "Maxwell's Silver Hammer", "Oh! Darling"}
|
||||
for i, title := range titles {
|
||||
rec, err := q.CreateRecordingFull(ctx, sqlcgen.CreateRecordingFullParams{
|
||||
Name: title,
|
||||
ArtistCreditID: ac.ID,
|
||||
TrackNumber: sql.NullInt64{Int64: int64(i + 1), Valid: true},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create recording: %v", err)
|
||||
}
|
||||
|
||||
if _, err := q.CreateReleaseGroupRecording(ctx, sqlcgen.CreateReleaseGroupRecordingParams{
|
||||
ReleaseGroupID: rg.ID,
|
||||
RecordingID: rec.ID,
|
||||
TrackNumber: sql.NullInt64{Int64: int64(i + 1), Valid: true},
|
||||
}); err != nil {
|
||||
t.Fatalf("link release group recording: %v", err)
|
||||
}
|
||||
|
||||
if _, err := q.CreateAudioFileWithGroupKey(ctx, sqlcgen.CreateAudioFileWithGroupKeyParams{
|
||||
FilePath: groupKey + "/" + title + ".mp3",
|
||||
LengthMilliseconds: 200000,
|
||||
FileTypeID: 0,
|
||||
RecordingID: rec.ID,
|
||||
Basename: title + ".mp3",
|
||||
LibraryID: libraryID,
|
||||
GroupKey: groupKey,
|
||||
TagStatus: "untagged",
|
||||
}); err != nil {
|
||||
t.Fatalf("create audio file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := db.ExecContext(`
|
||||
INSERT INTO tagging_items (group_key, library_id, track_count, album_name, album_artist, disc_number, status)
|
||||
VALUES (?, ?, 4, 'Abbey Road', 'The Beatles', 0, 'pending')
|
||||
`, groupKey, libraryID); err != nil {
|
||||
t.Fatalf("insert tagging item: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListPendingFolders_FlagsLikelyMixedBag(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db := database.NewTestDB(t)
|
||||
seedMixedBagFolder(t, db, "g-junk", 0)
|
||||
seedCoherentAlbum(t, db, "g-abbey-road", 0)
|
||||
|
||||
s := newTestService(t, db)
|
||||
|
||||
items, err := s.ListPendingFolders(0)
|
||||
if err != nil {
|
||||
t.Fatalf("ListPendingFolders: %v", err)
|
||||
}
|
||||
|
||||
got := make(map[string]bool, len(items))
|
||||
for _, it := range items {
|
||||
got[it.GroupKey] = it.LikelyMixedBag
|
||||
}
|
||||
|
||||
if !got["g-junk"] {
|
||||
t.Error("expected g-junk (no artist/album consensus) to be flagged LikelyMixedBag")
|
||||
}
|
||||
|
||||
if got["g-abbey-road"] {
|
||||
t.Error(
|
||||
"expected g-abbey-road (coherent single-artist album) to NOT be flagged LikelyMixedBag",
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitMixedFolder_CarvesOutClustersAndSingletons(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db := database.NewTestDB(t)
|
||||
seedMixedBagFolder(t, db, "g-junk", 0)
|
||||
|
||||
s := newTestService(t, db)
|
||||
|
||||
items, err := s.SplitMixedFolder("g-junk")
|
||||
if err != nil {
|
||||
t.Fatalf("SplitMixedFolder: %v", err)
|
||||
}
|
||||
|
||||
// Every track leaves the parent: 2 clustered groups (2 tracks
|
||||
// each) + 1 singleton for the unclustered "Lone Song" track. The
|
||||
// parent is now empty and must not survive as a 4th item.
|
||||
if len(items) != 3 { //nolint:mnd
|
||||
t.Fatalf("expected 3 resulting groups, got %d: %+v", len(items), items)
|
||||
}
|
||||
|
||||
for _, it := range items {
|
||||
if it.GroupKey == "g-junk" {
|
||||
t.Fatal("expected the original group to be fully drained and removed")
|
||||
}
|
||||
|
||||
if !it.Synthetic {
|
||||
t.Errorf("child group %q: Synthetic = false, want true", it.GroupKey)
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
clustered []PendingItem
|
||||
singleton *PendingItem
|
||||
)
|
||||
|
||||
for i, it := range items {
|
||||
if it.TrackCount == 1 {
|
||||
singleton = &items[i]
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
clustered = append(clustered, it)
|
||||
}
|
||||
|
||||
if singleton == nil {
|
||||
t.Fatal("expected a singleton child for the unclustered Lone Song track")
|
||||
}
|
||||
|
||||
if singleton.AlbumName != "" {
|
||||
t.Errorf(
|
||||
"singleton child album_name = %q, want empty (Lone Song had no album tag)",
|
||||
singleton.AlbumName,
|
||||
)
|
||||
}
|
||||
|
||||
if len(clustered) != 2 { //nolint:mnd
|
||||
t.Fatalf("expected 2 clustered children, got %d", len(clustered))
|
||||
}
|
||||
|
||||
seenAlbums := map[string]bool{}
|
||||
|
||||
for _, c := range clustered {
|
||||
if c.TrackCount != 2 { //nolint:mnd
|
||||
t.Errorf("child group %q: track_count = %d, want 2", c.GroupKey, c.TrackCount)
|
||||
}
|
||||
|
||||
seenAlbums[c.AlbumName] = true
|
||||
}
|
||||
|
||||
if !seenAlbums["Album One"] || !seenAlbums["Album Two"] {
|
||||
t.Errorf("expected children for Album One and Album Two, got %+v", clustered)
|
||||
}
|
||||
|
||||
// The physical file paths must be untouched — only group_key
|
||||
// reassignment happened, no files moved on disk.
|
||||
locals, err := s.scorer.LocalTracksForGroup(db.Ctx, clustered[0].GroupKey)
|
||||
if err != nil {
|
||||
t.Fatalf("load synthetic group tracks: %v", err)
|
||||
}
|
||||
|
||||
for _, l := range locals {
|
||||
if l.FilePath == "" {
|
||||
t.Error("expected non-empty file path preserved on the synthetic group's tracks")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitMixedFolder_NothingToClusterErrors(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db := database.NewTestDB(t)
|
||||
|
||||
if _, err := db.Queries.CreateAudioFileWithGroupKey(
|
||||
db.Ctx,
|
||||
sqlcgen.CreateAudioFileWithGroupKeyParams{
|
||||
FilePath: "/coherent/01.mp3",
|
||||
FileTypeID: 0,
|
||||
RecordingID: mustCreateRecording(t, db, "Track"),
|
||||
Basename: "01.mp3",
|
||||
LibraryID: 0,
|
||||
GroupKey: "g-coherent",
|
||||
TagStatus: "untagged",
|
||||
},
|
||||
); err != nil {
|
||||
t.Fatalf("create audio file: %v", err)
|
||||
}
|
||||
|
||||
if _, err := db.ExecContext(`
|
||||
INSERT INTO tagging_items (group_key, library_id, track_count, album_name, album_artist, disc_number, status)
|
||||
VALUES ('g-coherent', 0, 1, '', '', 0, 'pending')
|
||||
`); err != nil {
|
||||
t.Fatalf("insert tagging item: %v", err)
|
||||
}
|
||||
|
||||
s := newTestService(t, db)
|
||||
|
||||
if _, err := s.SplitMixedFolder("g-coherent"); !errors.Is(err, errNothingToSplit) {
|
||||
t.Fatalf("err = %v, want errNothingToSplit", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListPendingFolders_PrunesOrphanedEntries(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db := database.NewTestDB(t)
|
||||
|
||||
// A real, live folder — must survive.
|
||||
if _, err := db.Queries.CreateAudioFileWithGroupKey(
|
||||
db.Ctx,
|
||||
sqlcgen.CreateAudioFileWithGroupKeyParams{
|
||||
FilePath: "/live/01.mp3",
|
||||
FileTypeID: 0,
|
||||
RecordingID: mustCreateRecording(t, db, "Track"),
|
||||
Basename: "01.mp3",
|
||||
LibraryID: 0,
|
||||
GroupKey: "g-live",
|
||||
TagStatus: "untagged",
|
||||
},
|
||||
); err != nil {
|
||||
t.Fatalf("create audio file: %v", err)
|
||||
}
|
||||
|
||||
if _, err := db.ExecContext(`
|
||||
INSERT INTO tagging_items (group_key, library_id, track_count, album_name, album_artist, disc_number, status)
|
||||
VALUES ('g-live', 0, 1, '', '', 0, 'pending')
|
||||
`); err != nil {
|
||||
t.Fatalf("insert live tagging item: %v", err)
|
||||
}
|
||||
|
||||
// An orphaned row: no audio_files row points at this group_key
|
||||
// any more (the file was deleted/moved and the bookkeeping that's
|
||||
// supposed to clean this up never ran) — this is exactly the
|
||||
// "old/nonexistent" entry the review UI shouldn't show.
|
||||
if _, err := db.ExecContext(`
|
||||
INSERT INTO tagging_items (group_key, library_id, track_count, album_name, album_artist, disc_number, status)
|
||||
VALUES ('g-orphan', 0, 3, 'Ghost Album', 'Ghost Artist', 0, 'pending')
|
||||
`); err != nil {
|
||||
t.Fatalf("insert orphaned tagging item: %v", err)
|
||||
}
|
||||
|
||||
s := newTestService(t, db)
|
||||
|
||||
items, err := s.ListPendingFolders(0)
|
||||
if err != nil {
|
||||
t.Fatalf("ListPendingFolders: %v", err)
|
||||
}
|
||||
|
||||
got := make(map[string]bool, len(items))
|
||||
for _, it := range items {
|
||||
got[it.GroupKey] = true
|
||||
}
|
||||
|
||||
if !got["g-live"] {
|
||||
t.Error("expected g-live (has a real audio_files row) to remain listed")
|
||||
}
|
||||
|
||||
if got["g-orphan"] {
|
||||
t.Error("expected g-orphan (no matching audio_files rows) to be pruned, not listed")
|
||||
}
|
||||
|
||||
if _, err := db.Queries.GetTaggingItem(db.Ctx, "g-orphan"); !errors.Is(err, sql.ErrNoRows) {
|
||||
t.Errorf("expected g-orphan row to be deleted from tagging_items, got err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustCreateRecording(t *testing.T, db *database.DB, title string) int64 {
|
||||
t.Helper()
|
||||
|
||||
ac, err := db.Queries.UpsertArtistCredit(db.Ctx, "Artist")
|
||||
if err != nil {
|
||||
t.Fatalf("upsert artist credit: %v", err)
|
||||
}
|
||||
|
||||
rec, err := db.Queries.CreateRecordingFull(db.Ctx, sqlcgen.CreateRecordingFullParams{
|
||||
Name: title,
|
||||
ArtistCreditID: ac.ID,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create recording: %v", err)
|
||||
}
|
||||
|
||||
return rec.ID
|
||||
}
|
||||
+87
-44
@@ -10,8 +10,8 @@ import (
|
||||
"path"
|
||||
|
||||
"github.com/BurntSushi/toml"
|
||||
"github.com/wailsapp/wails/v2/pkg/runtime"
|
||||
|
||||
"yellowjacket/backend/download"
|
||||
"yellowjacket/backend/events"
|
||||
"yellowjacket/backend/favorites"
|
||||
"yellowjacket/backend/library"
|
||||
@@ -31,14 +31,15 @@ var errSaveBeforeLoad = errors.New("refusing to save: config not loaded from dis
|
||||
type Config struct {
|
||||
ctx context.Context
|
||||
logger *slog.Logger
|
||||
filePath string // required
|
||||
loaded bool // true once Load() succeeds
|
||||
Library *library.Config `toml:"Library"`
|
||||
Theme *theme.Config `toml:"Theme"`
|
||||
Window *WindowConfig `toml:"Window"`
|
||||
TrackList *tracklist.Config `toml:"TrackList"`
|
||||
Favorites *favorites.Config `toml:"Favorites"`
|
||||
Shortcuts *shortcuts.Config `toml:"Shortcuts"`
|
||||
filePath string // required
|
||||
loaded bool // true once Load() succeeds
|
||||
Library *library.Config `toml:"Library"`
|
||||
Theme *theme.Config `toml:"Theme"`
|
||||
Window *WindowConfig `toml:"Window"`
|
||||
TrackList *tracklist.Config `toml:"TrackList"`
|
||||
Favorites *favorites.Config `toml:"Favorites"`
|
||||
Shortcuts *shortcuts.Config `toml:"Shortcuts"`
|
||||
Downloads *download.UserConfig `toml:"Downloads"`
|
||||
}
|
||||
|
||||
// NewConfig creates a new config by loading it from disk.
|
||||
@@ -258,6 +259,12 @@ func (c *Config) applyDefaults() {
|
||||
}
|
||||
|
||||
c.Shortcuts.ApplyDefaults()
|
||||
|
||||
if c.Downloads == nil {
|
||||
c.Downloads = &download.UserConfig{}
|
||||
}
|
||||
|
||||
c.Downloads.ApplyDefaults()
|
||||
}
|
||||
|
||||
// SetContext sets the Wails runtime context for event emission.
|
||||
@@ -298,15 +305,13 @@ func (c *Config) SetLibraryDirectory(dir string) error {
|
||||
)
|
||||
}
|
||||
|
||||
if c.ctx != nil {
|
||||
runtime.EventsEmit(
|
||||
c.ctx,
|
||||
events.LibraryConfigChanged,
|
||||
map[string]any{
|
||||
"DirectoryPath": dir,
|
||||
},
|
||||
)
|
||||
}
|
||||
events.Emit(
|
||||
c.ctx,
|
||||
events.LibraryConfigChanged,
|
||||
map[string]any{
|
||||
"DirectoryPath": dir,
|
||||
},
|
||||
)
|
||||
|
||||
c.logger.Info(
|
||||
"library directory updated",
|
||||
@@ -356,6 +361,50 @@ func (c *Config) SetScanConcurrency(mode string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetDownloadPreferences returns the configured auto-download
|
||||
// guardrails.
|
||||
func (c *Config) GetDownloadPreferences() download.AutoDownloadPrefs {
|
||||
if c.Downloads == nil {
|
||||
return download.AutoDownloadPrefs{}
|
||||
}
|
||||
|
||||
return c.Downloads.AutoDownloadPrefs()
|
||||
}
|
||||
|
||||
// SetDownloadPreferences saves new auto-download guardrails. This only
|
||||
// persists them; the download package cannot depend on config (config
|
||||
// already depends on download for UserConfig), so making the change
|
||||
// live without a restart is the caller's job — the frontend settings
|
||||
// save calls this and download.Service.SetPreferences in the same
|
||||
// action, and app.go's initDownloadRuntime applies the saved value to
|
||||
// the running Manager at startup.
|
||||
func (c *Config) SetDownloadPreferences(prefs download.AutoDownloadPrefs) error {
|
||||
if c.Downloads == nil {
|
||||
c.Downloads = &download.UserConfig{}
|
||||
c.Downloads.ApplyDefaults()
|
||||
}
|
||||
|
||||
formats := make([]string, 0, len(prefs.AllowedFormats))
|
||||
for _, f := range prefs.AllowedFormats {
|
||||
formats = append(formats, string(f))
|
||||
}
|
||||
|
||||
c.Downloads.MinFileSizeMB = prefs.MinSizeMB
|
||||
c.Downloads.MaxFileSizeMB = prefs.MaxSizeMB
|
||||
c.Downloads.PreferredFileSizeMB = prefs.PreferredSizeMB
|
||||
c.Downloads.AllowedFormats = formats
|
||||
|
||||
if err := c.Save(); err != nil {
|
||||
return fmt.Errorf(
|
||||
"could not save config: %w", err,
|
||||
)
|
||||
}
|
||||
|
||||
c.logger.Info("download auto-pick preferences updated")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetThemeAccentColor returns the configured accent colour.
|
||||
func (c *Config) GetThemeAccentColor() string {
|
||||
if c.Theme == nil {
|
||||
@@ -442,11 +491,11 @@ func (c *Config) SetThemeBackgroundShade(
|
||||
|
||||
// emitThemeChanged sends the ThemeConfigChanged event to the frontend.
|
||||
func (c *Config) emitThemeChanged() {
|
||||
if c.ctx == nil || c.Theme == nil {
|
||||
if c.Theme == nil {
|
||||
return
|
||||
}
|
||||
|
||||
runtime.EventsEmit(
|
||||
events.Emit(
|
||||
c.ctx,
|
||||
events.ThemeConfigChanged,
|
||||
map[string]any{
|
||||
@@ -512,7 +561,7 @@ func (c *Config) emitTrackListChanged() {
|
||||
})
|
||||
}
|
||||
|
||||
runtime.EventsEmit(
|
||||
events.Emit(
|
||||
c.ctx,
|
||||
events.TrackListConfigChanged,
|
||||
map[string]any{
|
||||
@@ -636,11 +685,11 @@ func (c *Config) SetPinDefaultPlaylist(pin bool) error {
|
||||
// emitFavoritesChanged sends the FavoritesConfigChanged event
|
||||
// to the frontend.
|
||||
func (c *Config) emitFavoritesChanged() {
|
||||
if c.ctx == nil || c.Favorites == nil {
|
||||
if c.Favorites == nil {
|
||||
return
|
||||
}
|
||||
|
||||
runtime.EventsEmit(
|
||||
events.Emit(
|
||||
c.ctx,
|
||||
events.FavoritesConfigChanged,
|
||||
map[string]any{
|
||||
@@ -677,13 +726,11 @@ func (c *Config) SetShortcuts(
|
||||
)
|
||||
}
|
||||
|
||||
if c.ctx != nil {
|
||||
runtime.EventsEmit(
|
||||
c.ctx,
|
||||
events.ShortcutsConfigChanged,
|
||||
bindings,
|
||||
)
|
||||
}
|
||||
events.Emit(
|
||||
c.ctx,
|
||||
events.ShortcutsConfigChanged,
|
||||
bindings,
|
||||
)
|
||||
|
||||
c.logger.Info("shortcuts config updated")
|
||||
|
||||
@@ -707,13 +754,11 @@ func (c *Config) SetShortcut(
|
||||
)
|
||||
}
|
||||
|
||||
if c.ctx != nil {
|
||||
runtime.EventsEmit(
|
||||
c.ctx,
|
||||
events.ShortcutsConfigChanged,
|
||||
c.Shortcuts.Bindings,
|
||||
)
|
||||
}
|
||||
events.Emit(
|
||||
c.ctx,
|
||||
events.ShortcutsConfigChanged,
|
||||
c.Shortcuts.Bindings,
|
||||
)
|
||||
|
||||
c.logger.Info(
|
||||
"shortcut updated",
|
||||
@@ -736,13 +781,11 @@ func (c *Config) ResetShortcuts() error {
|
||||
)
|
||||
}
|
||||
|
||||
if c.ctx != nil {
|
||||
runtime.EventsEmit(
|
||||
c.ctx,
|
||||
events.ShortcutsConfigChanged,
|
||||
c.Shortcuts.Bindings,
|
||||
)
|
||||
}
|
||||
events.Emit(
|
||||
c.ctx,
|
||||
events.ShortcutsConfigChanged,
|
||||
c.Shortcuts.Bindings,
|
||||
)
|
||||
|
||||
c.logger.Info("shortcuts reset to defaults")
|
||||
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"yellowjacket/backend/events"
|
||||
)
|
||||
|
||||
// setupRecordedConfig builds a Config that saves to a temp directory
|
||||
// and records the events it would push to the frontend.
|
||||
func setupRecordedConfig(t *testing.T) (*Config, *events.Recorder) {
|
||||
t.Helper()
|
||||
|
||||
conf := &Config{
|
||||
logger: slog.Default(),
|
||||
filePath: filepath.Join(t.TempDir(), "config.toml"),
|
||||
}
|
||||
conf.applyDefaults()
|
||||
|
||||
// Load, not just applyDefaults: Save refuses to write a config that
|
||||
// was never hydrated from disk, so without this only the first
|
||||
// setter in a test succeeds.
|
||||
if err := conf.Load(); err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
|
||||
rec := events.NewRecorder()
|
||||
conf.SetContext(events.WithSink(context.Background(), rec))
|
||||
|
||||
return conf, rec
|
||||
}
|
||||
|
||||
// payloadMap returns the map payload of the most recent named event.
|
||||
func payloadMap(
|
||||
t *testing.T,
|
||||
rec *events.Recorder,
|
||||
name string,
|
||||
) map[string]any {
|
||||
t.Helper()
|
||||
|
||||
ev, ok := rec.Last(name)
|
||||
if !ok {
|
||||
t.Fatalf("no %s emitted; got %v", name, rec.Names())
|
||||
}
|
||||
|
||||
data, ok := ev.Payload().(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("%s payload is %T, want map[string]any", name, ev.Payload())
|
||||
}
|
||||
|
||||
return data
|
||||
}
|
||||
|
||||
// TestEmit_ThemeChangeCarriesBothFields pins that the theme event is a
|
||||
// snapshot of both fields, not a delta: the frontend applies the whole
|
||||
// colour ramp from it, so an accent change that omitted the shade would
|
||||
// re-derive the ramp against a default background.
|
||||
func TestEmit_ThemeChangeCarriesBothFields(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
conf, rec := setupRecordedConfig(t)
|
||||
|
||||
if err := conf.SetThemeBackgroundShade("light"); err != nil {
|
||||
t.Fatalf("SetThemeBackgroundShade: %v", err)
|
||||
}
|
||||
|
||||
if err := conf.SetThemeAccentColor("#ff0000"); err != nil {
|
||||
t.Fatalf("SetThemeAccentColor: %v", err)
|
||||
}
|
||||
|
||||
if got := rec.Count(events.ThemeConfigChanged); got != 2 {
|
||||
t.Errorf("emitted %d ThemeConfigChanged, want 2", got)
|
||||
}
|
||||
|
||||
data := payloadMap(t, rec, events.ThemeConfigChanged)
|
||||
if data["AccentColor"] != "#ff0000" {
|
||||
t.Errorf("AccentColor = %v, want #ff0000", data["AccentColor"])
|
||||
}
|
||||
|
||||
if data["BackgroundShade"] != "light" {
|
||||
t.Errorf("BackgroundShade = %v, want light", data["BackgroundShade"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestEmit_ThemeChangeIsNotEmittedOnRejectedValue pins that a rejected
|
||||
// write does not tell the frontend the theme changed.
|
||||
func TestEmit_ThemeChangeIsNotEmittedOnRejectedValue(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
conf, rec := setupRecordedConfig(t)
|
||||
|
||||
if err := conf.SetThemeAccentColor("not-a-colour"); err == nil {
|
||||
t.Fatal("SetThemeAccentColor accepted an invalid colour")
|
||||
}
|
||||
|
||||
if got := rec.Count(events.ThemeConfigChanged); got != 0 {
|
||||
t.Errorf("emitted %d ThemeConfigChanged for a rejected write, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestEmit_ShortcutChangeSendsWholeBindingMap covers the surface the
|
||||
// 357-line frontend shortcut service rebuilds itself from.
|
||||
func TestEmit_ShortcutChangeSendsWholeBindingMap(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
conf, rec := setupRecordedConfig(t)
|
||||
|
||||
if err := conf.SetShortcut("playPause", "k"); err != nil {
|
||||
t.Fatalf("SetShortcut: %v", err)
|
||||
}
|
||||
|
||||
ev, ok := rec.Last(events.ShortcutsConfigChanged)
|
||||
if !ok {
|
||||
t.Fatalf("no ShortcutsConfigChanged; got %v", rec.Names())
|
||||
}
|
||||
|
||||
bindings, ok := ev.Payload().(map[string]string)
|
||||
if !ok {
|
||||
t.Fatalf("payload is %T, want map[string]string", ev.Payload())
|
||||
}
|
||||
|
||||
if bindings["playPause"] != "k" {
|
||||
t.Errorf("playPause = %q, want k", bindings["playPause"])
|
||||
}
|
||||
|
||||
// The whole map, not just the changed key — the frontend replaces
|
||||
// its binding table wholesale on this event.
|
||||
if len(bindings) < 2 {
|
||||
t.Errorf("emitted %d bindings, want the full default set", len(bindings))
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmit_ResetShortcutsRepublishesDefaults(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
conf, rec := setupRecordedConfig(t)
|
||||
|
||||
if err := conf.SetShortcut("playPause", "k"); err != nil {
|
||||
t.Fatalf("SetShortcut: %v", err)
|
||||
}
|
||||
|
||||
rec.Reset()
|
||||
|
||||
if err := conf.ResetShortcuts(); err != nil {
|
||||
t.Fatalf("ResetShortcuts: %v", err)
|
||||
}
|
||||
|
||||
ev, ok := rec.Last(events.ShortcutsConfigChanged)
|
||||
if !ok {
|
||||
t.Fatalf("no ShortcutsConfigChanged after reset; got %v", rec.Names())
|
||||
}
|
||||
|
||||
bindings, ok := ev.Payload().(map[string]string)
|
||||
if !ok {
|
||||
t.Fatalf("payload is %T, want map[string]string", ev.Payload())
|
||||
}
|
||||
|
||||
if bindings["playPause"] == "k" {
|
||||
t.Error("reset emitted the overridden binding, not the default")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmit_FavoritesChangeCarriesFullConfig(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
conf, rec := setupRecordedConfig(t)
|
||||
|
||||
if err := conf.SetFavoritesPlaylistID(7); err != nil {
|
||||
t.Fatalf("SetFavoritesPlaylistID: %v", err)
|
||||
}
|
||||
|
||||
data := payloadMap(t, rec, events.FavoritesConfigChanged)
|
||||
if data["PlaylistID"] != int64(7) {
|
||||
t.Errorf("PlaylistID = %#v, want int64(7)", data["PlaylistID"])
|
||||
}
|
||||
|
||||
for _, key := range []string{"IconStyle", "PinDefault"} {
|
||||
if _, ok := data[key]; !ok {
|
||||
t.Errorf("payload is missing %q; the settings page reads it", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
+311
-3709
File diff suppressed because it is too large
Load Diff
@@ -12,7 +12,7 @@ import (
|
||||
// Migration 6 integration tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func TestMigration6FreshDB(t *testing.T) {
|
||||
func TestSchemaCreatesLibrariesTable(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db := NewTestDB(t)
|
||||
@@ -172,32 +172,6 @@ func TestMigration6FreshDB(t *testing.T) {
|
||||
t.Error("track_metadata VIEW does not contain library_id")
|
||||
}
|
||||
|
||||
// Verify user_version >= 7.
|
||||
var version int
|
||||
|
||||
verRows, err := db.QueryContext("PRAGMA user_version")
|
||||
if err != nil {
|
||||
t.Fatalf("PRAGMA user_version: %v", err)
|
||||
}
|
||||
|
||||
if !verRows.Next() {
|
||||
_ = verRows.Close()
|
||||
|
||||
t.Fatal("PRAGMA user_version: no row returned")
|
||||
}
|
||||
|
||||
if err := verRows.Scan(&version); err != nil {
|
||||
_ = verRows.Close()
|
||||
|
||||
t.Fatalf("scan user_version: %v", err)
|
||||
}
|
||||
|
||||
_ = verRows.Close()
|
||||
|
||||
if version < 7 {
|
||||
t.Errorf("user_version = %d, want >= 7", version)
|
||||
}
|
||||
|
||||
// Verify libraries table has only the sentinel row on fresh DB.
|
||||
count, err := db.Queries.CountLibraries(db.Ctx)
|
||||
if err != nil {
|
||||
@@ -213,7 +187,7 @@ func TestMigration6FreshDB(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigration6LibraryQueries(t *testing.T) {
|
||||
func TestLibraryQueries(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db := NewTestDB(t)
|
||||
@@ -331,7 +305,7 @@ func TestMigration6LibraryQueries(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigration6PhantomPlaylistTracks(t *testing.T) {
|
||||
func TestPhantomPlaylistTracksAreCleaned(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, libID := NewTestDBWithLibrary(t, "Test", "/test/music")
|
||||
@@ -462,7 +436,7 @@ func TestMigration6PhantomPlaylistTracks(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigration6AudioFilesLibraryFK(t *testing.T) {
|
||||
func TestAudioFilesLibraryForeignKey(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, libID := NewTestDBWithLibrary(t, "Test", "/test/fk-lib")
|
||||
@@ -523,7 +497,7 @@ func TestMigration6AudioFilesLibraryFK(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigration6TrackMetadataViewHasLibraryID(t *testing.T) {
|
||||
func TestTrackMetadataViewHasLibraryID(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, libID := NewTestDBWithLibrary(t, "Test", "/test/view-lib")
|
||||
@@ -592,37 +566,11 @@ func TestMigration6TrackMetadataViewHasLibraryID(t *testing.T) {
|
||||
// Migration 9 integration tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func TestMigration9SmartPlaylistColumns(t *testing.T) {
|
||||
func TestSmartPlaylistColumns(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db := NewTestDB(t)
|
||||
|
||||
// Verify user_version >= 9.
|
||||
var version int
|
||||
|
||||
verRows, err := db.QueryContext("PRAGMA user_version")
|
||||
if err != nil {
|
||||
t.Fatalf("PRAGMA user_version: %v", err)
|
||||
}
|
||||
|
||||
if !verRows.Next() {
|
||||
_ = verRows.Close()
|
||||
|
||||
t.Fatal("PRAGMA user_version: no row returned")
|
||||
}
|
||||
|
||||
if err := verRows.Scan(&version); err != nil {
|
||||
_ = verRows.Close()
|
||||
|
||||
t.Fatalf("scan user_version: %v", err)
|
||||
}
|
||||
|
||||
_ = verRows.Close()
|
||||
|
||||
if version < 9 {
|
||||
t.Errorf("user_version = %d, want >= 9", version)
|
||||
}
|
||||
|
||||
// Verify playlists table has is_smart and smart_rules columns.
|
||||
hasSmart := false
|
||||
hasRules := false
|
||||
@@ -774,37 +722,11 @@ func TestMigration9SmartPlaylistColumns(t *testing.T) {
|
||||
// Migration 10 — play history tracking
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func TestMigration10PlayHistory(t *testing.T) {
|
||||
func TestPlayHistoryTable(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db := NewTestDB(t)
|
||||
|
||||
// Verify user_version >= 10.
|
||||
var version int
|
||||
|
||||
verRows, err := db.QueryContext("PRAGMA user_version")
|
||||
if err != nil {
|
||||
t.Fatalf("PRAGMA user_version: %v", err)
|
||||
}
|
||||
|
||||
if !verRows.Next() {
|
||||
_ = verRows.Close()
|
||||
|
||||
t.Fatal("PRAGMA user_version: no row returned")
|
||||
}
|
||||
|
||||
if err := verRows.Scan(&version); err != nil {
|
||||
_ = verRows.Close()
|
||||
|
||||
t.Fatalf("scan user_version: %v", err)
|
||||
}
|
||||
|
||||
_ = verRows.Close()
|
||||
|
||||
if version < 10 {
|
||||
t.Errorf("user_version = %d, want >= 10", version)
|
||||
}
|
||||
|
||||
// Verify play_history table exists.
|
||||
var tableCount int64
|
||||
|
||||
@@ -1058,7 +980,7 @@ func TestMigration10PlayHistory(t *testing.T) {
|
||||
// Migration 11 — explore_cache table
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func TestMigration11ExploreCache(t *testing.T) {
|
||||
func TestHTTPCacheTable(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// explore_cache was split into http_cache + artist_metadata by
|
||||
@@ -1069,32 +991,6 @@ func TestMigration11ExploreCache(t *testing.T) {
|
||||
|
||||
db := NewTestDB(t)
|
||||
|
||||
// Verify user_version >= 11.
|
||||
var version int
|
||||
|
||||
verRows, err := db.QueryContext("PRAGMA user_version")
|
||||
if err != nil {
|
||||
t.Fatalf("PRAGMA user_version: %v", err)
|
||||
}
|
||||
|
||||
if !verRows.Next() {
|
||||
_ = verRows.Close()
|
||||
|
||||
t.Fatal("PRAGMA user_version: no row returned")
|
||||
}
|
||||
|
||||
if err := verRows.Scan(&version); err != nil {
|
||||
_ = verRows.Close()
|
||||
|
||||
t.Fatalf("scan user_version: %v", err)
|
||||
}
|
||||
|
||||
_ = verRows.Close()
|
||||
|
||||
if version < 11 {
|
||||
t.Errorf("user_version = %d, want >= 11", version)
|
||||
}
|
||||
|
||||
// Verify explore_cache table exists.
|
||||
var tableCount int64
|
||||
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// migrateDownloadRename performs the download subsystem's table rename
|
||||
// for existing databases that still carry the old table names: the
|
||||
// durable "I asked for this" record moved from download_wants to
|
||||
// download_requests, and the one-shot search-and-grab attempt moved
|
||||
// from download_requests to download_downloads (see CLAUDE.md and
|
||||
// .planning/NOTES.md for the full Want->Request / Request->Download
|
||||
// rename).
|
||||
//
|
||||
// This cannot be a plain sql/migrations file the way an ADD COLUMN
|
||||
// migration is. That pattern's tolerance for "duplicate column name"
|
||||
// works because a fresh database's sql/schemas pass already produces
|
||||
// the identical target shape under the identical table name, so
|
||||
// replaying the ALTER TABLE against it is a safe no-op. Here the name
|
||||
// "download_requests" is reused for a different table before and after
|
||||
// the rename, so a fresh database's schema pass creates a real, empty,
|
||||
// correctly-shaped download_downloads AND a real, empty,
|
||||
// correctly-shaped (new) download_requests before this ever runs.
|
||||
// Blindly replaying "ALTER TABLE download_requests RENAME TO
|
||||
// download_downloads" against that fresh database would rename the new,
|
||||
// empty Request table into Download's place, destroying the fresh
|
||||
// install rather than no-opping. Gating on whether the OLD
|
||||
// download_wants table still exists — a name nothing creates or
|
||||
// references once this has run — is what tells an old database and a
|
||||
// fresh (or already migrated) one apart without executing anything
|
||||
// destructive on the fresh path.
|
||||
func migrateDownloadRename(ctx context.Context, db *sql.DB) error {
|
||||
var name string
|
||||
|
||||
err := db.QueryRowContext(
|
||||
ctx,
|
||||
`SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'download_wants'`,
|
||||
).Scan(&name)
|
||||
|
||||
switch {
|
||||
case errors.Is(err, sql.ErrNoRows):
|
||||
// Nothing to migrate: either a fresh install (sql/schemas
|
||||
// already produced the target shape) or a database this has
|
||||
// already run against.
|
||||
case err != nil:
|
||||
return fmt.Errorf("check for download_wants table: %w", err)
|
||||
default:
|
||||
if err := runDownloadRename(ctx, db); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return ensureDownloadIndexes(ctx, db)
|
||||
}
|
||||
|
||||
// runDownloadRename performs the actual rename dance against a
|
||||
// database confirmed to still have the old download_wants table.
|
||||
func runDownloadRename(ctx context.Context, db *sql.DB) error {
|
||||
stmts := []string{
|
||||
// The schema pass already created an empty, correctly-shaped
|
||||
// download_downloads placeholder under this name (it never
|
||||
// existed under the old naming), which would otherwise collide
|
||||
// with the rename below.
|
||||
`DROP TABLE IF EXISTS download_downloads`,
|
||||
|
||||
// 1. Free the "download_requests" name: the old one-shot
|
||||
// attempt table becomes download_downloads.
|
||||
`ALTER TABLE download_requests RENAME TO download_downloads`,
|
||||
`ALTER TABLE download_downloads RENAME COLUMN want_id TO request_id`,
|
||||
|
||||
// 2. Claim the now-free "download_requests" name for the
|
||||
// durable-intent table.
|
||||
`ALTER TABLE download_wants RENAME TO download_requests`,
|
||||
|
||||
// 3. The transfer table's FK now points at download_downloads.
|
||||
`ALTER TABLE download_items RENAME COLUMN request_id TO download_id`,
|
||||
|
||||
// Named indexes survive a table/column rename attached to their
|
||||
// old name, so drop them here; ensureDownloadIndexes recreates
|
||||
// them under the names sql/schemas' comments describe.
|
||||
`DROP INDEX IF EXISTS idx_download_requests_created`,
|
||||
`DROP INDEX IF EXISTS idx_download_requests_state`,
|
||||
`DROP INDEX IF EXISTS idx_download_wants_due`,
|
||||
`DROP INDEX IF EXISTS idx_download_wants_entity`,
|
||||
`DROP INDEX IF EXISTS idx_download_wants_parent`,
|
||||
`DROP INDEX IF EXISTS idx_download_items_request`,
|
||||
}
|
||||
|
||||
tx, err := db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin download rename migration: %w", err)
|
||||
}
|
||||
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
for _, stmt := range stmts {
|
||||
if _, err := tx.ExecContext(ctx, stmt); err != nil {
|
||||
return fmt.Errorf("download rename migration %q: %w", stmt, err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("commit download rename migration: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ensureDownloadIndexes creates the indexes sql/schemas deliberately
|
||||
// omits inline for the renamed table/columns (see
|
||||
// migrateDownloadRename), under their final names. Safe to call
|
||||
// unconditionally: IF NOT EXISTS makes it a no-op once created, and by
|
||||
// the time this runs every column/table involved is guaranteed to be
|
||||
// in its final shape on both a fresh and a migrated database.
|
||||
func ensureDownloadIndexes(ctx context.Context, db *sql.DB) error {
|
||||
stmts := []string{
|
||||
`CREATE INDEX IF NOT EXISTS idx_download_downloads_created
|
||||
ON download_downloads(created_at DESC)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_download_downloads_state
|
||||
ON download_downloads(state)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_download_requests_due
|
||||
ON download_requests(next_try_at) WHERE state = 'wanted'`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_download_requests_entity
|
||||
ON download_requests(entity, state)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_download_requests_parent
|
||||
ON download_requests(parent_id)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_download_items_download
|
||||
ON download_items(download_id)`,
|
||||
}
|
||||
|
||||
for _, stmt := range stmts {
|
||||
if _, err := db.ExecContext(ctx, stmt); err != nil {
|
||||
return fmt.Errorf("ensure download index: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,365 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// oldDownloadRequestsDDL, oldDownloadWantsDDL and oldDownloadItemsDDL
|
||||
// are frozen snapshots of the download subsystem's tables exactly as
|
||||
// they read before the Want/Request rename (see
|
||||
// download_rename_migration.go) — i.e. what a real user's existing
|
||||
// database looks like today, before upgrading to a build that includes
|
||||
// this migration.
|
||||
const oldDownloadRequestsDDL = `
|
||||
CREATE TABLE IF NOT EXISTS download_requests (
|
||||
id TEXT PRIMARY KEY,
|
||||
library_id INTEGER NOT NULL,
|
||||
source TEXT NOT NULL DEFAULT 'manual',
|
||||
want_id INTEGER REFERENCES download_wants(id) ON DELETE SET NULL,
|
||||
release_mbid TEXT,
|
||||
release_group_mbid TEXT,
|
||||
recording_mbid TEXT,
|
||||
artist TEXT NOT NULL DEFAULT '',
|
||||
album TEXT NOT NULL DEFAULT '',
|
||||
query TEXT NOT NULL DEFAULT '',
|
||||
expected TEXT NOT NULL DEFAULT '[]',
|
||||
state TEXT NOT NULL DEFAULT 'searching'
|
||||
CHECK(state IN ('searching', 'found', 'queued', 'grabbing',
|
||||
'verifying', 'tagging', 'importing',
|
||||
'complete', 'cancelled', 'failed')),
|
||||
error TEXT NOT NULL DEFAULT '',
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY(library_id) REFERENCES libraries(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_download_requests_created
|
||||
ON download_requests(created_at DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_download_requests_state
|
||||
ON download_requests(state);
|
||||
`
|
||||
|
||||
const oldDownloadWantsDDL = `
|
||||
CREATE TABLE IF NOT EXISTS download_wants (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
mbid TEXT NOT NULL,
|
||||
entity TEXT NOT NULL
|
||||
CHECK(entity IN ('artist', 'release-group', 'release', 'recording')),
|
||||
library_id INTEGER NOT NULL,
|
||||
artist TEXT NOT NULL DEFAULT '',
|
||||
title TEXT NOT NULL DEFAULT '',
|
||||
scope TEXT NOT NULL DEFAULT 'future'
|
||||
CHECK(scope IN ('future', 'all')),
|
||||
secondary INTEGER NOT NULL DEFAULT 0,
|
||||
state TEXT NOT NULL DEFAULT 'wanted'
|
||||
CHECK(state IN ('wanted', 'satisfied', 'paused')),
|
||||
parent_id INTEGER,
|
||||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
last_error TEXT NOT NULL DEFAULT '',
|
||||
last_tried_at DATETIME,
|
||||
next_try_at DATETIME,
|
||||
external_ids TEXT NOT NULL DEFAULT '{}',
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(mbid, library_id),
|
||||
FOREIGN KEY(library_id) REFERENCES libraries(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY(parent_id) REFERENCES download_wants(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_download_wants_due
|
||||
ON download_wants(next_try_at)
|
||||
WHERE state = 'wanted';
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_download_wants_entity
|
||||
ON download_wants(entity, state);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_download_wants_parent
|
||||
ON download_wants(parent_id);
|
||||
`
|
||||
|
||||
const oldDownloadItemsDDL = `
|
||||
CREATE TABLE IF NOT EXISTS download_items (
|
||||
id TEXT PRIMARY KEY,
|
||||
request_id TEXT NOT NULL,
|
||||
provider_id INTEGER NOT NULL,
|
||||
transport_id INTEGER,
|
||||
external_id TEXT NOT NULL DEFAULT '',
|
||||
candidate TEXT NOT NULL DEFAULT '{}',
|
||||
state TEXT NOT NULL DEFAULT 'queued'
|
||||
CHECK(state IN ('searching', 'found', 'queued', 'grabbing',
|
||||
'verifying', 'tagging', 'importing',
|
||||
'complete', 'cancelled', 'failed')),
|
||||
staging_dir TEXT NOT NULL DEFAULT '',
|
||||
bytes_done INTEGER NOT NULL DEFAULT 0,
|
||||
bytes_total INTEGER NOT NULL DEFAULT 0,
|
||||
imported_paths TEXT NOT NULL DEFAULT '[]',
|
||||
error TEXT NOT NULL DEFAULT '',
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY(request_id) REFERENCES download_requests(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_download_items_live
|
||||
ON download_items(state)
|
||||
WHERE state NOT IN ('complete', 'cancelled', 'failed');
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_download_items_request
|
||||
ON download_items(request_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_download_items_state
|
||||
ON download_items(state);
|
||||
`
|
||||
|
||||
// seedOldDownloadSchema builds the pre-rename download tables and
|
||||
// inserts one row of real data into each, standing in for a real
|
||||
// user's database at the moment it upgrades.
|
||||
func seedOldDownloadSchema(t *testing.T, db *sql.DB) {
|
||||
t.Helper()
|
||||
|
||||
for _, ddl := range []string{
|
||||
oldDownloadWantsDDL, oldDownloadRequestsDDL, oldDownloadItemsDDL,
|
||||
} {
|
||||
if _, err := db.ExecContext(t.Context(), ddl); err != nil {
|
||||
t.Fatalf("create old download schema: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := db.ExecContext(
|
||||
t.Context(),
|
||||
`INSERT INTO libraries (id, name, path) VALUES (1, 'Test', '/music')`,
|
||||
); err != nil {
|
||||
t.Fatalf("seed library: %v", err)
|
||||
}
|
||||
|
||||
if _, err := db.ExecContext(
|
||||
t.Context(),
|
||||
`INSERT INTO download_wants
|
||||
(id, mbid, entity, library_id, artist, title, state)
|
||||
VALUES (1, 'artist-mbid', 'artist', 1, 'Radiohead', 'Radiohead', 'wanted')`,
|
||||
); err != nil {
|
||||
t.Fatalf("seed download_wants: %v", err)
|
||||
}
|
||||
|
||||
if _, err := db.ExecContext(
|
||||
t.Context(),
|
||||
`INSERT INTO download_requests
|
||||
(id, library_id, source, want_id, release_group_mbid, artist, album, state)
|
||||
VALUES ('dl-1', 1, 'wanted', 1, 'rg-mbid', 'Radiohead', 'OK Computer', 'complete')`,
|
||||
); err != nil {
|
||||
t.Fatalf("seed download_requests: %v", err)
|
||||
}
|
||||
|
||||
if _, err := db.ExecContext(
|
||||
t.Context(),
|
||||
`INSERT INTO download_items
|
||||
(id, request_id, provider_id, state)
|
||||
VALUES ('item-1', 'dl-1', 1, 'complete')`,
|
||||
); err != nil {
|
||||
t.Fatalf("seed download_items: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDownloadRename_FreshInstallUntouched confirms applySchema on a
|
||||
// brand-new database produces the target shape directly and that
|
||||
// migrateDownloadRename's gate (checking for the old download_wants
|
||||
// table) is a no-op there — the destructive path this test guards
|
||||
// against is exactly the one described in download_rename_migration.go:
|
||||
// blindly replaying the rename against a fresh database's already-
|
||||
// correct, empty download_requests/download_downloads tables.
|
||||
func TestDownloadRename_FreshInstallUntouched(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db := openMemDB(t)
|
||||
|
||||
if err := applySchema(t.Context(), db); err != nil {
|
||||
t.Fatalf("apply schema (fresh): %v", err)
|
||||
}
|
||||
|
||||
for _, table := range []string{"download_downloads", "download_requests", "download_items"} {
|
||||
var name string
|
||||
|
||||
err := db.QueryRowContext(
|
||||
t.Context(),
|
||||
`SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?`,
|
||||
table,
|
||||
).Scan(&name)
|
||||
if err != nil {
|
||||
t.Errorf("expected table %q to exist on a fresh install: %v", table, err)
|
||||
}
|
||||
}
|
||||
|
||||
var stray string
|
||||
|
||||
err := db.QueryRowContext(
|
||||
t.Context(),
|
||||
`SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'download_wants'`,
|
||||
).Scan(&stray)
|
||||
if !errors.Is(err, sql.ErrNoRows) {
|
||||
t.Errorf("old download_wants table should not exist on a fresh install, err=%v", err)
|
||||
}
|
||||
|
||||
// Both auto-download guardrail indexes sql/schemas deliberately
|
||||
// omits (see ensureDownloadIndexes) must still exist.
|
||||
for _, idx := range []string{
|
||||
"idx_download_requests_due",
|
||||
"idx_download_requests_entity",
|
||||
"idx_download_requests_parent",
|
||||
"idx_download_items_download",
|
||||
} {
|
||||
var name string
|
||||
|
||||
err := db.QueryRowContext(
|
||||
t.Context(),
|
||||
`SELECT name FROM sqlite_master WHERE type = 'index' AND name = ?`,
|
||||
idx,
|
||||
).Scan(&name)
|
||||
if err != nil {
|
||||
t.Errorf("expected index %q to exist on a fresh install: %v", idx, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestDownloadRename_UpgradesExistingDatabase is the regression test
|
||||
// for the rename itself: an old-shaped database (download_wants +
|
||||
// old-style download_requests, both with real rows) must end up with
|
||||
// the same table names, column names, and data a fresh install would
|
||||
// have — nothing dropped, nothing silently emptied.
|
||||
func TestDownloadRename_UpgradesExistingDatabase(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fresh := openMemDB(t)
|
||||
if err := applySchema(t.Context(), fresh); err != nil {
|
||||
t.Fatalf("apply schema (fresh): %v", err)
|
||||
}
|
||||
|
||||
upgraded := openMemDB(t)
|
||||
|
||||
librariesDDL, err := schemas.ReadFile("sql/schemas/libraries.sql")
|
||||
if err != nil {
|
||||
t.Fatalf("read libraries schema: %v", err)
|
||||
}
|
||||
|
||||
if _, err := upgraded.ExecContext(t.Context(), string(librariesDDL)); err != nil {
|
||||
t.Fatalf("create libraries table: %v", err)
|
||||
}
|
||||
|
||||
seedOldDownloadSchema(t, upgraded)
|
||||
|
||||
if err := applySchema(t.Context(), upgraded); err != nil {
|
||||
t.Fatalf("apply schema (upgrade path): %v", err)
|
||||
}
|
||||
|
||||
// Column order must match a fresh install's, for the same reason
|
||||
// TestMigrations_ColumnOrderMatchesFreshInstall checks tagging_items:
|
||||
// sqlc's `SELECT *` binds positionally.
|
||||
for _, table := range []string{"download_downloads", "download_requests", "download_items"} {
|
||||
freshCols := tableColumns(t, fresh, table)
|
||||
upgradedCols := tableColumns(t, upgraded, table)
|
||||
|
||||
if len(freshCols) != len(upgradedCols) {
|
||||
t.Fatalf(
|
||||
"%s: column count mismatch: fresh has %d (%v), upgraded has %d (%v)",
|
||||
table, len(freshCols), freshCols, len(upgradedCols), upgradedCols,
|
||||
)
|
||||
}
|
||||
|
||||
for i := range freshCols {
|
||||
if freshCols[i] != upgradedCols[i] {
|
||||
t.Errorf(
|
||||
"%s: column order mismatch at %d: fresh %q, upgraded %q\nfresh: %v\nupgraded: %v",
|
||||
table,
|
||||
i,
|
||||
freshCols[i],
|
||||
upgradedCols[i],
|
||||
freshCols,
|
||||
upgradedCols,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The seeded rows survived the rename under their new names.
|
||||
var (
|
||||
requestMBID string
|
||||
requestEntity string
|
||||
)
|
||||
|
||||
err = upgraded.QueryRowContext(
|
||||
t.Context(), `SELECT mbid, entity FROM download_requests WHERE id = 1`,
|
||||
).Scan(&requestMBID, &requestEntity)
|
||||
if err != nil {
|
||||
t.Fatalf("seeded request row missing after rename: %v", err)
|
||||
}
|
||||
|
||||
if requestMBID != "artist-mbid" || requestEntity != "artist" {
|
||||
t.Errorf("request row corrupted: mbid=%q entity=%q", requestMBID, requestEntity)
|
||||
}
|
||||
|
||||
var (
|
||||
downloadRequestID sql.NullInt64
|
||||
downloadAlbum string
|
||||
)
|
||||
|
||||
err = upgraded.QueryRowContext(
|
||||
t.Context(),
|
||||
`SELECT request_id, album FROM download_downloads WHERE id = 'dl-1'`,
|
||||
).Scan(&downloadRequestID, &downloadAlbum)
|
||||
if err != nil {
|
||||
t.Fatalf("seeded download row missing after rename: %v", err)
|
||||
}
|
||||
|
||||
if !downloadRequestID.Valid || downloadRequestID.Int64 != 1 {
|
||||
t.Errorf("download.request_id = %v, want 1 (renamed from want_id)", downloadRequestID)
|
||||
}
|
||||
|
||||
if downloadAlbum != "OK Computer" {
|
||||
t.Errorf("download.album = %q, want OK Computer", downloadAlbum)
|
||||
}
|
||||
|
||||
var itemDownloadID string
|
||||
|
||||
err = upgraded.QueryRowContext(
|
||||
t.Context(),
|
||||
`SELECT download_id FROM download_items WHERE id = 'item-1'`,
|
||||
).Scan(&itemDownloadID)
|
||||
if err != nil {
|
||||
t.Fatalf("seeded item row missing after rename: %v", err)
|
||||
}
|
||||
|
||||
if itemDownloadID != "dl-1" {
|
||||
t.Errorf("item.download_id = %q, want dl-1 (renamed from request_id)", itemDownloadID)
|
||||
}
|
||||
|
||||
// The old table is gone, not just emptied.
|
||||
var stray string
|
||||
|
||||
err = upgraded.QueryRowContext(
|
||||
t.Context(),
|
||||
`SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'download_wants'`,
|
||||
).Scan(&stray)
|
||||
if !errors.Is(err, sql.ErrNoRows) {
|
||||
t.Errorf("old download_wants table should be gone after migration, err=%v", err)
|
||||
}
|
||||
|
||||
// Running the whole thing again (as a second app startup would) is
|
||||
// a no-op: the gate sees no download_wants table and does nothing
|
||||
// further, so this must not error or duplicate anything.
|
||||
if err := applySchema(t.Context(), upgraded); err != nil {
|
||||
t.Fatalf("apply schema a second time: %v", err)
|
||||
}
|
||||
|
||||
var count int
|
||||
|
||||
if err := upgraded.QueryRowContext(
|
||||
t.Context(), `SELECT COUNT(*) FROM download_requests`,
|
||||
).Scan(&count); err != nil {
|
||||
t.Fatalf("count download_requests: %v", err)
|
||||
}
|
||||
|
||||
if count != 1 {
|
||||
t.Errorf("download_requests has %d rows after a second migration pass, want 1", count)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// seedExploreRow inserts one explore_index row.
|
||||
func seedExploreRow(t *testing.T, db *DB, mbid, title, artist string) {
|
||||
t.Helper()
|
||||
|
||||
if _, err := db.ExecContext(`
|
||||
INSERT INTO explore_index (entity_type, mbid, title, artist_name, artist_mbid)
|
||||
VALUES ('recording', ?, ?, ?, '')
|
||||
`, mbid, title, artist); err != nil {
|
||||
t.Fatalf("seed %s: %v", mbid, err)
|
||||
}
|
||||
}
|
||||
|
||||
// ftsMatches returns how many FTS rows match a query.
|
||||
func ftsMatches(t *testing.T, db *DB, query string) int {
|
||||
t.Helper()
|
||||
|
||||
rows, err := db.QueryContext(
|
||||
"SELECT COUNT(*) FROM explore_index_fts WHERE explore_index_fts MATCH ?", query,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("fts query %q: %v", query, err)
|
||||
}
|
||||
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
n := 0
|
||||
|
||||
if rows.Next() {
|
||||
if err := rows.Scan(&n); err != nil {
|
||||
t.Fatalf("scan fts count: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := rows.Err(); err != nil {
|
||||
t.Fatalf("fts rows: %v", err)
|
||||
}
|
||||
|
||||
return n
|
||||
}
|
||||
|
||||
// Rows written while FTS sync is suspended are invisible to search
|
||||
// until the window closes — and fully searchable afterwards. This is
|
||||
// the contract the dump import's bulk-load path depends on.
|
||||
func TestExploreFTSSuspendResumeIndexesBulkRows(t *testing.T) {
|
||||
db := NewTestDB(t)
|
||||
|
||||
seedExploreRow(t, db, "mbid-before", "Before Suspend", "Artist One")
|
||||
|
||||
if got := ftsMatches(t, db, "Before"); got != 1 {
|
||||
t.Fatalf("matches for pre-suspend row = %d, want 1", got)
|
||||
}
|
||||
|
||||
if err := db.SuspendExploreIndexFTS(); err != nil {
|
||||
t.Fatalf("suspend: %v", err)
|
||||
}
|
||||
|
||||
seedExploreRow(t, db, "mbid-during", "During Suspend", "Artist Two")
|
||||
|
||||
if got := ftsMatches(t, db, "During"); got != 0 {
|
||||
t.Errorf("matches while suspended = %d, want 0 (triggers should be off)", got)
|
||||
}
|
||||
|
||||
if err := db.ResumeExploreIndexFTS(); err != nil {
|
||||
t.Fatalf("resume: %v", err)
|
||||
}
|
||||
|
||||
if got := ftsMatches(t, db, "During"); got != 1 {
|
||||
t.Errorf("matches for bulk-loaded row after resume = %d, want 1", got)
|
||||
}
|
||||
|
||||
if got := ftsMatches(t, db, "Before"); got != 1 {
|
||||
t.Errorf("matches for pre-suspend row after resume = %d, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The import wipes explore_index before reassembling it. With the
|
||||
// triggers suspended that DELETE writes no FTS delete-markers, so the
|
||||
// rebuild must be what clears the old rows out of search.
|
||||
func TestExploreFTSResumeDropsDeletedRows(t *testing.T) {
|
||||
db := NewTestDB(t)
|
||||
|
||||
seedExploreRow(t, db, "mbid-stale", "Stale Recording", "Old Artist")
|
||||
|
||||
if err := db.SuspendExploreIndexFTS(); err != nil {
|
||||
t.Fatalf("suspend: %v", err)
|
||||
}
|
||||
|
||||
if _, err := db.ExecContext("DELETE FROM explore_index"); err != nil {
|
||||
t.Fatalf("wipe: %v", err)
|
||||
}
|
||||
|
||||
seedExploreRow(t, db, "mbid-fresh", "Fresh Recording", "New Artist")
|
||||
|
||||
if err := db.ResumeExploreIndexFTS(); err != nil {
|
||||
t.Fatalf("resume: %v", err)
|
||||
}
|
||||
|
||||
if got := ftsMatches(t, db, "Stale"); got != 0 {
|
||||
t.Errorf("matches for wiped row = %d, want 0", got)
|
||||
}
|
||||
|
||||
if got := ftsMatches(t, db, "Fresh"); got != 1 {
|
||||
t.Errorf("matches for reassembled row = %d, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
// resumeFTS runs from a defer as well as at its natural point in the
|
||||
// pipeline, so a second call must be harmless.
|
||||
func TestExploreFTSResumeIsIdempotent(t *testing.T) {
|
||||
db := NewTestDB(t)
|
||||
|
||||
if err := db.SuspendExploreIndexFTS(); err != nil {
|
||||
t.Fatalf("suspend: %v", err)
|
||||
}
|
||||
|
||||
seedExploreRow(t, db, "mbid-a", "Repeatable Resume", "Artist")
|
||||
|
||||
if err := db.ResumeExploreIndexFTS(); err != nil {
|
||||
t.Fatalf("first resume: %v", err)
|
||||
}
|
||||
|
||||
if err := db.ResumeExploreIndexFTS(); err != nil {
|
||||
t.Fatalf("second resume: %v", err)
|
||||
}
|
||||
|
||||
if got := ftsMatches(t, db, "Repeatable"); got != 1 {
|
||||
t.Errorf("matches after repeated resume = %d, want 1", got)
|
||||
}
|
||||
|
||||
// Triggers must still be live for ordinary writes after the window.
|
||||
seedExploreRow(t, db, "mbid-b", "Postwindow Row", "Artist")
|
||||
|
||||
if got := ftsMatches(t, db, "Postwindow"); got != 1 {
|
||||
t.Errorf("matches for row written after resume = %d, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Suspending twice must not fail — the triggers are simply already gone.
|
||||
func TestExploreFTSSuspendIsIdempotent(t *testing.T) {
|
||||
db := NewTestDB(t)
|
||||
|
||||
if err := db.SuspendExploreIndexFTS(); err != nil {
|
||||
t.Fatalf("first suspend: %v", err)
|
||||
}
|
||||
|
||||
if err := db.SuspendExploreIndexFTS(); err != nil {
|
||||
t.Fatalf("second suspend: %v", err)
|
||||
}
|
||||
|
||||
if err := db.ResumeExploreIndexFTS(); err != nil {
|
||||
t.Fatalf("resume: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// oldTaggingItemsDDL is a frozen snapshot of tagging_items exactly as
|
||||
// it read before sql/migrations/0001_tagging_items_synthetic.sql —
|
||||
// i.e. what a real user's existing database looks like today, before
|
||||
// upgrading to a build that includes that migration.
|
||||
const oldTaggingItemsDDL = `
|
||||
CREATE TABLE IF NOT EXISTS tagging_items (
|
||||
group_key TEXT PRIMARY KEY,
|
||||
library_id INTEGER NOT NULL,
|
||||
track_count INTEGER NOT NULL DEFAULT 0,
|
||||
album_name TEXT NOT NULL DEFAULT '',
|
||||
album_artist TEXT NOT NULL DEFAULT '',
|
||||
disc_number INTEGER NOT NULL DEFAULT 0,
|
||||
best_match_release_mbid TEXT,
|
||||
score REAL,
|
||||
last_checked_at DATETIME,
|
||||
status TEXT NOT NULL DEFAULT 'pending'
|
||||
CHECK(status IN ('pending', 'matched', 'confirmed', 'skipped')),
|
||||
cleared_at DATETIME,
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY(library_id) REFERENCES libraries(id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_tagging_items_library_status
|
||||
ON tagging_items(library_id, status);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_tagging_items_status_pending
|
||||
ON tagging_items(library_id) WHERE status = 'pending';
|
||||
`
|
||||
|
||||
// tableColumns returns the column names of a table in on-disk
|
||||
// (positional) order, via PRAGMA table_info — the order sqlc's
|
||||
// generated `SELECT *` scans bind to positionally.
|
||||
func tableColumns(t *testing.T, db *sql.DB, table string) []string {
|
||||
t.Helper()
|
||||
|
||||
rows, err := db.QueryContext(t.Context(), "PRAGMA table_info("+table+")")
|
||||
if err != nil {
|
||||
t.Fatalf("PRAGMA table_info(%s): %v", table, err)
|
||||
}
|
||||
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
var cols []string
|
||||
|
||||
for rows.Next() {
|
||||
var (
|
||||
cid int
|
||||
name string
|
||||
ctype string
|
||||
notnull int
|
||||
dfltValue sql.NullString
|
||||
primaryKey int
|
||||
)
|
||||
|
||||
if err := rows.Scan(&cid, &name, &ctype, ¬null, &dfltValue, &primaryKey); err != nil {
|
||||
t.Fatalf("scan table_info row: %v", err)
|
||||
}
|
||||
|
||||
cols = append(cols, name)
|
||||
}
|
||||
|
||||
if err := rows.Err(); err != nil {
|
||||
t.Fatalf("iterate table_info: %v", err)
|
||||
}
|
||||
|
||||
return cols
|
||||
}
|
||||
|
||||
func openMemDB(t *testing.T) *sql.DB {
|
||||
t.Helper()
|
||||
|
||||
db, err := sql.Open("sqlite", ":memory:?_busy_timeout=5000&_journal_mode=WAL")
|
||||
if err != nil {
|
||||
t.Fatalf("open in-memory db: %v", err)
|
||||
}
|
||||
|
||||
db.SetMaxOpenConns(1)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
|
||||
if err := applyPRAGMAs(t.Context(), db); err != nil {
|
||||
t.Fatalf("apply pragmas: %v", err)
|
||||
}
|
||||
|
||||
return db
|
||||
}
|
||||
|
||||
// TestMigrations_ColumnOrderMatchesFreshInstall is the regression
|
||||
// test for the exact failure mode that got the old 48-step migration
|
||||
// chain torn out (see .planning/NOTES.md, "No migration chain"):
|
||||
// sql/schemas drifting from what migrations actually produce, so
|
||||
// sqlc-generated code silently reads the wrong thing.
|
||||
//
|
||||
// A fresh install takes tagging_items straight from sql/schemas
|
||||
// (CREATE TABLE, columns in file order). An existing database takes
|
||||
// it from sql/schemas (the base shape, unchanged since the table
|
||||
// already existed) plus sql/migrations/0001 (`ALTER TABLE ADD
|
||||
// COLUMN`, which SQLite always appends at the END of the column
|
||||
// list, regardless of where the column sits in the CREATE TABLE
|
||||
// statement). If sql/schemas ever declares a migrated column
|
||||
// somewhere other than last, the two paths produce tables with the
|
||||
// SAME columns in a DIFFERENT order — invisible until a `SELECT *`
|
||||
// (e.g. GetTaggingItem) silently binds a value to the wrong field.
|
||||
func TestMigrations_ColumnOrderMatchesFreshInstall(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fresh := openMemDB(t)
|
||||
if err := applySchema(t.Context(), fresh); err != nil {
|
||||
t.Fatalf("apply schema (fresh): %v", err)
|
||||
}
|
||||
|
||||
upgraded := openMemDB(t)
|
||||
|
||||
librariesDDL, err := schemas.ReadFile("sql/schemas/libraries.sql")
|
||||
if err != nil {
|
||||
t.Fatalf("read libraries schema: %v", err)
|
||||
}
|
||||
|
||||
if _, err := upgraded.ExecContext(t.Context(), string(librariesDDL)); err != nil {
|
||||
t.Fatalf("create libraries table: %v", err)
|
||||
}
|
||||
|
||||
if _, err := upgraded.ExecContext(t.Context(), oldTaggingItemsDDL); err != nil {
|
||||
t.Fatalf("create pre-migration tagging_items: %v", err)
|
||||
}
|
||||
|
||||
// sql/schemas no-ops on the pre-existing tagging_items (IF NOT
|
||||
// EXISTS), then sql/migrations/0001's ALTER TABLE statements
|
||||
// actually add the missing columns for real this time.
|
||||
if err := applySchema(t.Context(), upgraded); err != nil {
|
||||
t.Fatalf("apply schema (upgrade path): %v", err)
|
||||
}
|
||||
|
||||
freshCols := tableColumns(t, fresh, "tagging_items")
|
||||
upgradedCols := tableColumns(t, upgraded, "tagging_items")
|
||||
|
||||
if len(freshCols) != len(upgradedCols) {
|
||||
t.Fatalf(
|
||||
"column count mismatch: fresh install has %d (%v), upgraded has %d (%v)",
|
||||
len(freshCols), freshCols, len(upgradedCols), upgradedCols,
|
||||
)
|
||||
}
|
||||
|
||||
for i := range freshCols {
|
||||
if freshCols[i] != upgradedCols[i] {
|
||||
t.Errorf(
|
||||
"column order mismatch at position %d: fresh install has %q, upgraded has %q\nfresh: %v\nupgraded: %v",
|
||||
i,
|
||||
freshCols[i],
|
||||
upgradedCols[i],
|
||||
freshCols,
|
||||
upgradedCols,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestMigrations_FreshDatabaseStillRecordsAndGetsIndex confirms a
|
||||
// brand-new database runs migration 0001 (tolerating "duplicate
|
||||
// column name" from its ALTER TABLE statements, since sql/schemas
|
||||
// already declared those columns), records it applied, AND still
|
||||
// gets the trailing CREATE INDEX statement sql/schemas deliberately
|
||||
// omits for migrated columns.
|
||||
func TestMigrations_FreshDatabaseStillRecordsAndGetsIndex(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fresh := openMemDB(t)
|
||||
if err := applySchema(t.Context(), fresh); err != nil {
|
||||
t.Fatalf("apply schema: %v", err)
|
||||
}
|
||||
|
||||
var version int
|
||||
|
||||
err := fresh.QueryRowContext(
|
||||
t.Context(), "SELECT version FROM schema_migrations WHERE version = 1",
|
||||
).Scan(&version)
|
||||
if err != nil {
|
||||
t.Fatalf("expected migration 1 to be recorded as applied on a fresh db: %v", err)
|
||||
}
|
||||
|
||||
var indexName string
|
||||
|
||||
err = fresh.QueryRowContext(
|
||||
t.Context(),
|
||||
"SELECT name FROM sqlite_master WHERE type = 'index' AND name = 'idx_tagging_items_parent_group_key'",
|
||||
).Scan(&indexName)
|
||||
if err != nil {
|
||||
t.Fatalf("expected idx_tagging_items_parent_group_key to exist on a fresh db: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -1062,45 +1062,17 @@ func TestClearSearchIndexPreservesSchema(t *testing.T) {
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Migration test
|
||||
// Schema constraint tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func TestMigrationsApplied(t *testing.T) {
|
||||
func TestSearchIndexSchema(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db := NewTestDB(t)
|
||||
|
||||
// Verify user_version >= 3 (all 3 migrations applied).
|
||||
// Use QueryContext + immediate Scan + Close to release the
|
||||
// single connection before subsequent ExecContext calls.
|
||||
var version int
|
||||
|
||||
rows, err := db.QueryContext("PRAGMA user_version")
|
||||
if err != nil {
|
||||
t.Fatalf("PRAGMA user_version: %v", err)
|
||||
}
|
||||
|
||||
if !rows.Next() {
|
||||
_ = rows.Close()
|
||||
|
||||
t.Fatal("PRAGMA user_version: no row returned")
|
||||
}
|
||||
|
||||
if err := rows.Scan(&version); err != nil {
|
||||
_ = rows.Close()
|
||||
|
||||
t.Fatalf("scan user_version: %v", err)
|
||||
}
|
||||
|
||||
_ = rows.Close()
|
||||
|
||||
if version < 3 {
|
||||
t.Errorf("user_version = %d, want >= 3", version)
|
||||
}
|
||||
|
||||
// Verify the UNIQUE index from migration 3 exists by attempting
|
||||
// a duplicate insert. First, create the prerequisite rows.
|
||||
_, err = db.ExecContext(
|
||||
// Verify the UNIQUE index on artist_credit_artist exists by
|
||||
// attempting a duplicate insert. First, create the prerequisites.
|
||||
_, err := db.ExecContext(
|
||||
"INSERT INTO artists (id, name) VALUES (1, 'Test')",
|
||||
)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
-- Adds SplitMixedFolder's synthetic-group bookkeeping to an
|
||||
-- existing tagging_items table. A fresh database never runs this
|
||||
-- file: sql/schemas/tagging_items.sql already declares these
|
||||
-- columns, so applySchema's isFreshDatabase check stamps this
|
||||
-- version as applied without executing it.
|
||||
ALTER TABLE tagging_items ADD COLUMN synthetic INTEGER NOT NULL DEFAULT 0;
|
||||
ALTER TABLE tagging_items ADD COLUMN parent_group_key TEXT NOT NULL DEFAULT '';
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_tagging_items_parent_group_key
|
||||
ON tagging_items(parent_group_key) WHERE parent_group_key != '';
|
||||
@@ -0,0 +1,24 @@
|
||||
-- Repairs tagging_items rows left behind by a library-scan bug: the
|
||||
-- rescan's orphan-cleanup phase deleted audio_files rows for files
|
||||
-- removed from disk without decrementing/clearing their tagging
|
||||
-- group, so a folder whose contents were fully replaced kept a
|
||||
-- phantom entry (stale track_count, no matching audio_files) in the
|
||||
-- autotag queue forever. The library scan code no longer has this
|
||||
-- gap, but a database written before the fix still carries the
|
||||
-- damage — this is a one-time repair, not ongoing bookkeeping.
|
||||
--
|
||||
-- Drop groups with no audio_files left at all.
|
||||
DELETE FROM tagging_items
|
||||
WHERE group_key NOT IN (
|
||||
SELECT DISTINCT group_key FROM audio_files WHERE group_key != ''
|
||||
);
|
||||
|
||||
-- Reconcile track_count for groups that are still alive but drifted
|
||||
-- (some, not all, of their tracks were removed without decrementing).
|
||||
UPDATE tagging_items
|
||||
SET track_count = (
|
||||
SELECT COUNT(*) FROM audio_files WHERE audio_files.group_key = tagging_items.group_key
|
||||
)
|
||||
WHERE track_count != (
|
||||
SELECT COUNT(*) FROM audio_files WHERE audio_files.group_key = tagging_items.group_key
|
||||
);
|
||||
@@ -32,3 +32,11 @@ SELECT
|
||||
(SELECT COUNT(*) FROM recordings WHERE artist_credit_id = ?1) +
|
||||
(SELECT COUNT(*) FROM release_groups WHERE album_artist_credit_id = ?1)
|
||||
AS total;
|
||||
|
||||
-- name: GetOrphanedArtistCreditIDs :many
|
||||
-- Artist credits no longer used by any recording or release group - run
|
||||
-- after orphaned recordings/release groups are deleted, so a credit
|
||||
-- that only existed for now-removed tracks is cleaned up too.
|
||||
SELECT ac.id FROM artist_credit ac
|
||||
WHERE NOT EXISTS (SELECT 1 FROM recordings r WHERE r.artist_credit_id = ac.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM release_groups rg WHERE rg.album_artist_credit_id = ac.id);
|
||||
|
||||
@@ -18,3 +18,7 @@ WHERE id =?;
|
||||
-- name: DeleteAllArtistCreditArtists :exec
|
||||
DELETE FROM artist_credit_artist;
|
||||
|
||||
-- name: DeleteArtistCreditArtistByCredit :exec
|
||||
DELETE FROM artist_credit_artist
|
||||
WHERE credit_id = ?;
|
||||
|
||||
|
||||
@@ -39,6 +39,15 @@ JOIN artist_credit ac ON ac.id = aca.credit_id
|
||||
JOIN release_groups rg ON rg.album_artist_credit_id = ac.id
|
||||
ORDER BY a.name;
|
||||
|
||||
-- name: GetOrphanedArtistIDs :many
|
||||
-- Artists no longer credited on any recording or release group - left
|
||||
-- behind when a scan's orphan cleanup removes the audio_files that used
|
||||
-- to justify them, since deleting an audio_files row doesn't cascade.
|
||||
SELECT a.id FROM artists a
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM artist_credit_artist aca WHERE aca.artist_id = a.id
|
||||
);
|
||||
|
||||
-- name: GetAlbumArtistsByLibrary :many
|
||||
SELECT DISTINCT a.id, a.name, a.mbid
|
||||
FROM artists a
|
||||
|
||||
@@ -6,8 +6,8 @@ RETURNING *;
|
||||
INSERT INTO audio_files (
|
||||
file_path, length_milliseconds, file_type_id, recording_id,
|
||||
sample_rate, bit_depth, channels, bitrate, file_size, basename,
|
||||
library_id, group_key, tag_status
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
library_id, group_key, tag_status, modified_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
RETURNING *;
|
||||
|
||||
-- name: GetAudioFileGroupKey :one
|
||||
@@ -32,9 +32,23 @@ WHERE id = ?;
|
||||
|
||||
-- name: UpdateAudioFileRecording :exec
|
||||
UPDATE audio_files
|
||||
SET recording_id = ?, sample_rate = ?, bit_depth = ?, channels = ?, bitrate = ?, file_size = ?
|
||||
SET recording_id = ?, sample_rate = ?, bit_depth = ?, channels = ?, bitrate = ?, file_size = ?, length_milliseconds = ?, modified_at = ?
|
||||
WHERE id = ?;
|
||||
|
||||
-- name: UpdateAudioFileStat :exec
|
||||
-- Records the on-disk mtime/size without re-reading tags. Used to
|
||||
-- backfill the staleness baseline for files the scan skipped, and to
|
||||
-- re-baseline after YellowJacket's own tag writer rewrites a file.
|
||||
UPDATE audio_files
|
||||
SET modified_at = ?, file_size = ?
|
||||
WHERE id = ?;
|
||||
|
||||
-- name: GetLibraryMaxModifiedAt :one
|
||||
-- Newest recorded mtime in a library, for the startup soft scan. 0 when
|
||||
-- the library is empty or no row has a baseline yet.
|
||||
SELECT CAST(COALESCE(MAX(modified_at), 0) AS INTEGER) FROM audio_files
|
||||
WHERE library_id = ?;
|
||||
|
||||
-- name: DeleteAudioFile :exec
|
||||
DELETE FROM audio_files
|
||||
WHERE id = ?;
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
-- name: ListDownloadProviders :many
|
||||
SELECT id, kind, name, enabled, priority, settings, created_at
|
||||
FROM download_providers
|
||||
ORDER BY priority DESC, name;
|
||||
|
||||
-- name: GetDownloadProvider :one
|
||||
SELECT id, kind, name, enabled, priority, settings, created_at
|
||||
FROM download_providers
|
||||
WHERE id = ?;
|
||||
|
||||
-- name: CreateDownloadProvider :one
|
||||
INSERT INTO download_providers (kind, name, enabled, priority, settings)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
RETURNING id;
|
||||
|
||||
-- name: UpdateDownloadProvider :exec
|
||||
UPDATE download_providers
|
||||
SET name = ?, enabled = ?, priority = ?, settings = ?
|
||||
WHERE id = ?;
|
||||
|
||||
-- name: DeleteDownloadProvider :exec
|
||||
DELETE FROM download_providers
|
||||
WHERE id = ?;
|
||||
|
||||
-- ---------------------------------------------------------------------
|
||||
-- Downloads (one-shot search+grab attempts)
|
||||
-- ---------------------------------------------------------------------
|
||||
|
||||
-- name: CreateDownload :exec
|
||||
INSERT INTO download_downloads (
|
||||
id, library_id, source, request_id, release_mbid, release_group_mbid,
|
||||
recording_mbid, artist, album, query, expected, state
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
|
||||
|
||||
-- name: GetDownload :one
|
||||
SELECT id, library_id, source, request_id, release_mbid, release_group_mbid,
|
||||
recording_mbid, artist, album, query, expected, state, error,
|
||||
created_at, updated_at
|
||||
FROM download_downloads
|
||||
WHERE id = ?;
|
||||
|
||||
-- name: ListDownloads :many
|
||||
SELECT id, library_id, source, request_id, release_mbid, release_group_mbid,
|
||||
recording_mbid, artist, album, query, expected, state, error,
|
||||
created_at, updated_at
|
||||
FROM download_downloads
|
||||
ORDER BY created_at DESC
|
||||
LIMIT ?;
|
||||
|
||||
-- name: ListLiveDownloads :many
|
||||
SELECT id, library_id, source, request_id, release_mbid, release_group_mbid,
|
||||
recording_mbid, artist, album, query, expected, state, error,
|
||||
created_at, updated_at
|
||||
FROM download_downloads
|
||||
WHERE state NOT IN ('complete', 'cancelled', 'failed')
|
||||
ORDER BY created_at;
|
||||
|
||||
-- name: SetDownloadState :exec
|
||||
UPDATE download_downloads
|
||||
SET state = ?, error = ?, updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?;
|
||||
|
||||
-- name: DeleteDownload :exec
|
||||
DELETE FROM download_downloads
|
||||
WHERE id = ?;
|
||||
|
||||
-- name: DeleteFinishedDownloads :exec
|
||||
DELETE FROM download_downloads
|
||||
WHERE state IN ('complete', 'cancelled', 'failed');
|
||||
|
||||
-- ---------------------------------------------------------------------
|
||||
-- Items (transfer records within a download)
|
||||
-- ---------------------------------------------------------------------
|
||||
|
||||
-- name: CreateDownloadItem :exec
|
||||
INSERT INTO download_items (
|
||||
id, download_id, provider_id, transport_id, external_id,
|
||||
candidate, state, staging_dir, bytes_total
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?);
|
||||
|
||||
-- name: GetDownloadItem :one
|
||||
SELECT id, download_id, provider_id, transport_id, external_id, candidate,
|
||||
state, staging_dir, bytes_done, bytes_total, imported_paths,
|
||||
error, created_at, updated_at
|
||||
FROM download_items
|
||||
WHERE id = ?;
|
||||
|
||||
-- name: ListDownloadItemsForDownload :many
|
||||
SELECT id, download_id, provider_id, transport_id, external_id, candidate,
|
||||
state, staging_dir, bytes_done, bytes_total, imported_paths,
|
||||
error, created_at, updated_at
|
||||
FROM download_items
|
||||
WHERE download_id = ?
|
||||
ORDER BY created_at;
|
||||
|
||||
-- name: ListLiveDownloadItems :many
|
||||
SELECT id, download_id, provider_id, transport_id, external_id, candidate,
|
||||
state, staging_dir, bytes_done, bytes_total, imported_paths,
|
||||
error, created_at, updated_at
|
||||
FROM download_items
|
||||
WHERE state NOT IN ('complete', 'cancelled', 'failed')
|
||||
ORDER BY created_at;
|
||||
|
||||
-- name: SetDownloadItemState :exec
|
||||
UPDATE download_items
|
||||
SET state = ?, error = ?, updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?;
|
||||
|
||||
-- name: SetDownloadItemProgress :exec
|
||||
UPDATE download_items
|
||||
SET bytes_done = ?, bytes_total = ?, updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?;
|
||||
|
||||
-- name: SetDownloadItemExternalID :exec
|
||||
UPDATE download_items
|
||||
SET external_id = ?, updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?;
|
||||
|
||||
-- name: SetDownloadItemImported :exec
|
||||
UPDATE download_items
|
||||
SET imported_paths = ?, state = 'complete', error = '',
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?;
|
||||
|
||||
-- ---------------------------------------------------------------------
|
||||
-- Requests (durable "I asked for this" records)
|
||||
-- ---------------------------------------------------------------------
|
||||
|
||||
-- name: UpsertDownloadRequest :one
|
||||
-- Adding something already requested is not an error and must not
|
||||
-- reset the retry clock, so the conflict path only refreshes display
|
||||
-- text and un-pauses nothing. scope and secondary are updated because
|
||||
-- asking again with a wider scope is a real change of intent.
|
||||
INSERT INTO download_requests (
|
||||
mbid, entity, library_id, artist, title, scope, secondary,
|
||||
parent_id, next_try_at
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(mbid, library_id) DO UPDATE SET
|
||||
artist = CASE WHEN excluded.artist <> '' THEN excluded.artist
|
||||
ELSE download_requests.artist END,
|
||||
title = CASE WHEN excluded.title <> '' THEN excluded.title
|
||||
ELSE download_requests.title END,
|
||||
scope = excluded.scope,
|
||||
secondary = excluded.secondary,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
RETURNING id;
|
||||
|
||||
-- name: GetDownloadRequest :one
|
||||
SELECT * FROM download_requests WHERE id = ?;
|
||||
|
||||
-- name: GetDownloadRequestByMBID :one
|
||||
SELECT * FROM download_requests WHERE mbid = ? AND library_id = ?;
|
||||
|
||||
-- name: ListDownloadRequests :many
|
||||
SELECT * FROM download_requests
|
||||
ORDER BY
|
||||
CASE state WHEN 'wanted' THEN 0 WHEN 'paused' THEN 1 ELSE 2 END,
|
||||
artist, title;
|
||||
|
||||
-- name: ListDownloadRequestsByEntity :many
|
||||
SELECT * FROM download_requests
|
||||
WHERE entity = ? AND state = ?
|
||||
ORDER BY id;
|
||||
|
||||
-- name: ListDueDownloadRequests :many
|
||||
-- Everything the reconciler should act on this pass: wanted, not an
|
||||
-- artist subscription (those expand rather than download), and either
|
||||
-- never tried or past its backoff.
|
||||
SELECT * FROM download_requests
|
||||
WHERE state = 'wanted'
|
||||
AND entity <> 'artist'
|
||||
AND (next_try_at IS NULL OR next_try_at <= CURRENT_TIMESTAMP)
|
||||
ORDER BY attempts, created_at
|
||||
LIMIT ?;
|
||||
|
||||
-- name: ListWantedDownloadRequests :many
|
||||
-- The same set ignoring the backoff, for a pass the user asked for by
|
||||
-- hand: "check now" that respected a six-hour retry schedule looked
|
||||
-- like a button that did nothing.
|
||||
SELECT * FROM download_requests
|
||||
WHERE state = 'wanted'
|
||||
AND entity <> 'artist'
|
||||
ORDER BY attempts, created_at
|
||||
LIMIT ?;
|
||||
|
||||
-- name: ListChildDownloadRequests :many
|
||||
SELECT * FROM download_requests WHERE parent_id = ? ORDER BY id;
|
||||
|
||||
-- name: SetDownloadRequestState :exec
|
||||
UPDATE download_requests
|
||||
SET state = ?, last_error = ?, updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?;
|
||||
|
||||
-- name: RecordDownloadRequestAttempt :exec
|
||||
UPDATE download_requests
|
||||
SET attempts = attempts + 1,
|
||||
last_error = ?,
|
||||
last_tried_at = CURRENT_TIMESTAMP,
|
||||
next_try_at = ?,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?;
|
||||
|
||||
-- name: SatisfyDownloadRequest :exec
|
||||
UPDATE download_requests
|
||||
SET state = 'satisfied', last_error = '', next_try_at = NULL,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?;
|
||||
|
||||
-- name: SetDownloadRequestExternalIDs :exec
|
||||
UPDATE download_requests
|
||||
SET external_ids = ?, updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?;
|
||||
|
||||
-- name: DeleteDownloadRequest :exec
|
||||
DELETE FROM download_requests WHERE id = ?;
|
||||
|
||||
-- name: DeleteSatisfiedDownloadRequests :exec
|
||||
DELETE FROM download_requests WHERE state = 'satisfied';
|
||||
@@ -0,0 +1,119 @@
|
||||
-- Queries behind the home page's "start listening" shelves.
|
||||
--
|
||||
-- Every one of these returns album ids and nothing else. The display
|
||||
-- columns (cover art, artist credit, year) already have exactly one
|
||||
-- correct expression of them, in GetAllAlbumsWithDetails, and a second
|
||||
-- copy per shelf would be six more places for that to drift. The home
|
||||
-- service joins the ids back to that one album list in Go.
|
||||
|
||||
-- name: HomeRecentlyPlayedAlbums :many
|
||||
-- Albums with the most recent play, newest first.
|
||||
SELECT rg.id AS album_id
|
||||
FROM release_groups rg
|
||||
JOIN release_group_recordings rgr ON rgr.release_group_id = rg.id
|
||||
JOIN audio_files af ON af.recording_id = rgr.recording_id
|
||||
WHERE af.last_played IS NOT NULL
|
||||
GROUP BY rg.id
|
||||
ORDER BY MAX(af.last_played) DESC
|
||||
LIMIT ?;
|
||||
|
||||
-- name: HomeRecentlyAddedAlbums :many
|
||||
-- Newest albums. audio_files has no import timestamp, so the row id
|
||||
-- stands in for one: it is monotonic and assigned at import, which is
|
||||
-- the same ordering an added_at column would give.
|
||||
SELECT rg.id AS album_id
|
||||
FROM release_groups rg
|
||||
JOIN release_group_recordings rgr ON rgr.release_group_id = rg.id
|
||||
JOIN audio_files af ON af.recording_id = rgr.recording_id
|
||||
GROUP BY rg.id
|
||||
ORDER BY MAX(af.id) DESC
|
||||
LIMIT ?;
|
||||
|
||||
-- name: HomeMostPlayedAlbums :many
|
||||
-- Albums by total plays across their tracks.
|
||||
SELECT rg.id AS album_id
|
||||
FROM release_groups rg
|
||||
JOIN release_group_recordings rgr ON rgr.release_group_id = rg.id
|
||||
JOIN audio_files af ON af.recording_id = rgr.recording_id
|
||||
GROUP BY rg.id
|
||||
HAVING SUM(af.play_count) > 0
|
||||
ORDER BY SUM(af.play_count) DESC
|
||||
LIMIT ?;
|
||||
|
||||
-- name: HomeUnplayedAlbums :many
|
||||
-- Albums nothing on has ever been played, sampled at random so the
|
||||
-- shelf is a different suggestion each time rather than the same
|
||||
-- alphabetical head of the list forever.
|
||||
SELECT rg.id AS album_id
|
||||
FROM release_groups rg
|
||||
JOIN release_group_recordings rgr ON rgr.release_group_id = rg.id
|
||||
JOIN audio_files af ON af.recording_id = rgr.recording_id
|
||||
GROUP BY rg.id
|
||||
HAVING SUM(af.play_count) = 0
|
||||
ORDER BY RANDOM()
|
||||
LIMIT ?;
|
||||
|
||||
-- name: HomeStaleAlbums :many
|
||||
-- Played before, but not for a long while.
|
||||
SELECT rg.id AS album_id
|
||||
FROM release_groups rg
|
||||
JOIN release_group_recordings rgr ON rgr.release_group_id = rg.id
|
||||
JOIN audio_files af ON af.recording_id = rgr.recording_id
|
||||
WHERE af.last_played IS NOT NULL
|
||||
GROUP BY rg.id
|
||||
HAVING MAX(af.last_played) < datetime('now', ?)
|
||||
ORDER BY MAX(af.last_played) ASC
|
||||
LIMIT ?;
|
||||
|
||||
-- name: HomeRandomAlbums :many
|
||||
SELECT rg.id AS album_id
|
||||
FROM release_groups rg
|
||||
JOIN release_group_recordings rgr ON rgr.release_group_id = rg.id
|
||||
JOIN audio_files af ON af.recording_id = rgr.recording_id
|
||||
GROUP BY rg.id
|
||||
ORDER BY RANDOM()
|
||||
LIMIT ?;
|
||||
|
||||
-- name: HomeAlbumsByGenre :many
|
||||
-- A random sample of albums carrying a genre, so the same genre shelf
|
||||
-- is not the same ten albums every time the page opens.
|
||||
SELECT rg.id AS album_id
|
||||
FROM release_groups rg
|
||||
JOIN release_group_recordings rgr ON rgr.release_group_id = rg.id
|
||||
JOIN recording_genres rgen ON rgen.recording_id = rgr.recording_id
|
||||
JOIN genres g ON g.id = rgen.genre_id
|
||||
WHERE g.name = ?
|
||||
GROUP BY rg.id
|
||||
ORDER BY RANDOM()
|
||||
LIMIT ?;
|
||||
|
||||
-- name: HomeTopGenres :many
|
||||
-- Genres ranked by how much of the library carries them, restricted to
|
||||
-- ones with at least a few albums: a shelf built from a genre one
|
||||
-- album carries is a shelf about that one album.
|
||||
SELECT
|
||||
g.name AS genre,
|
||||
COUNT(DISTINCT rgr.release_group_id) AS album_count
|
||||
FROM genres g
|
||||
JOIN recording_genres rgen ON rgen.genre_id = g.id
|
||||
JOIN release_group_recordings rgr ON rgr.recording_id = rgen.recording_id
|
||||
GROUP BY g.id
|
||||
HAVING album_count >= 3
|
||||
ORDER BY album_count DESC
|
||||
LIMIT ?;
|
||||
|
||||
-- name: HomeTopArtists :many
|
||||
-- Artists by total plays, as the album-artist credit text the album
|
||||
-- list already displays.
|
||||
SELECT
|
||||
COALESCE(ac.text, '') AS artist_name,
|
||||
SUM(af.play_count) AS plays
|
||||
FROM release_groups rg
|
||||
JOIN artist_credit ac ON ac.id = rg.album_artist_credit_id
|
||||
JOIN release_group_recordings rgr ON rgr.release_group_id = rg.id
|
||||
JOIN audio_files af ON af.recording_id = rgr.recording_id
|
||||
WHERE ac.text <> ''
|
||||
GROUP BY ac.text
|
||||
HAVING plays > 0
|
||||
ORDER BY plays DESC
|
||||
LIMIT ?;
|
||||
@@ -37,3 +37,11 @@ ORDER BY name;
|
||||
|
||||
-- name: CountRecordingsByArtistCredit :one
|
||||
SELECT COUNT(*) FROM recordings WHERE artist_credit_id = ?;
|
||||
|
||||
-- name: GetOrphanedRecordingIDs :many
|
||||
-- Recordings no longer backed by any audio_files row - left behind
|
||||
-- when a scan's orphan cleanup deletes the file that used to own them,
|
||||
-- since deleting audio_files doesn't cascade to recordings.
|
||||
SELECT r.id FROM recordings r
|
||||
LEFT JOIN audio_files af ON af.recording_id = r.id
|
||||
WHERE af.id IS NULL;
|
||||
|
||||
@@ -26,3 +26,7 @@ WHERE release_group_id = ? AND recording_id = ?;
|
||||
|
||||
-- name: DeleteAllReleaseGroupRecordings :exec
|
||||
DELETE FROM release_group_recordings;
|
||||
|
||||
-- name: DeleteReleaseGroupRecordingsByRecording :exec
|
||||
DELETE FROM release_group_recordings
|
||||
WHERE recording_id = ?;
|
||||
|
||||
@@ -167,6 +167,14 @@ ORDER BY rg.name;
|
||||
-- name: CountReleaseGroupRecordings :one
|
||||
SELECT COUNT(*) FROM release_group_recordings WHERE release_group_id = ?;
|
||||
|
||||
-- name: GetOrphanedReleaseGroupIDs :many
|
||||
-- Release groups with no recordings left in them - run after orphaned
|
||||
-- recordings (and their release_group_recordings rows) are deleted, so
|
||||
-- a release group whose last owned track was removed is cleaned up too.
|
||||
SELECT rg.id FROM release_groups rg
|
||||
LEFT JOIN release_group_recordings rgr ON rgr.release_group_id = rg.id
|
||||
WHERE rgr.id IS NULL;
|
||||
|
||||
-- name: GetAlbumsByArtistByLibrary :many
|
||||
SELECT
|
||||
rg.id,
|
||||
|
||||
@@ -24,11 +24,63 @@ WHERE group_key = ?;
|
||||
DELETE FROM tagging_items
|
||||
WHERE group_key = ? AND track_count <= 0;
|
||||
|
||||
-- name: PruneOrphanedTaggingItems :exec
|
||||
-- Self-healing sweep for rows whose track_count bookkeeping (scan
|
||||
-- orphan cleanup, maybeRebindTaggingGroup, SplitMixedFolder) never
|
||||
-- ran or drifted: a cancelled scan, a library move/rename the
|
||||
-- SoftScanAllLibraries disk-count/mtime heuristic did not catch, or
|
||||
-- a decrement that landed without its paired delete. Rather than
|
||||
-- trust track_count, this checks the ground truth directly: any
|
||||
-- group_key no audio_files row still points at is gone, and its
|
||||
-- tagging_items row (and cascaded tagging_candidates) should be too.
|
||||
-- Cheap: one indexed (idx_audio_files_group_key) existence check per
|
||||
-- row. Called opportunistically wherever the pending list is read,
|
||||
-- so stale entries cannot linger indefinitely between full rescans.
|
||||
DELETE FROM tagging_items
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM audio_files af WHERE af.group_key = tagging_items.group_key
|
||||
);
|
||||
|
||||
-- name: MarkTaggingItemSynthetic :exec
|
||||
-- Stamps a group as carved out of parent_group_key by
|
||||
-- SplitMixedFolder. Idempotent: safe to call every time a track
|
||||
-- is migrated into the synthetic group, not just on first creation.
|
||||
UPDATE tagging_items
|
||||
SET synthetic = 1,
|
||||
parent_group_key = ?
|
||||
WHERE group_key = ?;
|
||||
|
||||
-- name: GetTaggingItem :one
|
||||
SELECT * FROM tagging_items
|
||||
WHERE group_key = ?
|
||||
LIMIT 1;
|
||||
|
||||
-- name: ListLikelyMixedBagGroupKeys :many
|
||||
-- Cheap, whole-library triage pass for autotag.IsMixedBag: one
|
||||
-- grouped scan over audio_files (indexed on group_key) rather than
|
||||
-- hydrating every group's full track list in Go. LOWER/TRIM is an
|
||||
-- approximation of autotag.Normalize (no unicode fold, no qualifier
|
||||
-- stripping) so this can flag a false positive Normalize would
|
||||
-- clear, or miss a true one Normalize would catch. Treat it as a
|
||||
-- triage filter for which groups are worth a real autotag.
|
||||
-- IsMixedBag check, or a badge at minimum, not the final word.
|
||||
SELECT ti.group_key
|
||||
FROM tagging_items ti
|
||||
JOIN audio_files af ON af.group_key = ti.group_key
|
||||
LEFT JOIN recordings r ON af.recording_id = r.id
|
||||
LEFT JOIN artist_credit ac ON r.artist_credit_id = ac.id
|
||||
LEFT JOIN release_group_recordings rgr ON rgr.recording_id = r.id
|
||||
LEFT JOIN release_groups rg ON rg.id = rgr.release_group_id
|
||||
WHERE ti.synthetic = 0
|
||||
AND ti.track_count >= 4
|
||||
AND (
|
||||
ti.album_artist = ''
|
||||
OR LOWER(TRIM(ti.album_artist)) IN ('various artists', 'various', 'va', 'v.a.', 'v a', 'unknown')
|
||||
)
|
||||
GROUP BY ti.group_key
|
||||
HAVING COUNT(DISTINCT CASE WHEN ac.text != '' THEN LOWER(TRIM(ac.text)) END) > 1
|
||||
AND COUNT(DISTINCT CASE WHEN rg.name != '' THEN LOWER(TRIM(rg.name)) END) > 1;
|
||||
|
||||
-- name: CountPendingTaggingItems :one
|
||||
SELECT COUNT(*) FROM tagging_items
|
||||
WHERE status = 'pending'
|
||||
@@ -71,7 +123,8 @@ SELECT
|
||||
ti.score,
|
||||
ti.last_checked_at,
|
||||
ti.status,
|
||||
ti.created_at
|
||||
ti.created_at,
|
||||
ti.synthetic
|
||||
FROM tagging_items ti
|
||||
LEFT JOIN libraries lb ON lb.id = ti.library_id
|
||||
WHERE (CAST(@library_id AS INTEGER) = 0 OR ti.library_id = @library_id)
|
||||
@@ -102,7 +155,8 @@ SELECT
|
||||
ti.score,
|
||||
ti.last_checked_at,
|
||||
ti.status,
|
||||
ti.created_at
|
||||
ti.created_at,
|
||||
ti.synthetic
|
||||
FROM tagging_items ti
|
||||
LEFT JOIN libraries lb ON lb.id = ti.library_id
|
||||
WHERE ti.group_key = ?
|
||||
@@ -130,6 +184,10 @@ ORDER BY ti.created_at DESC, ti.group_key
|
||||
LIMIT @row_limit OFFSET @row_offset;
|
||||
|
||||
-- name: ListAudioFilesInTaggingGroup :many
|
||||
-- album_name/album_artist are the PER-TRACK tags (via each track's
|
||||
-- own release_group link), not the folder-level tagging_items
|
||||
-- values. SplitMixedFolder clusters on these to find sub-albums
|
||||
-- hiding inside a folder full of unrelated tracks.
|
||||
SELECT
|
||||
af.id,
|
||||
af.file_path,
|
||||
@@ -140,10 +198,15 @@ SELECT
|
||||
COALESCE(r.disc_number, 0) AS disc_number,
|
||||
COALESCE(r.name, '') AS title,
|
||||
COALESCE(ac.text, '') AS artist_name,
|
||||
COALESCE(r.mbid, '') AS recording_mbid
|
||||
COALESCE(r.mbid, '') AS recording_mbid,
|
||||
COALESCE(rg.name, '') AS album_name,
|
||||
COALESCE(rgac.text, '') AS album_artist
|
||||
FROM audio_files af
|
||||
LEFT JOIN recordings r ON af.recording_id = r.id
|
||||
LEFT JOIN artist_credit ac ON r.artist_credit_id = ac.id
|
||||
LEFT JOIN release_group_recordings rgr ON rgr.recording_id = r.id
|
||||
LEFT JOIN release_groups rg ON rg.id = rgr.release_group_id
|
||||
LEFT JOIN artist_credit rgac ON rg.album_artist_credit_id = rgac.id
|
||||
WHERE af.group_key = ?
|
||||
ORDER BY COALESCE(r.disc_number, 0),
|
||||
COALESCE(r.track_number, 0),
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
CREATE TABLE IF NOT EXISTS libraries (
|
||||
id INTEGER PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
path TEXT NOT NULL UNIQUE,
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
autotag_warning_acked INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
@@ -11,3 +11,6 @@ CREATE INDEX IF NOT EXISTS idx_artist_credit_artist_artist_id
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_artist_credit_artist_credit_id
|
||||
ON artist_credit_artist(credit_id);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_artist_credit_artist_unique
|
||||
ON artist_credit_artist(artist_id, credit_id);
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
CREATE TABLE IF NOT EXISTS artist_images (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
artist_mbid TEXT NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
source_url TEXT NOT NULL,
|
||||
file_path TEXT NOT NULL,
|
||||
is_primary INTEGER NOT NULL DEFAULT 0,
|
||||
sort_order INTEGER NOT NULL DEFAULT 0,
|
||||
width INTEGER,
|
||||
height INTEGER,
|
||||
file_size INTEGER,
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_artist_images_mbid
|
||||
ON artist_images(artist_mbid);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_artist_images_source
|
||||
ON artist_images(artist_mbid, source, source_url);
|
||||
@@ -2,6 +2,8 @@
|
||||
-- Sources: audiodb, fanart, wikidata-p18, wikipedia-lead, mb:artist-rels.
|
||||
-- No TTL — this data changes very rarely and is the backing store for
|
||||
-- the artist detail page.
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS artist_metadata (
|
||||
mbid TEXT NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
@@ -9,4 +11,5 @@ CREATE TABLE IF NOT EXISTS artist_metadata (
|
||||
fetched_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (mbid, source)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_artist_metadata_mbid ON artist_metadata(mbid);
|
||||
|
||||
@@ -3,3 +3,5 @@ CREATE TABLE IF NOT EXISTS artists (
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
mbid TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_artists_mbid ON artists(mbid) WHERE mbid IS NOT NULL;
|
||||
|
||||
@@ -18,22 +18,28 @@ CREATE TABLE IF NOT EXISTS audio_files (
|
||||
'untagged', 'auto_matched', 'user_confirmed', 'user_skipped_permanent'
|
||||
)),
|
||||
group_key TEXT NOT NULL DEFAULT '',
|
||||
-- File mtime as a Unix timestamp in seconds, captured at import.
|
||||
-- Compared against the on-disk mtime during a scan to detect files
|
||||
-- another application retagged in place. 0 means "never recorded"
|
||||
-- (rows predating migration 47) and is treated as not-stale so an
|
||||
-- upgrade does not re-import the whole library.
|
||||
modified_at int NOT NULL DEFAULT 0,
|
||||
FOREIGN KEY(file_type_id) REFERENCES file_types(id),
|
||||
FOREIGN KEY(recording_id) REFERENCES recordings(id),
|
||||
FOREIGN KEY(library_id) REFERENCES libraries(id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_audio_files_basename
|
||||
ON audio_files(basename);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_audio_files_group_key
|
||||
ON audio_files(group_key) WHERE group_key != '';
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_audio_files_library_id
|
||||
ON audio_files(library_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_audio_files_recording_id
|
||||
ON audio_files(recording_id);
|
||||
|
||||
-- idx_audio_files_library_id is created by migration 6 (not here) because
|
||||
-- on existing databases this schema file is a no-op (CREATE TABLE IF NOT EXISTS)
|
||||
-- and the library_id column doesn't exist until the migration adds it.
|
||||
--
|
||||
-- idx_audio_files_tag_status_untagged + idx_audio_files_group_key are
|
||||
-- created by migrations 31 and 32 for the same reason — on a pre-31
|
||||
-- database the partial index predicates (`WHERE tag_status = '...'`
|
||||
-- and `WHERE group_key != ''`) would reference columns that don't
|
||||
-- yet exist, since CREATE TABLE IF NOT EXISTS does not add columns
|
||||
-- to existing tables. sqlc still sees the columns above, and fresh
|
||||
-- DBs pick up the indexes inside the migrations.
|
||||
CREATE INDEX IF NOT EXISTS idx_audio_files_tag_status_untagged
|
||||
ON audio_files(library_id) WHERE tag_status = 'untagged';
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
-- One row per "go find me this", from the moment a search is fired
|
||||
-- until the files are in the library or the attempt is abandoned. A
|
||||
-- Download is one attempt: it searches, it grabs, it succeeds or
|
||||
-- fails, and then it is history. See download_requests.sql for the
|
||||
-- durable record a Download may be attached to.
|
||||
--
|
||||
-- release_mbid / release_group_mbid are the anchor: a download that
|
||||
-- carries one can be matched against a known tracklist at import time,
|
||||
-- which is what makes unattended completion safe. Free-text downloads
|
||||
-- (both NULL) are always presented to the user for confirmation.
|
||||
--
|
||||
-- `expected` caches the anchor's tracklist as JSON so ranking and
|
||||
-- import do not have to re-resolve it, and so a download survives the
|
||||
-- explore index being rebuilt underneath it.
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS download_downloads (
|
||||
id TEXT PRIMARY KEY,
|
||||
library_id INTEGER NOT NULL,
|
||||
-- source records where the download came from: 'explore-album',
|
||||
-- 'explore-artist', 'missing-album', 'wanted', 'manual'.
|
||||
source TEXT NOT NULL DEFAULT 'manual',
|
||||
-- request_id is set when this download is attached to a durable
|
||||
-- Request (see download_requests.sql), whether raised by the
|
||||
-- reconciler or attached to a manual anchored download. NULL for
|
||||
-- a free-text download with nothing stable to attach to.
|
||||
-- Requests are durable and downloads are disposable, so the delete
|
||||
-- is a SET NULL rather than a cascade in either direction.
|
||||
request_id INTEGER REFERENCES download_requests(id) ON DELETE SET NULL,
|
||||
release_mbid TEXT,
|
||||
release_group_mbid TEXT,
|
||||
-- recording_mbid anchors a single-track download raised from a
|
||||
-- track-level request.
|
||||
recording_mbid TEXT,
|
||||
artist TEXT NOT NULL DEFAULT '',
|
||||
album TEXT NOT NULL DEFAULT '',
|
||||
query TEXT NOT NULL DEFAULT '',
|
||||
expected TEXT NOT NULL DEFAULT '[]',
|
||||
state TEXT NOT NULL DEFAULT 'searching'
|
||||
CHECK(state IN ('searching', 'found', 'queued', 'grabbing',
|
||||
'verifying', 'tagging', 'importing',
|
||||
'complete', 'cancelled', 'failed')),
|
||||
error TEXT NOT NULL DEFAULT '',
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY(library_id) REFERENCES libraries(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_download_downloads_created
|
||||
ON download_downloads(created_at DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_download_downloads_state
|
||||
ON download_downloads(state);
|
||||
@@ -0,0 +1,52 @@
|
||||
-- One row per grab attempt against one candidate. A download can have
|
||||
-- several: the first pick stalls, the user picks another, or a
|
||||
-- search-only provider's candidate is fetched by a separate transport
|
||||
-- (in which case provider_id is the searcher and transport_id is the
|
||||
-- fetcher).
|
||||
--
|
||||
-- `candidate` is the full ranked Candidate as JSON. It is stored
|
||||
-- rather than re-derived because the provider's result set is
|
||||
-- ephemeral — a Soulseek peer that had the files an hour ago may be
|
||||
-- offline now, and the item still has to render in the UI and explain
|
||||
-- why it was chosen.
|
||||
--
|
||||
-- external_id holds a delegating manager's own identifier (a Lidarr
|
||||
-- queue id), which is how polling finds the record again after a
|
||||
-- restart.
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS download_items (
|
||||
id TEXT PRIMARY KEY,
|
||||
download_id TEXT NOT NULL,
|
||||
provider_id INTEGER NOT NULL,
|
||||
transport_id INTEGER,
|
||||
external_id TEXT NOT NULL DEFAULT '',
|
||||
candidate TEXT NOT NULL DEFAULT '{}',
|
||||
state TEXT NOT NULL DEFAULT 'queued'
|
||||
CHECK(state IN ('searching', 'found', 'queued', 'grabbing',
|
||||
'verifying', 'tagging', 'importing',
|
||||
'complete', 'cancelled', 'failed')),
|
||||
staging_dir TEXT NOT NULL DEFAULT '',
|
||||
bytes_done INTEGER NOT NULL DEFAULT 0,
|
||||
bytes_total INTEGER NOT NULL DEFAULT 0,
|
||||
-- imported_paths is a JSON array of the library paths the files
|
||||
-- ended up at, so an import can be undone without guessing.
|
||||
imported_paths TEXT NOT NULL DEFAULT '[]',
|
||||
error TEXT NOT NULL DEFAULT '',
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY(download_id) REFERENCES download_downloads(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_download_items_live
|
||||
ON download_items(state)
|
||||
WHERE state NOT IN ('complete', 'cancelled', 'failed');
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_download_items_state
|
||||
ON download_items(state);
|
||||
|
||||
-- idx_download_items_download is deliberately NOT declared here: on an
|
||||
-- existing database this table already exists at schema-pass time with
|
||||
-- its old column still named request_id, so an inline CREATE INDEX on
|
||||
-- download_id would fail outright. See ensureDownloadIndexes in
|
||||
-- backend/database/download_rename_migration.go.
|
||||
@@ -0,0 +1,28 @@
|
||||
-- Download clients the user has connected: an slskd daemon, a Lidarr
|
||||
-- instance, yt-dlp on PATH. One row per configured instance, so two
|
||||
-- Prowlarr servers or two Soulseek accounts coexist.
|
||||
--
|
||||
-- Secrets (API keys, passwords) are NOT stored here. They live in a
|
||||
-- 0600 file keyed by this row's id, so this table can be dumped into a
|
||||
-- bug report without redaction. `settings` holds only non-sensitive
|
||||
-- values (host, port, category, output format) as a JSON object.
|
||||
--
|
||||
-- `kind` names the adapter implementation and is looked up in the
|
||||
-- provider registry at startup; a row whose kind no longer exists is
|
||||
-- reported to the user rather than silently dropped.
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS download_providers (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
kind TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
enabled INTEGER NOT NULL DEFAULT 1,
|
||||
-- priority breaks ties between providers that found equally good
|
||||
-- candidates. Higher wins; 50 is the neutral default.
|
||||
priority INTEGER NOT NULL DEFAULT 50,
|
||||
settings TEXT NOT NULL DEFAULT '{}',
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_download_providers_enabled
|
||||
ON download_providers(enabled);
|
||||
@@ -0,0 +1,79 @@
|
||||
-- A Request is a persistent "I want this", stored as a MusicBrainz ID
|
||||
-- and almost nothing else. It outlives every download attempt made on
|
||||
-- its behalf: nothing being findable today is the normal case for
|
||||
-- obscure music, and the correct response is to try again next week,
|
||||
-- not to show the user a failed row they have to remember to retry.
|
||||
--
|
||||
-- Because a Request is only an MBID, it stays true when everything
|
||||
-- around it changes: the explore index is rebuilt, a provider is
|
||||
-- swapped out, the release the user originally saw is superseded by a
|
||||
-- remaster. The display fields are a cache for the list view and are
|
||||
-- never consulted for matching.
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS download_requests (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
mbid TEXT NOT NULL,
|
||||
entity TEXT NOT NULL
|
||||
CHECK(entity IN ('artist', 'release-group', 'release', 'recording')),
|
||||
library_id INTEGER NOT NULL,
|
||||
|
||||
-- Display text, cached so the list renders without touching the
|
||||
-- explore index. Neither is authoritative; the MBID is.
|
||||
artist TEXT NOT NULL DEFAULT '',
|
||||
title TEXT NOT NULL DEFAULT '',
|
||||
|
||||
scope TEXT NOT NULL DEFAULT 'future'
|
||||
CHECK(scope IN ('future', 'all')),
|
||||
|
||||
-- secondary controls whether an artist request's expansion includes
|
||||
-- compilations, live albums and remixes. Off by default: someone
|
||||
-- subscribing to an artist wants the albums, not six versions of
|
||||
-- the same greatest-hits package.
|
||||
secondary INTEGER NOT NULL DEFAULT 0,
|
||||
|
||||
state TEXT NOT NULL DEFAULT 'wanted'
|
||||
CHECK(state IN ('wanted', 'satisfied', 'paused')),
|
||||
|
||||
-- parent_id links a request the reconciler derived from an artist
|
||||
-- request. Deleting the artist takes its derived children with
|
||||
-- it, but children the user pinned themselves have no parent and
|
||||
-- stay.
|
||||
parent_id INTEGER,
|
||||
|
||||
-- Retry bookkeeping. attempts drives the backoff; last_error is
|
||||
-- the most recent reason it did not work out, which for a request
|
||||
-- is information rather than a failure.
|
||||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
last_error TEXT NOT NULL DEFAULT '',
|
||||
last_tried_at DATETIME,
|
||||
next_try_at DATETIME,
|
||||
|
||||
-- external_ids maps provider row ID to that provider's own
|
||||
-- identifier for this request, for clients that keep their own
|
||||
-- persistent list (a Lidarr artist ID). JSON object.
|
||||
external_ids TEXT NOT NULL DEFAULT '{}',
|
||||
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
-- One request per thing per library. Asking twice is not two
|
||||
-- requests, and this is what lets an artist expansion re-run every
|
||||
-- reconcile without accumulating duplicates.
|
||||
UNIQUE(mbid, library_id),
|
||||
|
||||
FOREIGN KEY(library_id) REFERENCES libraries(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY(parent_id) REFERENCES download_requests(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
-- idx_download_requests_{due,entity,parent} are deliberately NOT
|
||||
-- declared here. This table name is reused from the old one-shot
|
||||
-- attempt table (also called download_requests before the Want/Request
|
||||
-- rename), so on an existing database this CREATE TABLE is a no-op
|
||||
-- against a table that, at schema-pass time, is still shaped like the
|
||||
-- OLD attempts table and lacks these columns entirely — an inline
|
||||
-- CREATE INDEX here would fail outright rather than just no-op. See
|
||||
-- migrateDownloadRename/ensureDownloadIndexes in
|
||||
-- backend/database/download_rename_migration.go, which create these
|
||||
-- once the rename has actually happened (or immediately, on a fresh
|
||||
-- database where the columns exist from the start).
|
||||
@@ -0,0 +1,6 @@
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS explore_champion_fts USING fts5(
|
||||
title, artist_name, aliases,
|
||||
content='explore_index',
|
||||
content_rowid='id',
|
||||
tokenize='unicode61 remove_diacritics 2'
|
||||
);
|
||||
@@ -0,0 +1,65 @@
|
||||
CREATE TABLE IF NOT EXISTS explore_index (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
entity_type TEXT NOT NULL,
|
||||
mbid TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
artist_name TEXT NOT NULL,
|
||||
artist_mbid TEXT NOT NULL,
|
||||
aliases TEXT NOT NULL DEFAULT '',
|
||||
|
||||
-- Popularity signals, derived from the ListenBrainz listens dump.
|
||||
popularity INTEGER NOT NULL DEFAULT 0,
|
||||
listener_count INTEGER NOT NULL DEFAULT 0,
|
||||
|
||||
-- Recording-specific fields.
|
||||
duration INTEGER NOT NULL DEFAULT 0,
|
||||
caa_release_mbid TEXT NOT NULL DEFAULT '',
|
||||
release_name TEXT NOT NULL DEFAULT '',
|
||||
|
||||
-- Release-group-specific fields.
|
||||
primary_type TEXT NOT NULL DEFAULT '',
|
||||
secondary_types TEXT NOT NULL DEFAULT '',
|
||||
release_date TEXT NOT NULL DEFAULT '',
|
||||
|
||||
-- Artist-specific fields.
|
||||
artist_type TEXT NOT NULL DEFAULT '',
|
||||
country TEXT NOT NULL DEFAULT '',
|
||||
disambiguation TEXT NOT NULL DEFAULT '',
|
||||
sort_name TEXT NOT NULL DEFAULT '',
|
||||
|
||||
-- Personalization flags.
|
||||
in_library INTEGER NOT NULL DEFAULT 0,
|
||||
is_similar INTEGER NOT NULL DEFAULT 0,
|
||||
|
||||
-- Cross-reference to local library tables. NULL when the
|
||||
-- entity has no corresponding row in the library.
|
||||
local_artist_id INTEGER,
|
||||
local_release_group_id INTEGER,
|
||||
local_recording_id INTEGER,
|
||||
|
||||
-- Set once an artist's full discography (release groups +
|
||||
-- recordings) has been fetched, so EnsureArtistDiscography can
|
||||
-- skip artists the catalog already covers.
|
||||
discog_fetched INTEGER NOT NULL DEFAULT 0,
|
||||
|
||||
|
||||
UNIQUE(mbid)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_explore_artist_lower
|
||||
ON explore_index(LOWER(artist_name))
|
||||
WHERE popularity > 0;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_explore_caa_release
|
||||
ON explore_index(caa_release_mbid)
|
||||
WHERE entity_type = 'release_group' AND caa_release_mbid != '';
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_explore_index_artist_mbid
|
||||
ON explore_index(artist_mbid, entity_type, popularity DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_explore_index_entity_pop
|
||||
ON explore_index(entity_type, popularity DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_explore_title_lower
|
||||
ON explore_index(LOWER(title))
|
||||
WHERE popularity > 0;
|
||||
@@ -0,0 +1,6 @@
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS explore_index_fts USING fts5(
|
||||
title, artist_name, aliases,
|
||||
content='explore_index',
|
||||
content_rowid='id',
|
||||
tokenize='unicode61 remove_diacritics 2'
|
||||
);
|
||||
@@ -0,0 +1,4 @@
|
||||
CREATE TABLE IF NOT EXISTS explore_index_meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
@@ -3,6 +3,8 @@ CREATE TABLE IF NOT EXISTS file_types (
|
||||
extension text NOT NULL UNIQUE
|
||||
);
|
||||
|
||||
-- Seed rows: the supported audio formats, referenced by
|
||||
-- audio_files.file_type_id.
|
||||
INSERT OR IGNORE INTO file_types (id, extension) VALUES (0, '.mp3');
|
||||
INSERT OR IGNORE INTO file_types (id, extension) VALUES (1, '.flac');
|
||||
INSERT OR IGNORE INTO file_types (id, extension) VALUES (2, '.ogg');
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
-- Short-lived HTTP response cache (search results, MB/LB lookups, etc).
|
||||
-- For long-lived enrichment data keyed by MBID, see artist_metadata.sql.
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS http_cache (
|
||||
url_key TEXT PRIMARY KEY,
|
||||
response BLOB NOT NULL,
|
||||
@@ -7,5 +9,7 @@ CREATE TABLE IF NOT EXISTS http_cache (
|
||||
entity_mbid TEXT NOT NULL DEFAULT '',
|
||||
entity_type TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_http_cache_expires ON http_cache(expires_at);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_http_cache_mbid ON http_cache(entity_mbid);
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
-- Durable state for background jobs. Currently holds one row per job
|
||||
-- that the user paused, so a paused library scan or search index build
|
||||
-- comes back paused after a restart instead of silently resuming (or
|
||||
-- silently never running again).
|
||||
--
|
||||
-- Rows are written when a durable job enters the paused state and
|
||||
-- deleted on resume, cancel, or completion — this is not a job history
|
||||
-- table, and it stays at zero rows in the common case.
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS job_state (
|
||||
id TEXT PRIMARY KEY,
|
||||
kind TEXT NOT NULL,
|
||||
title TEXT NOT NULL DEFAULT '',
|
||||
subtitle TEXT NOT NULL DEFAULT '',
|
||||
paused_at TEXT NOT NULL
|
||||
);
|
||||
@@ -0,0 +1,20 @@
|
||||
-- One row per "go find me this", from the moment the user asks until
|
||||
-- the files are in the library or the attempt is abandoned.
|
||||
--
|
||||
-- release_mbid / release_group_mbid are the anchor: a request that
|
||||
-- carries one can be matched against a known tracklist at import time,
|
||||
-- which is what makes unattended completion safe. Free-text requests
|
||||
-- (both NULL) are always presented to the user for confirmation.
|
||||
--
|
||||
-- `expected` caches the anchor's tracklist as JSON so ranking and
|
||||
-- import do not have to re-resolve it, and so a request survives the
|
||||
-- explore index being rebuilt underneath it.
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS libraries (
|
||||
id INTEGER PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
path TEXT NOT NULL UNIQUE,
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
autotag_warning_acked INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
@@ -7,6 +7,8 @@
|
||||
-- tokenised inverted index, so it stays compact even for large
|
||||
-- libraries. contentless_delete=1 lets us delete/reinsert a single
|
||||
-- row when a track's lyrics change (scan update or LRCLIB backfill).
|
||||
|
||||
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS lyrics_index USING fts5(
|
||||
lyrics,
|
||||
content='',
|
||||
|
||||
@@ -6,4 +6,5 @@ CREATE TABLE IF NOT EXISTS player_state (
|
||||
last_position_seconds INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
-- Singleton row: player state is a single mutable record.
|
||||
INSERT OR IGNORE INTO player_state (id) VALUES (1);
|
||||
|
||||
@@ -1,21 +1,20 @@
|
||||
CREATE TABLE IF NOT EXISTS playlist_tracks (
|
||||
id INTEGER PRIMARY KEY,
|
||||
playlist_id INTEGER NOT NULL,
|
||||
audio_file_id INTEGER,
|
||||
position INTEGER NOT NULL,
|
||||
phantom_title TEXT,
|
||||
phantom_artist TEXT,
|
||||
phantom_album TEXT,
|
||||
phantom_duration_ms INTEGER,
|
||||
phantom_genre TEXT,
|
||||
phantom_cover_art_path TEXT,
|
||||
phantom_file_path TEXT,
|
||||
FOREIGN KEY(playlist_id) REFERENCES playlists(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY(audio_file_id) REFERENCES audio_files(id) ON DELETE SET NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_playlist_tracks_playlist_id
|
||||
ON playlist_tracks(playlist_id);
|
||||
CREATE TABLE IF NOT EXISTS "playlist_tracks" (
|
||||
id INTEGER PRIMARY KEY,
|
||||
playlist_id INTEGER NOT NULL,
|
||||
audio_file_id INTEGER,
|
||||
position INTEGER NOT NULL,
|
||||
phantom_title TEXT,
|
||||
phantom_artist TEXT,
|
||||
phantom_album TEXT,
|
||||
phantom_duration_ms INTEGER,
|
||||
phantom_genre TEXT,
|
||||
phantom_cover_art_path TEXT, phantom_file_path TEXT,
|
||||
FOREIGN KEY(playlist_id) REFERENCES playlists(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY(audio_file_id) REFERENCES audio_files(id) ON DELETE SET NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_playlist_tracks_audio_file_id
|
||||
ON playlist_tracks(audio_file_id);
|
||||
ON playlist_tracks(audio_file_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_playlist_tracks_playlist_id
|
||||
ON playlist_tracks(playlist_id);
|
||||
|
||||
@@ -8,4 +8,5 @@ CREATE TABLE IF NOT EXISTS queue (
|
||||
FOREIGN KEY(source_playlist_id) REFERENCES playlists(id) ON DELETE SET NULL
|
||||
);
|
||||
|
||||
-- Singleton row: there is exactly one playback queue.
|
||||
INSERT OR IGNORE INTO queue (id) VALUES (1);
|
||||
|
||||
+3
-3
@@ -7,8 +7,8 @@ CREATE TABLE IF NOT EXISTS recording_genres (
|
||||
UNIQUE(recording_id, genre_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_recording_genres_recording_id
|
||||
ON recording_genres(recording_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_recording_genres_genre_id
|
||||
ON recording_genres(genre_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_recording_genres_recording_id
|
||||
ON recording_genres(recording_id);
|
||||
@@ -15,3 +15,5 @@ CREATE TABLE IF NOT EXISTS recordings (
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_recordings_artist_credit_id
|
||||
ON recordings(artist_credit_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_recordings_mbid ON recordings(mbid) WHERE mbid IS NOT NULL;
|
||||
|
||||
@@ -1,30 +1,20 @@
|
||||
CREATE TABLE IF NOT EXISTS release_groups (
|
||||
CREATE TABLE IF NOT EXISTS "release_groups" (
|
||||
id INTEGER PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
cover_art_id INTEGER,
|
||||
album_artist_credit_id INTEGER,
|
||||
-- year is the *technical release year* of the album as it lives
|
||||
-- in the user's library — typically the file's ID3 year tag,
|
||||
-- which for remasters/reissues is the reissue year.
|
||||
year INTEGER,
|
||||
-- original_year is the album's *first-release-date* year sourced
|
||||
-- from MusicBrainz' release-group.first-release-date. For a 2010
|
||||
-- remaster of a 1973 album, year=2010 and original_year=1973.
|
||||
-- Populated by autotag apply; NULL until the user accepts a
|
||||
-- candidate (or for libraries that have never been autotagged).
|
||||
-- Reads should COALESCE(original_year, year) to get the
|
||||
-- preferred user-facing year.
|
||||
original_year INTEGER,
|
||||
total_tracks INTEGER,
|
||||
total_discs INTEGER,
|
||||
mbid TEXT,
|
||||
total_discs INTEGER, mbid TEXT, original_year INTEGER,
|
||||
FOREIGN KEY(cover_art_id) REFERENCES cover_art(id),
|
||||
FOREIGN KEY(album_artist_credit_id) REFERENCES artist_credit(id),
|
||||
UNIQUE(name, album_artist_credit_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_release_groups_cover_art_id
|
||||
ON release_groups(cover_art_id);
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_release_groups_album_artist_credit_id
|
||||
ON release_groups(album_artist_credit_id);
|
||||
ON release_groups(album_artist_credit_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_release_groups_cover_art_id
|
||||
ON release_groups(cover_art_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_release_groups_mbid ON release_groups(mbid) WHERE mbid IS NOT NULL;
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
CREATE TABLE IF NOT EXISTS release_to_rg (
|
||||
release_mbid TEXT PRIMARY KEY,
|
||||
rg_mbid TEXT NOT NULL
|
||||
) WITHOUT ROWID;
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user